Files
clutch/database/L2/L2_Builder.py

1244 lines
57 KiB
Python
Raw Permalink Normal View History

import sqlite3
import json
import os
import sys
import logging
from dataclasses import dataclass, field
from typing import List, Dict, Optional, Any, Tuple
from datetime import datetime
# Setup logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
# Constants
L1A_DB_PATH = 'database/L1/L1.db'
L2_DB_PATH = 'database/L2/L2.db'
SCHEMA_PATH = 'database/L2/schema.sql'
# --- Data Structures for Unification ---
@dataclass
class PlayerStats:
steam_id_64: str
team_id: int = 0
kills: int = 0
deaths: int = 0
assists: int = 0
headshot_count: int = 0
kd_ratio: float = 0.0
adr: float = 0.0
rating: float = 0.0
rating2: float = 0.0
rating3: float = 0.0
rws: float = 0.0
mvp_count: int = 0
elo_change: float = 0.0
origin_elo: float = 0.0
rank_score: int = 0
is_win: bool = False
# VIP Stats
kast: float = 0.0
entry_kills: int = 0
entry_deaths: int = 0
awp_kills: int = 0
clutch_1v1: int = 0
clutch_1v2: int = 0
clutch_1v3: int = 0
clutch_1v4: int = 0
clutch_1v5: int = 0
flash_assists: int = 0
flash_duration: float = 0.0
jump_count: int = 0
damage_total: int = 0
damage_received: int = 0
damage_receive: int = 0
damage_stats: int = 0
assisted_kill: int = 0
awp_kill: int = 0
awp_kill_ct: int = 0
awp_kill_t: int = 0
benefit_kill: int = 0
day: str = ""
defused_bomb: int = 0
end_1v1: int = 0
end_1v2: int = 0
end_1v3: int = 0
end_1v4: int = 0
end_1v5: int = 0
explode_bomb: int = 0
first_death: int = 0
fd_ct: int = 0
fd_t: int = 0
first_kill: int = 0
flash_enemy: int = 0
flash_team: int = 0
flash_team_time: float = 0.0
flash_time: float = 0.0
game_mode: str = ""
group_id: int = 0
hold_total: int = 0
id: int = 0
is_highlight: int = 0
is_most_1v2: int = 0
is_most_assist: int = 0
is_most_awp: int = 0
is_most_end: int = 0
is_most_first_kill: int = 0
is_most_headshot: int = 0
is_most_jump: int = 0
is_svp: int = 0
is_tie: int = 0
kill_1: int = 0
kill_2: int = 0
kill_3: int = 0
kill_4: int = 0
kill_5: int = 0
many_assists_cnt1: int = 0
many_assists_cnt2: int = 0
many_assists_cnt3: int = 0
many_assists_cnt4: int = 0
many_assists_cnt5: int = 0
map: str = ""
match_code: str = ""
match_mode: str = ""
match_team_id: int = 0
match_time: int = 0
per_headshot: float = 0.0
perfect_kill: int = 0
planted_bomb: int = 0
revenge_kill: int = 0
round_total: int = 0
season: str = ""
team_kill: int = 0
throw_harm: int = 0
throw_harm_enemy: int = 0
uid: int = 0
year: str = ""
sts_raw: str = ""
level_info_raw: str = ""
# Utility Usage
util_flash_usage: int = 0
util_smoke_usage: int = 0
util_molotov_usage: int = 0
util_he_usage: int = 0
util_decoy_usage: int = 0
@dataclass
class RoundEvent:
event_id: str
event_type: str # 'kill', 'bomb_plant', etc.
event_time: int
attacker_steam_id: Optional[str] = None
victim_steam_id: Optional[str] = None
assister_steam_id: Optional[str] = None
flash_assist_steam_id: Optional[str] = None
trade_killer_steam_id: Optional[str] = None
weapon: Optional[str] = None
is_headshot: bool = False
is_wallbang: bool = False
is_blind: bool = False
is_through_smoke: bool = False
is_noscope: bool = False
# Spatial
attacker_pos: Optional[Tuple[int, int, int]] = None
victim_pos: Optional[Tuple[int, int, int]] = None
# Score
score_change_attacker: float = 0.0
score_change_victim: float = 0.0
@dataclass
class PlayerEconomy:
steam_id_64: str
side: str
start_money: int = 0
equipment_value: int = 0
main_weapon: str = ""
has_helmet: bool = False
has_defuser: bool = False
has_zeus: bool = False
round_performance_score: float = 0.0
@dataclass
class RoundData:
round_num: int
winner_side: str
win_reason: int
win_reason_desc: str
duration: float
end_time_stamp: str
ct_score: int
t_score: int
ct_money_start: int = 0
t_money_start: int = 0
events: List[RoundEvent] = field(default_factory=list)
economies: List[PlayerEconomy] = field(default_factory=list)
@dataclass
class MatchTeamData:
group_id: int
group_all_score: int = 0
group_change_elo: float = 0.0
group_fh_role: int = 0
group_fh_score: int = 0
group_origin_elo: float = 0.0
group_sh_role: int = 0
group_sh_score: int = 0
group_tid: int = 0
group_uids: str = ""
@dataclass
class MatchData:
match_id: str
match_code: str = ""
map_name: str = ""
start_time: int = 0
end_time: int = 0
duration: int = 0
winner_team: int = 0
score_team1: int = 0
score_team2: int = 0
server_ip: str = ""
server_port: int = 0
location: str = ""
has_side_data_and_rating2: int = 0
match_main_id: int = 0
demo_url: str = ""
game_mode: int = 0
game_name: str = ""
map_desc: str = ""
location_full: str = ""
match_mode: int = 0
match_status: int = 0
match_flag: int = 0
status: int = 0
waiver: int = 0
year: int = 0
season: str = ""
round_total: int = 0
cs_type: int = 0
priority_show_type: int = 0
pug10m_show_type: int = 0
credit_match_status: int = 0
knife_winner: int = 0
knife_winner_role: int = 0
most_1v2_uid: int = 0
most_assist_uid: int = 0
most_awp_uid: int = 0
most_end_uid: int = 0
most_first_kill_uid: int = 0
most_headshot_uid: int = 0
most_jump_uid: int = 0
mvp_uid: int = 0
response_code: int = 0
response_message: str = ""
response_status: int = 0
response_timestamp: int = 0
response_trace_id: str = ""
response_success: int = 0
response_errcode: int = 0
treat_info_raw: str = ""
round_list_raw: str = ""
leetify_data_raw: str = ""
data_source_type: str = "unknown"
data_round_list: Dict = field(default_factory=dict) # Parsed round_list data for processors
data_leetify: Dict = field(default_factory=dict) # Parsed leetify data for processors
players: Dict[str, PlayerStats] = field(default_factory=dict) # Key: steam_id_64
players_t: Dict[str, PlayerStats] = field(default_factory=dict)
players_ct: Dict[str, PlayerStats] = field(default_factory=dict)
rounds: List[RoundData] = field(default_factory=list)
player_meta: Dict[str, Dict] = field(default_factory=dict) # steam_id -> {uid, name, avatar, ...}
teams: List[MatchTeamData] = field(default_factory=list)
# --- Database Helper ---
_WEAPON_PRICES = {
"glock": 200, "hkp2000": 200, "usp_silencer": 200, "elite": 300, "p250": 300,
"tec9": 500, "fiveseven": 500, "cz75a": 500, "revolver": 600, "deagle": 700,
"mac10": 1050, "mp9": 1250, "ump45": 1200, "bizon": 1400, "mp7": 1500, "mp5sd": 1500,
"nova": 1050, "mag7": 1300, "sawedoff": 1100, "xm1014": 2000,
"galilar": 1800, "famas": 2050, "ak47": 2700, "m4a1": 2900, "m4a1_silencer": 2900,
"aug": 3300, "sg556": 3300, "awp": 4750, "scar20": 5000, "g3sg1": 5000,
"negev": 1700, "m249": 5200,
"flashbang": 200, "hegrenade": 300, "smokegrenade": 300, "molotov": 400, "incgrenade": 600, "decoy": 50,
"taser": 200, "zeus": 200, "kevlar": 650, "assaultsuit": 1000, "defuser": 400,
"vest": 650, "vesthelm": 1000
}
def _get_equipment_value(items: List[str]) -> int:
total = 0
for item in items:
if not isinstance(item, str): continue
name = item.lower().replace("weapon_", "").replace("item_", "")
# normalize
if name in ["m4a4"]: name = "m4a1"
if name in ["m4a1-s", "m4a1s"]: name = "m4a1_silencer"
if name in ["sg553"]: name = "sg556"
if "kevlar" in name and "100" in name:
# Heuristic: kevlar(100) usually means just vest if no helmet mentioned?
# Or maybe it means full? Let's assume vest unless helmet is explicit?
# Actually, classic JSON often has "kevlar(100)" and sometimes "assaultsuit".
# Let's assume 650 for kevlar(100).
name = "kevlar"
price = _WEAPON_PRICES.get(name, 0)
# Fallback
if price == 0:
if "kevlar" in name: price = 650
if "assaultsuit" in name or "helmet" in name: price = 1000
total += price
return total
def init_db():
if os.path.exists(L2_DB_PATH):
logger.info(f"Removing existing L2 DB at {L2_DB_PATH}")
try:
os.remove(L2_DB_PATH)
except PermissionError:
logger.error("Cannot remove L2 DB, it might be open.")
return False
conn = sqlite3.connect(L2_DB_PATH)
with open(SCHEMA_PATH, 'r', encoding='utf-8') as f:
schema_sql = f.read()
conn.executescript(schema_sql)
conn.commit()
conn.close()
logger.info("L2 DB Initialized.")
return True
# --- Parsers ---
class MatchParser:
def __init__(self, match_id, raw_requests):
self.match_id = match_id
self.raw_requests = raw_requests
self.match_data = MatchData(match_id=match_id)
# Extracted JSON bodies
self.data_match = None
self.data_match_wrapper = None
self.data_vip = None
self.data_leetify = None
self.data_round_list = None
self._extract_payloads()
def _extract_payloads(self):
for req in self.raw_requests:
url = req.get('url', '')
body = req.get('body', {})
if not body:
continue
# Check URLs
if 'crane/http/api/data/match/' in url:
self.data_match_wrapper = body
self.data_match = body.get('data', {})
elif 'crane/http/api/data/vip_plus_match_data/' in url:
self.data_vip = body.get('data', {})
elif 'crane/http/api/match/leetify_rating/' in url:
self.data_leetify = body.get('data', {})
elif 'crane/http/api/match/round/' in url:
self.data_round_list = body.get('data', {})
def parse(self) -> MatchData:
if not self.data_match:
logger.warning(f"No base match data found for {self.match_id}")
return self.match_data
self._parse_base_info()
self._parse_players_base()
self._parse_players_vip()
# Decide which round source to use
if self.data_leetify and self.data_leetify.get('leetify_data'):
self.match_data.data_source_type = 'leetify'
self.match_data.data_leetify = self.data_leetify # Pass to processors
try:
self.match_data.leetify_data_raw = json.dumps(self.data_leetify.get('leetify_data', {}), ensure_ascii=False)
except:
self.match_data.leetify_data_raw = ""
self.match_data.round_list_raw = ""
self._parse_leetify_rounds()
elif self.data_round_list and self.data_round_list.get('round_list'):
self.match_data.data_source_type = 'classic'
self.match_data.data_round_list = self.data_round_list # Pass to processors
try:
self.match_data.round_list_raw = json.dumps(self.data_round_list.get('round_list', []), ensure_ascii=False)
except:
self.match_data.round_list_raw = ""
self.match_data.leetify_data_raw = ""
self._parse_classic_rounds()
else:
self.match_data.data_source_type = 'unknown'
self.match_data.round_list_raw = ""
self.match_data.leetify_data_raw = ""
logger.info(f"No round data found for {self.match_id}")
return self.match_data
def _parse_base_info(self):
m = self.data_match.get('main', {})
self.match_data.match_code = m.get('match_code', '')
self.match_data.map_name = m.get('map', '')
self.match_data.start_time = m.get('start_time', 0)
self.match_data.end_time = m.get('end_time', 0)
self.match_data.duration = self.match_data.end_time - self.match_data.start_time if self.match_data.end_time else 0
self.match_data.winner_team = m.get('match_winner', 0)
self.match_data.score_team1 = m.get('group1_all_score', 0)
self.match_data.score_team2 = m.get('group2_all_score', 0)
self.match_data.server_ip = m.get('server_ip', '')
# Port is sometimes string
try:
self.match_data.server_port = int(m.get('server_port', 0))
except:
self.match_data.server_port = 0
self.match_data.location = m.get('location', '')
def safe_int(val):
try:
return int(float(val)) if val is not None else 0
except:
return 0
def safe_float(val):
try:
return float(val) if val is not None else 0.0
except:
return 0.0
def safe_text(val):
return "" if val is None else str(val)
wrapper = self.data_match_wrapper or {}
self.match_data.response_code = safe_int(wrapper.get('code'))
self.match_data.response_message = safe_text(wrapper.get('message'))
self.match_data.response_status = safe_int(wrapper.get('status'))
self.match_data.response_timestamp = safe_int(wrapper.get('timeStamp') if wrapper.get('timeStamp') is not None else wrapper.get('timestamp'))
self.match_data.response_trace_id = safe_text(wrapper.get('traceId') if wrapper.get('traceId') is not None else wrapper.get('trace_id'))
self.match_data.response_success = safe_int(wrapper.get('success'))
self.match_data.response_errcode = safe_int(wrapper.get('errcode'))
self.match_data.has_side_data_and_rating2 = safe_int(self.data_match.get('has_side_data_and_rating2'))
self.match_data.match_main_id = safe_int(m.get('id'))
self.match_data.demo_url = safe_text(m.get('demo_url'))
self.match_data.game_mode = safe_int(m.get('game_mode'))
self.match_data.game_name = safe_text(m.get('game_name'))
self.match_data.map_desc = safe_text(m.get('map_desc'))
self.match_data.location_full = safe_text(m.get('location_full'))
self.match_data.match_mode = safe_int(m.get('match_mode'))
self.match_data.match_status = safe_int(m.get('match_status'))
self.match_data.match_flag = safe_int(m.get('match_flag'))
self.match_data.status = safe_int(m.get('status'))
self.match_data.waiver = safe_int(m.get('waiver'))
self.match_data.year = safe_int(m.get('year'))
self.match_data.season = safe_text(m.get('season'))
self.match_data.round_total = safe_int(m.get('round_total'))
self.match_data.cs_type = safe_int(m.get('cs_type'))
self.match_data.priority_show_type = safe_int(m.get('priority_show_type'))
self.match_data.pug10m_show_type = safe_int(m.get('pug10m_show_type'))
self.match_data.credit_match_status = safe_int(m.get('credit_match_status'))
self.match_data.knife_winner = safe_int(m.get('knife_winner'))
self.match_data.knife_winner_role = safe_int(m.get('knife_winner_role'))
self.match_data.most_1v2_uid = safe_int(m.get('most_1v2_uid'))
self.match_data.most_assist_uid = safe_int(m.get('most_assist_uid'))
self.match_data.most_awp_uid = safe_int(m.get('most_awp_uid'))
self.match_data.most_end_uid = safe_int(m.get('most_end_uid'))
self.match_data.most_first_kill_uid = safe_int(m.get('most_first_kill_uid'))
self.match_data.most_headshot_uid = safe_int(m.get('most_headshot_uid'))
self.match_data.most_jump_uid = safe_int(m.get('most_jump_uid'))
self.match_data.mvp_uid = safe_int(m.get('mvp_uid'))
treat_info = self.data_match.get('treat_info')
if treat_info is not None:
try:
self.match_data.treat_info_raw = json.dumps(treat_info, ensure_ascii=False)
except:
self.match_data.treat_info_raw = ""
self.match_data.teams = []
for idx in [1, 2]:
team = MatchTeamData(
group_id=idx,
group_all_score=safe_int(m.get(f"group{idx}_all_score")),
group_change_elo=safe_float(m.get(f"group{idx}_change_elo")),
group_fh_role=safe_int(m.get(f"group{idx}_fh_role")),
group_fh_score=safe_int(m.get(f"group{idx}_fh_score")),
group_origin_elo=safe_float(m.get(f"group{idx}_origin_elo")),
group_sh_role=safe_int(m.get(f"group{idx}_sh_role")),
group_sh_score=safe_int(m.get(f"group{idx}_sh_score")),
group_tid=safe_int(m.get(f"group{idx}_tid")),
group_uids=safe_text(m.get(f"group{idx}_uids"))
)
self.match_data.teams.append(team)
def _parse_players_base(self):
# Players are in group_1 and group_2 lists in data_match
groups = []
if 'group_1' in self.data_match: groups.extend(self.data_match['group_1'])
if 'group_2' in self.data_match: groups.extend(self.data_match['group_2'])
def safe_int(val):
try:
return int(float(val)) if val is not None else 0
except:
return 0
def safe_text(val):
return "" if val is None else str(val)
for p in groups:
# We need steam_id.
# Structure: user_info -> user_data -> steam -> steamId
user_info = p.get('user_info', {})
user_data = user_info.get('user_data', {})
steam_data = user_data.get('steam', {})
steam_id = str(steam_data.get('steamId', ''))
fight = p.get('fight', {})
fight_t = p.get('fight_t', {})
fight_ct = p.get('fight_ct', {})
uid = fight.get('uid')
# Store meta for dim_players
user_data = user_info.get('user_data', {})
profile = user_data.get('profile', {})
# If steam_id is empty, use temporary placeholder '5E:{uid}'
# Ideally we want steam_id_64.
if not steam_id and uid:
steam_id = f"5E:{uid}"
if not steam_id:
continue
status = user_data.get('status', {})
platform_exp = user_data.get('platformExp', {})
trusted = user_data.get('trusted', {})
certify = user_data.get('certify', {})
identity = user_data.get('identity', {})
plus_info = user_info.get('plus_info', {}) or p.get('plus_info', {})
user_info_raw = ""
try:
user_info_raw = json.dumps(user_info, ensure_ascii=False)
except:
user_info_raw = ""
self.match_data.player_meta[steam_id] = {
'uid': safe_int(uid),
'username': safe_text(user_data.get('username')),
'uuid': safe_text(user_data.get('uuid')),
'email': safe_text(user_data.get('email')),
'area': safe_text(user_data.get('area')),
'mobile': safe_text(user_data.get('mobile')),
'avatar_url': safe_text(profile.get('avatarUrl')),
'domain': safe_text(profile.get('domain')),
'user_domain': safe_text(user_data.get('domain')),
'created_at': safe_int(user_data.get('createdAt')),
'updated_at': safe_int(user_data.get('updatedAt')),
'username_audit_status': safe_int(user_data.get('usernameAuditStatus')),
'accid': safe_text(user_data.get('Accid')),
'team_id': safe_int(user_data.get('teamID')),
'trumpet_count': safe_int(user_data.get('trumpetCount')),
'profile_nickname': safe_text(profile.get('nickname')),
'profile_avatar_audit_status': safe_int(profile.get('avatarAuditStatus')),
'profile_rgb_avatar_url': safe_text(profile.get('rgbAvatarUrl')),
'profile_photo_url': safe_text(profile.get('photoUrl')),
'profile_gender': safe_int(profile.get('gender')),
'profile_birthday': safe_int(profile.get('birthday')),
'profile_country_id': safe_text(profile.get('countryId')),
'profile_region_id': safe_text(profile.get('regionId')),
'profile_city_id': safe_text(profile.get('cityId')),
'profile_language': safe_text(profile.get('language')),
'profile_recommend_url': safe_text(profile.get('recommendUrl')),
'profile_group_id': safe_int(profile.get('groupId')),
'profile_reg_source': safe_int(profile.get('regSource')),
'status_status': safe_int(status.get('status')),
'status_expire': safe_int(status.get('expire')),
'status_cancellation_status': safe_int(status.get('cancellationStatus')),
'status_new_user': safe_int(status.get('newUser')),
'status_login_banned_time': safe_int(status.get('loginBannedTime')),
'status_anticheat_type': safe_int(status.get('anticheatType')),
'status_flag_status1': safe_text(status.get('flagStatus1')),
'status_anticheat_status': safe_text(status.get('anticheatStatus')),
'status_flag_honor': safe_text(status.get('FlagHonor')),
'status_privacy_policy_status': safe_int(status.get('PrivacyPolicyStatus')),
'status_csgo_frozen_exptime': safe_int(status.get('csgoFrozenExptime')),
'platformexp_level': safe_int(platform_exp.get('level')),
'platformexp_exp': safe_int(platform_exp.get('exp')),
'steam_account': safe_text(steam_data.get('steamAccount')),
'steam_trade_url': safe_text(steam_data.get('tradeUrl')),
'steam_rent_id': safe_text(steam_data.get('rentSteamId')),
'trusted_credit': safe_int(trusted.get('credit')),
'trusted_credit_level': safe_int(trusted.get('creditLevel')),
'trusted_score': safe_int(trusted.get('score')),
'trusted_status': safe_int(trusted.get('status')),
'trusted_credit_status': safe_int(trusted.get('creditStatus')),
'certify_id_type': safe_int(certify.get('idType')),
'certify_status': safe_int(certify.get('status')),
'certify_age': safe_int(certify.get('age')),
'certify_real_name': safe_text(certify.get('realName')),
'certify_uid_list': safe_text(json.dumps(certify.get('uidList'), ensure_ascii=False)) if certify.get('uidList') is not None else "",
'certify_audit_status': safe_int(certify.get('auditStatus')),
'certify_gender': safe_int(certify.get('gender')),
'identity_type': safe_int(identity.get('type')),
'identity_extras': safe_text(identity.get('extras')),
'identity_status': safe_int(identity.get('status')),
'identity_slogan': safe_text(identity.get('slogan')),
'identity_list': safe_text(json.dumps(identity.get('identity_list'), ensure_ascii=False)) if identity.get('identity_list') is not None else "",
'identity_slogan_ext': safe_text(identity.get('slogan_ext')),
'identity_live_url': safe_text(identity.get('live_url')),
'identity_live_type': safe_int(identity.get('live_type')),
'plus_is_plus': safe_int(plus_info.get('is_plus')),
'user_info_raw': user_info_raw
}
stats = PlayerStats(steam_id_64=steam_id)
sts = p.get('sts', {})
level_info = p.get('level_info', {})
try:
# Use safe conversion helper
def safe_int(val):
try: return int(float(val)) if val is not None else 0
except: return 0
def safe_float(val):
try: return float(val) if val is not None else 0.0
except: return 0.0
def safe_text(val):
return "" if val is None else str(val)
if sts is not None:
try:
stats.sts_raw = json.dumps(sts, ensure_ascii=False)
except:
stats.sts_raw = ""
if level_info is not None:
try:
stats.level_info_raw = json.dumps(level_info, ensure_ascii=False)
except:
stats.level_info_raw = ""
def get_stat(key):
if key in fight and fight.get(key) not in [None, ""]:
return fight.get(key)
return 0
def build_side_stats(fight_side, team_id_value):
side_stats = PlayerStats(steam_id_64=steam_id)
side_stats.team_id = team_id_value
side_stats.kills = safe_int(fight_side.get('kill'))
side_stats.deaths = safe_int(fight_side.get('death'))
side_stats.assists = safe_int(fight_side.get('assist'))
side_stats.headshot_count = safe_int(fight_side.get('headshot'))
side_stats.adr = safe_float(fight_side.get('adr'))
# Use rating2 for side-specific rating (it's the actual rating for that side)
side_stats.rating = safe_float(fight_side.get('rating2'))
side_stats.rating2 = safe_float(fight_side.get('rating2'))
side_stats.rating3 = safe_float(fight_side.get('rating3'))
side_stats.rws = safe_float(fight_side.get('rws'))
side_stats.kast = safe_float(fight_side.get('kast'))
side_stats.mvp_count = safe_int(fight_side.get('is_mvp'))
side_stats.elo_change = safe_float(sts.get('change_elo'))
side_stats.origin_elo = safe_float(sts.get('origin_elo'))
side_stats.rank_score = safe_int(sts.get('rank'))
side_stats.flash_duration = safe_float(fight_side.get('flash_enemy_time'))
side_stats.jump_count = safe_int(fight_side.get('jump_total'))
side_stats.is_win = bool(safe_int(fight_side.get('is_win')))
side_stats.assisted_kill = safe_int(fight_side.get('assisted_kill'))
side_stats.awp_kill = safe_int(fight_side.get('awp_kill'))
side_stats.benefit_kill = safe_int(fight_side.get('benefit_kill'))
side_stats.day = safe_text(fight_side.get('day'))
side_stats.defused_bomb = safe_int(fight_side.get('defused_bomb'))
side_stats.end_1v1 = safe_int(fight_side.get('end_1v1'))
side_stats.end_1v2 = safe_int(fight_side.get('end_1v2'))
side_stats.end_1v3 = safe_int(fight_side.get('end_1v3'))
side_stats.end_1v4 = safe_int(fight_side.get('end_1v4'))
side_stats.end_1v5 = safe_int(fight_side.get('end_1v5'))
side_stats.explode_bomb = safe_int(fight_side.get('explode_bomb'))
side_stats.first_death = safe_int(fight_side.get('first_death'))
side_stats.first_kill = safe_int(fight_side.get('first_kill'))
side_stats.flash_enemy = safe_int(fight_side.get('flash_enemy'))
side_stats.flash_team = safe_int(fight_side.get('flash_team'))
side_stats.flash_team_time = safe_float(fight_side.get('flash_team_time'))
side_stats.flash_time = safe_float(fight_side.get('flash_time'))
side_stats.game_mode = safe_text(fight_side.get('game_mode'))
side_stats.group_id = safe_int(fight_side.get('group_id'))
side_stats.hold_total = safe_int(fight_side.get('hold_total'))
side_stats.id = safe_int(fight_side.get('id'))
side_stats.is_highlight = safe_int(fight_side.get('is_highlight'))
side_stats.is_most_1v2 = safe_int(fight_side.get('is_most_1v2'))
side_stats.is_most_assist = safe_int(fight_side.get('is_most_assist'))
side_stats.is_most_awp = safe_int(fight_side.get('is_most_awp'))
side_stats.is_most_end = safe_int(fight_side.get('is_most_end'))
side_stats.is_most_first_kill = safe_int(fight_side.get('is_most_first_kill'))
side_stats.is_most_headshot = safe_int(fight_side.get('is_most_headshot'))
side_stats.is_most_jump = safe_int(fight_side.get('is_most_jump'))
side_stats.is_svp = safe_int(fight_side.get('is_svp'))
side_stats.is_tie = safe_int(fight_side.get('is_tie'))
side_stats.kill_1 = safe_int(fight_side.get('kill_1'))
side_stats.kill_2 = safe_int(fight_side.get('kill_2'))
side_stats.kill_3 = safe_int(fight_side.get('kill_3'))
side_stats.kill_4 = safe_int(fight_side.get('kill_4'))
side_stats.kill_5 = safe_int(fight_side.get('kill_5'))
side_stats.many_assists_cnt1 = safe_int(fight_side.get('many_assists_cnt1'))
side_stats.many_assists_cnt2 = safe_int(fight_side.get('many_assists_cnt2'))
side_stats.many_assists_cnt3 = safe_int(fight_side.get('many_assists_cnt3'))
side_stats.many_assists_cnt4 = safe_int(fight_side.get('many_assists_cnt4'))
side_stats.many_assists_cnt5 = safe_int(fight_side.get('many_assists_cnt5'))
side_stats.map = safe_text(fight_side.get('map'))
side_stats.match_code = safe_text(fight_side.get('match_code'))
side_stats.match_mode = safe_text(fight_side.get('match_mode'))
side_stats.match_team_id = safe_int(fight_side.get('match_team_id'))
side_stats.match_time = safe_int(fight_side.get('match_time'))
side_stats.per_headshot = safe_float(fight_side.get('per_headshot'))
side_stats.perfect_kill = safe_int(fight_side.get('perfect_kill'))
side_stats.planted_bomb = safe_int(fight_side.get('planted_bomb'))
side_stats.revenge_kill = safe_int(fight_side.get('revenge_kill'))
side_stats.round_total = safe_int(fight_side.get('round_total'))
side_stats.season = safe_text(fight_side.get('season'))
side_stats.team_kill = safe_int(fight_side.get('team_kill'))
side_stats.throw_harm = safe_int(fight_side.get('throw_harm'))
side_stats.throw_harm_enemy = safe_int(fight_side.get('throw_harm_enemy'))
side_stats.uid = safe_int(fight_side.get('uid'))
side_stats.year = safe_text(fight_side.get('year'))
# Map missing fields
side_stats.clutch_1v1 = side_stats.end_1v1
side_stats.clutch_1v2 = side_stats.end_1v2
side_stats.clutch_1v3 = side_stats.end_1v3
side_stats.clutch_1v4 = side_stats.end_1v4
side_stats.clutch_1v5 = side_stats.end_1v5
side_stats.entry_kills = side_stats.first_kill
side_stats.entry_deaths = side_stats.first_death
return side_stats
team_id_value = safe_int(fight.get('match_team_id'))
stats.team_id = team_id_value
stats.kills = safe_int(get_stat('kill'))
stats.deaths = safe_int(get_stat('death'))
# Force calculate K/D
if stats.deaths > 0:
stats.kd_ratio = stats.kills / stats.deaths
else:
stats.kd_ratio = float(stats.kills)
stats.assists = safe_int(get_stat('assist'))
stats.headshot_count = safe_int(get_stat('headshot'))
stats.adr = safe_float(get_stat('adr'))
stats.rating = safe_float(get_stat('rating'))
stats.rating2 = safe_float(get_stat('rating2'))
stats.rating3 = safe_float(get_stat('rating3'))
stats.rws = safe_float(get_stat('rws'))
# is_mvp might be string "1" or int 1
stats.mvp_count = safe_int(get_stat('is_mvp'))
stats.flash_duration = safe_float(get_stat('flash_enemy_time'))
stats.jump_count = safe_int(get_stat('jump_total'))
stats.is_win = bool(safe_int(get_stat('is_win')))
stats.elo_change = safe_float(sts.get('change_elo'))
stats.origin_elo = safe_float(sts.get('origin_elo'))
stats.rank_score = safe_int(sts.get('rank'))
stats.assisted_kill = safe_int(fight.get('assisted_kill'))
stats.awp_kill = safe_int(fight.get('awp_kill'))
stats.benefit_kill = safe_int(fight.get('benefit_kill'))
stats.day = safe_text(fight.get('day'))
stats.defused_bomb = safe_int(fight.get('defused_bomb'))
stats.end_1v1 = safe_int(fight.get('end_1v1'))
stats.end_1v2 = safe_int(fight.get('end_1v2'))
stats.end_1v3 = safe_int(fight.get('end_1v3'))
stats.end_1v4 = safe_int(fight.get('end_1v4'))
stats.end_1v5 = safe_int(fight.get('end_1v5'))
stats.explode_bomb = safe_int(fight.get('explode_bomb'))
stats.first_death = safe_int(fight.get('first_death'))
stats.first_kill = safe_int(fight.get('first_kill'))
stats.flash_enemy = safe_int(fight.get('flash_enemy'))
stats.flash_team = safe_int(fight.get('flash_team'))
stats.flash_team_time = safe_float(fight.get('flash_team_time'))
stats.flash_time = safe_float(fight.get('flash_time'))
stats.game_mode = safe_text(fight.get('game_mode'))
stats.group_id = safe_int(fight.get('group_id'))
stats.hold_total = safe_int(fight.get('hold_total'))
stats.id = safe_int(fight.get('id'))
stats.is_highlight = safe_int(fight.get('is_highlight'))
stats.is_most_1v2 = safe_int(fight.get('is_most_1v2'))
stats.is_most_assist = safe_int(fight.get('is_most_assist'))
stats.is_most_awp = safe_int(fight.get('is_most_awp'))
stats.is_most_end = safe_int(fight.get('is_most_end'))
stats.is_most_first_kill = safe_int(fight.get('is_most_first_kill'))
stats.is_most_headshot = safe_int(fight.get('is_most_headshot'))
stats.is_most_jump = safe_int(fight.get('is_most_jump'))
stats.is_svp = safe_int(fight.get('is_svp'))
stats.is_tie = safe_int(fight.get('is_tie'))
stats.kill_1 = safe_int(fight.get('kill_1'))
stats.kill_2 = safe_int(fight.get('kill_2'))
stats.kill_3 = safe_int(fight.get('kill_3'))
stats.kill_4 = safe_int(fight.get('kill_4'))
stats.kill_5 = safe_int(fight.get('kill_5'))
stats.many_assists_cnt1 = safe_int(fight.get('many_assists_cnt1'))
stats.many_assists_cnt2 = safe_int(fight.get('many_assists_cnt2'))
stats.many_assists_cnt3 = safe_int(fight.get('many_assists_cnt3'))
stats.many_assists_cnt4 = safe_int(fight.get('many_assists_cnt4'))
stats.many_assists_cnt5 = safe_int(fight.get('many_assists_cnt5'))
stats.map = safe_text(fight.get('map'))
stats.match_code = safe_text(fight.get('match_code'))
stats.match_mode = safe_text(fight.get('match_mode'))
stats.match_team_id = safe_int(fight.get('match_team_id'))
stats.match_time = safe_int(fight.get('match_time'))
stats.per_headshot = safe_float(fight.get('per_headshot'))
stats.perfect_kill = safe_int(fight.get('perfect_kill'))
stats.planted_bomb = safe_int(fight.get('planted_bomb'))
stats.revenge_kill = safe_int(fight.get('revenge_kill'))
stats.round_total = safe_int(fight.get('round_total'))
stats.season = safe_text(fight.get('season'))
stats.team_kill = safe_int(fight.get('team_kill'))
stats.throw_harm = safe_int(fight.get('throw_harm'))
stats.throw_harm_enemy = safe_int(fight.get('throw_harm_enemy'))
stats.uid = safe_int(fight.get('uid'))
stats.year = safe_text(fight.get('year'))
# Fix missing damage_total
if stats.round_total == 0 and len(self.match_data.rounds) > 0:
stats.round_total = len(self.match_data.rounds)
stats.damage_total = safe_int(fight.get('damage_total'))
if stats.damage_total == 0 and stats.adr > 0 and stats.round_total > 0:
stats.damage_total = int(stats.adr * stats.round_total)
# Map missing fields
stats.clutch_1v1 = stats.end_1v1
stats.clutch_1v2 = stats.end_1v2
stats.clutch_1v3 = stats.end_1v3
stats.clutch_1v4 = stats.end_1v4
stats.clutch_1v5 = stats.end_1v5
stats.entry_kills = stats.first_kill
stats.entry_deaths = stats.first_death
except Exception as e:
logger.error(f"Error parsing stats for {steam_id} in {self.match_id}: {e}")
pass
self.match_data.players[steam_id] = stats
if isinstance(fight_t, dict) and fight_t:
t_team_id = team_id_value or safe_int(fight_t.get('match_team_id'))
self.match_data.players_t[steam_id] = build_side_stats(fight_t, t_team_id)
if isinstance(fight_ct, dict) and fight_ct:
ct_team_id = team_id_value or safe_int(fight_ct.get('match_team_id'))
self.match_data.players_ct[steam_id] = build_side_stats(fight_ct, ct_team_id)
def _parse_players_vip(self):
if not self.data_vip:
return
# Structure: data_vip -> steamid (key) -> dict
for sid, vdata in self.data_vip.items():
# SID might be steam_id_64 directly
if sid in self.match_data.players:
p = self.match_data.players[sid]
p.kast = float(vdata.get('kast', 0))
p.awp_kills = int(vdata.get('awp_kill', 0))
p.awp_kill_ct = int(vdata.get('awp_kill_ct', 0))
p.awp_kill_t = int(vdata.get('awp_kill_t', 0))
p.fd_ct = int(vdata.get('fd_ct', 0))
p.fd_t = int(vdata.get('fd_t', 0))
if int(vdata.get('damage_receive', 0)) > 0: p.damage_receive = int(vdata.get('damage_receive', 0))
if int(vdata.get('damage_stats', 0)) > 0: p.damage_stats = int(vdata.get('damage_stats', 0))
if int(vdata.get('damage_total', 0)) > 0: p.damage_total = int(vdata.get('damage_total', 0))
if int(vdata.get('damage_received', 0)) > 0: p.damage_received = int(vdata.get('damage_received', 0))
if int(vdata.get('flash_assists', 0)) > 0: p.flash_assists = int(vdata.get('flash_assists', 0))
else:
# Try to match by 5E ID if possible, but here keys are steamids usually
pass
for sid, p in self.match_data.players.items():
if sid in self.match_data.players_t:
self.match_data.players_t[sid].awp_kill_t = p.awp_kill_t
self.match_data.players_t[sid].fd_t = p.fd_t
if sid in self.match_data.players_ct:
self.match_data.players_ct[sid].awp_kill_ct = p.awp_kill_ct
self.match_data.players_ct[sid].fd_ct = p.fd_ct
def _parse_leetify_rounds(self):
l_data = self.data_leetify.get('leetify_data', {})
round_list = l_data.get('round_stat', [])
for idx, r in enumerate(round_list):
# Utility Usage (Leetify)
bron = r.get('bron_equipment', {})
for sid, items in bron.items():
sid = str(sid)
if sid in self.match_data.players:
p = self.match_data.players[sid]
if isinstance(items, list):
for item in items:
if not isinstance(item, dict): continue
name = item.get('WeaponName', '')
if name == 'weapon_flashbang': p.util_flash_usage += 1
elif name == 'weapon_smokegrenade': p.util_smoke_usage += 1
elif name in ['weapon_molotov', 'weapon_incgrenade']: p.util_molotov_usage += 1
elif name == 'weapon_hegrenade': p.util_he_usage += 1
elif name == 'weapon_decoy': p.util_decoy_usage += 1
rd = RoundData(
round_num=r.get('round', idx + 1),
winner_side='CT' if r.get('win_reason') in [7, 8, 9] else 'T', # Approximate logic, need real enum
win_reason=r.get('win_reason', 0),
win_reason_desc=str(r.get('win_reason', 0)),
duration=0, # Leetify might not have exact duration easily
end_time_stamp=r.get('end_ts', ''),
ct_score=r.get('sfui_event', {}).get('score_ct', 0),
t_score=r.get('sfui_event', {}).get('score_t', 0),
ct_money_start=r.get('ct_money_group', 0),
t_money_start=r.get('t_money_group', 0)
)
# Events
# Leetify has 'show_event' list
events = r.get('show_event', [])
for evt in events:
e_type_code = evt.get('event_type')
# Mapping needed for event types.
# Assuming 3 is kill based on schema 'kill_event' presence
if evt.get('kill_event'):
k = evt['kill_event']
re = RoundEvent(
event_id=f"{self.match_id}_{rd.round_num}_{k.get('Ts', '')}_{k.get('Killer')}",
event_type='kill',
event_time=evt.get('ts', 0),
attacker_steam_id=k.get('Killer'),
victim_steam_id=k.get('Victim'),
weapon=k.get('WeaponName'),
is_headshot=k.get('Headshot', False),
is_wallbang=k.get('Penetrated', False),
is_blind=k.get('AttackerBlind', False),
is_through_smoke=k.get('ThroughSmoke', False),
is_noscope=k.get('NoScope', False)
)
# Leetify specifics
# Trade?
if evt.get('trade_score_change'):
re.trade_killer_steam_id = list(evt['trade_score_change'].keys())[0]
if evt.get('assist_killer_score_change'):
re.assister_steam_id = list(evt['assist_killer_score_change'].keys())[0]
if evt.get('flash_assist_killer_score_change'):
re.flash_assist_steam_id = list(evt['flash_assist_killer_score_change'].keys())[0]
# Score changes
if evt.get('killer_score_change'):
# e.g. {'<steamid>': {'score': 17.0}}
vals = list(evt['killer_score_change'].values())
if vals: re.score_change_attacker = vals[0].get('score', 0)
if evt.get('victim_score_change'):
vals = list(evt['victim_score_change'].values())
if vals: re.score_change_victim = vals[0].get('score', 0)
rd.events.append(re)
bron_equipment = r.get('bron_equipment') or {}
player_t_score = r.get('player_t_score') or {}
player_ct_score = r.get('player_ct_score') or {}
player_bron_crash = r.get('player_bron_crash') or {}
def pick_main_weapon(items):
if not isinstance(items, list):
return ""
ignore = {
"weapon_knife",
"weapon_knife_t",
"weapon_knife_gg",
"weapon_knife_ct",
"weapon_c4",
"weapon_flashbang",
"weapon_hegrenade",
"weapon_smokegrenade",
"weapon_molotov",
"weapon_incgrenade",
"weapon_decoy"
}
for it in items:
if not isinstance(it, dict):
continue
name = it.get('WeaponName')
if name and name not in ignore:
return name
for it in items:
if not isinstance(it, dict):
continue
name = it.get('WeaponName')
if name:
return name
return ""
def pick_money(items):
if not isinstance(items, list):
return 0
vals = []
for it in items:
if isinstance(it, dict) and it.get('Money') is not None:
vals.append(it.get('Money'))
return int(max(vals)) if vals else 0
side_scores = {}
for sid, val in player_t_score.items():
side_scores[str(sid)] = ("T", float(val) if val is not None else 0.0)
for sid, val in player_ct_score.items():
side_scores[str(sid)] = ("CT", float(val) if val is not None else 0.0)
for sid in set(list(side_scores.keys()) + [str(k) for k in bron_equipment.keys()]):
if sid not in side_scores:
continue
side, score = side_scores[sid]
items = bron_equipment.get(sid) or bron_equipment.get(str(sid)) or []
start_money = pick_money(items)
equipment_value = player_bron_crash.get(sid)
if equipment_value is None:
equipment_value = player_bron_crash.get(str(sid))
equipment_value = int(equipment_value) if equipment_value is not None else 0
main_weapon = pick_main_weapon(items)
has_helmet = False
has_defuser = False
has_zeus = False
if isinstance(items, list):
for it in items:
if isinstance(it, dict):
name = it.get('WeaponName', '')
if name == 'item_assaultsuit':
has_helmet = True
elif name == 'item_defuser':
has_defuser = True
elif name and ('taser' in name or 'zeus' in name):
has_zeus = True
rd.economies.append(PlayerEconomy(
steam_id_64=str(sid),
side=side,
start_money=start_money,
equipment_value=equipment_value,
main_weapon=main_weapon,
has_helmet=has_helmet,
has_defuser=has_defuser,
has_zeus=has_zeus,
round_performance_score=float(score)
))
self.match_data.rounds.append(rd)
def _parse_classic_rounds(self):
r_list = self.data_round_list.get('round_list', [])
for idx, r in enumerate(r_list):
# Classic round data often lacks score/winner in the list root?
# Check schema: 'current_score' -> ct/t
cur_score = r.get('current_score', {})
rd = RoundData(
round_num=idx + 1,
winner_side='None', # Default to None if unknown
win_reason=0,
win_reason_desc='',
duration=float(cur_score.get('final_round_time', 0)),
end_time_stamp='',
ct_score=cur_score.get('ct', 0),
t_score=cur_score.get('t', 0)
)
# Utility Usage (Classic) & Economy
equiped = r.get('equiped', {})
for sid, items in equiped.items():
# Ensure sid is string
sid = str(sid)
# Utility
if sid in self.match_data.players:
p = self.match_data.players[sid]
if isinstance(items, list):
for item in items:
if item == 'flashbang': p.util_flash_usage += 1
elif item == 'smokegrenade': p.util_smoke_usage += 1
elif item in ['molotov', 'incgrenade']: p.util_molotov_usage += 1
elif item == 'hegrenade': p.util_he_usage += 1
elif item == 'decoy': p.util_decoy_usage += 1
# Economy
if isinstance(items, list):
equipment_value = _get_equipment_value(items)
has_zeus = any('taser' in str(i).lower() or 'zeus' in str(i).lower() for i in items)
has_helmet = any('helmet' in str(i).lower() or 'assaultsuit' in str(i).lower() for i in items)
has_defuser = any('defuser' in str(i).lower() for i in items)
# Determine Main Weapon
main_weapon = ""
# Simplified logic: pick most expensive non-grenade/knife
best_price = 0
for item in items:
if not isinstance(item, str): continue
name = item.lower().replace("weapon_", "").replace("item_", "")
if name in ['knife', 'c4', 'flashbang', 'hegrenade', 'smokegrenade', 'molotov', 'incgrenade', 'decoy', 'taser', 'zeus', 'kevlar', 'assaultsuit', 'defuser']:
continue
price = _WEAPON_PRICES.get(name, 0)
if price > best_price:
best_price = price
main_weapon = item
# Determine Side
side = "Unknown"
for item in items:
if "usp" in str(item) or "m4a1" in str(item) or "famas" in str(item) or "defuser" in str(item):
side = "CT"
break
if "glock" in str(item) or "ak47" in str(item) or "galil" in str(item) or "mac10" in str(item):
side = "T"
break
rd.economies.append(PlayerEconomy(
steam_id_64=sid,
side=side,
start_money=0, # Classic often doesn't give start money
equipment_value=equipment_value,
main_weapon=main_weapon,
has_helmet=has_helmet,
has_defuser=has_defuser,
has_zeus=has_zeus,
round_performance_score=0.0
))
# Kills
# Classic has 'all_kill' list
kills = r.get('all_kill', [])
for k in kills:
attacker = k.get('attacker', {})
victim = k.get('victim', {})
# Pos extraction
apos = attacker.get('pos', {})
vpos = victim.get('pos', {})
re = RoundEvent(
event_id=f"{self.match_id}_{rd.round_num}_{k.get('pasttime')}_{attacker.get('steamid_64')}",
event_type='kill',
event_time=k.get('pasttime', 0),
attacker_steam_id=str(attacker.get('steamid_64', '')),
victim_steam_id=str(victim.get('steamid_64', '')),
weapon=k.get('weapon', ''),
is_headshot=k.get('headshot', False),
is_wallbang=k.get('penetrated', False),
is_blind=k.get('attackerblind', False),
is_through_smoke=k.get('throughsmoke', False),
is_noscope=k.get('noscope', False),
attacker_pos=(apos.get('x', 0), apos.get('y', 0), apos.get('z', 0)),
victim_pos=(vpos.get('x', 0), vpos.get('y', 0), vpos.get('z', 0))
)
rd.events.append(re)
c4_events = r.get('c4_event', [])
for e in c4_events:
if not isinstance(e, dict):
continue
event_name = str(e.get('event_name') or '').lower()
if not event_name:
continue
if 'plant' in event_name:
etype = 'bomb_plant'
elif 'defus' in event_name:
etype = 'bomb_defuse'
else:
continue
sid = e.get('steamid_64')
re = RoundEvent(
event_id=f"{self.match_id}_{rd.round_num}_{etype}_{e.get('pasttime', 0)}_{sid}",
event_type=etype,
event_time=int(e.get('pasttime', 0) or 0),
attacker_steam_id=str(sid) if sid is not None else None,
)
rd.events.append(re)
self.match_data.rounds.append(rd)
# --- Main Execution ---
def process_matches():
"""
Main ETL pipeline: L1 L2 using modular processor architecture
"""
if not init_db():
return
# Import processors (handle both script and module import)
try:
from .processors import match_processor, player_processor, round_processor
except ImportError:
# Running as script, use absolute import
import sys
import os
sys.path.insert(0, os.path.dirname(__file__))
from processors import match_processor, player_processor, round_processor
l1_conn = sqlite3.connect(L1A_DB_PATH)
l1_cursor = l1_conn.cursor()
l2_conn = sqlite3.connect(L2_DB_PATH)
logger.info("Reading from L1...")
l1_cursor.execute("SELECT match_id, content FROM raw_iframe_network")
count = 0
success_count = 0
error_count = 0
while True:
rows = l1_cursor.fetchmany(10)
if not rows:
break
for row in rows:
match_id, content = row
try:
# Parse JSON from L1
raw_requests = json.loads(content)
parser = MatchParser(match_id, raw_requests)
match_data = parser.parse()
# Process dim_maps (lightweight, stays in main flow)
if match_data.map_name:
cursor = l2_conn.cursor()
cursor.execute("""
INSERT INTO dim_maps (map_name, map_desc)
VALUES (?, ?)
ON CONFLICT(map_name) DO UPDATE SET map_desc=excluded.map_desc
""", (match_data.map_name, match_data.map_desc))
# Delegate to specialized processors
match_success = match_processor.MatchProcessor.process(match_data, l2_conn)
player_success = player_processor.PlayerProcessor.process(match_data, l2_conn)
round_success = round_processor.RoundProcessor.process(match_data, l2_conn)
if match_success and player_success and round_success:
success_count += 1
else:
error_count += 1
logger.warning(f"Partial failure for match {match_id}")
count += 1
if count % 10 == 0:
l2_conn.commit()
print(f"Processed {count} matches ({success_count} success, {error_count} errors)...", end='\r')
except Exception as e:
error_count += 1
logger.error(f"Error processing match {match_id}: {e}")
import traceback
traceback.print_exc()
l2_conn.commit()
l1_conn.close()
l2_conn.close()
logger.info(f"\nDone. Processed {count} matches ({success_count} success, {error_count} errors).")
if __name__ == "__main__":
process_matches()