import sqlite3 import json import os import sys import logging from dataclasses import dataclass, field from typing import List, Dict, Optional, Any, Tuple from datetime import datetime # Setup logging logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) # Constants L1A_DB_PATH = 'database/L1/L1.db' L2_DB_PATH = 'database/L2/L2.db' SCHEMA_PATH = 'database/L2/schema.sql' # --- Data Structures for Unification --- @dataclass class PlayerStats: steam_id_64: str team_id: int = 0 kills: int = 0 deaths: int = 0 assists: int = 0 headshot_count: int = 0 kd_ratio: float = 0.0 adr: float = 0.0 rating: float = 0.0 rating2: float = 0.0 rating3: float = 0.0 rws: float = 0.0 mvp_count: int = 0 elo_change: float = 0.0 origin_elo: float = 0.0 rank_score: int = 0 is_win: bool = False # VIP Stats kast: float = 0.0 entry_kills: int = 0 entry_deaths: int = 0 awp_kills: int = 0 clutch_1v1: int = 0 clutch_1v2: int = 0 clutch_1v3: int = 0 clutch_1v4: int = 0 clutch_1v5: int = 0 flash_assists: int = 0 flash_duration: float = 0.0 jump_count: int = 0 damage_total: int = 0 damage_received: int = 0 damage_receive: int = 0 damage_stats: int = 0 assisted_kill: int = 0 awp_kill: int = 0 awp_kill_ct: int = 0 awp_kill_t: int = 0 benefit_kill: int = 0 day: str = "" defused_bomb: int = 0 end_1v1: int = 0 end_1v2: int = 0 end_1v3: int = 0 end_1v4: int = 0 end_1v5: int = 0 explode_bomb: int = 0 first_death: int = 0 fd_ct: int = 0 fd_t: int = 0 first_kill: int = 0 flash_enemy: int = 0 flash_team: int = 0 flash_team_time: float = 0.0 flash_time: float = 0.0 game_mode: str = "" group_id: int = 0 hold_total: int = 0 id: int = 0 is_highlight: int = 0 is_most_1v2: int = 0 is_most_assist: int = 0 is_most_awp: int = 0 is_most_end: int = 0 is_most_first_kill: int = 0 is_most_headshot: int = 0 is_most_jump: int = 0 is_svp: int = 0 is_tie: int = 0 kill_1: int = 0 kill_2: int = 0 kill_3: int = 0 kill_4: int = 0 kill_5: int = 0 many_assists_cnt1: int = 0 many_assists_cnt2: int = 0 many_assists_cnt3: int = 0 many_assists_cnt4: int = 0 many_assists_cnt5: int = 0 map: str = "" match_code: str = "" match_mode: str = "" match_team_id: int = 0 match_time: int = 0 per_headshot: float = 0.0 perfect_kill: int = 0 planted_bomb: int = 0 revenge_kill: int = 0 round_total: int = 0 season: str = "" team_kill: int = 0 throw_harm: int = 0 throw_harm_enemy: int = 0 uid: int = 0 year: str = "" sts_raw: str = "" level_info_raw: str = "" # Utility Usage util_flash_usage: int = 0 util_smoke_usage: int = 0 util_molotov_usage: int = 0 util_he_usage: int = 0 util_decoy_usage: int = 0 @dataclass class RoundEvent: event_id: str event_type: str # 'kill', 'bomb_plant', etc. event_time: int attacker_steam_id: Optional[str] = None victim_steam_id: Optional[str] = None assister_steam_id: Optional[str] = None flash_assist_steam_id: Optional[str] = None trade_killer_steam_id: Optional[str] = None weapon: Optional[str] = None is_headshot: bool = False is_wallbang: bool = False is_blind: bool = False is_through_smoke: bool = False is_noscope: bool = False # Spatial attacker_pos: Optional[Tuple[int, int, int]] = None victim_pos: Optional[Tuple[int, int, int]] = None # Score score_change_attacker: float = 0.0 score_change_victim: float = 0.0 @dataclass class PlayerEconomy: steam_id_64: str side: str start_money: int = 0 equipment_value: int = 0 main_weapon: str = "" has_helmet: bool = False has_defuser: bool = False has_zeus: bool = False round_performance_score: float = 0.0 @dataclass class RoundData: round_num: int winner_side: str win_reason: int win_reason_desc: str duration: float end_time_stamp: str ct_score: int t_score: int ct_money_start: int = 0 t_money_start: int = 0 events: List[RoundEvent] = field(default_factory=list) economies: List[PlayerEconomy] = field(default_factory=list) @dataclass class MatchTeamData: group_id: int group_all_score: int = 0 group_change_elo: float = 0.0 group_fh_role: int = 0 group_fh_score: int = 0 group_origin_elo: float = 0.0 group_sh_role: int = 0 group_sh_score: int = 0 group_tid: int = 0 group_uids: str = "" @dataclass class MatchData: match_id: str match_code: str = "" map_name: str = "" start_time: int = 0 end_time: int = 0 duration: int = 0 winner_team: int = 0 score_team1: int = 0 score_team2: int = 0 server_ip: str = "" server_port: int = 0 location: str = "" has_side_data_and_rating2: int = 0 match_main_id: int = 0 demo_url: str = "" game_mode: int = 0 game_name: str = "" map_desc: str = "" location_full: str = "" match_mode: int = 0 match_status: int = 0 match_flag: int = 0 status: int = 0 waiver: int = 0 year: int = 0 season: str = "" round_total: int = 0 cs_type: int = 0 priority_show_type: int = 0 pug10m_show_type: int = 0 credit_match_status: int = 0 knife_winner: int = 0 knife_winner_role: int = 0 most_1v2_uid: int = 0 most_assist_uid: int = 0 most_awp_uid: int = 0 most_end_uid: int = 0 most_first_kill_uid: int = 0 most_headshot_uid: int = 0 most_jump_uid: int = 0 mvp_uid: int = 0 response_code: int = 0 response_message: str = "" response_status: int = 0 response_timestamp: int = 0 response_trace_id: str = "" response_success: int = 0 response_errcode: int = 0 treat_info_raw: str = "" round_list_raw: str = "" leetify_data_raw: str = "" data_source_type: str = "unknown" data_round_list: Dict = field(default_factory=dict) # Parsed round_list data for processors data_leetify: Dict = field(default_factory=dict) # Parsed leetify data for processors players: Dict[str, PlayerStats] = field(default_factory=dict) # Key: steam_id_64 players_t: Dict[str, PlayerStats] = field(default_factory=dict) players_ct: Dict[str, PlayerStats] = field(default_factory=dict) rounds: List[RoundData] = field(default_factory=list) player_meta: Dict[str, Dict] = field(default_factory=dict) # steam_id -> {uid, name, avatar, ...} teams: List[MatchTeamData] = field(default_factory=list) # --- Database Helper --- _WEAPON_PRICES = { "glock": 200, "hkp2000": 200, "usp_silencer": 200, "elite": 300, "p250": 300, "tec9": 500, "fiveseven": 500, "cz75a": 500, "revolver": 600, "deagle": 700, "mac10": 1050, "mp9": 1250, "ump45": 1200, "bizon": 1400, "mp7": 1500, "mp5sd": 1500, "nova": 1050, "mag7": 1300, "sawedoff": 1100, "xm1014": 2000, "galilar": 1800, "famas": 2050, "ak47": 2700, "m4a1": 2900, "m4a1_silencer": 2900, "aug": 3300, "sg556": 3300, "awp": 4750, "scar20": 5000, "g3sg1": 5000, "negev": 1700, "m249": 5200, "flashbang": 200, "hegrenade": 300, "smokegrenade": 300, "molotov": 400, "incgrenade": 600, "decoy": 50, "taser": 200, "zeus": 200, "kevlar": 650, "assaultsuit": 1000, "defuser": 400, "vest": 650, "vesthelm": 1000 } def _get_equipment_value(items: List[str]) -> int: total = 0 for item in items: if not isinstance(item, str): continue name = item.lower().replace("weapon_", "").replace("item_", "") # normalize if name in ["m4a4"]: name = "m4a1" if name in ["m4a1-s", "m4a1s"]: name = "m4a1_silencer" if name in ["sg553"]: name = "sg556" if "kevlar" in name and "100" in name: # Heuristic: kevlar(100) usually means just vest if no helmet mentioned? # Or maybe it means full? Let's assume vest unless helmet is explicit? # Actually, classic JSON often has "kevlar(100)" and sometimes "assaultsuit". # Let's assume 650 for kevlar(100). name = "kevlar" price = _WEAPON_PRICES.get(name, 0) # Fallback if price == 0: if "kevlar" in name: price = 650 if "assaultsuit" in name or "helmet" in name: price = 1000 total += price return total def init_db(): if os.path.exists(L2_DB_PATH): logger.info(f"Removing existing L2 DB at {L2_DB_PATH}") try: os.remove(L2_DB_PATH) except PermissionError: logger.error("Cannot remove L2 DB, it might be open.") return False conn = sqlite3.connect(L2_DB_PATH) with open(SCHEMA_PATH, 'r', encoding='utf-8') as f: schema_sql = f.read() conn.executescript(schema_sql) conn.commit() conn.close() logger.info("L2 DB Initialized.") return True # --- Parsers --- class MatchParser: def __init__(self, match_id, raw_requests): self.match_id = match_id self.raw_requests = raw_requests self.match_data = MatchData(match_id=match_id) # Extracted JSON bodies self.data_match = None self.data_match_wrapper = None self.data_vip = None self.data_leetify = None self.data_round_list = None self._extract_payloads() def _extract_payloads(self): for req in self.raw_requests: url = req.get('url', '') body = req.get('body', {}) if not body: continue # Check URLs if 'crane/http/api/data/match/' in url: self.data_match_wrapper = body self.data_match = body.get('data', {}) elif 'crane/http/api/data/vip_plus_match_data/' in url: self.data_vip = body.get('data', {}) elif 'crane/http/api/match/leetify_rating/' in url: self.data_leetify = body.get('data', {}) elif 'crane/http/api/match/round/' in url: self.data_round_list = body.get('data', {}) def parse(self) -> MatchData: if not self.data_match: logger.warning(f"No base match data found for {self.match_id}") return self.match_data self._parse_base_info() self._parse_players_base() self._parse_players_vip() # Decide which round source to use if self.data_leetify and self.data_leetify.get('leetify_data'): self.match_data.data_source_type = 'leetify' self.match_data.data_leetify = self.data_leetify # Pass to processors try: self.match_data.leetify_data_raw = json.dumps(self.data_leetify.get('leetify_data', {}), ensure_ascii=False) except: self.match_data.leetify_data_raw = "" self.match_data.round_list_raw = "" self._parse_leetify_rounds() elif self.data_round_list and self.data_round_list.get('round_list'): self.match_data.data_source_type = 'classic' self.match_data.data_round_list = self.data_round_list # Pass to processors try: self.match_data.round_list_raw = json.dumps(self.data_round_list.get('round_list', []), ensure_ascii=False) except: self.match_data.round_list_raw = "" self.match_data.leetify_data_raw = "" self._parse_classic_rounds() else: self.match_data.data_source_type = 'unknown' self.match_data.round_list_raw = "" self.match_data.leetify_data_raw = "" logger.info(f"No round data found for {self.match_id}") return self.match_data def _parse_base_info(self): m = self.data_match.get('main', {}) self.match_data.match_code = m.get('match_code', '') self.match_data.map_name = m.get('map', '') self.match_data.start_time = m.get('start_time', 0) self.match_data.end_time = m.get('end_time', 0) self.match_data.duration = self.match_data.end_time - self.match_data.start_time if self.match_data.end_time else 0 self.match_data.winner_team = m.get('match_winner', 0) self.match_data.score_team1 = m.get('group1_all_score', 0) self.match_data.score_team2 = m.get('group2_all_score', 0) self.match_data.server_ip = m.get('server_ip', '') # Port is sometimes string try: self.match_data.server_port = int(m.get('server_port', 0)) except: self.match_data.server_port = 0 self.match_data.location = m.get('location', '') def safe_int(val): try: return int(float(val)) if val is not None else 0 except: return 0 def safe_float(val): try: return float(val) if val is not None else 0.0 except: return 0.0 def safe_text(val): return "" if val is None else str(val) wrapper = self.data_match_wrapper or {} self.match_data.response_code = safe_int(wrapper.get('code')) self.match_data.response_message = safe_text(wrapper.get('message')) self.match_data.response_status = safe_int(wrapper.get('status')) self.match_data.response_timestamp = safe_int(wrapper.get('timeStamp') if wrapper.get('timeStamp') is not None else wrapper.get('timestamp')) self.match_data.response_trace_id = safe_text(wrapper.get('traceId') if wrapper.get('traceId') is not None else wrapper.get('trace_id')) self.match_data.response_success = safe_int(wrapper.get('success')) self.match_data.response_errcode = safe_int(wrapper.get('errcode')) self.match_data.has_side_data_and_rating2 = safe_int(self.data_match.get('has_side_data_and_rating2')) self.match_data.match_main_id = safe_int(m.get('id')) self.match_data.demo_url = safe_text(m.get('demo_url')) self.match_data.game_mode = safe_int(m.get('game_mode')) self.match_data.game_name = safe_text(m.get('game_name')) self.match_data.map_desc = safe_text(m.get('map_desc')) self.match_data.location_full = safe_text(m.get('location_full')) self.match_data.match_mode = safe_int(m.get('match_mode')) self.match_data.match_status = safe_int(m.get('match_status')) self.match_data.match_flag = safe_int(m.get('match_flag')) self.match_data.status = safe_int(m.get('status')) self.match_data.waiver = safe_int(m.get('waiver')) self.match_data.year = safe_int(m.get('year')) self.match_data.season = safe_text(m.get('season')) self.match_data.round_total = safe_int(m.get('round_total')) self.match_data.cs_type = safe_int(m.get('cs_type')) self.match_data.priority_show_type = safe_int(m.get('priority_show_type')) self.match_data.pug10m_show_type = safe_int(m.get('pug10m_show_type')) self.match_data.credit_match_status = safe_int(m.get('credit_match_status')) self.match_data.knife_winner = safe_int(m.get('knife_winner')) self.match_data.knife_winner_role = safe_int(m.get('knife_winner_role')) self.match_data.most_1v2_uid = safe_int(m.get('most_1v2_uid')) self.match_data.most_assist_uid = safe_int(m.get('most_assist_uid')) self.match_data.most_awp_uid = safe_int(m.get('most_awp_uid')) self.match_data.most_end_uid = safe_int(m.get('most_end_uid')) self.match_data.most_first_kill_uid = safe_int(m.get('most_first_kill_uid')) self.match_data.most_headshot_uid = safe_int(m.get('most_headshot_uid')) self.match_data.most_jump_uid = safe_int(m.get('most_jump_uid')) self.match_data.mvp_uid = safe_int(m.get('mvp_uid')) treat_info = self.data_match.get('treat_info') if treat_info is not None: try: self.match_data.treat_info_raw = json.dumps(treat_info, ensure_ascii=False) except: self.match_data.treat_info_raw = "" self.match_data.teams = [] for idx in [1, 2]: team = MatchTeamData( group_id=idx, group_all_score=safe_int(m.get(f"group{idx}_all_score")), group_change_elo=safe_float(m.get(f"group{idx}_change_elo")), group_fh_role=safe_int(m.get(f"group{idx}_fh_role")), group_fh_score=safe_int(m.get(f"group{idx}_fh_score")), group_origin_elo=safe_float(m.get(f"group{idx}_origin_elo")), group_sh_role=safe_int(m.get(f"group{idx}_sh_role")), group_sh_score=safe_int(m.get(f"group{idx}_sh_score")), group_tid=safe_int(m.get(f"group{idx}_tid")), group_uids=safe_text(m.get(f"group{idx}_uids")) ) self.match_data.teams.append(team) def _parse_players_base(self): # Players are in group_1 and group_2 lists in data_match groups = [] if 'group_1' in self.data_match: groups.extend(self.data_match['group_1']) if 'group_2' in self.data_match: groups.extend(self.data_match['group_2']) def safe_int(val): try: return int(float(val)) if val is not None else 0 except: return 0 def safe_text(val): return "" if val is None else str(val) for p in groups: # We need steam_id. # Structure: user_info -> user_data -> steam -> steamId user_info = p.get('user_info', {}) user_data = user_info.get('user_data', {}) steam_data = user_data.get('steam', {}) steam_id = str(steam_data.get('steamId', '')) fight = p.get('fight', {}) fight_t = p.get('fight_t', {}) fight_ct = p.get('fight_ct', {}) uid = fight.get('uid') # Store meta for dim_players user_data = user_info.get('user_data', {}) profile = user_data.get('profile', {}) # If steam_id is empty, use temporary placeholder '5E:{uid}' # Ideally we want steam_id_64. if not steam_id and uid: steam_id = f"5E:{uid}" if not steam_id: continue status = user_data.get('status', {}) platform_exp = user_data.get('platformExp', {}) trusted = user_data.get('trusted', {}) certify = user_data.get('certify', {}) identity = user_data.get('identity', {}) plus_info = user_info.get('plus_info', {}) or p.get('plus_info', {}) user_info_raw = "" try: user_info_raw = json.dumps(user_info, ensure_ascii=False) except: user_info_raw = "" self.match_data.player_meta[steam_id] = { 'uid': safe_int(uid), 'username': safe_text(user_data.get('username')), 'uuid': safe_text(user_data.get('uuid')), 'email': safe_text(user_data.get('email')), 'area': safe_text(user_data.get('area')), 'mobile': safe_text(user_data.get('mobile')), 'avatar_url': safe_text(profile.get('avatarUrl')), 'domain': safe_text(profile.get('domain')), 'user_domain': safe_text(user_data.get('domain')), 'created_at': safe_int(user_data.get('createdAt')), 'updated_at': safe_int(user_data.get('updatedAt')), 'username_audit_status': safe_int(user_data.get('usernameAuditStatus')), 'accid': safe_text(user_data.get('Accid')), 'team_id': safe_int(user_data.get('teamID')), 'trumpet_count': safe_int(user_data.get('trumpetCount')), 'profile_nickname': safe_text(profile.get('nickname')), 'profile_avatar_audit_status': safe_int(profile.get('avatarAuditStatus')), 'profile_rgb_avatar_url': safe_text(profile.get('rgbAvatarUrl')), 'profile_photo_url': safe_text(profile.get('photoUrl')), 'profile_gender': safe_int(profile.get('gender')), 'profile_birthday': safe_int(profile.get('birthday')), 'profile_country_id': safe_text(profile.get('countryId')), 'profile_region_id': safe_text(profile.get('regionId')), 'profile_city_id': safe_text(profile.get('cityId')), 'profile_language': safe_text(profile.get('language')), 'profile_recommend_url': safe_text(profile.get('recommendUrl')), 'profile_group_id': safe_int(profile.get('groupId')), 'profile_reg_source': safe_int(profile.get('regSource')), 'status_status': safe_int(status.get('status')), 'status_expire': safe_int(status.get('expire')), 'status_cancellation_status': safe_int(status.get('cancellationStatus')), 'status_new_user': safe_int(status.get('newUser')), 'status_login_banned_time': safe_int(status.get('loginBannedTime')), 'status_anticheat_type': safe_int(status.get('anticheatType')), 'status_flag_status1': safe_text(status.get('flagStatus1')), 'status_anticheat_status': safe_text(status.get('anticheatStatus')), 'status_flag_honor': safe_text(status.get('FlagHonor')), 'status_privacy_policy_status': safe_int(status.get('PrivacyPolicyStatus')), 'status_csgo_frozen_exptime': safe_int(status.get('csgoFrozenExptime')), 'platformexp_level': safe_int(platform_exp.get('level')), 'platformexp_exp': safe_int(platform_exp.get('exp')), 'steam_account': safe_text(steam_data.get('steamAccount')), 'steam_trade_url': safe_text(steam_data.get('tradeUrl')), 'steam_rent_id': safe_text(steam_data.get('rentSteamId')), 'trusted_credit': safe_int(trusted.get('credit')), 'trusted_credit_level': safe_int(trusted.get('creditLevel')), 'trusted_score': safe_int(trusted.get('score')), 'trusted_status': safe_int(trusted.get('status')), 'trusted_credit_status': safe_int(trusted.get('creditStatus')), 'certify_id_type': safe_int(certify.get('idType')), 'certify_status': safe_int(certify.get('status')), 'certify_age': safe_int(certify.get('age')), 'certify_real_name': safe_text(certify.get('realName')), 'certify_uid_list': safe_text(json.dumps(certify.get('uidList'), ensure_ascii=False)) if certify.get('uidList') is not None else "", 'certify_audit_status': safe_int(certify.get('auditStatus')), 'certify_gender': safe_int(certify.get('gender')), 'identity_type': safe_int(identity.get('type')), 'identity_extras': safe_text(identity.get('extras')), 'identity_status': safe_int(identity.get('status')), 'identity_slogan': safe_text(identity.get('slogan')), 'identity_list': safe_text(json.dumps(identity.get('identity_list'), ensure_ascii=False)) if identity.get('identity_list') is not None else "", 'identity_slogan_ext': safe_text(identity.get('slogan_ext')), 'identity_live_url': safe_text(identity.get('live_url')), 'identity_live_type': safe_int(identity.get('live_type')), 'plus_is_plus': safe_int(plus_info.get('is_plus')), 'user_info_raw': user_info_raw } stats = PlayerStats(steam_id_64=steam_id) sts = p.get('sts', {}) level_info = p.get('level_info', {}) try: # Use safe conversion helper def safe_int(val): try: return int(float(val)) if val is not None else 0 except: return 0 def safe_float(val): try: return float(val) if val is not None else 0.0 except: return 0.0 def safe_text(val): return "" if val is None else str(val) if sts is not None: try: stats.sts_raw = json.dumps(sts, ensure_ascii=False) except: stats.sts_raw = "" if level_info is not None: try: stats.level_info_raw = json.dumps(level_info, ensure_ascii=False) except: stats.level_info_raw = "" def get_stat(key): if key in fight and fight.get(key) not in [None, ""]: return fight.get(key) return 0 def build_side_stats(fight_side, team_id_value): side_stats = PlayerStats(steam_id_64=steam_id) side_stats.team_id = team_id_value side_stats.kills = safe_int(fight_side.get('kill')) side_stats.deaths = safe_int(fight_side.get('death')) side_stats.assists = safe_int(fight_side.get('assist')) side_stats.headshot_count = safe_int(fight_side.get('headshot')) side_stats.adr = safe_float(fight_side.get('adr')) # Use rating2 for side-specific rating (it's the actual rating for that side) side_stats.rating = safe_float(fight_side.get('rating2')) side_stats.rating2 = safe_float(fight_side.get('rating2')) side_stats.rating3 = safe_float(fight_side.get('rating3')) side_stats.rws = safe_float(fight_side.get('rws')) side_stats.kast = safe_float(fight_side.get('kast')) side_stats.mvp_count = safe_int(fight_side.get('is_mvp')) side_stats.elo_change = safe_float(sts.get('change_elo')) side_stats.origin_elo = safe_float(sts.get('origin_elo')) side_stats.rank_score = safe_int(sts.get('rank')) side_stats.flash_duration = safe_float(fight_side.get('flash_enemy_time')) side_stats.jump_count = safe_int(fight_side.get('jump_total')) side_stats.is_win = bool(safe_int(fight_side.get('is_win'))) side_stats.assisted_kill = safe_int(fight_side.get('assisted_kill')) side_stats.awp_kill = safe_int(fight_side.get('awp_kill')) side_stats.benefit_kill = safe_int(fight_side.get('benefit_kill')) side_stats.day = safe_text(fight_side.get('day')) side_stats.defused_bomb = safe_int(fight_side.get('defused_bomb')) side_stats.end_1v1 = safe_int(fight_side.get('end_1v1')) side_stats.end_1v2 = safe_int(fight_side.get('end_1v2')) side_stats.end_1v3 = safe_int(fight_side.get('end_1v3')) side_stats.end_1v4 = safe_int(fight_side.get('end_1v4')) side_stats.end_1v5 = safe_int(fight_side.get('end_1v5')) side_stats.explode_bomb = safe_int(fight_side.get('explode_bomb')) side_stats.first_death = safe_int(fight_side.get('first_death')) side_stats.first_kill = safe_int(fight_side.get('first_kill')) side_stats.flash_enemy = safe_int(fight_side.get('flash_enemy')) side_stats.flash_team = safe_int(fight_side.get('flash_team')) side_stats.flash_team_time = safe_float(fight_side.get('flash_team_time')) side_stats.flash_time = safe_float(fight_side.get('flash_time')) side_stats.game_mode = safe_text(fight_side.get('game_mode')) side_stats.group_id = safe_int(fight_side.get('group_id')) side_stats.hold_total = safe_int(fight_side.get('hold_total')) side_stats.id = safe_int(fight_side.get('id')) side_stats.is_highlight = safe_int(fight_side.get('is_highlight')) side_stats.is_most_1v2 = safe_int(fight_side.get('is_most_1v2')) side_stats.is_most_assist = safe_int(fight_side.get('is_most_assist')) side_stats.is_most_awp = safe_int(fight_side.get('is_most_awp')) side_stats.is_most_end = safe_int(fight_side.get('is_most_end')) side_stats.is_most_first_kill = safe_int(fight_side.get('is_most_first_kill')) side_stats.is_most_headshot = safe_int(fight_side.get('is_most_headshot')) side_stats.is_most_jump = safe_int(fight_side.get('is_most_jump')) side_stats.is_svp = safe_int(fight_side.get('is_svp')) side_stats.is_tie = safe_int(fight_side.get('is_tie')) side_stats.kill_1 = safe_int(fight_side.get('kill_1')) side_stats.kill_2 = safe_int(fight_side.get('kill_2')) side_stats.kill_3 = safe_int(fight_side.get('kill_3')) side_stats.kill_4 = safe_int(fight_side.get('kill_4')) side_stats.kill_5 = safe_int(fight_side.get('kill_5')) side_stats.many_assists_cnt1 = safe_int(fight_side.get('many_assists_cnt1')) side_stats.many_assists_cnt2 = safe_int(fight_side.get('many_assists_cnt2')) side_stats.many_assists_cnt3 = safe_int(fight_side.get('many_assists_cnt3')) side_stats.many_assists_cnt4 = safe_int(fight_side.get('many_assists_cnt4')) side_stats.many_assists_cnt5 = safe_int(fight_side.get('many_assists_cnt5')) side_stats.map = safe_text(fight_side.get('map')) side_stats.match_code = safe_text(fight_side.get('match_code')) side_stats.match_mode = safe_text(fight_side.get('match_mode')) side_stats.match_team_id = safe_int(fight_side.get('match_team_id')) side_stats.match_time = safe_int(fight_side.get('match_time')) side_stats.per_headshot = safe_float(fight_side.get('per_headshot')) side_stats.perfect_kill = safe_int(fight_side.get('perfect_kill')) side_stats.planted_bomb = safe_int(fight_side.get('planted_bomb')) side_stats.revenge_kill = safe_int(fight_side.get('revenge_kill')) side_stats.round_total = safe_int(fight_side.get('round_total')) side_stats.season = safe_text(fight_side.get('season')) side_stats.team_kill = safe_int(fight_side.get('team_kill')) side_stats.throw_harm = safe_int(fight_side.get('throw_harm')) side_stats.throw_harm_enemy = safe_int(fight_side.get('throw_harm_enemy')) side_stats.uid = safe_int(fight_side.get('uid')) side_stats.year = safe_text(fight_side.get('year')) # Map missing fields side_stats.clutch_1v1 = side_stats.end_1v1 side_stats.clutch_1v2 = side_stats.end_1v2 side_stats.clutch_1v3 = side_stats.end_1v3 side_stats.clutch_1v4 = side_stats.end_1v4 side_stats.clutch_1v5 = side_stats.end_1v5 side_stats.entry_kills = side_stats.first_kill side_stats.entry_deaths = side_stats.first_death return side_stats team_id_value = safe_int(fight.get('match_team_id')) stats.team_id = team_id_value stats.kills = safe_int(get_stat('kill')) stats.deaths = safe_int(get_stat('death')) # Force calculate K/D if stats.deaths > 0: stats.kd_ratio = stats.kills / stats.deaths else: stats.kd_ratio = float(stats.kills) stats.assists = safe_int(get_stat('assist')) stats.headshot_count = safe_int(get_stat('headshot')) stats.adr = safe_float(get_stat('adr')) stats.rating = safe_float(get_stat('rating')) stats.rating2 = safe_float(get_stat('rating2')) stats.rating3 = safe_float(get_stat('rating3')) stats.rws = safe_float(get_stat('rws')) # is_mvp might be string "1" or int 1 stats.mvp_count = safe_int(get_stat('is_mvp')) stats.flash_duration = safe_float(get_stat('flash_enemy_time')) stats.jump_count = safe_int(get_stat('jump_total')) stats.is_win = bool(safe_int(get_stat('is_win'))) stats.elo_change = safe_float(sts.get('change_elo')) stats.origin_elo = safe_float(sts.get('origin_elo')) stats.rank_score = safe_int(sts.get('rank')) stats.assisted_kill = safe_int(fight.get('assisted_kill')) stats.awp_kill = safe_int(fight.get('awp_kill')) stats.benefit_kill = safe_int(fight.get('benefit_kill')) stats.day = safe_text(fight.get('day')) stats.defused_bomb = safe_int(fight.get('defused_bomb')) stats.end_1v1 = safe_int(fight.get('end_1v1')) stats.end_1v2 = safe_int(fight.get('end_1v2')) stats.end_1v3 = safe_int(fight.get('end_1v3')) stats.end_1v4 = safe_int(fight.get('end_1v4')) stats.end_1v5 = safe_int(fight.get('end_1v5')) stats.explode_bomb = safe_int(fight.get('explode_bomb')) stats.first_death = safe_int(fight.get('first_death')) stats.first_kill = safe_int(fight.get('first_kill')) stats.flash_enemy = safe_int(fight.get('flash_enemy')) stats.flash_team = safe_int(fight.get('flash_team')) stats.flash_team_time = safe_float(fight.get('flash_team_time')) stats.flash_time = safe_float(fight.get('flash_time')) stats.game_mode = safe_text(fight.get('game_mode')) stats.group_id = safe_int(fight.get('group_id')) stats.hold_total = safe_int(fight.get('hold_total')) stats.id = safe_int(fight.get('id')) stats.is_highlight = safe_int(fight.get('is_highlight')) stats.is_most_1v2 = safe_int(fight.get('is_most_1v2')) stats.is_most_assist = safe_int(fight.get('is_most_assist')) stats.is_most_awp = safe_int(fight.get('is_most_awp')) stats.is_most_end = safe_int(fight.get('is_most_end')) stats.is_most_first_kill = safe_int(fight.get('is_most_first_kill')) stats.is_most_headshot = safe_int(fight.get('is_most_headshot')) stats.is_most_jump = safe_int(fight.get('is_most_jump')) stats.is_svp = safe_int(fight.get('is_svp')) stats.is_tie = safe_int(fight.get('is_tie')) stats.kill_1 = safe_int(fight.get('kill_1')) stats.kill_2 = safe_int(fight.get('kill_2')) stats.kill_3 = safe_int(fight.get('kill_3')) stats.kill_4 = safe_int(fight.get('kill_4')) stats.kill_5 = safe_int(fight.get('kill_5')) stats.many_assists_cnt1 = safe_int(fight.get('many_assists_cnt1')) stats.many_assists_cnt2 = safe_int(fight.get('many_assists_cnt2')) stats.many_assists_cnt3 = safe_int(fight.get('many_assists_cnt3')) stats.many_assists_cnt4 = safe_int(fight.get('many_assists_cnt4')) stats.many_assists_cnt5 = safe_int(fight.get('many_assists_cnt5')) stats.map = safe_text(fight.get('map')) stats.match_code = safe_text(fight.get('match_code')) stats.match_mode = safe_text(fight.get('match_mode')) stats.match_team_id = safe_int(fight.get('match_team_id')) stats.match_time = safe_int(fight.get('match_time')) stats.per_headshot = safe_float(fight.get('per_headshot')) stats.perfect_kill = safe_int(fight.get('perfect_kill')) stats.planted_bomb = safe_int(fight.get('planted_bomb')) stats.revenge_kill = safe_int(fight.get('revenge_kill')) stats.round_total = safe_int(fight.get('round_total')) stats.season = safe_text(fight.get('season')) stats.team_kill = safe_int(fight.get('team_kill')) stats.throw_harm = safe_int(fight.get('throw_harm')) stats.throw_harm_enemy = safe_int(fight.get('throw_harm_enemy')) stats.uid = safe_int(fight.get('uid')) stats.year = safe_text(fight.get('year')) # Fix missing damage_total if stats.round_total == 0 and len(self.match_data.rounds) > 0: stats.round_total = len(self.match_data.rounds) stats.damage_total = safe_int(fight.get('damage_total')) if stats.damage_total == 0 and stats.adr > 0 and stats.round_total > 0: stats.damage_total = int(stats.adr * stats.round_total) # Map missing fields stats.clutch_1v1 = stats.end_1v1 stats.clutch_1v2 = stats.end_1v2 stats.clutch_1v3 = stats.end_1v3 stats.clutch_1v4 = stats.end_1v4 stats.clutch_1v5 = stats.end_1v5 stats.entry_kills = stats.first_kill stats.entry_deaths = stats.first_death except Exception as e: logger.error(f"Error parsing stats for {steam_id} in {self.match_id}: {e}") pass self.match_data.players[steam_id] = stats if isinstance(fight_t, dict) and fight_t: t_team_id = team_id_value or safe_int(fight_t.get('match_team_id')) self.match_data.players_t[steam_id] = build_side_stats(fight_t, t_team_id) if isinstance(fight_ct, dict) and fight_ct: ct_team_id = team_id_value or safe_int(fight_ct.get('match_team_id')) self.match_data.players_ct[steam_id] = build_side_stats(fight_ct, ct_team_id) def _parse_players_vip(self): if not self.data_vip: return # Structure: data_vip -> steamid (key) -> dict for sid, vdata in self.data_vip.items(): # SID might be steam_id_64 directly if sid in self.match_data.players: p = self.match_data.players[sid] p.kast = float(vdata.get('kast', 0)) p.awp_kills = int(vdata.get('awp_kill', 0)) p.awp_kill_ct = int(vdata.get('awp_kill_ct', 0)) p.awp_kill_t = int(vdata.get('awp_kill_t', 0)) p.fd_ct = int(vdata.get('fd_ct', 0)) p.fd_t = int(vdata.get('fd_t', 0)) if int(vdata.get('damage_receive', 0)) > 0: p.damage_receive = int(vdata.get('damage_receive', 0)) if int(vdata.get('damage_stats', 0)) > 0: p.damage_stats = int(vdata.get('damage_stats', 0)) if int(vdata.get('damage_total', 0)) > 0: p.damage_total = int(vdata.get('damage_total', 0)) if int(vdata.get('damage_received', 0)) > 0: p.damage_received = int(vdata.get('damage_received', 0)) if int(vdata.get('flash_assists', 0)) > 0: p.flash_assists = int(vdata.get('flash_assists', 0)) else: # Try to match by 5E ID if possible, but here keys are steamids usually pass for sid, p in self.match_data.players.items(): if sid in self.match_data.players_t: self.match_data.players_t[sid].awp_kill_t = p.awp_kill_t self.match_data.players_t[sid].fd_t = p.fd_t if sid in self.match_data.players_ct: self.match_data.players_ct[sid].awp_kill_ct = p.awp_kill_ct self.match_data.players_ct[sid].fd_ct = p.fd_ct def _parse_leetify_rounds(self): l_data = self.data_leetify.get('leetify_data', {}) round_list = l_data.get('round_stat', []) for idx, r in enumerate(round_list): # Utility Usage (Leetify) bron = r.get('bron_equipment', {}) for sid, items in bron.items(): sid = str(sid) if sid in self.match_data.players: p = self.match_data.players[sid] if isinstance(items, list): for item in items: if not isinstance(item, dict): continue name = item.get('WeaponName', '') if name == 'weapon_flashbang': p.util_flash_usage += 1 elif name == 'weapon_smokegrenade': p.util_smoke_usage += 1 elif name in ['weapon_molotov', 'weapon_incgrenade']: p.util_molotov_usage += 1 elif name == 'weapon_hegrenade': p.util_he_usage += 1 elif name == 'weapon_decoy': p.util_decoy_usage += 1 rd = RoundData( round_num=r.get('round', idx + 1), winner_side='CT' if r.get('win_reason') in [7, 8, 9] else 'T', # Approximate logic, need real enum win_reason=r.get('win_reason', 0), win_reason_desc=str(r.get('win_reason', 0)), duration=0, # Leetify might not have exact duration easily end_time_stamp=r.get('end_ts', ''), ct_score=r.get('sfui_event', {}).get('score_ct', 0), t_score=r.get('sfui_event', {}).get('score_t', 0), ct_money_start=r.get('ct_money_group', 0), t_money_start=r.get('t_money_group', 0) ) # Events # Leetify has 'show_event' list events = r.get('show_event', []) for evt in events: e_type_code = evt.get('event_type') # Mapping needed for event types. # Assuming 3 is kill based on schema 'kill_event' presence if evt.get('kill_event'): k = evt['kill_event'] re = RoundEvent( event_id=f"{self.match_id}_{rd.round_num}_{k.get('Ts', '')}_{k.get('Killer')}", event_type='kill', event_time=evt.get('ts', 0), attacker_steam_id=k.get('Killer'), victim_steam_id=k.get('Victim'), weapon=k.get('WeaponName'), is_headshot=k.get('Headshot', False), is_wallbang=k.get('Penetrated', False), is_blind=k.get('AttackerBlind', False), is_through_smoke=k.get('ThroughSmoke', False), is_noscope=k.get('NoScope', False) ) # Leetify specifics # Trade? if evt.get('trade_score_change'): re.trade_killer_steam_id = list(evt['trade_score_change'].keys())[0] if evt.get('assist_killer_score_change'): re.assister_steam_id = list(evt['assist_killer_score_change'].keys())[0] if evt.get('flash_assist_killer_score_change'): re.flash_assist_steam_id = list(evt['flash_assist_killer_score_change'].keys())[0] # Score changes if evt.get('killer_score_change'): # e.g. {'': {'score': 17.0}} vals = list(evt['killer_score_change'].values()) if vals: re.score_change_attacker = vals[0].get('score', 0) if evt.get('victim_score_change'): vals = list(evt['victim_score_change'].values()) if vals: re.score_change_victim = vals[0].get('score', 0) rd.events.append(re) bron_equipment = r.get('bron_equipment') or {} player_t_score = r.get('player_t_score') or {} player_ct_score = r.get('player_ct_score') or {} player_bron_crash = r.get('player_bron_crash') or {} def pick_main_weapon(items): if not isinstance(items, list): return "" ignore = { "weapon_knife", "weapon_knife_t", "weapon_knife_gg", "weapon_knife_ct", "weapon_c4", "weapon_flashbang", "weapon_hegrenade", "weapon_smokegrenade", "weapon_molotov", "weapon_incgrenade", "weapon_decoy" } for it in items: if not isinstance(it, dict): continue name = it.get('WeaponName') if name and name not in ignore: return name for it in items: if not isinstance(it, dict): continue name = it.get('WeaponName') if name: return name return "" def pick_money(items): if not isinstance(items, list): return 0 vals = [] for it in items: if isinstance(it, dict) and it.get('Money') is not None: vals.append(it.get('Money')) return int(max(vals)) if vals else 0 side_scores = {} for sid, val in player_t_score.items(): side_scores[str(sid)] = ("T", float(val) if val is not None else 0.0) for sid, val in player_ct_score.items(): side_scores[str(sid)] = ("CT", float(val) if val is not None else 0.0) for sid in set(list(side_scores.keys()) + [str(k) for k in bron_equipment.keys()]): if sid not in side_scores: continue side, score = side_scores[sid] items = bron_equipment.get(sid) or bron_equipment.get(str(sid)) or [] start_money = pick_money(items) equipment_value = player_bron_crash.get(sid) if equipment_value is None: equipment_value = player_bron_crash.get(str(sid)) equipment_value = int(equipment_value) if equipment_value is not None else 0 main_weapon = pick_main_weapon(items) has_helmet = False has_defuser = False has_zeus = False if isinstance(items, list): for it in items: if isinstance(it, dict): name = it.get('WeaponName', '') if name == 'item_assaultsuit': has_helmet = True elif name == 'item_defuser': has_defuser = True elif name and ('taser' in name or 'zeus' in name): has_zeus = True rd.economies.append(PlayerEconomy( steam_id_64=str(sid), side=side, start_money=start_money, equipment_value=equipment_value, main_weapon=main_weapon, has_helmet=has_helmet, has_defuser=has_defuser, has_zeus=has_zeus, round_performance_score=float(score) )) self.match_data.rounds.append(rd) def _parse_classic_rounds(self): r_list = self.data_round_list.get('round_list', []) for idx, r in enumerate(r_list): # Classic round data often lacks score/winner in the list root? # Check schema: 'current_score' -> ct/t cur_score = r.get('current_score', {}) rd = RoundData( round_num=idx + 1, winner_side='None', # Default to None if unknown win_reason=0, win_reason_desc='', duration=float(cur_score.get('final_round_time', 0)), end_time_stamp='', ct_score=cur_score.get('ct', 0), t_score=cur_score.get('t', 0) ) # Utility Usage (Classic) & Economy equiped = r.get('equiped', {}) for sid, items in equiped.items(): # Ensure sid is string sid = str(sid) # Utility if sid in self.match_data.players: p = self.match_data.players[sid] if isinstance(items, list): for item in items: if item == 'flashbang': p.util_flash_usage += 1 elif item == 'smokegrenade': p.util_smoke_usage += 1 elif item in ['molotov', 'incgrenade']: p.util_molotov_usage += 1 elif item == 'hegrenade': p.util_he_usage += 1 elif item == 'decoy': p.util_decoy_usage += 1 # Economy if isinstance(items, list): equipment_value = _get_equipment_value(items) has_zeus = any('taser' in str(i).lower() or 'zeus' in str(i).lower() for i in items) has_helmet = any('helmet' in str(i).lower() or 'assaultsuit' in str(i).lower() for i in items) has_defuser = any('defuser' in str(i).lower() for i in items) # Determine Main Weapon main_weapon = "" # Simplified logic: pick most expensive non-grenade/knife best_price = 0 for item in items: if not isinstance(item, str): continue name = item.lower().replace("weapon_", "").replace("item_", "") if name in ['knife', 'c4', 'flashbang', 'hegrenade', 'smokegrenade', 'molotov', 'incgrenade', 'decoy', 'taser', 'zeus', 'kevlar', 'assaultsuit', 'defuser']: continue price = _WEAPON_PRICES.get(name, 0) if price > best_price: best_price = price main_weapon = item # Determine Side side = "Unknown" for item in items: if "usp" in str(item) or "m4a1" in str(item) or "famas" in str(item) or "defuser" in str(item): side = "CT" break if "glock" in str(item) or "ak47" in str(item) or "galil" in str(item) or "mac10" in str(item): side = "T" break rd.economies.append(PlayerEconomy( steam_id_64=sid, side=side, start_money=0, # Classic often doesn't give start money equipment_value=equipment_value, main_weapon=main_weapon, has_helmet=has_helmet, has_defuser=has_defuser, has_zeus=has_zeus, round_performance_score=0.0 )) # Kills # Classic has 'all_kill' list kills = r.get('all_kill', []) for k in kills: attacker = k.get('attacker', {}) victim = k.get('victim', {}) # Pos extraction apos = attacker.get('pos', {}) vpos = victim.get('pos', {}) re = RoundEvent( event_id=f"{self.match_id}_{rd.round_num}_{k.get('pasttime')}_{attacker.get('steamid_64')}", event_type='kill', event_time=k.get('pasttime', 0), attacker_steam_id=str(attacker.get('steamid_64', '')), victim_steam_id=str(victim.get('steamid_64', '')), weapon=k.get('weapon', ''), is_headshot=k.get('headshot', False), is_wallbang=k.get('penetrated', False), is_blind=k.get('attackerblind', False), is_through_smoke=k.get('throughsmoke', False), is_noscope=k.get('noscope', False), attacker_pos=(apos.get('x', 0), apos.get('y', 0), apos.get('z', 0)), victim_pos=(vpos.get('x', 0), vpos.get('y', 0), vpos.get('z', 0)) ) rd.events.append(re) c4_events = r.get('c4_event', []) for e in c4_events: if not isinstance(e, dict): continue event_name = str(e.get('event_name') or '').lower() if not event_name: continue if 'plant' in event_name: etype = 'bomb_plant' elif 'defus' in event_name: etype = 'bomb_defuse' else: continue sid = e.get('steamid_64') re = RoundEvent( event_id=f"{self.match_id}_{rd.round_num}_{etype}_{e.get('pasttime', 0)}_{sid}", event_type=etype, event_time=int(e.get('pasttime', 0) or 0), attacker_steam_id=str(sid) if sid is not None else None, ) rd.events.append(re) self.match_data.rounds.append(rd) # --- Main Execution --- def process_matches(): """ Main ETL pipeline: L1 → L2 using modular processor architecture """ if not init_db(): return # Import processors (handle both script and module import) try: from .processors import match_processor, player_processor, round_processor except ImportError: # Running as script, use absolute import import sys import os sys.path.insert(0, os.path.dirname(__file__)) from processors import match_processor, player_processor, round_processor l1_conn = sqlite3.connect(L1A_DB_PATH) l1_cursor = l1_conn.cursor() l2_conn = sqlite3.connect(L2_DB_PATH) logger.info("Reading from L1...") l1_cursor.execute("SELECT match_id, content FROM raw_iframe_network") count = 0 success_count = 0 error_count = 0 while True: rows = l1_cursor.fetchmany(10) if not rows: break for row in rows: match_id, content = row try: # Parse JSON from L1 raw_requests = json.loads(content) parser = MatchParser(match_id, raw_requests) match_data = parser.parse() # Process dim_maps (lightweight, stays in main flow) if match_data.map_name: cursor = l2_conn.cursor() cursor.execute(""" INSERT INTO dim_maps (map_name, map_desc) VALUES (?, ?) ON CONFLICT(map_name) DO UPDATE SET map_desc=excluded.map_desc """, (match_data.map_name, match_data.map_desc)) # Delegate to specialized processors match_success = match_processor.MatchProcessor.process(match_data, l2_conn) player_success = player_processor.PlayerProcessor.process(match_data, l2_conn) round_success = round_processor.RoundProcessor.process(match_data, l2_conn) if match_success and player_success and round_success: success_count += 1 else: error_count += 1 logger.warning(f"Partial failure for match {match_id}") count += 1 if count % 10 == 0: l2_conn.commit() print(f"Processed {count} matches ({success_count} success, {error_count} errors)...", end='\r') except Exception as e: error_count += 1 logger.error(f"Error processing match {match_id}: {e}") import traceback traceback.print_exc() l2_conn.commit() l1_conn.close() l2_conn.close() logger.info(f"\nDone. Processed {count} matches ({success_count} success, {error_count} errors).") if __name__ == "__main__": process_matches()