import sqlite3 import json import os import sys import logging from dataclasses import dataclass, field from typing import List, Dict, Optional, Any, Tuple from datetime import datetime # Setup logging logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) # Constants L1A_DB_PATH = 'database/L1A/L1A.sqlite' L2_DB_PATH = 'database/L2/L2_Main.sqlite' SCHEMA_PATH = 'database/L2/schema.sql' # --- Data Structures for Unification --- @dataclass class PlayerStats: steam_id_64: str team_id: int = 0 kills: int = 0 deaths: int = 0 assists: int = 0 headshot_count: int = 0 kd_ratio: float = 0.0 adr: float = 0.0 rating: float = 0.0 rating2: float = 0.0 rating3: float = 0.0 rws: float = 0.0 mvp_count: int = 0 elo_change: float = 0.0 rank_score: int = 0 is_win: bool = False # VIP Stats kast: float = 0.0 entry_kills: int = 0 entry_deaths: int = 0 awp_kills: int = 0 clutch_1v1: int = 0 clutch_1v2: int = 0 clutch_1v3: int = 0 clutch_1v4: int = 0 clutch_1v5: int = 0 flash_assists: int = 0 flash_duration: float = 0.0 jump_count: int = 0 damage_total: int = 0 damage_received: int = 0 damage_receive: int = 0 damage_stats: int = 0 assisted_kill: int = 0 awp_kill: int = 0 awp_kill_ct: int = 0 awp_kill_t: int = 0 benefit_kill: int = 0 day: str = "" defused_bomb: int = 0 end_1v1: int = 0 end_1v2: int = 0 end_1v3: int = 0 end_1v4: int = 0 end_1v5: int = 0 explode_bomb: int = 0 first_death: int = 0 fd_ct: int = 0 fd_t: int = 0 first_kill: int = 0 flash_enemy: int = 0 flash_team: int = 0 flash_team_time: float = 0.0 flash_time: float = 0.0 game_mode: str = "" group_id: int = 0 hold_total: int = 0 id: int = 0 is_highlight: int = 0 is_most_1v2: int = 0 is_most_assist: int = 0 is_most_awp: int = 0 is_most_end: int = 0 is_most_first_kill: int = 0 is_most_headshot: int = 0 is_most_jump: int = 0 is_svp: int = 0 is_tie: int = 0 kill_1: int = 0 kill_2: int = 0 kill_3: int = 0 kill_4: int = 0 kill_5: int = 0 many_assists_cnt1: int = 0 many_assists_cnt2: int = 0 many_assists_cnt3: int = 0 many_assists_cnt4: int = 0 many_assists_cnt5: int = 0 map: str = "" match_code: str = "" match_mode: str = "" match_team_id: int = 0 match_time: int = 0 per_headshot: float = 0.0 perfect_kill: int = 0 planted_bomb: int = 0 revenge_kill: int = 0 round_total: int = 0 season: str = "" team_kill: int = 0 throw_harm: int = 0 throw_harm_enemy: int = 0 uid: int = 0 year: str = "" sts_raw: str = "" level_info_raw: str = "" # Utility Usage util_flash_usage: int = 0 util_smoke_usage: int = 0 util_molotov_usage: int = 0 util_he_usage: int = 0 util_decoy_usage: int = 0 @dataclass class RoundEvent: event_id: str event_type: str # 'kill', 'bomb_plant', etc. event_time: int attacker_steam_id: Optional[str] = None victim_steam_id: Optional[str] = None assister_steam_id: Optional[str] = None flash_assist_steam_id: Optional[str] = None trade_killer_steam_id: Optional[str] = None weapon: Optional[str] = None is_headshot: bool = False is_wallbang: bool = False is_blind: bool = False is_through_smoke: bool = False is_noscope: bool = False # Spatial attacker_pos: Optional[Tuple[int, int, int]] = None victim_pos: Optional[Tuple[int, int, int]] = None # Score score_change_attacker: float = 0.0 score_change_victim: float = 0.0 @dataclass class PlayerEconomy: steam_id_64: str side: str start_money: int = 0 equipment_value: int = 0 main_weapon: str = "" has_helmet: bool = False has_defuser: bool = False round_performance_score: float = 0.0 @dataclass class RoundData: round_num: int winner_side: str win_reason: int win_reason_desc: str duration: float end_time_stamp: str ct_score: int t_score: int ct_money_start: int = 0 t_money_start: int = 0 events: List[RoundEvent] = field(default_factory=list) economies: List[PlayerEconomy] = field(default_factory=list) @dataclass class MatchTeamData: group_id: int group_all_score: int = 0 group_change_elo: float = 0.0 group_fh_role: int = 0 group_fh_score: int = 0 group_origin_elo: float = 0.0 group_sh_role: int = 0 group_sh_score: int = 0 group_tid: int = 0 group_uids: str = "" @dataclass class MatchData: match_id: str match_code: str = "" map_name: str = "" start_time: int = 0 end_time: int = 0 duration: int = 0 winner_team: int = 0 score_team1: int = 0 score_team2: int = 0 server_ip: str = "" server_port: int = 0 location: str = "" has_side_data_and_rating2: int = 0 match_main_id: int = 0 demo_url: str = "" game_mode: int = 0 game_name: str = "" map_desc: str = "" location_full: str = "" match_mode: int = 0 match_status: int = 0 match_flag: int = 0 status: int = 0 waiver: int = 0 year: int = 0 season: str = "" round_total: int = 0 cs_type: int = 0 priority_show_type: int = 0 pug10m_show_type: int = 0 credit_match_status: int = 0 knife_winner: int = 0 knife_winner_role: int = 0 most_1v2_uid: int = 0 most_assist_uid: int = 0 most_awp_uid: int = 0 most_end_uid: int = 0 most_first_kill_uid: int = 0 most_headshot_uid: int = 0 most_jump_uid: int = 0 mvp_uid: int = 0 response_code: int = 0 response_message: str = "" response_status: int = 0 response_timestamp: int = 0 response_trace_id: str = "" response_success: int = 0 response_errcode: int = 0 treat_info_raw: str = "" round_list_raw: str = "" leetify_data_raw: str = "" data_source_type: str = "unknown" players: Dict[str, PlayerStats] = field(default_factory=dict) # Key: steam_id_64 players_t: Dict[str, PlayerStats] = field(default_factory=dict) players_ct: Dict[str, PlayerStats] = field(default_factory=dict) rounds: List[RoundData] = field(default_factory=list) player_meta: Dict[str, Dict] = field(default_factory=dict) # steam_id -> {uid, name, avatar, ...} teams: List[MatchTeamData] = field(default_factory=list) # --- Database Helper --- def init_db(): if os.path.exists(L2_DB_PATH): logger.info(f"Removing existing L2 DB at {L2_DB_PATH}") try: os.remove(L2_DB_PATH) except PermissionError: logger.error("Cannot remove L2 DB, it might be open.") return False conn = sqlite3.connect(L2_DB_PATH) with open(SCHEMA_PATH, 'r', encoding='utf-8') as f: schema_sql = f.read() conn.executescript(schema_sql) conn.commit() conn.close() logger.info("L2 DB Initialized.") return True # --- Parsers --- class MatchParser: def __init__(self, match_id, raw_requests): self.match_id = match_id self.raw_requests = raw_requests self.match_data = MatchData(match_id=match_id) # Extracted JSON bodies self.data_match = None self.data_match_wrapper = None self.data_vip = None self.data_leetify = None self.data_round_list = None self._extract_payloads() def _extract_payloads(self): for req in self.raw_requests: url = req.get('url', '') body = req.get('body', {}) if not body: continue # Check URLs if 'crane/http/api/data/match/' in url: self.data_match_wrapper = body self.data_match = body.get('data', {}) elif 'crane/http/api/data/vip_plus_match_data/' in url: self.data_vip = body.get('data', {}) elif 'crane/http/api/match/leetify_rating/' in url: self.data_leetify = body.get('data', {}) elif 'crane/http/api/match/round/' in url: self.data_round_list = body.get('data', {}) def parse(self) -> MatchData: if not self.data_match: logger.warning(f"No base match data found for {self.match_id}") return self.match_data self._parse_base_info() self._parse_players_base() self._parse_players_vip() # Decide which round source to use if self.data_leetify and self.data_leetify.get('leetify_data'): self.match_data.data_source_type = 'leetify' try: self.match_data.leetify_data_raw = json.dumps(self.data_leetify.get('leetify_data', {}), ensure_ascii=False) except: self.match_data.leetify_data_raw = "" self.match_data.round_list_raw = "" self._parse_leetify_rounds() elif self.data_round_list and self.data_round_list.get('round_list'): self.match_data.data_source_type = 'classic' try: self.match_data.round_list_raw = json.dumps(self.data_round_list.get('round_list', []), ensure_ascii=False) except: self.match_data.round_list_raw = "" self.match_data.leetify_data_raw = "" self._parse_classic_rounds() else: self.match_data.data_source_type = 'unknown' self.match_data.round_list_raw = "" self.match_data.leetify_data_raw = "" logger.info(f"No round data found for {self.match_id}") return self.match_data def _parse_base_info(self): m = self.data_match.get('main', {}) self.match_data.match_code = m.get('match_code', '') self.match_data.map_name = m.get('map', '') self.match_data.start_time = m.get('start_time', 0) self.match_data.end_time = m.get('end_time', 0) self.match_data.duration = self.match_data.end_time - self.match_data.start_time if self.match_data.end_time else 0 self.match_data.winner_team = m.get('match_winner', 0) self.match_data.score_team1 = m.get('group1_all_score', 0) self.match_data.score_team2 = m.get('group2_all_score', 0) self.match_data.server_ip = m.get('server_ip', '') # Port is sometimes string try: self.match_data.server_port = int(m.get('server_port', 0)) except: self.match_data.server_port = 0 self.match_data.location = m.get('location', '') def safe_int(val): try: return int(float(val)) if val is not None else 0 except: return 0 def safe_float(val): try: return float(val) if val is not None else 0.0 except: return 0.0 def safe_text(val): return "" if val is None else str(val) wrapper = self.data_match_wrapper or {} self.match_data.response_code = safe_int(wrapper.get('code')) self.match_data.response_message = safe_text(wrapper.get('message')) self.match_data.response_status = safe_int(wrapper.get('status')) self.match_data.response_timestamp = safe_int(wrapper.get('timeStamp') if wrapper.get('timeStamp') is not None else wrapper.get('timestamp')) self.match_data.response_trace_id = safe_text(wrapper.get('traceId') if wrapper.get('traceId') is not None else wrapper.get('trace_id')) self.match_data.response_success = safe_int(wrapper.get('success')) self.match_data.response_errcode = safe_int(wrapper.get('errcode')) self.match_data.has_side_data_and_rating2 = safe_int(self.data_match.get('has_side_data_and_rating2')) self.match_data.match_main_id = safe_int(m.get('id')) self.match_data.demo_url = safe_text(m.get('demo_url')) self.match_data.game_mode = safe_int(m.get('game_mode')) self.match_data.game_name = safe_text(m.get('game_name')) self.match_data.map_desc = safe_text(m.get('map_desc')) self.match_data.location_full = safe_text(m.get('location_full')) self.match_data.match_mode = safe_int(m.get('match_mode')) self.match_data.match_status = safe_int(m.get('match_status')) self.match_data.match_flag = safe_int(m.get('match_flag')) self.match_data.status = safe_int(m.get('status')) self.match_data.waiver = safe_int(m.get('waiver')) self.match_data.year = safe_int(m.get('year')) self.match_data.season = safe_text(m.get('season')) self.match_data.round_total = safe_int(m.get('round_total')) self.match_data.cs_type = safe_int(m.get('cs_type')) self.match_data.priority_show_type = safe_int(m.get('priority_show_type')) self.match_data.pug10m_show_type = safe_int(m.get('pug10m_show_type')) self.match_data.credit_match_status = safe_int(m.get('credit_match_status')) self.match_data.knife_winner = safe_int(m.get('knife_winner')) self.match_data.knife_winner_role = safe_int(m.get('knife_winner_role')) self.match_data.most_1v2_uid = safe_int(m.get('most_1v2_uid')) self.match_data.most_assist_uid = safe_int(m.get('most_assist_uid')) self.match_data.most_awp_uid = safe_int(m.get('most_awp_uid')) self.match_data.most_end_uid = safe_int(m.get('most_end_uid')) self.match_data.most_first_kill_uid = safe_int(m.get('most_first_kill_uid')) self.match_data.most_headshot_uid = safe_int(m.get('most_headshot_uid')) self.match_data.most_jump_uid = safe_int(m.get('most_jump_uid')) self.match_data.mvp_uid = safe_int(m.get('mvp_uid')) treat_info = self.data_match.get('treat_info') if treat_info is not None: try: self.match_data.treat_info_raw = json.dumps(treat_info, ensure_ascii=False) except: self.match_data.treat_info_raw = "" self.match_data.teams = [] for idx in [1, 2]: team = MatchTeamData( group_id=idx, group_all_score=safe_int(m.get(f"group{idx}_all_score")), group_change_elo=safe_float(m.get(f"group{idx}_change_elo")), group_fh_role=safe_int(m.get(f"group{idx}_fh_role")), group_fh_score=safe_int(m.get(f"group{idx}_fh_score")), group_origin_elo=safe_float(m.get(f"group{idx}_origin_elo")), group_sh_role=safe_int(m.get(f"group{idx}_sh_role")), group_sh_score=safe_int(m.get(f"group{idx}_sh_score")), group_tid=safe_int(m.get(f"group{idx}_tid")), group_uids=safe_text(m.get(f"group{idx}_uids")) ) self.match_data.teams.append(team) def _parse_players_base(self): # Players are in group_1 and group_2 lists in data_match groups = [] if 'group_1' in self.data_match: groups.extend(self.data_match['group_1']) if 'group_2' in self.data_match: groups.extend(self.data_match['group_2']) def safe_int(val): try: return int(float(val)) if val is not None else 0 except: return 0 def safe_text(val): return "" if val is None else str(val) for p in groups: # We need steam_id. # Structure: user_info -> user_data -> steam -> steamId user_info = p.get('user_info', {}) user_data = user_info.get('user_data', {}) steam_data = user_data.get('steam', {}) steam_id = str(steam_data.get('steamId', '')) fight = p.get('fight', {}) fight_t = p.get('fight_t', {}) fight_ct = p.get('fight_ct', {}) uid = fight.get('uid') # Store meta for dim_players user_data = user_info.get('user_data', {}) profile = user_data.get('profile', {}) # If steam_id is empty, use temporary placeholder '5E:{uid}' # Ideally we want steam_id_64. if not steam_id and uid: steam_id = f"5E:{uid}" if not steam_id: continue status = user_data.get('status', {}) platform_exp = user_data.get('platformExp', {}) trusted = user_data.get('trusted', {}) certify = user_data.get('certify', {}) identity = user_data.get('identity', {}) plus_info = user_info.get('plus_info', {}) or p.get('plus_info', {}) user_info_raw = "" try: user_info_raw = json.dumps(user_info, ensure_ascii=False) except: user_info_raw = "" self.match_data.player_meta[steam_id] = { 'uid': safe_int(uid), 'username': safe_text(user_data.get('username')), 'uuid': safe_text(user_data.get('uuid')), 'email': safe_text(user_data.get('email')), 'area': safe_text(user_data.get('area')), 'mobile': safe_text(user_data.get('mobile')), 'avatar_url': safe_text(profile.get('avatarUrl')), 'domain': safe_text(profile.get('domain')), 'user_domain': safe_text(user_data.get('domain')), 'created_at': safe_int(user_data.get('createdAt')), 'updated_at': safe_int(user_data.get('updatedAt')), 'username_audit_status': safe_int(user_data.get('usernameAuditStatus')), 'accid': safe_text(user_data.get('Accid')), 'team_id': safe_int(user_data.get('teamID')), 'trumpet_count': safe_int(user_data.get('trumpetCount')), 'profile_nickname': safe_text(profile.get('nickname')), 'profile_avatar_audit_status': safe_int(profile.get('avatarAuditStatus')), 'profile_rgb_avatar_url': safe_text(profile.get('rgbAvatarUrl')), 'profile_photo_url': safe_text(profile.get('photoUrl')), 'profile_gender': safe_int(profile.get('gender')), 'profile_birthday': safe_int(profile.get('birthday')), 'profile_country_id': safe_text(profile.get('countryId')), 'profile_region_id': safe_text(profile.get('regionId')), 'profile_city_id': safe_text(profile.get('cityId')), 'profile_language': safe_text(profile.get('language')), 'profile_recommend_url': safe_text(profile.get('recommendUrl')), 'profile_group_id': safe_int(profile.get('groupId')), 'profile_reg_source': safe_int(profile.get('regSource')), 'status_status': safe_int(status.get('status')), 'status_expire': safe_int(status.get('expire')), 'status_cancellation_status': safe_int(status.get('cancellationStatus')), 'status_new_user': safe_int(status.get('newUser')), 'status_login_banned_time': safe_int(status.get('loginBannedTime')), 'status_anticheat_type': safe_int(status.get('anticheatType')), 'status_flag_status1': safe_text(status.get('flagStatus1')), 'status_anticheat_status': safe_text(status.get('anticheatStatus')), 'status_flag_honor': safe_text(status.get('FlagHonor')), 'status_privacy_policy_status': safe_int(status.get('PrivacyPolicyStatus')), 'status_csgo_frozen_exptime': safe_int(status.get('csgoFrozenExptime')), 'platformexp_level': safe_int(platform_exp.get('level')), 'platformexp_exp': safe_int(platform_exp.get('exp')), 'steam_account': safe_text(steam_data.get('steamAccount')), 'steam_trade_url': safe_text(steam_data.get('tradeUrl')), 'steam_rent_id': safe_text(steam_data.get('rentSteamId')), 'trusted_credit': safe_int(trusted.get('credit')), 'trusted_credit_level': safe_int(trusted.get('creditLevel')), 'trusted_score': safe_int(trusted.get('score')), 'trusted_status': safe_int(trusted.get('status')), 'trusted_credit_status': safe_int(trusted.get('creditStatus')), 'certify_id_type': safe_int(certify.get('idType')), 'certify_status': safe_int(certify.get('status')), 'certify_age': safe_int(certify.get('age')), 'certify_real_name': safe_text(certify.get('realName')), 'certify_uid_list': safe_text(json.dumps(certify.get('uidList'), ensure_ascii=False)) if certify.get('uidList') is not None else "", 'certify_audit_status': safe_int(certify.get('auditStatus')), 'certify_gender': safe_int(certify.get('gender')), 'identity_type': safe_int(identity.get('type')), 'identity_extras': safe_text(identity.get('extras')), 'identity_status': safe_int(identity.get('status')), 'identity_slogan': safe_text(identity.get('slogan')), 'identity_list': safe_text(json.dumps(identity.get('identity_list'), ensure_ascii=False)) if identity.get('identity_list') is not None else "", 'identity_slogan_ext': safe_text(identity.get('slogan_ext')), 'identity_live_url': safe_text(identity.get('live_url')), 'identity_live_type': safe_int(identity.get('live_type')), 'plus_is_plus': safe_int(plus_info.get('is_plus')), 'user_info_raw': user_info_raw } stats = PlayerStats(steam_id_64=steam_id) sts = p.get('sts', {}) level_info = p.get('level_info', {}) try: # Use safe conversion helper def safe_int(val): try: return int(float(val)) if val is not None else 0 except: return 0 def safe_float(val): try: return float(val) if val is not None else 0.0 except: return 0.0 def safe_text(val): return "" if val is None else str(val) if sts is not None: try: stats.sts_raw = json.dumps(sts, ensure_ascii=False) except: stats.sts_raw = "" if level_info is not None: try: stats.level_info_raw = json.dumps(level_info, ensure_ascii=False) except: stats.level_info_raw = "" def get_stat(key): if key in fight and fight.get(key) not in [None, ""]: return fight.get(key) return 0 def build_side_stats(fight_side, team_id_value): side_stats = PlayerStats(steam_id_64=steam_id) side_stats.team_id = team_id_value side_stats.kills = safe_int(fight_side.get('kill')) side_stats.deaths = safe_int(fight_side.get('death')) side_stats.assists = safe_int(fight_side.get('assist')) side_stats.headshot_count = safe_int(fight_side.get('headshot')) side_stats.adr = safe_float(fight_side.get('adr')) side_stats.rating = safe_float(fight_side.get('rating')) side_stats.rating2 = safe_float(fight_side.get('rating2')) side_stats.rating3 = safe_float(fight_side.get('rating3')) side_stats.rws = safe_float(fight_side.get('rws')) side_stats.kast = safe_float(fight_side.get('kast')) side_stats.mvp_count = safe_int(fight_side.get('is_mvp')) side_stats.flash_duration = safe_float(fight_side.get('flash_enemy_time')) side_stats.jump_count = safe_int(fight_side.get('jump_total')) side_stats.is_win = bool(safe_int(fight_side.get('is_win'))) side_stats.assisted_kill = safe_int(fight_side.get('assisted_kill')) side_stats.awp_kill = safe_int(fight_side.get('awp_kill')) side_stats.benefit_kill = safe_int(fight_side.get('benefit_kill')) side_stats.day = safe_text(fight_side.get('day')) side_stats.defused_bomb = safe_int(fight_side.get('defused_bomb')) side_stats.end_1v1 = safe_int(fight_side.get('end_1v1')) side_stats.end_1v2 = safe_int(fight_side.get('end_1v2')) side_stats.end_1v3 = safe_int(fight_side.get('end_1v3')) side_stats.end_1v4 = safe_int(fight_side.get('end_1v4')) side_stats.end_1v5 = safe_int(fight_side.get('end_1v5')) side_stats.explode_bomb = safe_int(fight_side.get('explode_bomb')) side_stats.first_death = safe_int(fight_side.get('first_death')) side_stats.first_kill = safe_int(fight_side.get('first_kill')) side_stats.flash_enemy = safe_int(fight_side.get('flash_enemy')) side_stats.flash_team = safe_int(fight_side.get('flash_team')) side_stats.flash_team_time = safe_float(fight_side.get('flash_team_time')) side_stats.flash_time = safe_float(fight_side.get('flash_time')) side_stats.game_mode = safe_text(fight_side.get('game_mode')) side_stats.group_id = safe_int(fight_side.get('group_id')) side_stats.hold_total = safe_int(fight_side.get('hold_total')) side_stats.id = safe_int(fight_side.get('id')) side_stats.is_highlight = safe_int(fight_side.get('is_highlight')) side_stats.is_most_1v2 = safe_int(fight_side.get('is_most_1v2')) side_stats.is_most_assist = safe_int(fight_side.get('is_most_assist')) side_stats.is_most_awp = safe_int(fight_side.get('is_most_awp')) side_stats.is_most_end = safe_int(fight_side.get('is_most_end')) side_stats.is_most_first_kill = safe_int(fight_side.get('is_most_first_kill')) side_stats.is_most_headshot = safe_int(fight_side.get('is_most_headshot')) side_stats.is_most_jump = safe_int(fight_side.get('is_most_jump')) side_stats.is_svp = safe_int(fight_side.get('is_svp')) side_stats.is_tie = safe_int(fight_side.get('is_tie')) side_stats.kill_1 = safe_int(fight_side.get('kill_1')) side_stats.kill_2 = safe_int(fight_side.get('kill_2')) side_stats.kill_3 = safe_int(fight_side.get('kill_3')) side_stats.kill_4 = safe_int(fight_side.get('kill_4')) side_stats.kill_5 = safe_int(fight_side.get('kill_5')) side_stats.many_assists_cnt1 = safe_int(fight_side.get('many_assists_cnt1')) side_stats.many_assists_cnt2 = safe_int(fight_side.get('many_assists_cnt2')) side_stats.many_assists_cnt3 = safe_int(fight_side.get('many_assists_cnt3')) side_stats.many_assists_cnt4 = safe_int(fight_side.get('many_assists_cnt4')) side_stats.many_assists_cnt5 = safe_int(fight_side.get('many_assists_cnt5')) side_stats.map = safe_text(fight_side.get('map')) side_stats.match_code = safe_text(fight_side.get('match_code')) side_stats.match_mode = safe_text(fight_side.get('match_mode')) side_stats.match_team_id = safe_int(fight_side.get('match_team_id')) side_stats.match_time = safe_int(fight_side.get('match_time')) side_stats.per_headshot = safe_float(fight_side.get('per_headshot')) side_stats.perfect_kill = safe_int(fight_side.get('perfect_kill')) side_stats.planted_bomb = safe_int(fight_side.get('planted_bomb')) side_stats.revenge_kill = safe_int(fight_side.get('revenge_kill')) side_stats.round_total = safe_int(fight_side.get('round_total')) side_stats.season = safe_text(fight_side.get('season')) side_stats.team_kill = safe_int(fight_side.get('team_kill')) side_stats.throw_harm = safe_int(fight_side.get('throw_harm')) side_stats.throw_harm_enemy = safe_int(fight_side.get('throw_harm_enemy')) side_stats.uid = safe_int(fight_side.get('uid')) side_stats.year = safe_text(fight_side.get('year')) # Map missing fields side_stats.clutch_1v1 = side_stats.end_1v1 side_stats.clutch_1v2 = side_stats.end_1v2 side_stats.clutch_1v3 = side_stats.end_1v3 side_stats.clutch_1v4 = side_stats.end_1v4 side_stats.clutch_1v5 = side_stats.end_1v5 side_stats.entry_kills = side_stats.first_kill side_stats.entry_deaths = side_stats.first_death return side_stats team_id_value = safe_int(fight.get('match_team_id')) stats.team_id = team_id_value stats.kills = safe_int(get_stat('kill')) stats.deaths = safe_int(get_stat('death')) # Force calculate K/D if stats.deaths > 0: stats.kd_ratio = stats.kills / stats.deaths else: stats.kd_ratio = float(stats.kills) stats.assists = safe_int(get_stat('assist')) stats.headshot_count = safe_int(get_stat('headshot')) stats.adr = safe_float(get_stat('adr')) stats.rating = safe_float(get_stat('rating')) stats.rating2 = safe_float(get_stat('rating2')) stats.rating3 = safe_float(get_stat('rating3')) stats.rws = safe_float(get_stat('rws')) # is_mvp might be string "1" or int 1 stats.mvp_count = safe_int(get_stat('is_mvp')) stats.flash_duration = safe_float(get_stat('flash_enemy_time')) stats.jump_count = safe_int(get_stat('jump_total')) stats.is_win = bool(safe_int(get_stat('is_win'))) stats.elo_change = safe_float(sts.get('change_elo')) stats.rank_score = safe_int(sts.get('rank')) stats.assisted_kill = safe_int(fight.get('assisted_kill')) stats.awp_kill = safe_int(fight.get('awp_kill')) stats.benefit_kill = safe_int(fight.get('benefit_kill')) stats.day = safe_text(fight.get('day')) stats.defused_bomb = safe_int(fight.get('defused_bomb')) stats.end_1v1 = safe_int(fight.get('end_1v1')) stats.end_1v2 = safe_int(fight.get('end_1v2')) stats.end_1v3 = safe_int(fight.get('end_1v3')) stats.end_1v4 = safe_int(fight.get('end_1v4')) stats.end_1v5 = safe_int(fight.get('end_1v5')) stats.explode_bomb = safe_int(fight.get('explode_bomb')) stats.first_death = safe_int(fight.get('first_death')) stats.first_kill = safe_int(fight.get('first_kill')) stats.flash_enemy = safe_int(fight.get('flash_enemy')) stats.flash_team = safe_int(fight.get('flash_team')) stats.flash_team_time = safe_float(fight.get('flash_team_time')) stats.flash_time = safe_float(fight.get('flash_time')) stats.game_mode = safe_text(fight.get('game_mode')) stats.group_id = safe_int(fight.get('group_id')) stats.hold_total = safe_int(fight.get('hold_total')) stats.id = safe_int(fight.get('id')) stats.is_highlight = safe_int(fight.get('is_highlight')) stats.is_most_1v2 = safe_int(fight.get('is_most_1v2')) stats.is_most_assist = safe_int(fight.get('is_most_assist')) stats.is_most_awp = safe_int(fight.get('is_most_awp')) stats.is_most_end = safe_int(fight.get('is_most_end')) stats.is_most_first_kill = safe_int(fight.get('is_most_first_kill')) stats.is_most_headshot = safe_int(fight.get('is_most_headshot')) stats.is_most_jump = safe_int(fight.get('is_most_jump')) stats.is_svp = safe_int(fight.get('is_svp')) stats.is_tie = safe_int(fight.get('is_tie')) stats.kill_1 = safe_int(fight.get('kill_1')) stats.kill_2 = safe_int(fight.get('kill_2')) stats.kill_3 = safe_int(fight.get('kill_3')) stats.kill_4 = safe_int(fight.get('kill_4')) stats.kill_5 = safe_int(fight.get('kill_5')) stats.many_assists_cnt1 = safe_int(fight.get('many_assists_cnt1')) stats.many_assists_cnt2 = safe_int(fight.get('many_assists_cnt2')) stats.many_assists_cnt3 = safe_int(fight.get('many_assists_cnt3')) stats.many_assists_cnt4 = safe_int(fight.get('many_assists_cnt4')) stats.many_assists_cnt5 = safe_int(fight.get('many_assists_cnt5')) stats.map = safe_text(fight.get('map')) stats.match_code = safe_text(fight.get('match_code')) stats.match_mode = safe_text(fight.get('match_mode')) stats.match_team_id = safe_int(fight.get('match_team_id')) stats.match_time = safe_int(fight.get('match_time')) stats.per_headshot = safe_float(fight.get('per_headshot')) stats.perfect_kill = safe_int(fight.get('perfect_kill')) stats.planted_bomb = safe_int(fight.get('planted_bomb')) stats.revenge_kill = safe_int(fight.get('revenge_kill')) stats.round_total = safe_int(fight.get('round_total')) stats.season = safe_text(fight.get('season')) stats.team_kill = safe_int(fight.get('team_kill')) stats.throw_harm = safe_int(fight.get('throw_harm')) stats.throw_harm_enemy = safe_int(fight.get('throw_harm_enemy')) stats.uid = safe_int(fight.get('uid')) stats.year = safe_text(fight.get('year')) # Map missing fields stats.clutch_1v1 = stats.end_1v1 stats.clutch_1v2 = stats.end_1v2 stats.clutch_1v3 = stats.end_1v3 stats.clutch_1v4 = stats.end_1v4 stats.clutch_1v5 = stats.end_1v5 stats.entry_kills = stats.first_kill stats.entry_deaths = stats.first_death except Exception as e: logger.error(f"Error parsing stats for {steam_id} in {self.match_id}: {e}") pass self.match_data.players[steam_id] = stats if isinstance(fight_t, dict) and fight_t: t_team_id = team_id_value or safe_int(fight_t.get('match_team_id')) self.match_data.players_t[steam_id] = build_side_stats(fight_t, t_team_id) if isinstance(fight_ct, dict) and fight_ct: ct_team_id = team_id_value or safe_int(fight_ct.get('match_team_id')) self.match_data.players_ct[steam_id] = build_side_stats(fight_ct, ct_team_id) def _parse_players_vip(self): if not self.data_vip: return # Structure: data_vip -> steamid (key) -> dict for sid, vdata in self.data_vip.items(): # SID might be steam_id_64 directly if sid in self.match_data.players: p = self.match_data.players[sid] p.kast = float(vdata.get('kast', 0)) p.awp_kills = int(vdata.get('awp_kill', 0)) p.awp_kill_ct = int(vdata.get('awp_kill_ct', 0)) p.awp_kill_t = int(vdata.get('awp_kill_t', 0)) p.fd_ct = int(vdata.get('fd_ct', 0)) p.fd_t = int(vdata.get('fd_t', 0)) p.damage_receive = int(vdata.get('damage_receive', 0)) p.damage_stats = int(vdata.get('damage_stats', 0)) p.damage_total = int(vdata.get('damage_total', 0)) p.damage_received = int(vdata.get('damage_received', 0)) p.flash_assists = int(vdata.get('flash_assists', 0)) else: # Try to match by 5E ID if possible, but here keys are steamids usually pass for sid, p in self.match_data.players.items(): if sid in self.match_data.players_t: self.match_data.players_t[sid].awp_kill_t = p.awp_kill_t self.match_data.players_t[sid].fd_t = p.fd_t if sid in self.match_data.players_ct: self.match_data.players_ct[sid].awp_kill_ct = p.awp_kill_ct self.match_data.players_ct[sid].fd_ct = p.fd_ct def _parse_leetify_rounds(self): l_data = self.data_leetify.get('leetify_data', {}) round_list = l_data.get('round_stat', []) for idx, r in enumerate(round_list): # Utility Usage (Leetify) bron = r.get('bron_equipment', {}) for sid, items in bron.items(): sid = str(sid) if sid in self.match_data.players: p = self.match_data.players[sid] if isinstance(items, list): for item in items: if not isinstance(item, dict): continue name = item.get('WeaponName', '') if name == 'weapon_flashbang': p.util_flash_usage += 1 elif name == 'weapon_smokegrenade': p.util_smoke_usage += 1 elif name in ['weapon_molotov', 'weapon_incgrenade']: p.util_molotov_usage += 1 elif name == 'weapon_hegrenade': p.util_he_usage += 1 elif name == 'weapon_decoy': p.util_decoy_usage += 1 rd = RoundData( round_num=r.get('round', idx + 1), winner_side='CT' if r.get('win_reason') in [7, 8, 9] else 'T', # Approximate logic, need real enum win_reason=r.get('win_reason', 0), win_reason_desc=str(r.get('win_reason', 0)), duration=0, # Leetify might not have exact duration easily end_time_stamp=r.get('end_ts', ''), ct_score=r.get('sfui_event', {}).get('score_ct', 0), t_score=r.get('sfui_event', {}).get('score_t', 0), ct_money_start=r.get('ct_money_group', 0), t_money_start=r.get('t_money_group', 0) ) # Events # Leetify has 'show_event' list events = r.get('show_event', []) for evt in events: e_type_code = evt.get('event_type') # Mapping needed for event types. # Assuming 3 is kill based on schema 'kill_event' presence if evt.get('kill_event'): k = evt['kill_event'] re = RoundEvent( event_id=f"{self.match_id}_{rd.round_num}_{k.get('Ts', '')}_{k.get('Killer')}", event_type='kill', event_time=evt.get('ts', 0), attacker_steam_id=k.get('Killer'), victim_steam_id=k.get('Victim'), weapon=k.get('WeaponName'), is_headshot=k.get('Headshot', False), is_wallbang=k.get('Penetrated', False), is_blind=k.get('AttackerBlind', False), is_through_smoke=k.get('ThroughSmoke', False), is_noscope=k.get('NoScope', False) ) # Leetify specifics # Trade? if evt.get('trade_score_change'): re.trade_killer_steam_id = list(evt['trade_score_change'].keys())[0] if evt.get('flash_assist_killer_score_change'): re.flash_assist_steam_id = list(evt['flash_assist_killer_score_change'].keys())[0] # Score changes if evt.get('killer_score_change'): # e.g. {'': {'score': 17.0}} vals = list(evt['killer_score_change'].values()) if vals: re.score_change_attacker = vals[0].get('score', 0) if evt.get('victim_score_change'): vals = list(evt['victim_score_change'].values()) if vals: re.score_change_victim = vals[0].get('score', 0) rd.events.append(re) bron_equipment = r.get('bron_equipment') or {} player_t_score = r.get('player_t_score') or {} player_ct_score = r.get('player_ct_score') or {} player_bron_crash = r.get('player_bron_crash') or {} def pick_main_weapon(items): if not isinstance(items, list): return "" ignore = { "weapon_knife", "weapon_knife_t", "weapon_knife_gg", "weapon_knife_ct", "weapon_c4", "weapon_flashbang", "weapon_hegrenade", "weapon_smokegrenade", "weapon_molotov", "weapon_incgrenade", "weapon_decoy" } for it in items: if not isinstance(it, dict): continue name = it.get('WeaponName') if name and name not in ignore: return name for it in items: if not isinstance(it, dict): continue name = it.get('WeaponName') if name: return name return "" def pick_money(items): if not isinstance(items, list): return 0 vals = [] for it in items: if isinstance(it, dict) and it.get('Money') is not None: vals.append(it.get('Money')) return int(max(vals)) if vals else 0 side_scores = {} for sid, val in player_t_score.items(): side_scores[str(sid)] = ("T", float(val) if val is not None else 0.0) for sid, val in player_ct_score.items(): side_scores[str(sid)] = ("CT", float(val) if val is not None else 0.0) for sid in set(list(side_scores.keys()) + [str(k) for k in bron_equipment.keys()]): if sid not in side_scores: continue side, score = side_scores[sid] items = bron_equipment.get(sid) or bron_equipment.get(str(sid)) or [] start_money = pick_money(items) equipment_value = player_bron_crash.get(sid) if equipment_value is None: equipment_value = player_bron_crash.get(str(sid)) equipment_value = int(equipment_value) if equipment_value is not None else 0 main_weapon = pick_main_weapon(items) has_helmet = False has_defuser = False if isinstance(items, list): for it in items: if isinstance(it, dict): name = it.get('WeaponName', '') if name == 'item_assaultsuit': has_helmet = True elif name == 'item_defuser': has_defuser = True rd.economies.append(PlayerEconomy( steam_id_64=str(sid), side=side, start_money=start_money, equipment_value=equipment_value, main_weapon=main_weapon, has_helmet=has_helmet, has_defuser=has_defuser, round_performance_score=float(score) )) self.match_data.rounds.append(rd) def _parse_classic_rounds(self): r_list = self.data_round_list.get('round_list', []) for idx, r in enumerate(r_list): # Classic round data often lacks score/winner in the list root? # Check schema: 'current_score' -> ct/t cur_score = r.get('current_score', {}) # Utility Usage (Classic) equiped = r.get('equiped', {}) for sid, items in equiped.items(): # Ensure sid is string sid = str(sid) if sid in self.match_data.players: p = self.match_data.players[sid] if isinstance(items, list): for item in items: if item == 'flashbang': p.util_flash_usage += 1 elif item == 'smokegrenade': p.util_smoke_usage += 1 elif item in ['molotov', 'incgrenade']: p.util_molotov_usage += 1 elif item == 'hegrenade': p.util_he_usage += 1 elif item == 'decoy': p.util_decoy_usage += 1 rd = RoundData( round_num=idx + 1, winner_side='None', # Default to None if unknown win_reason=0, win_reason_desc='', duration=float(cur_score.get('final_round_time', 0)), end_time_stamp='', ct_score=cur_score.get('ct', 0), t_score=cur_score.get('t', 0) ) # Kills # Classic has 'all_kill' list kills = r.get('all_kill', []) for k in kills: attacker = k.get('attacker', {}) victim = k.get('victim', {}) # Pos extraction apos = attacker.get('pos', {}) vpos = victim.get('pos', {}) re = RoundEvent( event_id=f"{self.match_id}_{rd.round_num}_{k.get('pasttime')}_{attacker.get('steamid_64')}", event_type='kill', event_time=k.get('pasttime', 0), attacker_steam_id=str(attacker.get('steamid_64', '')), victim_steam_id=str(victim.get('steamid_64', '')), weapon=k.get('weapon', ''), is_headshot=k.get('headshot', False), is_wallbang=k.get('penetrated', False), is_blind=k.get('attackerblind', False), is_through_smoke=k.get('throughsmoke', False), is_noscope=k.get('noscope', False), attacker_pos=(apos.get('x', 0), apos.get('y', 0), apos.get('z', 0)), victim_pos=(vpos.get('x', 0), vpos.get('y', 0), vpos.get('z', 0)) ) rd.events.append(re) self.match_data.rounds.append(rd) # --- Main Execution --- def process_matches(): if not init_db(): return l1_conn = sqlite3.connect(L1A_DB_PATH) l1_cursor = l1_conn.cursor() l2_conn = sqlite3.connect(L2_DB_PATH) l2_cursor = l2_conn.cursor() logger.info("Reading from L1A...") l1_cursor.execute("SELECT match_id, content FROM raw_iframe_network") count = 0 while True: rows = l1_cursor.fetchmany(10) if not rows: break for row in rows: match_id, content = row try: raw_requests = json.loads(content) parser = MatchParser(match_id, raw_requests) match_data = parser.parse() save_match(l2_cursor, match_data) count += 1 if count % 10 == 0: l2_conn.commit() print(f"Processed {count} matches...", end='\r') except Exception as e: logger.error(f"Error processing match {match_id}: {e}") # continue l2_conn.commit() l1_conn.close() l2_conn.close() logger.info(f"\nDone. Processed {count} matches.") def save_match(cursor, m: MatchData): # 1. Dim Players (Upsert) player_meta_columns = [ "steam_id_64", "uid", "username", "avatar_url", "domain", "created_at", "updated_at", "last_seen_match_id", "uuid", "email", "area", "mobile", "user_domain", "username_audit_status", "accid", "team_id", "trumpet_count", "profile_nickname", "profile_avatar_audit_status", "profile_rgb_avatar_url", "profile_photo_url", "profile_gender", "profile_birthday", "profile_country_id", "profile_region_id", "profile_city_id", "profile_language", "profile_recommend_url", "profile_group_id", "profile_reg_source", "status_status", "status_expire", "status_cancellation_status", "status_new_user", "status_login_banned_time", "status_anticheat_type", "status_flag_status1", "status_anticheat_status", "status_flag_honor", "status_privacy_policy_status", "status_csgo_frozen_exptime", "platformexp_level", "platformexp_exp", "steam_account", "steam_trade_url", "steam_rent_id", "trusted_credit", "trusted_credit_level", "trusted_score", "trusted_status", "trusted_credit_status", "certify_id_type", "certify_status", "certify_age", "certify_real_name", "certify_uid_list", "certify_audit_status", "certify_gender", "identity_type", "identity_extras", "identity_status", "identity_slogan", "identity_list", "identity_slogan_ext", "identity_live_url", "identity_live_type", "plus_is_plus", "user_info_raw" ] player_meta_placeholders = ",".join(["?"] * len(player_meta_columns)) player_meta_columns_sql = ",".join(player_meta_columns) for sid, meta in m.player_meta.items(): cursor.execute(""" INSERT INTO dim_players (""" + player_meta_columns_sql + """) VALUES (""" + player_meta_placeholders + """) ON CONFLICT(steam_id_64) DO UPDATE SET uid=excluded.uid, username=excluded.username, avatar_url=excluded.avatar_url, domain=excluded.domain, created_at=excluded.created_at, updated_at=excluded.updated_at, last_seen_match_id=excluded.last_seen_match_id, uuid=excluded.uuid, email=excluded.email, area=excluded.area, mobile=excluded.mobile, user_domain=excluded.user_domain, username_audit_status=excluded.username_audit_status, accid=excluded.accid, team_id=excluded.team_id, trumpet_count=excluded.trumpet_count, profile_nickname=excluded.profile_nickname, profile_avatar_audit_status=excluded.profile_avatar_audit_status, profile_rgb_avatar_url=excluded.profile_rgb_avatar_url, profile_photo_url=excluded.profile_photo_url, profile_gender=excluded.profile_gender, profile_birthday=excluded.profile_birthday, profile_country_id=excluded.profile_country_id, profile_region_id=excluded.profile_region_id, profile_city_id=excluded.profile_city_id, profile_language=excluded.profile_language, profile_recommend_url=excluded.profile_recommend_url, profile_group_id=excluded.profile_group_id, profile_reg_source=excluded.profile_reg_source, status_status=excluded.status_status, status_expire=excluded.status_expire, status_cancellation_status=excluded.status_cancellation_status, status_new_user=excluded.status_new_user, status_login_banned_time=excluded.status_login_banned_time, status_anticheat_type=excluded.status_anticheat_type, status_flag_status1=excluded.status_flag_status1, status_anticheat_status=excluded.status_anticheat_status, status_flag_honor=excluded.status_flag_honor, status_privacy_policy_status=excluded.status_privacy_policy_status, status_csgo_frozen_exptime=excluded.status_csgo_frozen_exptime, platformexp_level=excluded.platformexp_level, platformexp_exp=excluded.platformexp_exp, steam_account=excluded.steam_account, steam_trade_url=excluded.steam_trade_url, steam_rent_id=excluded.steam_rent_id, trusted_credit=excluded.trusted_credit, trusted_credit_level=excluded.trusted_credit_level, trusted_score=excluded.trusted_score, trusted_status=excluded.trusted_status, trusted_credit_status=excluded.trusted_credit_status, certify_id_type=excluded.certify_id_type, certify_status=excluded.certify_status, certify_age=excluded.certify_age, certify_real_name=excluded.certify_real_name, certify_uid_list=excluded.certify_uid_list, certify_audit_status=excluded.certify_audit_status, certify_gender=excluded.certify_gender, identity_type=excluded.identity_type, identity_extras=excluded.identity_extras, identity_status=excluded.identity_status, identity_slogan=excluded.identity_slogan, identity_list=excluded.identity_list, identity_slogan_ext=excluded.identity_slogan_ext, identity_live_url=excluded.identity_live_url, identity_live_type=excluded.identity_live_type, plus_is_plus=excluded.plus_is_plus, user_info_raw=excluded.user_info_raw """, ( sid, meta.get('uid'), meta.get('username'), meta.get('avatar_url'), meta.get('domain'), meta.get('created_at'), meta.get('updated_at'), m.match_id, meta.get('uuid'), meta.get('email'), meta.get('area'), meta.get('mobile'), meta.get('user_domain'), meta.get('username_audit_status'), meta.get('accid'), meta.get('team_id'), meta.get('trumpet_count'), meta.get('profile_nickname'), meta.get('profile_avatar_audit_status'), meta.get('profile_rgb_avatar_url'), meta.get('profile_photo_url'), meta.get('profile_gender'), meta.get('profile_birthday'), meta.get('profile_country_id'), meta.get('profile_region_id'), meta.get('profile_city_id'), meta.get('profile_language'), meta.get('profile_recommend_url'), meta.get('profile_group_id'), meta.get('profile_reg_source'), meta.get('status_status'), meta.get('status_expire'), meta.get('status_cancellation_status'), meta.get('status_new_user'), meta.get('status_login_banned_time'), meta.get('status_anticheat_type'), meta.get('status_flag_status1'), meta.get('status_anticheat_status'), meta.get('status_flag_honor'), meta.get('status_privacy_policy_status'), meta.get('status_csgo_frozen_exptime'), meta.get('platformexp_level'), meta.get('platformexp_exp'), meta.get('steam_account'), meta.get('steam_trade_url'), meta.get('steam_rent_id'), meta.get('trusted_credit'), meta.get('trusted_credit_level'), meta.get('trusted_score'), meta.get('trusted_status'), meta.get('trusted_credit_status'), meta.get('certify_id_type'), meta.get('certify_status'), meta.get('certify_age'), meta.get('certify_real_name'), meta.get('certify_uid_list'), meta.get('certify_audit_status'), meta.get('certify_gender'), meta.get('identity_type'), meta.get('identity_extras'), meta.get('identity_status'), meta.get('identity_slogan'), meta.get('identity_list'), meta.get('identity_slogan_ext'), meta.get('identity_live_url'), meta.get('identity_live_type'), meta.get('plus_is_plus'), meta.get('user_info_raw') )) # 2. Dim Maps (Ignore if exists) if m.map_name: cursor.execute(""" INSERT INTO dim_maps (map_name, map_desc) VALUES (?, ?) ON CONFLICT(map_name) DO UPDATE SET map_desc=excluded.map_desc """, (m.map_name, m.map_desc)) # 3. Fact Matches cursor.execute(""" INSERT OR REPLACE INTO fact_matches (match_id, match_code, map_name, start_time, end_time, duration, winner_team, score_team1, score_team2, server_ip, server_port, location, has_side_data_and_rating2, match_main_id, demo_url, game_mode, game_name, map_desc, location_full, match_mode, match_status, match_flag, status, waiver, year, season, round_total, cs_type, priority_show_type, pug10m_show_type, credit_match_status, knife_winner, knife_winner_role, most_1v2_uid, most_assist_uid, most_awp_uid, most_end_uid, most_first_kill_uid, most_headshot_uid, most_jump_uid, mvp_uid, response_code, response_message, response_status, response_timestamp, response_trace_id, response_success, response_errcode, treat_info_raw, round_list_raw, leetify_data_raw, data_source_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( m.match_id, m.match_code, m.map_name, m.start_time, m.end_time, m.duration, m.winner_team, m.score_team1, m.score_team2, m.server_ip, m.server_port, m.location, m.has_side_data_and_rating2, m.match_main_id, m.demo_url, m.game_mode, m.game_name, m.map_desc, m.location_full, m.match_mode, m.match_status, m.match_flag, m.status, m.waiver, m.year, m.season, m.round_total, m.cs_type, m.priority_show_type, m.pug10m_show_type, m.credit_match_status, m.knife_winner, m.knife_winner_role, m.most_1v2_uid, m.most_assist_uid, m.most_awp_uid, m.most_end_uid, m.most_first_kill_uid, m.most_headshot_uid, m.most_jump_uid, m.mvp_uid, m.response_code, m.response_message, m.response_status, m.response_timestamp, m.response_trace_id, m.response_success, m.response_errcode, m.treat_info_raw, m.round_list_raw, m.leetify_data_raw, m.data_source_type )) for t in m.teams: cursor.execute(""" INSERT OR REPLACE INTO fact_match_teams (match_id, group_id, group_all_score, group_change_elo, group_fh_role, group_fh_score, group_origin_elo, group_sh_role, group_sh_score, group_tid, group_uids) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( m.match_id, t.group_id, t.group_all_score, t.group_change_elo, t.group_fh_role, t.group_fh_score, t.group_origin_elo, t.group_sh_role, t.group_sh_score, t.group_tid, t.group_uids )) # 4. Fact Match Players player_columns = [ "match_id", "steam_id_64", "team_id", "kills", "deaths", "assists", "headshot_count", "kd_ratio", "adr", "rating", "rating2", "rating3", "rws", "mvp_count", "elo_change", "rank_score", "is_win", "kast", "entry_kills", "entry_deaths", "awp_kills", "clutch_1v1", "clutch_1v2", "clutch_1v3", "clutch_1v4", "clutch_1v5", "flash_assists", "flash_duration", "jump_count", "damage_total", "damage_received", "damage_receive", "damage_stats", "assisted_kill", "awp_kill", "awp_kill_ct", "awp_kill_t", "benefit_kill", "day", "defused_bomb", "end_1v1", "end_1v2", "end_1v3", "end_1v4", "end_1v5", "explode_bomb", "first_death", "fd_ct", "fd_t", "first_kill", "flash_enemy", "flash_team", "flash_team_time", "flash_time", "game_mode", "group_id", "hold_total", "id", "is_highlight", "is_most_1v2", "is_most_assist", "is_most_awp", "is_most_end", "is_most_first_kill", "is_most_headshot", "is_most_jump", "is_svp", "is_tie", "kill_1", "kill_2", "kill_3", "kill_4", "kill_5", "many_assists_cnt1", "many_assists_cnt2", "many_assists_cnt3", "many_assists_cnt4", "many_assists_cnt5", "map", "match_code", "match_mode", "match_team_id", "match_time", "per_headshot", "perfect_kill", "planted_bomb", "revenge_kill", "round_total", "season", "team_kill", "throw_harm", "throw_harm_enemy", "uid", "year", "sts_raw", "level_info_raw", "util_flash_usage", "util_smoke_usage", "util_molotov_usage", "util_he_usage", "util_decoy_usage" ] player_placeholders = ",".join(["?"] * len(player_columns)) player_columns_sql = ",".join(player_columns) def player_values(sid, p): return [ m.match_id, sid, p.team_id, p.kills, p.deaths, p.assists, p.headshot_count, p.kd_ratio, p.adr, p.rating, p.rating2, p.rating3, p.rws, p.mvp_count, p.elo_change, p.rank_score, p.is_win, p.kast, p.entry_kills, p.entry_deaths, p.awp_kills, p.clutch_1v1, p.clutch_1v2, p.clutch_1v3, p.clutch_1v4, p.clutch_1v5, p.flash_assists, p.flash_duration, p.jump_count, p.damage_total, p.damage_received, p.damage_receive, p.damage_stats, p.assisted_kill, p.awp_kill, p.awp_kill_ct, p.awp_kill_t, p.benefit_kill, p.day, p.defused_bomb, p.end_1v1, p.end_1v2, p.end_1v3, p.end_1v4, p.end_1v5, p.explode_bomb, p.first_death, p.fd_ct, p.fd_t, p.first_kill, p.flash_enemy, p.flash_team, p.flash_team_time, p.flash_time, p.game_mode, p.group_id, p.hold_total, p.id, p.is_highlight, p.is_most_1v2, p.is_most_assist, p.is_most_awp, p.is_most_end, p.is_most_first_kill, p.is_most_headshot, p.is_most_jump, p.is_svp, p.is_tie, p.kill_1, p.kill_2, p.kill_3, p.kill_4, p.kill_5, p.many_assists_cnt1, p.many_assists_cnt2, p.many_assists_cnt3, p.many_assists_cnt4, p.many_assists_cnt5, p.map, p.match_code, p.match_mode, p.match_team_id, p.match_time, p.per_headshot, p.perfect_kill, p.planted_bomb, p.revenge_kill, p.round_total, p.season, p.team_kill, p.throw_harm, p.throw_harm_enemy, p.uid, p.year, p.sts_raw, p.level_info_raw, p.util_flash_usage, p.util_smoke_usage, p.util_molotov_usage, p.util_he_usage, p.util_decoy_usage ] for sid, p in m.players.items(): cursor.execute( f"INSERT OR REPLACE INTO fact_match_players ({player_columns_sql}) VALUES ({player_placeholders})", player_values(sid, p) ) for sid, p in m.players_t.items(): cursor.execute( f"INSERT OR REPLACE INTO fact_match_players_t ({player_columns_sql}) VALUES ({player_placeholders})", player_values(sid, p) ) for sid, p in m.players_ct.items(): cursor.execute( f"INSERT OR REPLACE INTO fact_match_players_ct ({player_columns_sql}) VALUES ({player_placeholders})", player_values(sid, p) ) # 5. Rounds & Events for r in m.rounds: cursor.execute(""" INSERT OR REPLACE INTO fact_rounds (match_id, round_num, winner_side, win_reason, win_reason_desc, duration, end_time_stamp, ct_score, t_score, ct_money_start, t_money_start) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( m.match_id, r.round_num, r.winner_side, r.win_reason, r.win_reason_desc, r.duration, r.end_time_stamp, r.ct_score, r.t_score, r.ct_money_start, r.t_money_start )) for e in r.events: # Handle Pos ax, ay, az = e.attacker_pos if e.attacker_pos else (None, None, None) vx, vy, vz = e.victim_pos if e.victim_pos else (None, None, None) # Use uuid for event_id to ensure uniqueness if logic fails import uuid if not e.event_id: e.event_id = str(uuid.uuid4()) cursor.execute(""" INSERT OR REPLACE INTO fact_round_events (event_id, match_id, round_num, event_type, event_time, attacker_steam_id, victim_steam_id, weapon, is_headshot, is_wallbang, is_blind, is_through_smoke, is_noscope, trade_killer_steam_id, flash_assist_steam_id, score_change_attacker, score_change_victim, attacker_pos_x, attacker_pos_y, attacker_pos_z, victim_pos_x, victim_pos_y, victim_pos_z) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( e.event_id, m.match_id, r.round_num, e.event_type, e.event_time, e.attacker_steam_id, e.victim_steam_id, e.weapon, e.is_headshot, e.is_wallbang, e.is_blind, e.is_through_smoke, e.is_noscope, e.trade_killer_steam_id, e.flash_assist_steam_id, e.score_change_attacker, e.score_change_victim, ax, ay, az, vx, vy, vz )) for pe in r.economies: cursor.execute(""" INSERT OR REPLACE INTO fact_round_player_economy (match_id, round_num, steam_id_64, side, start_money, equipment_value, main_weapon, has_helmet, has_defuser, round_performance_score) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( m.match_id, r.round_num, pe.steam_id_64, pe.side, pe.start_money, pe.equipment_value, pe.main_weapon, pe.has_helmet, pe.has_defuser, pe.round_performance_score )) # 6. Calculate & Save Clutch Attempts _calculate_and_save_clutch_attempts(cursor, m.match_id, m.round_list_raw) def _calculate_and_save_clutch_attempts(cursor, match_id, round_list_raw): if not round_list_raw: return try: round_list = json.loads(round_list_raw) except: return player_attempts = {} for round_data in round_list: all_kills = round_data.get('all_kill', []) if not all_kills: continue team_members = {1: set(), 2: set()} # Scan for team members for k in all_kills: if k.get('attacker') and k['attacker'].get('steamid_64'): tid = k['attacker'].get('team') if tid in [1, 2]: team_members[tid].add(k['attacker']['steamid_64']) if k.get('victim') and k['victim'].get('steamid_64'): tid = k['victim'].get('team') if tid in [1, 2]: team_members[tid].add(k['victim']['steamid_64']) if not team_members[1] or not team_members[2]: continue alive = {1: team_members[1].copy(), 2: team_members[2].copy()} clutch_triggered_players = set() # Sort kills by time sorted_kills = sorted(all_kills, key=lambda x: x.get('pasttime', 0)) for k in sorted_kills: victim = k.get('victim') if not victim: continue v_sid = victim.get('steamid_64') v_team = victim.get('team') if v_team not in [1, 2] or v_sid not in alive[v_team]: continue alive[v_team].remove(v_sid) if len(alive[v_team]) == 1: survivor_sid = list(alive[v_team])[0] if survivor_sid not in clutch_triggered_players: opponent_team = 3 - v_team opponents_alive_count = len(alive[opponent_team]) if opponents_alive_count >= 1: if survivor_sid not in player_attempts: player_attempts[survivor_sid] = {'1v1': 0, '1v2': 0, '1v3': 0, '1v4': 0, '1v5': 0} n = min(opponents_alive_count, 5) key = f'1v{n}' player_attempts[survivor_sid][key] += 1 clutch_triggered_players.add(survivor_sid) # Save to DB cursor.execute(""" CREATE TABLE IF NOT EXISTS fact_match_clutch_attempts ( match_id TEXT, steam_id_64 TEXT, attempt_1v1 INTEGER DEFAULT 0, attempt_1v2 INTEGER DEFAULT 0, attempt_1v3 INTEGER DEFAULT 0, attempt_1v4 INTEGER DEFAULT 0, attempt_1v5 INTEGER DEFAULT 0, PRIMARY KEY (match_id, steam_id_64) ) """) for pid, att in player_attempts.items(): cursor.execute(""" INSERT OR REPLACE INTO fact_match_clutch_attempts (match_id, steam_id_64, attempt_1v1, attempt_1v2, attempt_1v3, attempt_1v4, attempt_1v5) VALUES (?, ?, ?, ?, ?, ?, ?) """, (match_id, pid, att['1v1'], att['1v2'], att['1v3'], att['1v4'], att['1v5'])) if __name__ == "__main__": process_matches()