yrtv/ETL/L2_Builder.py

import sqlite3
import json
import os
import sys
import logging
from dataclasses import dataclass, field
from typing import List, Dict, Optional, Any, Tuple
from datetime import datetime

# Setup logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

# Constants
L1A_DB_PATH = 'database/L1A/L1A.sqlite'
L2_DB_PATH = 'database/L2/L2_Main.sqlite'
SCHEMA_PATH = 'database/L2/schema.sql'

# --- Data Structures for Unification ---

@dataclass
class PlayerStats:
    steam_id_64: str
    team_id: int = 0
    kills: int = 0
    deaths: int = 0
    assists: int = 0
    headshot_count: int = 0
    kd_ratio: float = 0.0
    adr: float = 0.0
    rating: float = 0.0
    rating2: float = 0.0
    rating3: float = 0.0
    rws: float = 0.0
    mvp_count: int = 0
    elo_change: float = 0.0
    rank_score: int = 0
    is_win: bool = False
    
    # VIP Stats
    kast: float = 0.0
    entry_kills: int = 0
    entry_deaths: int = 0
    awp_kills: int = 0
    clutch_1v1: int = 0
    clutch_1v2: int = 0
    clutch_1v3: int = 0
    clutch_1v4: int = 0
    clutch_1v5: int = 0
    flash_assists: int = 0
    flash_duration: float = 0.0
    jump_count: int = 0
    damage_total: int = 0
    damage_received: int = 0
    damage_receive: int = 0
    damage_stats: int = 0
    assisted_kill: int = 0
    awp_kill: int = 0
    awp_kill_ct: int = 0
    awp_kill_t: int = 0
    benefit_kill: int = 0
    day: str = ""
    defused_bomb: int = 0
    end_1v1: int = 0
    end_1v2: int = 0
    end_1v3: int = 0
    end_1v4: int = 0
    end_1v5: int = 0
    explode_bomb: int = 0
    first_death: int = 0
    fd_ct: int = 0
    fd_t: int = 0
    first_kill: int = 0
    flash_enemy: int = 0
    flash_team: int = 0
    flash_team_time: float = 0.0
    flash_time: float = 0.0
    game_mode: str = ""
    group_id: int = 0
    hold_total: int = 0
    id: int = 0
    is_highlight: int = 0
    is_most_1v2: int = 0
    is_most_assist: int = 0
    is_most_awp: int = 0
    is_most_end: int = 0
    is_most_first_kill: int = 0
    is_most_headshot: int = 0
    is_most_jump: int = 0
    is_svp: int = 0
    is_tie: int = 0
    kill_1: int = 0
    kill_2: int = 0
    kill_3: int = 0
    kill_4: int = 0
    kill_5: int = 0
    many_assists_cnt1: int = 0
    many_assists_cnt2: int = 0
    many_assists_cnt3: int = 0
    many_assists_cnt4: int = 0
    many_assists_cnt5: int = 0
    map: str = ""
    match_code: str = ""
    match_mode: str = ""
    match_team_id: int = 0
    match_time: int = 0
    per_headshot: float = 0.0
    perfect_kill: int = 0
    planted_bomb: int = 0
    revenge_kill: int = 0
    round_total: int = 0
    season: str = ""
    team_kill: int = 0
    throw_harm: int = 0
    throw_harm_enemy: int = 0
    uid: int = 0
    year: str = ""
    sts_raw: str = ""
    level_info_raw: str = ""
    
    # Utility Usage
    util_flash_usage: int = 0
    util_smoke_usage: int = 0
    util_molotov_usage: int = 0
    util_he_usage: int = 0
    util_decoy_usage: int = 0

@dataclass
class RoundEvent:
    event_id: str
    event_type: str # 'kill', 'bomb_plant', etc.
    event_time: int
    attacker_steam_id: Optional[str] = None
    victim_steam_id: Optional[str] = None
    assister_steam_id: Optional[str] = None
    flash_assist_steam_id: Optional[str] = None
    trade_killer_steam_id: Optional[str] = None
    weapon: Optional[str] = None
    is_headshot: bool = False
    is_wallbang: bool = False
    is_blind: bool = False
    is_through_smoke: bool = False
    is_noscope: bool = False
    # Spatial
    attacker_pos: Optional[Tuple[int, int, int]] = None
    victim_pos: Optional[Tuple[int, int, int]] = None
    # Score
    score_change_attacker: float = 0.0
    score_change_victim: float = 0.0

@dataclass
class PlayerEconomy:
    steam_id_64: str
    side: str
    start_money: int = 0
    equipment_value: int = 0
    main_weapon: str = ""
    has_helmet: bool = False
    has_defuser: bool = False
    has_zeus: bool = False
    round_performance_score: float = 0.0

@dataclass
class RoundData:
    round_num: int
    winner_side: str
    win_reason: int
    win_reason_desc: str
    duration: float
    end_time_stamp: str
    ct_score: int
    t_score: int
    ct_money_start: int = 0
    t_money_start: int = 0
    events: List[RoundEvent] = field(default_factory=list)
    economies: List[PlayerEconomy] = field(default_factory=list)

@dataclass
class MatchTeamData:
    group_id: int
    group_all_score: int = 0
    group_change_elo: float = 0.0
    group_fh_role: int = 0
    group_fh_score: int = 0
    group_origin_elo: float = 0.0
    group_sh_role: int = 0
    group_sh_score: int = 0
    group_tid: int = 0
    group_uids: str = ""

@dataclass
class MatchData:
    match_id: str
    match_code: str = ""
    map_name: str = ""
    start_time: int = 0
    end_time: int = 0
    duration: int = 0
    winner_team: int = 0
    score_team1: int = 0
    score_team2: int = 0
    server_ip: str = ""
    server_port: int = 0
    location: str = ""
    has_side_data_and_rating2: int = 0
    match_main_id: int = 0
    demo_url: str = ""
    game_mode: int = 0
    game_name: str = ""
    map_desc: str = ""
    location_full: str = ""
    match_mode: int = 0
    match_status: int = 0
    match_flag: int = 0
    status: int = 0
    waiver: int = 0
    year: int = 0
    season: str = ""
    round_total: int = 0
    cs_type: int = 0
    priority_show_type: int = 0
    pug10m_show_type: int = 0
    credit_match_status: int = 0
    knife_winner: int = 0
    knife_winner_role: int = 0
    most_1v2_uid: int = 0
    most_assist_uid: int = 0
    most_awp_uid: int = 0
    most_end_uid: int = 0
    most_first_kill_uid: int = 0
    most_headshot_uid: int = 0
    most_jump_uid: int = 0
    mvp_uid: int = 0
    response_code: int = 0
    response_message: str = ""
    response_status: int = 0
    response_timestamp: int = 0
    response_trace_id: str = ""
    response_success: int = 0
    response_errcode: int = 0
    treat_info_raw: str = ""
    round_list_raw: str = ""
    leetify_data_raw: str = ""
    data_source_type: str = "unknown"
    players: Dict[str, PlayerStats] = field(default_factory=dict) # Key: steam_id_64
    players_t: Dict[str, PlayerStats] = field(default_factory=dict)
    players_ct: Dict[str, PlayerStats] = field(default_factory=dict)
    rounds: List[RoundData] = field(default_factory=list)
    player_meta: Dict[str, Dict] = field(default_factory=dict) # steam_id -> {uid, name, avatar, ...}
    teams: List[MatchTeamData] = field(default_factory=list)

# --- Database Helper ---

def init_db():
    if os.path.exists(L2_DB_PATH):
        logger.info(f"Removing existing L2 DB at {L2_DB_PATH}")
        try:
            os.remove(L2_DB_PATH)
        except PermissionError:
            logger.error("Cannot remove L2 DB, it might be open.")
            return False

    conn = sqlite3.connect(L2_DB_PATH)
    with open(SCHEMA_PATH, 'r', encoding='utf-8') as f:
        schema_sql = f.read()
        conn.executescript(schema_sql)
    conn.commit()
    conn.close()
    logger.info("L2 DB Initialized.")
    return True

# --- Parsers ---

class MatchParser:
    def __init__(self, match_id, raw_requests):
        self.match_id = match_id
        self.raw_requests = raw_requests
        self.match_data = MatchData(match_id=match_id)
        
        # Extracted JSON bodies
        self.data_match = None
        self.data_match_wrapper = None
        self.data_vip = None
        self.data_leetify = None
        self.data_round_list = None
        
        self._extract_payloads()

    def _extract_payloads(self):
        for req in self.raw_requests:
            url = req.get('url', '')
            body = req.get('body', {})
            
            if not body:
                continue

            # Check URLs
            if 'crane/http/api/data/match/' in url:
                self.data_match_wrapper = body
                self.data_match = body.get('data', {})
            elif 'crane/http/api/data/vip_plus_match_data/' in url:
                self.data_vip = body.get('data', {})
            elif 'crane/http/api/match/leetify_rating/' in url:
                self.data_leetify = body.get('data', {})
            elif 'crane/http/api/match/round/' in url:
                self.data_round_list = body.get('data', {})

    def parse(self) -> MatchData:
        if not self.data_match:
            logger.warning(f"No base match data found for {self.match_id}")
            return self.match_data

        self._parse_base_info()
        self._parse_players_base()
        self._parse_players_vip()
        
        # Decide which round source to use
        if self.data_leetify and self.data_leetify.get('leetify_data'):
            self.match_data.data_source_type = 'leetify'
            try:
                self.match_data.leetify_data_raw = json.dumps(self.data_leetify.get('leetify_data', {}), ensure_ascii=False)
            except:
                self.match_data.leetify_data_raw = ""
            self.match_data.round_list_raw = ""
            self._parse_leetify_rounds()
        elif self.data_round_list and self.data_round_list.get('round_list'):
            self.match_data.data_source_type = 'classic'
            try:
                self.match_data.round_list_raw = json.dumps(self.data_round_list.get('round_list', []), ensure_ascii=False)
            except:
                self.match_data.round_list_raw = ""
            self.match_data.leetify_data_raw = ""
            self._parse_classic_rounds()
        else:
            self.match_data.data_source_type = 'unknown'
            self.match_data.round_list_raw = ""
            self.match_data.leetify_data_raw = ""
            logger.info(f"No round data found for {self.match_id}")

        return self.match_data

    def _parse_base_info(self):
        m = self.data_match.get('main', {})
        self.match_data.match_code = m.get('match_code', '')
        self.match_data.map_name = m.get('map', '')
        self.match_data.start_time = m.get('start_time', 0)
        self.match_data.end_time = m.get('end_time', 0)
        self.match_data.duration = self.match_data.end_time - self.match_data.start_time if self.match_data.end_time else 0
        self.match_data.winner_team = m.get('match_winner', 0)
        self.match_data.score_team1 = m.get('group1_all_score', 0)
        self.match_data.score_team2 = m.get('group2_all_score', 0)
        self.match_data.server_ip = m.get('server_ip', '')
        # Port is sometimes string
        try:
            self.match_data.server_port = int(m.get('server_port', 0))
        except:
            self.match_data.server_port = 0
        self.match_data.location = m.get('location', '')
        def safe_int(val):
            try:
                return int(float(val)) if val is not None else 0
            except:
                return 0
        def safe_float(val):
            try:
                return float(val) if val is not None else 0.0
            except:
                return 0.0
        def safe_text(val):
            return "" if val is None else str(val)
        wrapper = self.data_match_wrapper or {}
        self.match_data.response_code = safe_int(wrapper.get('code'))
        self.match_data.response_message = safe_text(wrapper.get('message'))
        self.match_data.response_status = safe_int(wrapper.get('status'))
        self.match_data.response_timestamp = safe_int(wrapper.get('timeStamp') if wrapper.get('timeStamp') is not None else wrapper.get('timestamp'))
        self.match_data.response_trace_id = safe_text(wrapper.get('traceId') if wrapper.get('traceId') is not None else wrapper.get('trace_id'))
        self.match_data.response_success = safe_int(wrapper.get('success'))
        self.match_data.response_errcode = safe_int(wrapper.get('errcode'))
        self.match_data.has_side_data_and_rating2 = safe_int(self.data_match.get('has_side_data_and_rating2'))
        self.match_data.match_main_id = safe_int(m.get('id'))
        self.match_data.demo_url = safe_text(m.get('demo_url'))
        self.match_data.game_mode = safe_int(m.get('game_mode'))
        self.match_data.game_name = safe_text(m.get('game_name'))
        self.match_data.map_desc = safe_text(m.get('map_desc'))
        self.match_data.location_full = safe_text(m.get('location_full'))
        self.match_data.match_mode = safe_int(m.get('match_mode'))
        self.match_data.match_status = safe_int(m.get('match_status'))
        self.match_data.match_flag = safe_int(m.get('match_flag'))
        self.match_data.status = safe_int(m.get('status'))
        self.match_data.waiver = safe_int(m.get('waiver'))
        self.match_data.year = safe_int(m.get('year'))
        self.match_data.season = safe_text(m.get('season'))
        self.match_data.round_total = safe_int(m.get('round_total'))
        self.match_data.cs_type = safe_int(m.get('cs_type'))
        self.match_data.priority_show_type = safe_int(m.get('priority_show_type'))
        self.match_data.pug10m_show_type = safe_int(m.get('pug10m_show_type'))
        self.match_data.credit_match_status = safe_int(m.get('credit_match_status'))
        self.match_data.knife_winner = safe_int(m.get('knife_winner'))
        self.match_data.knife_winner_role = safe_int(m.get('knife_winner_role'))
        self.match_data.most_1v2_uid = safe_int(m.get('most_1v2_uid'))
        self.match_data.most_assist_uid = safe_int(m.get('most_assist_uid'))
        self.match_data.most_awp_uid = safe_int(m.get('most_awp_uid'))
        self.match_data.most_end_uid = safe_int(m.get('most_end_uid'))
        self.match_data.most_first_kill_uid = safe_int(m.get('most_first_kill_uid'))
        self.match_data.most_headshot_uid = safe_int(m.get('most_headshot_uid'))
        self.match_data.most_jump_uid = safe_int(m.get('most_jump_uid'))
        self.match_data.mvp_uid = safe_int(m.get('mvp_uid'))
        treat_info = self.data_match.get('treat_info')
        if treat_info is not None:
            try:
                self.match_data.treat_info_raw = json.dumps(treat_info, ensure_ascii=False)
            except:
                self.match_data.treat_info_raw = ""
        self.match_data.teams = []
        for idx in [1, 2]:
            team = MatchTeamData(
                group_id=idx,
                group_all_score=safe_int(m.get(f"group{idx}_all_score")),
                group_change_elo=safe_float(m.get(f"group{idx}_change_elo")),
                group_fh_role=safe_int(m.get(f"group{idx}_fh_role")),
                group_fh_score=safe_int(m.get(f"group{idx}_fh_score")),
                group_origin_elo=safe_float(m.get(f"group{idx}_origin_elo")),
                group_sh_role=safe_int(m.get(f"group{idx}_sh_role")),
                group_sh_score=safe_int(m.get(f"group{idx}_sh_score")),
                group_tid=safe_int(m.get(f"group{idx}_tid")),
                group_uids=safe_text(m.get(f"group{idx}_uids"))
            )
            self.match_data.teams.append(team)

    def _parse_players_base(self):
        # Players are in group_1 and group_2 lists in data_match
        groups = []
        if 'group_1' in self.data_match: groups.extend(self.data_match['group_1'])
        if 'group_2' in self.data_match: groups.extend(self.data_match['group_2'])
        def safe_int(val):
            try:
                return int(float(val)) if val is not None else 0
            except:
                return 0
        def safe_text(val):
            return "" if val is None else str(val)

        for p in groups:
            # We need steam_id. 
            # Structure: user_info -> user_data -> steam -> steamId
            user_info = p.get('user_info', {})
            user_data = user_info.get('user_data', {})
            steam_data = user_data.get('steam', {})
            steam_id = str(steam_data.get('steamId', ''))
            
            fight = p.get('fight', {})
            fight_t = p.get('fight_t', {})
            fight_ct = p.get('fight_ct', {})
            uid = fight.get('uid')
            
            # Store meta for dim_players
            user_data = user_info.get('user_data', {})
            profile = user_data.get('profile', {})
            
            # If steam_id is empty, use temporary placeholder '5E:{uid}'
            # Ideally we want steam_id_64.
            if not steam_id and uid:
                steam_id = f"5E:{uid}"
            
            if not steam_id:
                continue

            status = user_data.get('status', {})
            platform_exp = user_data.get('platformExp', {})
            trusted = user_data.get('trusted', {})
            certify = user_data.get('certify', {})
            identity = user_data.get('identity', {})
            plus_info = user_info.get('plus_info', {}) or p.get('plus_info', {})
            user_info_raw = ""
            try:
                user_info_raw = json.dumps(user_info, ensure_ascii=False)
            except:
                user_info_raw = ""

            self.match_data.player_meta[steam_id] = {
                'uid': safe_int(uid),
                'username': safe_text(user_data.get('username')),
                'uuid': safe_text(user_data.get('uuid')),
                'email': safe_text(user_data.get('email')),
                'area': safe_text(user_data.get('area')),
                'mobile': safe_text(user_data.get('mobile')),
                'avatar_url': safe_text(profile.get('avatarUrl')),
                'domain': safe_text(profile.get('domain')),
                'user_domain': safe_text(user_data.get('domain')),
                'created_at': safe_int(user_data.get('createdAt')),
                'updated_at': safe_int(user_data.get('updatedAt')),
                'username_audit_status': safe_int(user_data.get('usernameAuditStatus')),
                'accid': safe_text(user_data.get('Accid')),
                'team_id': safe_int(user_data.get('teamID')),
                'trumpet_count': safe_int(user_data.get('trumpetCount')),
                'profile_nickname': safe_text(profile.get('nickname')),
                'profile_avatar_audit_status': safe_int(profile.get('avatarAuditStatus')),
                'profile_rgb_avatar_url': safe_text(profile.get('rgbAvatarUrl')),
                'profile_photo_url': safe_text(profile.get('photoUrl')),
                'profile_gender': safe_int(profile.get('gender')),
                'profile_birthday': safe_int(profile.get('birthday')),
                'profile_country_id': safe_text(profile.get('countryId')),
                'profile_region_id': safe_text(profile.get('regionId')),
                'profile_city_id': safe_text(profile.get('cityId')),
                'profile_language': safe_text(profile.get('language')),
                'profile_recommend_url': safe_text(profile.get('recommendUrl')),
                'profile_group_id': safe_int(profile.get('groupId')),
                'profile_reg_source': safe_int(profile.get('regSource')),
                'status_status': safe_int(status.get('status')),
                'status_expire': safe_int(status.get('expire')),
                'status_cancellation_status': safe_int(status.get('cancellationStatus')),
                'status_new_user': safe_int(status.get('newUser')),
                'status_login_banned_time': safe_int(status.get('loginBannedTime')),
                'status_anticheat_type': safe_int(status.get('anticheatType')),
                'status_flag_status1': safe_text(status.get('flagStatus1')),
                'status_anticheat_status': safe_text(status.get('anticheatStatus')),
                'status_flag_honor': safe_text(status.get('FlagHonor')),
                'status_privacy_policy_status': safe_int(status.get('PrivacyPolicyStatus')),
                'status_csgo_frozen_exptime': safe_int(status.get('csgoFrozenExptime')),
                'platformexp_level': safe_int(platform_exp.get('level')),
                'platformexp_exp': safe_int(platform_exp.get('exp')),
                'steam_account': safe_text(steam_data.get('steamAccount')),
                'steam_trade_url': safe_text(steam_data.get('tradeUrl')),
                'steam_rent_id': safe_text(steam_data.get('rentSteamId')),
                'trusted_credit': safe_int(trusted.get('credit')),
                'trusted_credit_level': safe_int(trusted.get('creditLevel')),
                'trusted_score': safe_int(trusted.get('score')),
                'trusted_status': safe_int(trusted.get('status')),
                'trusted_credit_status': safe_int(trusted.get('creditStatus')),
                'certify_id_type': safe_int(certify.get('idType')),
                'certify_status': safe_int(certify.get('status')),
                'certify_age': safe_int(certify.get('age')),
                'certify_real_name': safe_text(certify.get('realName')),
                'certify_uid_list': safe_text(json.dumps(certify.get('uidList'), ensure_ascii=False)) if certify.get('uidList') is not None else "",
                'certify_audit_status': safe_int(certify.get('auditStatus')),
                'certify_gender': safe_int(certify.get('gender')),
                'identity_type': safe_int(identity.get('type')),
                'identity_extras': safe_text(identity.get('extras')),
                'identity_status': safe_int(identity.get('status')),
                'identity_slogan': safe_text(identity.get('slogan')),
                'identity_list': safe_text(json.dumps(identity.get('identity_list'), ensure_ascii=False)) if identity.get('identity_list') is not None else "",
                'identity_slogan_ext': safe_text(identity.get('slogan_ext')),
                'identity_live_url': safe_text(identity.get('live_url')),
                'identity_live_type': safe_int(identity.get('live_type')),
                'plus_is_plus': safe_int(plus_info.get('is_plus')),
                'user_info_raw': user_info_raw
            }

            stats = PlayerStats(steam_id_64=steam_id)
            sts = p.get('sts', {})
            level_info = p.get('level_info', {})

            try:
                # Use safe conversion helper
                def safe_int(val):
                    try: return int(float(val)) if val is not None else 0
                    except: return 0
                
                def safe_float(val):
                    try: return float(val) if val is not None else 0.0
                    except: return 0.0
                
                def safe_text(val):
                    return "" if val is None else str(val)
                if sts is not None:
                    try:
                        stats.sts_raw = json.dumps(sts, ensure_ascii=False)
                    except:
                        stats.sts_raw = ""
                if level_info is not None:
                    try:
                        stats.level_info_raw = json.dumps(level_info, ensure_ascii=False)
                    except:
                        stats.level_info_raw = ""

                def get_stat(key):
                    if key in fight and fight.get(key) not in [None, ""]:
                        return fight.get(key)
                    return 0

                def build_side_stats(fight_side, team_id_value):
                    side_stats = PlayerStats(steam_id_64=steam_id)
                    side_stats.team_id = team_id_value
                    side_stats.kills = safe_int(fight_side.get('kill'))
                    side_stats.deaths = safe_int(fight_side.get('death'))
                    side_stats.assists = safe_int(fight_side.get('assist'))
                    side_stats.headshot_count = safe_int(fight_side.get('headshot'))
                    side_stats.adr = safe_float(fight_side.get('adr'))
                    side_stats.rating = safe_float(fight_side.get('rating'))
                    side_stats.rating2 = safe_float(fight_side.get('rating2'))
                    side_stats.rating3 = safe_float(fight_side.get('rating3'))
                    side_stats.rws = safe_float(fight_side.get('rws'))
                    side_stats.kast = safe_float(fight_side.get('kast'))
                    side_stats.mvp_count = safe_int(fight_side.get('is_mvp'))
                    side_stats.flash_duration = safe_float(fight_side.get('flash_enemy_time'))
                    side_stats.jump_count = safe_int(fight_side.get('jump_total'))
                    side_stats.is_win = bool(safe_int(fight_side.get('is_win')))
                    side_stats.assisted_kill = safe_int(fight_side.get('assisted_kill'))
                    side_stats.awp_kill = safe_int(fight_side.get('awp_kill'))
                    side_stats.benefit_kill = safe_int(fight_side.get('benefit_kill'))
                    side_stats.day = safe_text(fight_side.get('day'))
                    side_stats.defused_bomb = safe_int(fight_side.get('defused_bomb'))
                    side_stats.end_1v1 = safe_int(fight_side.get('end_1v1'))
                    side_stats.end_1v2 = safe_int(fight_side.get('end_1v2'))
                    side_stats.end_1v3 = safe_int(fight_side.get('end_1v3'))
                    side_stats.end_1v4 = safe_int(fight_side.get('end_1v4'))
                    side_stats.end_1v5 = safe_int(fight_side.get('end_1v5'))
                    side_stats.explode_bomb = safe_int(fight_side.get('explode_bomb'))
                    side_stats.first_death = safe_int(fight_side.get('first_death'))
                    side_stats.first_kill = safe_int(fight_side.get('first_kill'))
                    side_stats.flash_enemy = safe_int(fight_side.get('flash_enemy'))
                    side_stats.flash_team = safe_int(fight_side.get('flash_team'))
                    side_stats.flash_team_time = safe_float(fight_side.get('flash_team_time'))
                    side_stats.flash_time = safe_float(fight_side.get('flash_time'))
                    side_stats.game_mode = safe_text(fight_side.get('game_mode'))
                    side_stats.group_id = safe_int(fight_side.get('group_id'))
                    side_stats.hold_total = safe_int(fight_side.get('hold_total'))
                    side_stats.id = safe_int(fight_side.get('id'))
                    side_stats.is_highlight = safe_int(fight_side.get('is_highlight'))
                    side_stats.is_most_1v2 = safe_int(fight_side.get('is_most_1v2'))
                    side_stats.is_most_assist = safe_int(fight_side.get('is_most_assist'))
                    side_stats.is_most_awp = safe_int(fight_side.get('is_most_awp'))
                    side_stats.is_most_end = safe_int(fight_side.get('is_most_end'))
                    side_stats.is_most_first_kill = safe_int(fight_side.get('is_most_first_kill'))
                    side_stats.is_most_headshot = safe_int(fight_side.get('is_most_headshot'))
                    side_stats.is_most_jump = safe_int(fight_side.get('is_most_jump'))
                    side_stats.is_svp = safe_int(fight_side.get('is_svp'))
                    side_stats.is_tie = safe_int(fight_side.get('is_tie'))
                    side_stats.kill_1 = safe_int(fight_side.get('kill_1'))
                    side_stats.kill_2 = safe_int(fight_side.get('kill_2'))
                    side_stats.kill_3 = safe_int(fight_side.get('kill_3'))
                    side_stats.kill_4 = safe_int(fight_side.get('kill_4'))
                    side_stats.kill_5 = safe_int(fight_side.get('kill_5'))
                    side_stats.many_assists_cnt1 = safe_int(fight_side.get('many_assists_cnt1'))
                    side_stats.many_assists_cnt2 = safe_int(fight_side.get('many_assists_cnt2'))
                    side_stats.many_assists_cnt3 = safe_int(fight_side.get('many_assists_cnt3'))
                    side_stats.many_assists_cnt4 = safe_int(fight_side.get('many_assists_cnt4'))
                    side_stats.many_assists_cnt5 = safe_int(fight_side.get('many_assists_cnt5'))
                    side_stats.map = safe_text(fight_side.get('map'))
                    side_stats.match_code = safe_text(fight_side.get('match_code'))
                    side_stats.match_mode = safe_text(fight_side.get('match_mode'))
                    side_stats.match_team_id = safe_int(fight_side.get('match_team_id'))
                    side_stats.match_time = safe_int(fight_side.get('match_time'))
                    side_stats.per_headshot = safe_float(fight_side.get('per_headshot'))
                    side_stats.perfect_kill = safe_int(fight_side.get('perfect_kill'))
                    side_stats.planted_bomb = safe_int(fight_side.get('planted_bomb'))
                    side_stats.revenge_kill = safe_int(fight_side.get('revenge_kill'))
                    side_stats.round_total = safe_int(fight_side.get('round_total'))
                    side_stats.season = safe_text(fight_side.get('season'))
                    side_stats.team_kill = safe_int(fight_side.get('team_kill'))
                    side_stats.throw_harm = safe_int(fight_side.get('throw_harm'))
                    side_stats.throw_harm_enemy = safe_int(fight_side.get('throw_harm_enemy'))
                    side_stats.uid = safe_int(fight_side.get('uid'))
                    side_stats.year = safe_text(fight_side.get('year'))

                    # Map missing fields
                    side_stats.clutch_1v1 = side_stats.end_1v1
                    side_stats.clutch_1v2 = side_stats.end_1v2
                    side_stats.clutch_1v3 = side_stats.end_1v3
                    side_stats.clutch_1v4 = side_stats.end_1v4
                    side_stats.clutch_1v5 = side_stats.end_1v5
                    side_stats.entry_kills = side_stats.first_kill
                    side_stats.entry_deaths = side_stats.first_death

                    return side_stats

                team_id_value = safe_int(fight.get('match_team_id'))
                stats.team_id = team_id_value
                stats.kills = safe_int(get_stat('kill'))
                stats.deaths = safe_int(get_stat('death'))
                
                # Force calculate K/D
                if stats.deaths > 0:
                    stats.kd_ratio = stats.kills / stats.deaths
                else:
                    stats.kd_ratio = float(stats.kills)
                    
                stats.assists = safe_int(get_stat('assist'))
                stats.headshot_count = safe_int(get_stat('headshot'))
                
                stats.adr = safe_float(get_stat('adr'))
                stats.rating = safe_float(get_stat('rating'))
                stats.rating2 = safe_float(get_stat('rating2'))
                stats.rating3 = safe_float(get_stat('rating3'))
                stats.rws = safe_float(get_stat('rws'))
                
                # is_mvp might be string "1" or int 1
                stats.mvp_count = safe_int(get_stat('is_mvp'))
                
                stats.flash_duration = safe_float(get_stat('flash_enemy_time'))
                stats.jump_count = safe_int(get_stat('jump_total'))
                stats.is_win = bool(safe_int(get_stat('is_win')))
                
                stats.elo_change = safe_float(sts.get('change_elo'))
                stats.rank_score = safe_int(sts.get('rank'))
                stats.assisted_kill = safe_int(fight.get('assisted_kill'))
                stats.awp_kill = safe_int(fight.get('awp_kill'))
                stats.benefit_kill = safe_int(fight.get('benefit_kill'))
                stats.day = safe_text(fight.get('day'))
                stats.defused_bomb = safe_int(fight.get('defused_bomb'))
                stats.end_1v1 = safe_int(fight.get('end_1v1'))
                stats.end_1v2 = safe_int(fight.get('end_1v2'))
                stats.end_1v3 = safe_int(fight.get('end_1v3'))
                stats.end_1v4 = safe_int(fight.get('end_1v4'))
                stats.end_1v5 = safe_int(fight.get('end_1v5'))
                stats.explode_bomb = safe_int(fight.get('explode_bomb'))
                stats.first_death = safe_int(fight.get('first_death'))
                stats.first_kill = safe_int(fight.get('first_kill'))
                stats.flash_enemy = safe_int(fight.get('flash_enemy'))
                stats.flash_team = safe_int(fight.get('flash_team'))
                stats.flash_team_time = safe_float(fight.get('flash_team_time'))
                stats.flash_time = safe_float(fight.get('flash_time'))
                stats.game_mode = safe_text(fight.get('game_mode'))
                stats.group_id = safe_int(fight.get('group_id'))
                stats.hold_total = safe_int(fight.get('hold_total'))
                stats.id = safe_int(fight.get('id'))
                stats.is_highlight = safe_int(fight.get('is_highlight'))
                stats.is_most_1v2 = safe_int(fight.get('is_most_1v2'))
                stats.is_most_assist = safe_int(fight.get('is_most_assist'))
                stats.is_most_awp = safe_int(fight.get('is_most_awp'))
                stats.is_most_end = safe_int(fight.get('is_most_end'))
                stats.is_most_first_kill = safe_int(fight.get('is_most_first_kill'))
                stats.is_most_headshot = safe_int(fight.get('is_most_headshot'))
                stats.is_most_jump = safe_int(fight.get('is_most_jump'))
                stats.is_svp = safe_int(fight.get('is_svp'))
                stats.is_tie = safe_int(fight.get('is_tie'))
                stats.kill_1 = safe_int(fight.get('kill_1'))
                stats.kill_2 = safe_int(fight.get('kill_2'))
                stats.kill_3 = safe_int(fight.get('kill_3'))
                stats.kill_4 = safe_int(fight.get('kill_4'))
                stats.kill_5 = safe_int(fight.get('kill_5'))
                stats.many_assists_cnt1 = safe_int(fight.get('many_assists_cnt1'))
                stats.many_assists_cnt2 = safe_int(fight.get('many_assists_cnt2'))
                stats.many_assists_cnt3 = safe_int(fight.get('many_assists_cnt3'))
                stats.many_assists_cnt4 = safe_int(fight.get('many_assists_cnt4'))
                stats.many_assists_cnt5 = safe_int(fight.get('many_assists_cnt5'))
                stats.map = safe_text(fight.get('map'))
                stats.match_code = safe_text(fight.get('match_code'))
                stats.match_mode = safe_text(fight.get('match_mode'))
                stats.match_team_id = safe_int(fight.get('match_team_id'))
                stats.match_time = safe_int(fight.get('match_time'))
                stats.per_headshot = safe_float(fight.get('per_headshot'))
                stats.perfect_kill = safe_int(fight.get('perfect_kill'))
                stats.planted_bomb = safe_int(fight.get('planted_bomb'))
                stats.revenge_kill = safe_int(fight.get('revenge_kill'))
                stats.round_total = safe_int(fight.get('round_total'))
                stats.season = safe_text(fight.get('season'))
                stats.team_kill = safe_int(fight.get('team_kill'))
                stats.throw_harm = safe_int(fight.get('throw_harm'))
                stats.throw_harm_enemy = safe_int(fight.get('throw_harm_enemy'))
                stats.uid = safe_int(fight.get('uid'))
                stats.year = safe_text(fight.get('year'))
                
                # Map missing fields
                stats.clutch_1v1 = stats.end_1v1
                stats.clutch_1v2 = stats.end_1v2
                stats.clutch_1v3 = stats.end_1v3
                stats.clutch_1v4 = stats.end_1v4
                stats.clutch_1v5 = stats.end_1v5
                stats.entry_kills = stats.first_kill
                stats.entry_deaths = stats.first_death

            except Exception as e:
                logger.error(f"Error parsing stats for {steam_id} in {self.match_id}: {e}")
                pass

            self.match_data.players[steam_id] = stats
            if isinstance(fight_t, dict) and fight_t:
                t_team_id = team_id_value or safe_int(fight_t.get('match_team_id'))
                self.match_data.players_t[steam_id] = build_side_stats(fight_t, t_team_id)
            if isinstance(fight_ct, dict) and fight_ct:
                ct_team_id = team_id_value or safe_int(fight_ct.get('match_team_id'))
                self.match_data.players_ct[steam_id] = build_side_stats(fight_ct, ct_team_id)

    def _parse_players_vip(self):
        if not self.data_vip:
            return

        # Structure: data_vip -> steamid (key) -> dict
        for sid, vdata in self.data_vip.items():
            # SID might be steam_id_64 directly
            if sid in self.match_data.players:
                p = self.match_data.players[sid]
                p.kast = float(vdata.get('kast', 0))
                p.awp_kills = int(vdata.get('awp_kill', 0))
                p.awp_kill_ct = int(vdata.get('awp_kill_ct', 0))
                p.awp_kill_t = int(vdata.get('awp_kill_t', 0))
                p.fd_ct = int(vdata.get('fd_ct', 0))
                p.fd_t = int(vdata.get('fd_t', 0))
                p.damage_receive = int(vdata.get('damage_receive', 0))
                p.damage_stats = int(vdata.get('damage_stats', 0))
                p.damage_total = int(vdata.get('damage_total', 0))
                p.damage_received = int(vdata.get('damage_received', 0))
                p.flash_assists = int(vdata.get('flash_assists', 0))
            else:
                # Try to match by 5E ID if possible, but here keys are steamids usually
                pass
        for sid, p in self.match_data.players.items():
            if sid in self.match_data.players_t:
                self.match_data.players_t[sid].awp_kill_t = p.awp_kill_t
                self.match_data.players_t[sid].fd_t = p.fd_t
            if sid in self.match_data.players_ct:
                self.match_data.players_ct[sid].awp_kill_ct = p.awp_kill_ct
                self.match_data.players_ct[sid].fd_ct = p.fd_ct

    def _parse_leetify_rounds(self):
        l_data = self.data_leetify.get('leetify_data', {})
        round_list = l_data.get('round_stat', [])
        
        for idx, r in enumerate(round_list):
            # Utility Usage (Leetify)
            bron = r.get('bron_equipment', {})
            for sid, items in bron.items():
                sid = str(sid)
                if sid in self.match_data.players:
                    p = self.match_data.players[sid]
                    if isinstance(items, list):
                        for item in items:
                            if not isinstance(item, dict): continue
                            name = item.get('WeaponName', '')
                            if name == 'weapon_flashbang': p.util_flash_usage += 1
                            elif name == 'weapon_smokegrenade': p.util_smoke_usage += 1
                            elif name in ['weapon_molotov', 'weapon_incgrenade']: p.util_molotov_usage += 1
                            elif name == 'weapon_hegrenade': p.util_he_usage += 1
                            elif name == 'weapon_decoy': p.util_decoy_usage += 1

            rd = RoundData(
                round_num=r.get('round', idx + 1),
                winner_side='CT' if r.get('win_reason') in [7, 8, 9] else 'T', # Approximate logic, need real enum
                win_reason=r.get('win_reason', 0),
                win_reason_desc=str(r.get('win_reason', 0)),
                duration=0, # Leetify might not have exact duration easily
                end_time_stamp=r.get('end_ts', ''),
                ct_score=r.get('sfui_event', {}).get('score_ct', 0),
                t_score=r.get('sfui_event', {}).get('score_t', 0),
                ct_money_start=r.get('ct_money_group', 0),
                t_money_start=r.get('t_money_group', 0)
            )
            
            # Events
            # Leetify has 'show_event' list
            events = r.get('show_event', [])
            for evt in events:
                e_type_code = evt.get('event_type')
                # Mapping needed for event types. 
                # Assuming 3 is kill based on schema 'kill_event' presence
                
                if evt.get('kill_event'):
                    k = evt['kill_event']
                    re = RoundEvent(
                        event_id=f"{self.match_id}_{rd.round_num}_{k.get('Ts', '')}_{k.get('Killer')}",
                        event_type='kill',
                        event_time=evt.get('ts', 0),
                        attacker_steam_id=k.get('Killer'),
                        victim_steam_id=k.get('Victim'),
                        weapon=k.get('WeaponName'),
                        is_headshot=k.get('Headshot', False),
                        is_wallbang=k.get('Penetrated', False),
                        is_blind=k.get('AttackerBlind', False),
                        is_through_smoke=k.get('ThroughSmoke', False),
                        is_noscope=k.get('NoScope', False)
                    )
                    
                    # Leetify specifics
                    # Trade?
                    if evt.get('trade_score_change'):
                        re.trade_killer_steam_id = list(evt['trade_score_change'].keys())[0]
                    
                    if evt.get('assist_killer_score_change'):
                        re.assister_steam_id = list(evt['assist_killer_score_change'].keys())[0]

                    if evt.get('flash_assist_killer_score_change'):
                        re.flash_assist_steam_id = list(evt['flash_assist_killer_score_change'].keys())[0]
                        
                    # Score changes
                    if evt.get('killer_score_change'):
                        # e.g. {'<steamid>': {'score': 17.0}}
                        vals = list(evt['killer_score_change'].values())
                        if vals: re.score_change_attacker = vals[0].get('score', 0)
                    
                    if evt.get('victim_score_change'):
                        vals = list(evt['victim_score_change'].values())
                        if vals: re.score_change_victim = vals[0].get('score', 0)

                    rd.events.append(re)

            bron_equipment = r.get('bron_equipment') or {}
            player_t_score = r.get('player_t_score') or {}
            player_ct_score = r.get('player_ct_score') or {}
            player_bron_crash = r.get('player_bron_crash') or {}

            def pick_main_weapon(items):
                if not isinstance(items, list):
                    return ""
                ignore = {
                    "weapon_knife",
                    "weapon_knife_t",
                    "weapon_knife_gg",
                    "weapon_knife_ct",
                    "weapon_c4",
                    "weapon_flashbang",
                    "weapon_hegrenade",
                    "weapon_smokegrenade",
                    "weapon_molotov",
                    "weapon_incgrenade",
                    "weapon_decoy"
                }
                for it in items:
                    if not isinstance(it, dict):
                        continue
                    name = it.get('WeaponName')
                    if name and name not in ignore:
                        return name
                for it in items:
                    if not isinstance(it, dict):
                        continue
                    name = it.get('WeaponName')
                    if name:
                        return name
                return ""

            def pick_money(items):
                if not isinstance(items, list):
                    return 0
                vals = []
                for it in items:
                    if isinstance(it, dict) and it.get('Money') is not None:
                        vals.append(it.get('Money'))
                return int(max(vals)) if vals else 0

            side_scores = {}
            for sid, val in player_t_score.items():
                side_scores[str(sid)] = ("T", float(val) if val is not None else 0.0)
            for sid, val in player_ct_score.items():
                side_scores[str(sid)] = ("CT", float(val) if val is not None else 0.0)

            for sid in set(list(side_scores.keys()) + [str(k) for k in bron_equipment.keys()]):
                if sid not in side_scores:
                    continue
                side, score = side_scores[sid]
                items = bron_equipment.get(sid) or bron_equipment.get(str(sid)) or []
                start_money = pick_money(items)
                equipment_value = player_bron_crash.get(sid)
                if equipment_value is None:
                    equipment_value = player_bron_crash.get(str(sid))
                equipment_value = int(equipment_value) if equipment_value is not None else 0
                main_weapon = pick_main_weapon(items)
                
                has_helmet = False
                has_defuser = False
                has_zeus = False
                if isinstance(items, list):
                    for it in items:
                        if isinstance(it, dict):
                            name = it.get('WeaponName', '')
                            if name == 'item_assaultsuit':
                                has_helmet = True
                            elif name == 'item_defuser':
                                has_defuser = True
                            elif name and ('taser' in name or 'zeus' in name):
                                has_zeus = True

                rd.economies.append(PlayerEconomy(
                    steam_id_64=str(sid),
                    side=side,
                    start_money=start_money,
                    equipment_value=equipment_value,
                    main_weapon=main_weapon,
                    has_helmet=has_helmet,
                    has_defuser=has_defuser,
                    has_zeus=has_zeus,
                    round_performance_score=float(score)
                ))

            self.match_data.rounds.append(rd)

    def _parse_classic_rounds(self):
        r_list = self.data_round_list.get('round_list', [])
        for idx, r in enumerate(r_list):
            # Classic round data often lacks score/winner in the list root?
            # Check schema: 'current_score' -> ct/t
            cur_score = r.get('current_score', {})
            
            # Utility Usage (Classic)
            equiped = r.get('equiped', {})
            for sid, items in equiped.items():
                # Ensure sid is string
                sid = str(sid)
                if sid in self.match_data.players:
                    p = self.match_data.players[sid]
                    if isinstance(items, list):
                        for item in items:
                            if item == 'flashbang': p.util_flash_usage += 1
                            elif item == 'smokegrenade': p.util_smoke_usage += 1
                            elif item in ['molotov', 'incgrenade']: p.util_molotov_usage += 1
                            elif item == 'hegrenade': p.util_he_usage += 1
                            elif item == 'decoy': p.util_decoy_usage += 1

            rd = RoundData(
                round_num=idx + 1,
                winner_side='None', # Default to None if unknown
                win_reason=0,
                win_reason_desc='',
                duration=float(cur_score.get('final_round_time', 0)),
                end_time_stamp='',
                ct_score=cur_score.get('ct', 0),
                t_score=cur_score.get('t', 0)
            )

            # Kills
            # Classic has 'all_kill' list
            kills = r.get('all_kill', [])
            for k in kills:
                attacker = k.get('attacker', {})
                victim = k.get('victim', {})
                
                # Pos extraction
                apos = attacker.get('pos', {})
                vpos = victim.get('pos', {})
                
                re = RoundEvent(
                    event_id=f"{self.match_id}_{rd.round_num}_{k.get('pasttime')}_{attacker.get('steamid_64')}",
                    event_type='kill',
                    event_time=k.get('pasttime', 0),
                    attacker_steam_id=str(attacker.get('steamid_64', '')),
                    victim_steam_id=str(victim.get('steamid_64', '')),
                    weapon=k.get('weapon', ''),
                    is_headshot=k.get('headshot', False),
                    is_wallbang=k.get('penetrated', False),
                    is_blind=k.get('attackerblind', False),
                    is_through_smoke=k.get('throughsmoke', False),
                    is_noscope=k.get('noscope', False),
                    attacker_pos=(apos.get('x', 0), apos.get('y', 0), apos.get('z', 0)),
                    victim_pos=(vpos.get('x', 0), vpos.get('y', 0), vpos.get('z', 0))
                )
                rd.events.append(re)

            c4_events = r.get('c4_event', [])
            for e in c4_events:
                if not isinstance(e, dict):
                    continue
                event_name = str(e.get('event_name') or '').lower()
                if not event_name:
                    continue
                if 'plant' in event_name:
                    etype = 'bomb_plant'
                elif 'defus' in event_name:
                    etype = 'bomb_defuse'
                else:
                    continue
                sid = e.get('steamid_64')
                re = RoundEvent(
                    event_id=f"{self.match_id}_{rd.round_num}_{etype}_{e.get('pasttime', 0)}_{sid}",
                    event_type=etype,
                    event_time=int(e.get('pasttime', 0) or 0),
                    attacker_steam_id=str(sid) if sid is not None else None,
                )
                rd.events.append(re)
            
            self.match_data.rounds.append(rd)

# --- Main Execution ---

def process_matches():
    if not init_db():
        return

    l1_conn = sqlite3.connect(L1A_DB_PATH)
    l1_cursor = l1_conn.cursor()
    
    l2_conn = sqlite3.connect(L2_DB_PATH)
    l2_cursor = l2_conn.cursor()

    logger.info("Reading from L1A...")
    l1_cursor.execute("SELECT match_id, content FROM raw_iframe_network")
    
    count = 0
    while True:
        rows = l1_cursor.fetchmany(10)
        if not rows:
            break
            
        for row in rows:
            match_id, content = row
            try:
                raw_requests = json.loads(content)
                parser = MatchParser(match_id, raw_requests)
                match_data = parser.parse()
                save_match(l2_cursor, match_data)
                count += 1
                if count % 10 == 0:
                    l2_conn.commit()
                    print(f"Processed {count} matches...", end='\r')
            except Exception as e:
                logger.error(f"Error processing match {match_id}: {e}")
                # continue

    l2_conn.commit()
    l1_conn.close()
    l2_conn.close()
    logger.info(f"\nDone. Processed {count} matches.")

def save_match(cursor, m: MatchData):
    # 1. Dim Players (Upsert)
    player_meta_columns = [
        "steam_id_64", "uid", "username", "avatar_url", "domain", "created_at", "updated_at",
        "last_seen_match_id", "uuid", "email", "area", "mobile", "user_domain",
        "username_audit_status", "accid", "team_id", "trumpet_count",
        "profile_nickname", "profile_avatar_audit_status", "profile_rgb_avatar_url",
        "profile_photo_url", "profile_gender", "profile_birthday", "profile_country_id",
        "profile_region_id", "profile_city_id", "profile_language", "profile_recommend_url",
        "profile_group_id", "profile_reg_source", "status_status", "status_expire",
        "status_cancellation_status", "status_new_user", "status_login_banned_time",
        "status_anticheat_type", "status_flag_status1", "status_anticheat_status",
        "status_flag_honor", "status_privacy_policy_status", "status_csgo_frozen_exptime",
        "platformexp_level", "platformexp_exp", "steam_account", "steam_trade_url",
        "steam_rent_id", "trusted_credit", "trusted_credit_level", "trusted_score",
        "trusted_status", "trusted_credit_status", "certify_id_type", "certify_status",
        "certify_age", "certify_real_name", "certify_uid_list", "certify_audit_status",
        "certify_gender", "identity_type", "identity_extras", "identity_status",
        "identity_slogan", "identity_list", "identity_slogan_ext", "identity_live_url",
        "identity_live_type", "plus_is_plus", "user_info_raw"
    ]
    player_meta_placeholders = ",".join(["?"] * len(player_meta_columns))
    player_meta_columns_sql = ",".join(player_meta_columns)
    for sid, meta in m.player_meta.items():
        cursor.execute("""
            INSERT INTO dim_players (""" + player_meta_columns_sql + """)
            VALUES (""" + player_meta_placeholders + """)
            ON CONFLICT(steam_id_64) DO UPDATE SET
                uid=excluded.uid,
                username=excluded.username,
                avatar_url=CASE 
                    WHEN excluded.avatar_url IS NOT NULL AND excluded.avatar_url != '' 
                    THEN excluded.avatar_url 
                    ELSE dim_players.avatar_url 
                END,
                domain=excluded.domain,
                created_at=excluded.created_at,
                updated_at=excluded.updated_at,
                last_seen_match_id=excluded.last_seen_match_id,
                uuid=excluded.uuid,
                email=excluded.email,
                area=excluded.area,
                mobile=excluded.mobile,
                user_domain=excluded.user_domain,
                username_audit_status=excluded.username_audit_status,
                accid=excluded.accid,
                team_id=excluded.team_id,
                trumpet_count=excluded.trumpet_count,
                profile_nickname=excluded.profile_nickname,
                profile_avatar_audit_status=excluded.profile_avatar_audit_status,
                profile_rgb_avatar_url=excluded.profile_rgb_avatar_url,
                profile_photo_url=excluded.profile_photo_url,
                profile_gender=excluded.profile_gender,
                profile_birthday=excluded.profile_birthday,
                profile_country_id=excluded.profile_country_id,
                profile_region_id=excluded.profile_region_id,
                profile_city_id=excluded.profile_city_id,
                profile_language=excluded.profile_language,
                profile_recommend_url=excluded.profile_recommend_url,
                profile_group_id=excluded.profile_group_id,
                profile_reg_source=excluded.profile_reg_source,
                status_status=excluded.status_status,
                status_expire=excluded.status_expire,
                status_cancellation_status=excluded.status_cancellation_status,
                status_new_user=excluded.status_new_user,
                status_login_banned_time=excluded.status_login_banned_time,
                status_anticheat_type=excluded.status_anticheat_type,
                status_flag_status1=excluded.status_flag_status1,
                status_anticheat_status=excluded.status_anticheat_status,
                status_flag_honor=excluded.status_flag_honor,
                status_privacy_policy_status=excluded.status_privacy_policy_status,
                status_csgo_frozen_exptime=excluded.status_csgo_frozen_exptime,
                platformexp_level=excluded.platformexp_level,
                platformexp_exp=excluded.platformexp_exp,
                steam_account=excluded.steam_account,
                steam_trade_url=excluded.steam_trade_url,
                steam_rent_id=excluded.steam_rent_id,
                trusted_credit=excluded.trusted_credit,
                trusted_credit_level=excluded.trusted_credit_level,
                trusted_score=excluded.trusted_score,
                trusted_status=excluded.trusted_status,
                trusted_credit_status=excluded.trusted_credit_status,
                certify_id_type=excluded.certify_id_type,
                certify_status=excluded.certify_status,
                certify_age=excluded.certify_age,
                certify_real_name=excluded.certify_real_name,
                certify_uid_list=excluded.certify_uid_list,
                certify_audit_status=excluded.certify_audit_status,
                certify_gender=excluded.certify_gender,
                identity_type=excluded.identity_type,
                identity_extras=excluded.identity_extras,
                identity_status=excluded.identity_status,
                identity_slogan=excluded.identity_slogan,
                identity_list=excluded.identity_list,
                identity_slogan_ext=excluded.identity_slogan_ext,
                identity_live_url=excluded.identity_live_url,
                identity_live_type=excluded.identity_live_type,
                plus_is_plus=excluded.plus_is_plus,
                user_info_raw=excluded.user_info_raw
        """, (
            sid, meta.get('uid'), meta.get('username'), meta.get('avatar_url'),
            meta.get('domain'), meta.get('created_at'), meta.get('updated_at'),
            m.match_id, meta.get('uuid'), meta.get('email'), meta.get('area'),
            meta.get('mobile'), meta.get('user_domain'), meta.get('username_audit_status'),
            meta.get('accid'), meta.get('team_id'), meta.get('trumpet_count'),
            meta.get('profile_nickname'), meta.get('profile_avatar_audit_status'),
            meta.get('profile_rgb_avatar_url'), meta.get('profile_photo_url'),
            meta.get('profile_gender'), meta.get('profile_birthday'),
            meta.get('profile_country_id'), meta.get('profile_region_id'),
            meta.get('profile_city_id'), meta.get('profile_language'),
            meta.get('profile_recommend_url'), meta.get('profile_group_id'),
            meta.get('profile_reg_source'), meta.get('status_status'),
            meta.get('status_expire'), meta.get('status_cancellation_status'),
            meta.get('status_new_user'), meta.get('status_login_banned_time'),
            meta.get('status_anticheat_type'), meta.get('status_flag_status1'),
            meta.get('status_anticheat_status'), meta.get('status_flag_honor'),
            meta.get('status_privacy_policy_status'), meta.get('status_csgo_frozen_exptime'),
            meta.get('platformexp_level'), meta.get('platformexp_exp'),
            meta.get('steam_account'), meta.get('steam_trade_url'),
            meta.get('steam_rent_id'), meta.get('trusted_credit'),
            meta.get('trusted_credit_level'), meta.get('trusted_score'),
            meta.get('trusted_status'), meta.get('trusted_credit_status'),
            meta.get('certify_id_type'), meta.get('certify_status'),
            meta.get('certify_age'), meta.get('certify_real_name'),
            meta.get('certify_uid_list'), meta.get('certify_audit_status'),
            meta.get('certify_gender'), meta.get('identity_type'),
            meta.get('identity_extras'), meta.get('identity_status'),
            meta.get('identity_slogan'), meta.get('identity_list'),
            meta.get('identity_slogan_ext'), meta.get('identity_live_url'),
            meta.get('identity_live_type'), meta.get('plus_is_plus'),
            meta.get('user_info_raw')
        ))

    # 2. Dim Maps (Ignore if exists)
    if m.map_name:
        cursor.execute("""
            INSERT INTO dim_maps (map_name, map_desc)
            VALUES (?, ?)
            ON CONFLICT(map_name) DO UPDATE SET
                map_desc=excluded.map_desc
        """, (m.map_name, m.map_desc))

    # 3. Fact Matches
    cursor.execute("""
        INSERT OR REPLACE INTO fact_matches 
        (match_id, match_code, map_name, start_time, end_time, duration, winner_team, score_team1, score_team2, server_ip, server_port, location, has_side_data_and_rating2, match_main_id, demo_url, game_mode, game_name, map_desc, location_full, match_mode, match_status, match_flag, status, waiver, year, season, round_total, cs_type, priority_show_type, pug10m_show_type, credit_match_status, knife_winner, knife_winner_role, most_1v2_uid, most_assist_uid, most_awp_uid, most_end_uid, most_first_kill_uid, most_headshot_uid, most_jump_uid, mvp_uid, response_code, response_message, response_status, response_timestamp, response_trace_id, response_success, response_errcode, treat_info_raw, round_list_raw, leetify_data_raw, data_source_type)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    """, (
        m.match_id, m.match_code, m.map_name, m.start_time, m.end_time, m.duration, 
        m.winner_team, m.score_team1, m.score_team2, m.server_ip, m.server_port, m.location,
        m.has_side_data_and_rating2, m.match_main_id, m.demo_url, m.game_mode, m.game_name, m.map_desc,
        m.location_full, m.match_mode, m.match_status, m.match_flag, m.status, m.waiver, m.year, m.season,
        m.round_total, m.cs_type, m.priority_show_type, m.pug10m_show_type, m.credit_match_status,
        m.knife_winner, m.knife_winner_role, m.most_1v2_uid, m.most_assist_uid, m.most_awp_uid,
        m.most_end_uid, m.most_first_kill_uid, m.most_headshot_uid, m.most_jump_uid, m.mvp_uid,
        m.response_code, m.response_message, m.response_status, m.response_timestamp, m.response_trace_id,
        m.response_success, m.response_errcode, m.treat_info_raw, m.round_list_raw, m.leetify_data_raw, m.data_source_type
    ))

    for t in m.teams:
        cursor.execute("""
            INSERT OR REPLACE INTO fact_match_teams
            (match_id, group_id, group_all_score, group_change_elo, group_fh_role, group_fh_score, group_origin_elo, group_sh_role, group_sh_score, group_tid, group_uids)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            m.match_id, t.group_id, t.group_all_score, t.group_change_elo, t.group_fh_role, t.group_fh_score,
            t.group_origin_elo, t.group_sh_role, t.group_sh_score, t.group_tid, t.group_uids
        ))

    # 4. Fact Match Players
    player_columns = [
        "match_id", "steam_id_64", "team_id", "kills", "deaths", "assists", "headshot_count",
        "kd_ratio", "adr", "rating", "rating2", "rating3", "rws", "mvp_count", "elo_change",
        "rank_score", "is_win", "kast", "entry_kills", "entry_deaths", "awp_kills",
        "clutch_1v1", "clutch_1v2", "clutch_1v3", "clutch_1v4", "clutch_1v5",
        "flash_assists", "flash_duration", "jump_count", "damage_total", "damage_received",
        "damage_receive", "damage_stats", "assisted_kill", "awp_kill", "awp_kill_ct",
        "awp_kill_t", "benefit_kill", "day", "defused_bomb", "end_1v1",
        "end_1v2", "end_1v3", "end_1v4", "end_1v5", "explode_bomb", "first_death",
        "fd_ct", "fd_t", "first_kill", "flash_enemy", "flash_team", "flash_team_time", "flash_time",
        "game_mode", "group_id", "hold_total", "id", "is_highlight", "is_most_1v2",
        "is_most_assist", "is_most_awp", "is_most_end", "is_most_first_kill",
        "is_most_headshot", "is_most_jump", "is_svp", "is_tie", "kill_1", "kill_2",
        "kill_3", "kill_4", "kill_5", "many_assists_cnt1", "many_assists_cnt2",
        "many_assists_cnt3", "many_assists_cnt4", "many_assists_cnt5", "map",
        "match_code", "match_mode", "match_team_id", "match_time", "per_headshot",
        "perfect_kill", "planted_bomb", "revenge_kill", "round_total", "season",
        "team_kill", "throw_harm", "throw_harm_enemy", "uid", "year", "sts_raw", "level_info_raw",
        "util_flash_usage", "util_smoke_usage", "util_molotov_usage", "util_he_usage", "util_decoy_usage"
    ]
    player_placeholders = ",".join(["?"] * len(player_columns))
    player_columns_sql = ",".join(player_columns)

    def player_values(sid, p):
        return [
            m.match_id, sid, p.team_id, p.kills, p.deaths, p.assists, p.headshot_count,
            p.kd_ratio, p.adr, p.rating, p.rating2, p.rating3, p.rws, p.mvp_count,
            p.elo_change, p.rank_score, p.is_win, p.kast, p.entry_kills, p.entry_deaths,
            p.awp_kills, p.clutch_1v1, p.clutch_1v2, p.clutch_1v3, p.clutch_1v4,
            p.clutch_1v5, p.flash_assists, p.flash_duration, p.jump_count, p.damage_total,
            p.damage_received, p.damage_receive, p.damage_stats, p.assisted_kill, p.awp_kill,
            p.awp_kill_ct, p.awp_kill_t, p.benefit_kill, p.day, p.defused_bomb, p.end_1v1,
            p.end_1v2, p.end_1v3, p.end_1v4, p.end_1v5, p.explode_bomb, p.first_death,
            p.fd_ct, p.fd_t, p.first_kill, p.flash_enemy, p.flash_team,
            p.flash_team_time, p.flash_time, p.game_mode, p.group_id, p.hold_total,
            p.id, p.is_highlight, p.is_most_1v2, p.is_most_assist, p.is_most_awp,
            p.is_most_end, p.is_most_first_kill, p.is_most_headshot, p.is_most_jump,
            p.is_svp, p.is_tie, p.kill_1, p.kill_2, p.kill_3, p.kill_4, p.kill_5,
            p.many_assists_cnt1, p.many_assists_cnt2, p.many_assists_cnt3, p.many_assists_cnt4,
            p.many_assists_cnt5, p.map, p.match_code, p.match_mode, p.match_team_id,
            p.match_time, p.per_headshot, p.perfect_kill, p.planted_bomb, p.revenge_kill,
            p.round_total, p.season, p.team_kill, p.throw_harm, p.throw_harm_enemy,
            p.uid, p.year, p.sts_raw, p.level_info_raw,
            p.util_flash_usage, p.util_smoke_usage, p.util_molotov_usage, p.util_he_usage, p.util_decoy_usage
        ]

    for sid, p in m.players.items():
        cursor.execute(
            f"INSERT OR REPLACE INTO fact_match_players ({player_columns_sql}) VALUES ({player_placeholders})",
            player_values(sid, p)
        )
    for sid, p in m.players_t.items():
        cursor.execute(
            f"INSERT OR REPLACE INTO fact_match_players_t ({player_columns_sql}) VALUES ({player_placeholders})",
            player_values(sid, p)
        )
    for sid, p in m.players_ct.items():
        cursor.execute(
            f"INSERT OR REPLACE INTO fact_match_players_ct ({player_columns_sql}) VALUES ({player_placeholders})",
            player_values(sid, p)
        )

    # 5. Rounds & Events
    for r in m.rounds:
        cursor.execute("""
            INSERT OR REPLACE INTO fact_rounds
            (match_id, round_num, winner_side, win_reason, win_reason_desc, duration, end_time_stamp, ct_score, t_score, ct_money_start, t_money_start)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            m.match_id, r.round_num, r.winner_side, r.win_reason, r.win_reason_desc, 
            r.duration, r.end_time_stamp, r.ct_score, r.t_score, r.ct_money_start, r.t_money_start
        ))
        
        for e in r.events:
            # Handle Pos
            ax, ay, az = e.attacker_pos if e.attacker_pos else (None, None, None)
            vx, vy, vz = e.victim_pos if e.victim_pos else (None, None, None)
            
            # Use uuid for event_id to ensure uniqueness if logic fails
            import uuid
            if not e.event_id:
                e.event_id = str(uuid.uuid4())
                
            cursor.execute("""
                INSERT OR REPLACE INTO fact_round_events
                (event_id, match_id, round_num, event_type, event_time, attacker_steam_id, victim_steam_id, assister_steam_id,
                weapon, is_headshot, is_wallbang, is_blind, is_through_smoke, is_noscope,
                trade_killer_steam_id, flash_assist_steam_id, score_change_attacker, score_change_victim,
                attacker_pos_x, attacker_pos_y, attacker_pos_z, victim_pos_x, victim_pos_y, victim_pos_z)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """, (
                e.event_id, m.match_id, r.round_num, e.event_type, e.event_time, e.attacker_steam_id, e.victim_steam_id,
                e.assister_steam_id, e.weapon, e.is_headshot, e.is_wallbang, e.is_blind, e.is_through_smoke, e.is_noscope,
                e.trade_killer_steam_id, e.flash_assist_steam_id, e.score_change_attacker, e.score_change_victim,
                ax, ay, az, vx, vy, vz
            ))

        for pe in r.economies:
            cursor.execute("""
                INSERT OR REPLACE INTO fact_round_player_economy
                (match_id, round_num, steam_id_64, side, start_money, equipment_value, main_weapon, has_helmet, has_defuser, has_zeus, round_performance_score)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """, (
                m.match_id, r.round_num, pe.steam_id_64, pe.side, pe.start_money, pe.equipment_value, pe.main_weapon, pe.has_helmet, pe.has_defuser, pe.has_zeus, pe.round_performance_score
            ))

    # 6. Calculate & Save Clutch Attempts
    _calculate_and_save_clutch_attempts(cursor, m.match_id, m.round_list_raw)

def _calculate_and_save_clutch_attempts(cursor, match_id, round_list_raw):
    if not round_list_raw:
        return

    try:
        round_list = json.loads(round_list_raw)
    except:
        return

    player_attempts = {}

    for round_data in round_list:
        all_kills = round_data.get('all_kill', [])
        if not all_kills:
            continue
            
        team_members = {1: set(), 2: set()}
        
        # Scan for team members
        for k in all_kills:
            if k.get('attacker') and k['attacker'].get('steamid_64'):
                tid = k['attacker'].get('team')
                if tid in [1, 2]:
                    team_members[tid].add(k['attacker']['steamid_64'])
            if k.get('victim') and k['victim'].get('steamid_64'):
                tid = k['victim'].get('team')
                if tid in [1, 2]:
                    team_members[tid].add(k['victim']['steamid_64'])
        
        if not team_members[1] or not team_members[2]:
            continue

        alive = {1: team_members[1].copy(), 2: team_members[2].copy()}
        clutch_triggered_players = set()
        
        # Sort kills by time
        sorted_kills = sorted(all_kills, key=lambda x: x.get('pasttime', 0))
        
        for k in sorted_kills:
            victim = k.get('victim')
            if not victim: continue
            
            v_sid = victim.get('steamid_64')
            v_team = victim.get('team')
            
            if v_team not in [1, 2] or v_sid not in alive[v_team]:
                continue
                
            alive[v_team].remove(v_sid)
            
            if len(alive[v_team]) == 1:
                survivor_sid = list(alive[v_team])[0]
                if survivor_sid not in clutch_triggered_players:
                    opponent_team = 3 - v_team
                    opponents_alive_count = len(alive[opponent_team])
                    
                    if opponents_alive_count >= 1:
                        if survivor_sid not in player_attempts:
                            player_attempts[survivor_sid] = {'1v1': 0, '1v2': 0, '1v3': 0, '1v4': 0, '1v5': 0}
                        
                        n = min(opponents_alive_count, 5)
                        key = f'1v{n}'
                        player_attempts[survivor_sid][key] += 1
                        clutch_triggered_players.add(survivor_sid)

    # Save to DB
    cursor.execute("""
        CREATE TABLE IF NOT EXISTS fact_match_clutch_attempts (
            match_id TEXT,
            steam_id_64 TEXT,
            attempt_1v1 INTEGER DEFAULT 0,
            attempt_1v2 INTEGER DEFAULT 0,
            attempt_1v3 INTEGER DEFAULT 0,
            attempt_1v4 INTEGER DEFAULT 0,
            attempt_1v5 INTEGER DEFAULT 0,
            PRIMARY KEY (match_id, steam_id_64)
        )
    """)
    
    for pid, att in player_attempts.items():
        cursor.execute("""
            INSERT OR REPLACE INTO fact_match_clutch_attempts 
            (match_id, steam_id_64, attempt_1v1, attempt_1v2, attempt_1v3, attempt_1v4, attempt_1v5)
            VALUES (?, ?, ?, ?, ?, ?, ?)
        """, (match_id, pid, att['1v1'], att['1v2'], att['1v3'], att['1v4'], att['1v5']))

if __name__ == "__main__":
    process_matches()