xunyulin230420/clutch
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Initial commit of Clutch-IQ project

Browse Source
This commit is contained in:
xunyulin230420
2026-02-05 23:26:03 +08:00
commit a355239861
66 changed files with 12922 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
database/L2/processors/__init__.py Normal file
View File

@@ -0,0 +1,20 @@
"""
L2 Processor Modules
This package contains specialized processors for L2 database construction:
- match_processor: Handles fact_matches and fact_match_teams
- player_processor: Handles dim_players and fact_match_players (all variants)
- round_processor: Dispatches round data processing based on data_source_type
- economy_processor: Processes leetify economic data
- event_processor: Processes kill and bomb events
- spatial_processor: Processes classic spatial (xyz) data
"""
__all__ = [
'match_processor',
'player_processor',
'round_processor',
'economy_processor',
'event_processor',
'spatial_processor'
]

271
database/L2/processors/economy_processor.py Normal file
View File

@@ -0,0 +1,271 @@
"""
Economy Processor - Handles leetify economic data
Responsibilities:
- Parse bron_equipment (equipment lists)
- Parse player_bron_crash (starting money)
- Calculate equipment_value
- Write to fact_round_player_economy and update fact_rounds
"""
import sqlite3
import json
import logging
import uuid
logger = logging.getLogger(__name__)
class EconomyProcessor:
@staticmethod
def process_classic(match_data, conn: sqlite3.Connection) -> bool:
"""
Process classic economy data (extracted from round_list equiped)
"""
try:
cursor = conn.cursor()
for r in match_data.rounds:
if not r.economies:
continue
for eco in r.economies:
if eco.side not in ['CT', 'T']:
# Skip rounds where side cannot be determined (avoids CHECK constraint failure)
continue
cursor.execute('''
INSERT OR REPLACE INTO fact_round_player_economy (
match_id, round_num, steam_id_64, side, start_money,
equipment_value, main_weapon, has_helmet, has_defuser,
has_zeus, round_performance_score, data_source_type
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (
match_data.match_id, r.round_num, eco.steam_id_64, eco.side, eco.start_money,
eco.equipment_value, eco.main_weapon, eco.has_helmet, eco.has_defuser,
eco.has_zeus, eco.round_performance_score, 'classic'
))
return True
except Exception as e:
logger.error(f"Error processing classic economy for match {match_data.match_id}: {e}")
import traceback
traceback.print_exc()
return False
@staticmethod
def process_leetify(match_data, conn: sqlite3.Connection) -> bool:
"""
Process leetify economy and round data
Args:
match_data: MatchData object with leetify_data parsed
conn: L2 database connection
Returns:
bool: True if successful
"""
try:
if not hasattr(match_data, 'data_leetify') or not match_data.data_leetify:
return True
leetify_data = match_data.data_leetify.get('leetify_data', {})
round_stats = leetify_data.get('round_stat', [])
if not round_stats:
return True
cursor = conn.cursor()
for r in round_stats:
round_num = r.get('round', 0)
# Extract round-level data
ct_money_start = r.get('ct_money_group', 0)
t_money_start = r.get('t_money_group', 0)
win_reason = r.get('win_reason', 0)
# Get timestamps
begin_ts = r.get('begin_ts', '')
end_ts = r.get('end_ts', '')
# Get sfui_event for scores
sfui = r.get('sfui_event', {})
ct_score = sfui.get('score_ct', 0)
t_score = sfui.get('score_t', 0)
# Determine winner_side based on show_event
show_events = r.get('show_event', [])
winner_side = 'None'
duration = 0.0
if show_events:
last_event = show_events[-1]
# Check if there's a win_reason in the last event
if last_event.get('win_reason'):
win_reason = last_event.get('win_reason', 0)
# Map win_reason to winner_side
# Typical mappings: 1=T_Win, 2=CT_Win, etc.
winner_side = _map_win_reason_to_side(win_reason)
# Calculate duration from event timestamps
if 'ts' in last_event:
duration = float(last_event.get('ts', 0))
# Insert/update fact_rounds
cursor.execute('''
INSERT OR REPLACE INTO fact_rounds (
match_id, round_num, winner_side, win_reason, win_reason_desc,
duration, ct_score, t_score, ct_money_start, t_money_start,
begin_ts, end_ts, data_source_type
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (
match_data.match_id, round_num, winner_side, win_reason,
_map_win_reason_desc(win_reason), duration, ct_score, t_score,
ct_money_start, t_money_start, begin_ts, end_ts, 'leetify'
))
# Process economy data
bron_equipment = r.get('bron_equipment', {})
player_t_score = r.get('player_t_score', {})
player_ct_score = r.get('player_ct_score', {})
player_bron_crash = r.get('player_bron_crash', {})
# Build side mapping
side_scores = {}
for sid, val in player_t_score.items():
side_scores[str(sid)] = ("T", float(val) if val is not None else 0.0)
for sid, val in player_ct_score.items():
side_scores[str(sid)] = ("CT", float(val) if val is not None else 0.0)
# Process each player's economy
for sid in set(list(side_scores.keys()) + [str(k) for k in bron_equipment.keys()]):
if sid not in side_scores:
continue
side, perf_score = side_scores[sid]
items = bron_equipment.get(sid) or bron_equipment.get(str(sid)) or []
start_money = _pick_money(items)
equipment_value = player_bron_crash.get(sid) or player_bron_crash.get(str(sid))
equipment_value = int(equipment_value) if equipment_value is not None else 0
main_weapon = _pick_main_weapon(items)
has_helmet = _has_item_type(items, ['weapon_vest', 'item_assaultsuit', 'item_kevlar'])
has_defuser = _has_item_type(items, ['item_defuser'])
has_zeus = _has_item_type(items, ['weapon_taser'])
cursor.execute('''
INSERT OR REPLACE INTO fact_round_player_economy (
match_id, round_num, steam_id_64, side, start_money,
equipment_value, main_weapon, has_helmet, has_defuser,
has_zeus, round_performance_score, data_source_type
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (
match_data.match_id, round_num, sid, side, start_money,
equipment_value, main_weapon, has_helmet, has_defuser,
has_zeus, perf_score, 'leetify'
))
logger.debug(f"Processed {len(round_stats)} leetify rounds for match {match_data.match_id}")
return True
except Exception as e:
logger.error(f"Error processing leetify economy for match {match_data.match_id}: {e}")
import traceback
traceback.print_exc()
return False
def _pick_main_weapon(items):
"""Extract main weapon from equipment list"""
if not isinstance(items, list):
return ""
ignore = {
"weapon_knife", "weapon_knife_t", "weapon_knife_gg", "weapon_knife_ct",
"weapon_c4", "weapon_flashbang", "weapon_hegrenade", "weapon_smokegrenade",
"weapon_molotov", "weapon_incgrenade", "weapon_decoy"
}
# First pass: ignore utility
for it in items:
if not isinstance(it, dict):
continue
name = it.get('WeaponName')
if name and name not in ignore:
return name
# Second pass: any weapon
for it in items:
if not isinstance(it, dict):
continue
name = it.get('WeaponName')
if name:
return name
return ""
def _pick_money(items):
"""Extract starting money from equipment list"""
if not isinstance(items, list):
return 0
vals = []
for it in items:
if isinstance(it, dict) and it.get('Money') is not None:
vals.append(it.get('Money'))
return int(max(vals)) if vals else 0
def _has_item_type(items, keywords):
"""Check if equipment list contains item matching keywords"""
if not isinstance(items, list):
return False
for it in items:
if not isinstance(it, dict):
continue
name = it.get('WeaponName', '')
if any(kw in name for kw in keywords):
return True
return False
def _map_win_reason_to_side(win_reason):
"""Map win_reason integer to winner_side"""
# Common mappings from CS:GO/CS2:
# 1 = Target_Bombed (T wins)
# 2 = Bomb_Defused (CT wins)
# 7 = CTs_Win (CT eliminates T)
# 8 = Terrorists_Win (T eliminates CT)
# 9 = Target_Saved (CT wins, time runs out)
# etc.
t_win_reasons = {1, 8, 12, 17}
ct_win_reasons = {2, 7, 9, 11}
if win_reason in t_win_reasons:
return 'T'
elif win_reason in ct_win_reasons:
return 'CT'
else:
return 'None'
def _map_win_reason_desc(win_reason):
"""Map win_reason integer to description"""
reason_map = {
0: 'None',
1: 'TargetBombed',
2: 'BombDefused',
7: 'CTsWin',
8: 'TerroristsWin',
9: 'TargetSaved',
11: 'CTSurrender',
12: 'TSurrender',
17: 'TerroristsPlanted'
}
return reason_map.get(win_reason, f'Unknown_{win_reason}')

293
database/L2/processors/event_processor.py Normal file
View File

@@ -0,0 +1,293 @@
"""
Event Processor - Handles kill and bomb events
Responsibilities:
- Process leetify show_event data (kills with score impacts)
- Process classic all_kill and c4_event data
- Generate unique event_ids
- Store twin probability changes (leetify only)
- Handle bomb plant/defuse events
"""
import sqlite3
import json
import logging
import uuid
logger = logging.getLogger(__name__)
class EventProcessor:
@staticmethod
def process_leetify_events(match_data, conn: sqlite3.Connection) -> bool:
"""
Process leetify event data
Args:
match_data: MatchData object with leetify_data parsed
conn: L2 database connection
Returns:
bool: True if successful
"""
try:
if not hasattr(match_data, 'data_leetify') or not match_data.data_leetify:
return True
leetify_data = match_data.data_leetify.get('leetify_data', {})
round_stats = leetify_data.get('round_stat', [])
if not round_stats:
return True
cursor = conn.cursor()
event_count = 0
for r in round_stats:
round_num = r.get('round', 0)
show_events = r.get('show_event', [])
for evt in show_events:
event_type_code = evt.get('event_type', 0)
# event_type: 3=kill, others for bomb/etc
if event_type_code == 3 and evt.get('kill_event'):
# Process kill event
k = evt['kill_event']
event_id = str(uuid.uuid4())
event_time = evt.get('ts', 0)
attacker_steam_id = str(k.get('Killer', ''))
victim_steam_id = str(k.get('Victim', ''))
weapon = k.get('WeaponName', '')
is_headshot = bool(k.get('Headshot', False))
is_wallbang = bool(k.get('Penetrated', False))
is_blind = bool(k.get('AttackerBlind', False))
is_through_smoke = bool(k.get('ThroughSmoke', False))
is_noscope = bool(k.get('NoScope', False))
# Extract assist info
assister_steam_id = None
flash_assist_steam_id = None
trade_killer_steam_id = None
if evt.get('assist_killer_score_change'):
assister_steam_id = str(list(evt['assist_killer_score_change'].keys())[0])
if evt.get('flash_assist_killer_score_change'):
flash_assist_steam_id = str(list(evt['flash_assist_killer_score_change'].keys())[0])
if evt.get('trade_score_change'):
trade_killer_steam_id = str(list(evt['trade_score_change'].keys())[0])
# Extract score changes
score_change_attacker = 0.0
score_change_victim = 0.0
if evt.get('killer_score_change'):
vals = list(evt['killer_score_change'].values())
if vals and isinstance(vals[0], dict):
score_change_attacker = float(vals[0].get('score', 0))
if evt.get('victim_score_change'):
vals = list(evt['victim_score_change'].values())
if vals and isinstance(vals[0], dict):
score_change_victim = float(vals[0].get('score', 0))
# Extract twin (team win probability) changes
twin = evt.get('twin', 0.0)
c_twin = evt.get('c_twin', 0.0)
twin_change = evt.get('twin_change', 0.0)
c_twin_change = evt.get('c_twin_change', 0.0)
cursor.execute('''
INSERT OR REPLACE INTO fact_round_events (
event_id, match_id, round_num, event_type, event_time,
attacker_steam_id, victim_steam_id, assister_steam_id,
flash_assist_steam_id, trade_killer_steam_id, weapon,
is_headshot, is_wallbang, is_blind, is_through_smoke,
is_noscope, score_change_attacker, score_change_victim,
twin, c_twin, twin_change, c_twin_change, data_source_type
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (
event_id, match_data.match_id, round_num, 'kill', event_time,
attacker_steam_id, victim_steam_id, assister_steam_id,
flash_assist_steam_id, trade_killer_steam_id, weapon,
is_headshot, is_wallbang, is_blind, is_through_smoke,
is_noscope, score_change_attacker, score_change_victim,
twin, c_twin, twin_change, c_twin_change, 'leetify'
))
event_count += 1
logger.debug(f"Processed {event_count} leetify events for match {match_data.match_id}")
return True
except Exception as e:
logger.error(f"Error processing leetify events for match {match_data.match_id}: {e}")
import traceback
traceback.print_exc()
return False
@staticmethod
def process_classic_events(match_data, conn: sqlite3.Connection) -> bool:
"""
Process classic event data (all_kill, c4_event)
Args:
match_data: MatchData object with round_list parsed
conn: L2 database connection
Returns:
bool: True if successful
"""
try:
if not hasattr(match_data, 'data_round_list') or not match_data.data_round_list:
return True
round_list = match_data.data_round_list.get('round_list', [])
if not round_list:
return True
cursor = conn.cursor()
event_count = 0
for idx, rd in enumerate(round_list, start=1):
round_num = idx
# Extract round basic info for fact_rounds
current_score = rd.get('current_score', {})
ct_score = current_score.get('ct', 0)
t_score = current_score.get('t', 0)
win_type = current_score.get('type', 0)
pasttime = current_score.get('pasttime', 0)
final_round_time = current_score.get('final_round_time', 0)
# Determine winner_side from win_type
winner_side = _map_win_type_to_side(win_type)
# Insert/update fact_rounds
cursor.execute('''
INSERT OR REPLACE INTO fact_rounds (
match_id, round_num, winner_side, win_reason, win_reason_desc,
duration, ct_score, t_score, end_time_stamp, final_round_time,
pasttime, data_source_type
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (
match_data.match_id, round_num, winner_side, win_type,
_map_win_type_desc(win_type), float(pasttime), ct_score, t_score,
'', final_round_time, pasttime, 'classic'
))
# Process kill events
all_kill = rd.get('all_kill', [])
for kill in all_kill:
event_id = str(uuid.uuid4())
event_time = kill.get('pasttime', 0)
attacker = kill.get('attacker', {})
victim = kill.get('victim', {})
attacker_steam_id = str(attacker.get('steamid_64', ''))
victim_steam_id = str(victim.get('steamid_64', ''))
weapon = kill.get('weapon', '')
is_headshot = bool(kill.get('headshot', False))
is_wallbang = bool(kill.get('penetrated', False))
is_blind = bool(kill.get('attackerblind', False))
is_through_smoke = bool(kill.get('throughsmoke', False))
is_noscope = bool(kill.get('noscope', False))
# Classic has spatial data - will be filled by spatial_processor
# But we still need to insert the event
cursor.execute('''
INSERT OR REPLACE INTO fact_round_events (
event_id, match_id, round_num, event_type, event_time,
attacker_steam_id, victim_steam_id, weapon, is_headshot,
is_wallbang, is_blind, is_through_smoke, is_noscope,
data_source_type
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (
event_id, match_data.match_id, round_num, 'kill', event_time,
attacker_steam_id, victim_steam_id, weapon, is_headshot,
is_wallbang, is_blind, is_through_smoke, is_noscope, 'classic'
))
event_count += 1
# Process bomb events
c4_events = rd.get('c4_event', [])
for c4 in c4_events:
event_id = str(uuid.uuid4())
event_name = c4.get('event_name', '')
event_time = c4.get('pasttime', 0)
steam_id = str(c4.get('steamid_64', ''))
# Map event_name to event_type
if 'plant' in event_name.lower():
event_type = 'bomb_plant'
attacker_steam_id = steam_id
victim_steam_id = None
elif 'defuse' in event_name.lower():
event_type = 'bomb_defuse'
attacker_steam_id = steam_id
victim_steam_id = None
else:
event_type = 'unknown'
attacker_steam_id = steam_id
victim_steam_id = None
cursor.execute('''
INSERT OR REPLACE INTO fact_round_events (
event_id, match_id, round_num, event_type, event_time,
attacker_steam_id, victim_steam_id, data_source_type
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
''', (
event_id, match_data.match_id, round_num, event_type,
event_time, attacker_steam_id, victim_steam_id, 'classic'
))
event_count += 1
logger.debug(f"Processed {event_count} classic events for match {match_data.match_id}")
return True
except Exception as e:
logger.error(f"Error processing classic events for match {match_data.match_id}: {e}")
import traceback
traceback.print_exc()
return False
def _map_win_type_to_side(win_type):
"""Map win_type to winner_side for classic data"""
# Based on CS:GO win types
t_win_types = {1, 8, 12, 17}
ct_win_types = {2, 7, 9, 11}
if win_type in t_win_types:
return 'T'
elif win_type in ct_win_types:
return 'CT'
else:
return 'None'
def _map_win_type_desc(win_type):
"""Map win_type to description"""
type_map = {
0: 'None',
1: 'TargetBombed',
2: 'BombDefused',
7: 'CTsWin',
8: 'TerroristsWin',
9: 'TargetSaved',
11: 'CTSurrender',
12: 'TSurrender',
17: 'TerroristsPlanted'
}
return type_map.get(win_type, f'Unknown_{win_type}')

128
database/L2/processors/match_processor.py Normal file
View File

@@ -0,0 +1,128 @@
"""
Match Processor - Handles fact_matches and fact_match_teams
Responsibilities:
- Extract match basic information from JSON
- Process team data (group1/group2)
- Store raw JSON fields (treat_info, response metadata)
- Set data_source_type marker
"""
import sqlite3
import json
import logging
from typing import Any, Dict
logger = logging.getLogger(__name__)
def safe_int(val):
"""Safely convert value to integer"""
try:
return int(float(val)) if val is not None else 0
except:
return 0
def safe_float(val):
"""Safely convert value to float"""
try:
return float(val) if val is not None else 0.0
except:
return 0.0
def safe_text(val):
"""Safely convert value to text"""
return "" if val is None else str(val)
class MatchProcessor:
@staticmethod
def process(match_data, conn: sqlite3.Connection) -> bool:
"""
Process match basic info and team data
Args:
match_data: MatchData object containing parsed JSON
conn: L2 database connection
Returns:
bool: True if successful
"""
try:
cursor = conn.cursor()
# Build column list and values dynamically to avoid count mismatches
columns = [
'match_id', 'match_code', 'map_name', 'start_time', 'end_time', 'duration',
'winner_team', 'score_team1', 'score_team2', 'server_ip', 'server_port', 'location',
'has_side_data_and_rating2', 'match_main_id', 'demo_url', 'game_mode', 'game_name',
'map_desc', 'location_full', 'match_mode', 'match_status', 'match_flag', 'status', 'waiver',
'year', 'season', 'round_total', 'cs_type', 'priority_show_type', 'pug10m_show_type',
'credit_match_status', 'knife_winner', 'knife_winner_role', 'most_1v2_uid',
'most_assist_uid', 'most_awp_uid', 'most_end_uid', 'most_first_kill_uid',
'most_headshot_uid', 'most_jump_uid', 'mvp_uid', 'response_code', 'response_message',
'response_status', 'response_timestamp', 'response_trace_id', 'response_success',
'response_errcode', 'treat_info_raw', 'round_list_raw', 'leetify_data_raw',
'data_source_type'
]
values = [
match_data.match_id, match_data.match_code, match_data.map_name, match_data.start_time,
match_data.end_time, match_data.duration, match_data.winner_team, match_data.score_team1,
match_data.score_team2, match_data.server_ip, match_data.server_port, match_data.location,
match_data.has_side_data_and_rating2, match_data.match_main_id, match_data.demo_url,
match_data.game_mode, match_data.game_name, match_data.map_desc, match_data.location_full,
match_data.match_mode, match_data.match_status, match_data.match_flag, match_data.status,
match_data.waiver, match_data.year, match_data.season, match_data.round_total,
match_data.cs_type, match_data.priority_show_type, match_data.pug10m_show_type,
match_data.credit_match_status, match_data.knife_winner, match_data.knife_winner_role,
match_data.most_1v2_uid, match_data.most_assist_uid, match_data.most_awp_uid,
match_data.most_end_uid, match_data.most_first_kill_uid, match_data.most_headshot_uid,
match_data.most_jump_uid, match_data.mvp_uid, match_data.response_code,
match_data.response_message, match_data.response_status, match_data.response_timestamp,
match_data.response_trace_id, match_data.response_success, match_data.response_errcode,
match_data.treat_info_raw, match_data.round_list_raw, match_data.leetify_data_raw,
match_data.data_source_type
]
# Build SQL dynamically
placeholders = ','.join(['?' for _ in columns])
columns_sql = ','.join(columns)
sql = f"INSERT OR REPLACE INTO fact_matches ({columns_sql}) VALUES ({placeholders})"
cursor.execute(sql, values)
# Process team data
for team in match_data.teams:
team_row = (
match_data.match_id,
team.group_id,
team.group_all_score,
team.group_change_elo,
team.group_fh_role,
team.group_fh_score,
team.group_origin_elo,
team.group_sh_role,
team.group_sh_score,
team.group_tid,
team.group_uids
)
cursor.execute('''
INSERT OR REPLACE INTO fact_match_teams (
match_id, group_id, group_all_score, group_change_elo,
group_fh_role, group_fh_score, group_origin_elo,
group_sh_role, group_sh_score, group_tid, group_uids
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', team_row)
logger.debug(f"Processed match {match_data.match_id}")
return True
except Exception as e:
logger.error(f"Error processing match {match_data.match_id}: {e}")
import traceback
traceback.print_exc()
return False

272
database/L2/processors/player_processor.py Normal file
View File

@@ -0,0 +1,272 @@
"""
Player Processor - Handles dim_players and fact_match_players
Responsibilities:
- Process player dimension table (UPSERT to avoid duplicates)
- Merge fight/fight_t/fight_ct data
- Process VIP+ advanced statistics
- Handle all player match statistics tables
"""
import sqlite3
import json
import logging
from typing import Any, Dict
logger = logging.getLogger(__name__)
def safe_int(val):
"""Safely convert value to integer"""
try:
return int(float(val)) if val is not None else 0
except:
return 0
def safe_float(val):
"""Safely convert value to float"""
try:
return float(val) if val is not None else 0.0
except:
return 0.0
def safe_text(val):
"""Safely convert value to text"""
return "" if val is None else str(val)
class PlayerProcessor:
@staticmethod
def process(match_data, conn: sqlite3.Connection) -> bool:
"""
Process all player-related data
Args:
match_data: MatchData object containing parsed JSON
conn: L2 database connection
Returns:
bool: True if successful
"""
try:
cursor = conn.cursor()
# Process dim_players (UPSERT) - using dynamic column building
for steam_id, meta in match_data.player_meta.items():
# Define columns (must match schema exactly)
player_columns = [
'steam_id_64', 'uid', 'username', 'avatar_url', 'domain', 'created_at', 'updated_at',
'last_seen_match_id', 'uuid', 'email', 'area', 'mobile', 'user_domain',
'username_audit_status', 'accid', 'team_id', 'trumpet_count', 'profile_nickname',
'profile_avatar_audit_status', 'profile_rgb_avatar_url', 'profile_photo_url',
'profile_gender', 'profile_birthday', 'profile_country_id', 'profile_region_id',
'profile_city_id', 'profile_language', 'profile_recommend_url', 'profile_group_id',
'profile_reg_source', 'status_status', 'status_expire', 'status_cancellation_status',
'status_new_user', 'status_login_banned_time', 'status_anticheat_type',
'status_flag_status1', 'status_anticheat_status', 'status_flag_honor',
'status_privacy_policy_status', 'status_csgo_frozen_exptime', 'platformexp_level',
'platformexp_exp', 'steam_account', 'steam_trade_url', 'steam_rent_id',
'trusted_credit', 'trusted_credit_level', 'trusted_score', 'trusted_status',
'trusted_credit_status', 'certify_id_type', 'certify_status', 'certify_age',
'certify_real_name', 'certify_uid_list', 'certify_audit_status', 'certify_gender',
'identity_type', 'identity_extras', 'identity_status', 'identity_slogan',
'identity_list', 'identity_slogan_ext', 'identity_live_url', 'identity_live_type',
'plus_is_plus', 'user_info_raw'
]
player_values = [
steam_id, meta['uid'], meta['username'], meta['avatar_url'], meta['domain'],
meta['created_at'], meta['updated_at'], match_data.match_id, meta['uuid'],
meta['email'], meta['area'], meta['mobile'], meta['user_domain'],
meta['username_audit_status'], meta['accid'], meta['team_id'],
meta['trumpet_count'], meta['profile_nickname'],
meta['profile_avatar_audit_status'], meta['profile_rgb_avatar_url'],
meta['profile_photo_url'], meta['profile_gender'], meta['profile_birthday'],
meta['profile_country_id'], meta['profile_region_id'], meta['profile_city_id'],
meta['profile_language'], meta['profile_recommend_url'], meta['profile_group_id'],
meta['profile_reg_source'], meta['status_status'], meta['status_expire'],
meta['status_cancellation_status'], meta['status_new_user'],
meta['status_login_banned_time'], meta['status_anticheat_type'],
meta['status_flag_status1'], meta['status_anticheat_status'],
meta['status_flag_honor'], meta['status_privacy_policy_status'],
meta['status_csgo_frozen_exptime'], meta['platformexp_level'],
meta['platformexp_exp'], meta['steam_account'], meta['steam_trade_url'],
meta['steam_rent_id'], meta['trusted_credit'], meta['trusted_credit_level'],
meta['trusted_score'], meta['trusted_status'], meta['trusted_credit_status'],
meta['certify_id_type'], meta['certify_status'], meta['certify_age'],
meta['certify_real_name'], meta['certify_uid_list'],
meta['certify_audit_status'], meta['certify_gender'], meta['identity_type'],
meta['identity_extras'], meta['identity_status'], meta['identity_slogan'],
meta['identity_list'], meta['identity_slogan_ext'], meta['identity_live_url'],
meta['identity_live_type'], meta['plus_is_plus'], meta['user_info_raw']
]
# Build SQL dynamically
placeholders = ','.join(['?' for _ in player_columns])
columns_sql = ','.join(player_columns)
sql = f"INSERT OR REPLACE INTO dim_players ({columns_sql}) VALUES ({placeholders})"
cursor.execute(sql, player_values)
# Process fact_match_players
for steam_id, stats in match_data.players.items():
player_stats_row = _build_player_stats_tuple(match_data.match_id, stats)
cursor.execute(_get_fact_match_players_insert_sql(), player_stats_row)
# Process fact_match_players_t
for steam_id, stats in match_data.players_t.items():
player_stats_row = _build_player_stats_tuple(match_data.match_id, stats)
cursor.execute(_get_fact_match_players_insert_sql('fact_match_players_t'), player_stats_row)
# Process fact_match_players_ct
for steam_id, stats in match_data.players_ct.items():
player_stats_row = _build_player_stats_tuple(match_data.match_id, stats)
cursor.execute(_get_fact_match_players_insert_sql('fact_match_players_ct'), player_stats_row)
logger.debug(f"Processed {len(match_data.players)} players for match {match_data.match_id}")
return True
except Exception as e:
logger.error(f"Error processing players for match {match_data.match_id}: {e}")
import traceback
traceback.print_exc()
return False
def _build_player_stats_tuple(match_id, stats):
"""Build tuple for player stats insertion"""
return (
match_id,
stats.steam_id_64,
stats.team_id,
stats.kills,
stats.deaths,
stats.assists,
stats.headshot_count,
stats.kd_ratio,
stats.adr,
stats.rating,
stats.rating2,
stats.rating3,
stats.rws,
stats.mvp_count,
stats.elo_change,
stats.origin_elo,
stats.rank_score,
stats.is_win,
stats.kast,
stats.entry_kills,
stats.entry_deaths,
stats.awp_kills,
stats.clutch_1v1,
stats.clutch_1v2,
stats.clutch_1v3,
stats.clutch_1v4,
stats.clutch_1v5,
stats.flash_assists,
stats.flash_duration,
stats.jump_count,
stats.util_flash_usage,
stats.util_smoke_usage,
stats.util_molotov_usage,
stats.util_he_usage,
stats.util_decoy_usage,
stats.damage_total,
stats.damage_received,
stats.damage_receive,
stats.damage_stats,
stats.assisted_kill,
stats.awp_kill,
stats.awp_kill_ct,
stats.awp_kill_t,
stats.benefit_kill,
stats.day,
stats.defused_bomb,
stats.end_1v1,
stats.end_1v2,
stats.end_1v3,
stats.end_1v4,
stats.end_1v5,
stats.explode_bomb,
stats.first_death,
stats.fd_ct,
stats.fd_t,
stats.first_kill,
stats.flash_enemy,
stats.flash_team,
stats.flash_team_time,
stats.flash_time,
stats.game_mode,
stats.group_id,
stats.hold_total,
stats.id,
stats.is_highlight,
stats.is_most_1v2,
stats.is_most_assist,
stats.is_most_awp,
stats.is_most_end,
stats.is_most_first_kill,
stats.is_most_headshot,
stats.is_most_jump,
stats.is_svp,
stats.is_tie,
stats.kill_1,
stats.kill_2,
stats.kill_3,
stats.kill_4,
stats.kill_5,
stats.many_assists_cnt1,
stats.many_assists_cnt2,
stats.many_assists_cnt3,
stats.many_assists_cnt4,
stats.many_assists_cnt5,
stats.map,
stats.match_code,
stats.match_mode,
stats.match_team_id,
stats.match_time,
stats.per_headshot,
stats.perfect_kill,
stats.planted_bomb,
stats.revenge_kill,
stats.round_total,
stats.season,
stats.team_kill,
stats.throw_harm,
stats.throw_harm_enemy,
stats.uid,
stats.year,
stats.sts_raw,
stats.level_info_raw
)
def _get_fact_match_players_insert_sql(table='fact_match_players'):
"""Get INSERT SQL for player stats table - dynamically generated"""
# Define columns explicitly to ensure exact match with schema
columns = [
'match_id', 'steam_id_64', 'team_id', 'kills', 'deaths', 'assists', 'headshot_count',
'kd_ratio', 'adr', 'rating', 'rating2', 'rating3', 'rws', 'mvp_count', 'elo_change',
'origin_elo', 'rank_score', 'is_win', 'kast', 'entry_kills', 'entry_deaths', 'awp_kills',
'clutch_1v1', 'clutch_1v2', 'clutch_1v3', 'clutch_1v4', 'clutch_1v5',
'flash_assists', 'flash_duration', 'jump_count', 'util_flash_usage',
'util_smoke_usage', 'util_molotov_usage', 'util_he_usage', 'util_decoy_usage',
'damage_total', 'damage_received', 'damage_receive', 'damage_stats',
'assisted_kill', 'awp_kill', 'awp_kill_ct', 'awp_kill_t', 'benefit_kill',
'day', 'defused_bomb', 'end_1v1', 'end_1v2', 'end_1v3', 'end_1v4', 'end_1v5',
'explode_bomb', 'first_death', 'fd_ct', 'fd_t', 'first_kill', 'flash_enemy',
'flash_team', 'flash_team_time', 'flash_time', 'game_mode', 'group_id',
'hold_total', 'id', 'is_highlight', 'is_most_1v2', 'is_most_assist',
'is_most_awp', 'is_most_end', 'is_most_first_kill', 'is_most_headshot',
'is_most_jump', 'is_svp', 'is_tie', 'kill_1', 'kill_2', 'kill_3', 'kill_4', 'kill_5',
'many_assists_cnt1', 'many_assists_cnt2', 'many_assists_cnt3',
'many_assists_cnt4', 'many_assists_cnt5', 'map', 'match_code', 'match_mode',
'match_team_id', 'match_time', 'per_headshot', 'perfect_kill', 'planted_bomb',
'revenge_kill', 'round_total', 'season', 'team_kill', 'throw_harm',
'throw_harm_enemy', 'uid', 'year', 'sts_raw', 'level_info_raw'
]
placeholders = ','.join(['?' for _ in columns])
columns_sql = ','.join(columns)
return f'INSERT OR REPLACE INTO {table} ({columns_sql}) VALUES ({placeholders})'

97
database/L2/processors/round_processor.py Normal file
View File

@@ -0,0 +1,97 @@
"""
Round Processor - Dispatches round data processing based on data_source_type
Responsibilities:
- Act as the unified entry point for round data processing
- Determine data source type (leetify vs classic)
- Dispatch to appropriate specialized processors
- Coordinate economy, event, and spatial processors
"""
import sqlite3
import logging
logger = logging.getLogger(__name__)
class RoundProcessor:
@staticmethod
def process(match_data, conn: sqlite3.Connection) -> bool:
"""
Process round data by dispatching to specialized processors
Args:
match_data: MatchData object containing parsed JSON
conn: L2 database connection
Returns:
bool: True if successful
"""
try:
# Import specialized processors
from . import economy_processor
from . import event_processor
from . import spatial_processor
if match_data.data_source_type == 'leetify':
logger.debug(f"Processing leetify data for match {match_data.match_id}")
# Process leetify rounds
success = economy_processor.EconomyProcessor.process_leetify(match_data, conn)
if not success:
logger.warning(f"Failed to process leetify economy for match {match_data.match_id}")
# Process leetify events
success = event_processor.EventProcessor.process_leetify_events(match_data, conn)
if not success:
logger.warning(f"Failed to process leetify events for match {match_data.match_id}")
elif match_data.data_source_type == 'classic':
logger.debug(f"Processing classic data for match {match_data.match_id}")
# Process classic rounds (basic round info)
success = _process_classic_rounds(match_data, conn)
if not success:
logger.warning(f"Failed to process classic rounds for match {match_data.match_id}")
# Process classic economy (NEW)
success = economy_processor.EconomyProcessor.process_classic(match_data, conn)
if not success:
logger.warning(f"Failed to process classic economy for match {match_data.match_id}")
# Process classic events (kills, bombs)
success = event_processor.EventProcessor.process_classic_events(match_data, conn)
if not success:
logger.warning(f"Failed to process classic events for match {match_data.match_id}")
# Process spatial data (xyz coordinates)
success = spatial_processor.SpatialProcessor.process(match_data, conn)
if not success:
logger.warning(f"Failed to process spatial data for match {match_data.match_id}")
else:
logger.info(f"No round data to process for match {match_data.match_id} (data_source_type={match_data.data_source_type})")
return True
except Exception as e:
logger.error(f"Error in round processor for match {match_data.match_id}: {e}")
import traceback
traceback.print_exc()
return False
def _process_classic_rounds(match_data, conn: sqlite3.Connection) -> bool:
"""
Process basic round information for classic data source
Classic round data contains:
- current_score (ct/t scores, type, pasttime, final_round_time)
- But lacks economy data
"""
try:
# This is handled by event_processor for classic
# Classic rounds are extracted from round_list structure
# which is processed in event_processor.process_classic_events
return True
except Exception as e:
logger.error(f"Error processing classic rounds: {e}")
return False

100
database/L2/processors/spatial_processor.py Normal file
View File

@@ -0,0 +1,100 @@
"""
Spatial Processor - Handles classic spatial (xyz) data
Responsibilities:
- Extract attacker/victim position data from classic round_list
- Update fact_round_events with spatial coordinates
- Prepare data for future heatmap/tactical board analysis
"""
import sqlite3
import logging
logger = logging.getLogger(__name__)
class SpatialProcessor:
@staticmethod
def process(match_data, conn: sqlite3.Connection) -> bool:
"""
Process spatial data from classic round_list
Args:
match_data: MatchData object with round_list parsed
conn: L2 database connection
Returns:
bool: True if successful
"""
try:
if not hasattr(match_data, 'data_round_list') or not match_data.data_round_list:
return True
round_list = match_data.data_round_list.get('round_list', [])
if not round_list:
return True
cursor = conn.cursor()
update_count = 0
for idx, rd in enumerate(round_list, start=1):
round_num = idx
# Process kill events with spatial data
all_kill = rd.get('all_kill', [])
for kill in all_kill:
attacker = kill.get('attacker', {})
victim = kill.get('victim', {})
attacker_steam_id = str(attacker.get('steamid_64', ''))
victim_steam_id = str(victim.get('steamid_64', ''))
event_time = kill.get('pasttime', 0)
# Extract positions
attacker_pos = attacker.get('pos', {})
victim_pos = victim.get('pos', {})
attacker_pos_x = attacker_pos.get('x', 0) if isinstance(attacker_pos, dict) else 0
attacker_pos_y = attacker_pos.get('y', 0) if isinstance(attacker_pos, dict) else 0
attacker_pos_z = attacker_pos.get('z', 0) if isinstance(attacker_pos, dict) else 0
victim_pos_x = victim_pos.get('x', 0) if isinstance(victim_pos, dict) else 0
victim_pos_y = victim_pos.get('y', 0) if isinstance(victim_pos, dict) else 0
victim_pos_z = victim_pos.get('z', 0) if isinstance(victim_pos, dict) else 0
# Update existing event with spatial data
# We match by match_id, round_num, attacker, victim, and event_time
cursor.execute('''
UPDATE fact_round_events
SET attacker_pos_x = ?,
attacker_pos_y = ?,
attacker_pos_z = ?,
victim_pos_x = ?,
victim_pos_y = ?,
victim_pos_z = ?
WHERE match_id = ?
AND round_num = ?
AND attacker_steam_id = ?
AND victim_steam_id = ?
AND event_time = ?
AND event_type = 'kill'
AND data_source_type = 'classic'
''', (
attacker_pos_x, attacker_pos_y, attacker_pos_z,
victim_pos_x, victim_pos_y, victim_pos_z,
match_data.match_id, round_num, attacker_steam_id,
victim_steam_id, event_time
))
if cursor.rowcount > 0:
update_count += 1
logger.debug(f"Updated {update_count} events with spatial data for match {match_data.match_id}")
return True
except Exception as e:
logger.error(f"Error processing spatial data for match {match_data.match_id}: {e}")
import traceback
traceback.print_exc()
return False