#!/usr/bin/env node /** * Daily soccer intelligence prefetch — run once per day. * * cron: 0 5 * * * (5am UTC, ~midnight ET — before US fixtures) * call: node scripts/soccer-data-prefetch.js [--leagues=WC,PL] [--dry-run] * * Why: football-data.org caps at 10 req/min and ~10/day for some * endpoints. We can't read these on the user request path. This script * batches the reads, transforms them into the per-player / per-team * aggregates the feature extractor consumes, and persists them to * Redis with conservative TTLs. * * Writes: * soccer:{league}:standings — raw standings from API * soccer:{league}:scorers — top-scorers list (projected) * soccer:player:{normalizedName} — per-player aggregate (per-90 rates) * soccer:teamdefense:{league}:{team} — team defensive aggregate + normalized rank * * Does NOT write next-match / last-fixture pointers — those are the * job of the poller (poller/soccer.js), which runs more frequently * since fixture state changes faster. * * xG data (`xg_per_90`, `xg_delta`) is left null on Day 1 — sourcing * it requires a soccerdata-Python bridge that's a follow-up. The * downstream feature extractor handles null xG gracefully. * * No DB writes. Graceful exit (code 0) when API keys are missing — the * script logs "skipped" and the feature extractor continues with the * static-data-only path. */ const fbd = require('../src/services/adapters/footballDataAdapter'); const { cacheSet } = require('../src/utils/redis'); const { normalizeName } = require('../src/utils/normalize'); const PLAYER_TTL_SEC = 24 * 3600; const STANDINGS_TTL_SEC = 12 * 3600; const SCORERS_TTL_SEC = 6 * 3600; const DEFENSE_TTL_SEC = 12 * 3600; function parseArgs(argv) { const args = { leagues: ['WC'], dryRun: false }; for (const a of argv.slice(2)) { if (a.startsWith('--leagues=')) { args.leagues = a.slice('--leagues='.length).split(',').map((s) => s.trim().toUpperCase()).filter(Boolean); } else if (a === '--dry-run') { args.dryRun = true; } } // env override falls through if no CLI value was given. if (!process.argv.some((a) => a.startsWith('--leagues='))) { const env = process.env.SOCCER_LEAGUES; if (env) args.leagues = env.split(',').map((s) => s.trim().toUpperCase()).filter(Boolean); } return args; } // Project a single team's standings row into the defensive aggregate // the feature extractor reads. defensive_rank_norm is on a 0..1 scale // (0 = best defense, 1 = worst) so it slots into engine1's opp_rank_stat. function aggregateTeamDefense(standingsRow, allRows) { const playedGames = standingsRow.playedGames || standingsRow.played || 0; const goalsAgainst = standingsRow.goalsAgainst ?? null; if (!playedGames || goalsAgainst == null) return null; const goalsConcededPerGame = goalsAgainst / playedGames; // Normalize against the rest of the table — defensive_rank_norm = the // team's goals-conceded percentile (0 best, 1 worst). const allRates = allRows .map((r) => { const pg = r.playedGames || r.played || 0; if (!pg) return null; return (r.goalsAgainst ?? 0) / pg; }) .filter((v) => Number.isFinite(v)) .sort((a, b) => a - b); let rank = allRates.findIndex((v) => v >= goalsConcededPerGame); if (rank === -1) rank = allRates.length - 1; const rankNorm = allRates.length > 1 ? rank / (allRates.length - 1) : 0; // Clean sheets (not on the football-data row in the free tier — null is OK). const cleanSheets = standingsRow.cleanSheets ?? null; const cleanSheetRate = cleanSheets != null && playedGames > 0 ? cleanSheets / playedGames : null; return { goals_conceded_per_game: Math.round(goalsConcededPerGame * 1000) / 1000, clean_sheet_rate: cleanSheetRate, defensive_rank: rank + 1, // 1-indexed for human reasoning defensive_rank_norm: rankNorm, // 0..1 for engine1 played_games: playedGames, }; } // Project a single scorer row into the per-player aggregate. function aggregatePlayerFromScorer(scorerRow) { // Number(null) is 0 — explicit null check so a missing minutes field // doesn't pretend the player played 0 minutes (which would still // satisfy Number.isFinite and break the per-90 fallback). const minutes = scorerRow.minutesPlayed == null ? null : Number(scorerRow.minutesPlayed); const goals = Number(scorerRow.goals) || 0; const assists = Number(scorerRow.assists) || 0; const played = Number(scorerRow.playedMatches) || 0; // Per-90 rates need minutes. The free tier sometimes omits minutes — // fall back to (goals / played) when missing. const goalsPer90 = Number.isFinite(minutes) && minutes > 0 ? Math.round((goals / (minutes / 90)) * 1000) / 1000 : (played > 0 ? Math.round((goals / played) * 1000) / 1000 : null); const assistsPer90 = Number.isFinite(minutes) && minutes > 0 ? Math.round((assists / (minutes / 90)) * 1000) / 1000 : (played > 0 ? Math.round((assists / played) * 1000) / 1000 : null); const minutesPerGame = Number.isFinite(minutes) && played > 0 ? Math.round(minutes / played) : null; return { team: scorerRow.team, position: scorerRow.position, nationality: scorerRow.nationality, goals, assists, played, minutes: Number.isFinite(minutes) ? minutes : null, goals_per_90: goalsPer90, assists_per_90: assistsPer90, minutes_per_game: minutesPerGame, // Day 1 — no rolling 5-match form, no xG. The feature extractor // falls back to season_per_90 when recent_form_per_90 is null. recent_form_per_90: null, season_per_90: goalsPer90, start_rate: null, xg_per_90: null, xa_per_90: null, xg_delta: null, }; } async function processLeague(league, { dryRun }) { const summary = { league, standings: 0, scorers: 0, players: 0, teamDefense: 0, skipped: false }; const [standings, scorers] = await Promise.all([ fbd.getLeagueStandings(league), fbd.getLeagueScorers(league), ]); // Either null means "API unavailable" — log + bail for this league. if (standings === null && scorers === null) { summary.skipped = true; return summary; } // ---- Standings → team defensive aggregates ---- // football-data wraps standings in groups (type === 'TOTAL' has the // table). Flatten all `table` rows so a competition with multiple // groups (e.g. World Cup group stage) feeds one combined rank table. if (Array.isArray(standings)) { const allRows = []; for (const group of standings) { if (Array.isArray(group?.table)) { for (const row of group.table) { if (row?.team?.name) allRows.push({ ...row, teamName: row.team.name }); } } } summary.standings = allRows.length; for (const row of allRows) { const agg = aggregateTeamDefense(row, allRows); if (!agg) continue; const key = `soccer:teamdefense:${league.toLowerCase()}:${row.teamName}`; if (!dryRun) await cacheSet(key, agg, DEFENSE_TTL_SEC); summary.teamDefense += 1; } if (!dryRun) await cacheSet(`soccer:${league.toLowerCase()}:standings`, standings, STANDINGS_TTL_SEC); } // ---- Scorers → per-player aggregates ---- if (Array.isArray(scorers)) { summary.scorers = scorers.length; for (const s of scorers) { if (!s?.name) continue; const profile = aggregatePlayerFromScorer(s); const key = `soccer:player:${normalizeName(s.name)}`; if (!dryRun) await cacheSet(key, profile, PLAYER_TTL_SEC); summary.players += 1; } if (!dryRun) await cacheSet(`soccer:${league.toLowerCase()}:scorers`, scorers, SCORERS_TTL_SEC); } return summary; } async function main(argv = process.argv) { const args = parseArgs(argv); const startTs = Date.now(); console.log(`[soccer-prefetch] starting — leagues=${args.leagues.join(',')} dry_run=${args.dryRun}`); if (!fbd.hasApiKey()) { console.warn('[soccer-prefetch] FOOTBALL_DATA_API_KEY not set — skipping. WC fixtures still flow via the OSS API in poller/soccer.js; non-WC leagues are no-ops until the key is configured.'); return { skipped: true }; } const results = []; for (const league of args.leagues) { try { const r = await processLeague(league, args); results.push(r); console.log(`[soccer-prefetch] ${league}: standings=${r.standings} scorers=${r.scorers} players=${r.players} teamDefense=${r.teamDefense} ${r.skipped ? '(skipped: no_data)' : ''}`); } catch (err) { console.warn(`[soccer-prefetch] ${league} failed:`, err.message); results.push({ league, error: err.message }); } } const elapsed = Math.round((Date.now() - startTs) / 1000); console.log(`[soccer-prefetch] done in ${elapsed}s — ${results.length} leagues processed`); return { results, elapsedSec: elapsed }; } if (require.main === module) { main().then(() => process.exit(0)).catch((err) => { console.error('[soccer-prefetch] fatal:', err); process.exit(1); }); } module.exports = { main, __internals: { parseArgs, aggregateTeamDefense, aggregatePlayerFromScorer, processLeague, PLAYER_TTL_SEC, STANDINGS_TTL_SEC, SCORERS_TTL_SEC, DEFENSE_TTL_SEC, }, };