#!/usr/bin/env node 'use strict'; /** * PHASE 2 — put a number on the resolution ceiling. * * Murphy's decomposition: Brier = reliability - resolution + uncertainty. * * reliability how far each bin's realized rate sits from its forecast (lower * is better; this is what calibration fixes) * resolution how far the bins' realized rates spread from the base rate * (HIGHER is better; this is discrimination, and NO amount of * calibration can create it) * uncertainty the base rate's own variance -- a property of the event * * Calibration moves reliability and leaves resolution untouched by construction: * a monotone map relabels bins without re-sorting the rows inside them. So if * resolution is near zero, honest numbers are all calibration can ever deliver. */ require('dotenv').config(); const fs = require('fs'); const path = require('path'); const { createClient } = require('@supabase/supabase-js'); const lp = require('../src/services/model/lowParamCalibrator'); const guards = require('../src/services/model/calibrationGuards'); const { knownNumber } = require('../src/utils/known'); const BOX = path.join(process.cwd(), '.seq-cache', 'batting-lines.json'); const STATS = ['hits', 'total_bases', 'rbi', 'runs']; const DEPLOYED = ['hits', 'total_bases']; const PAGE = 1000; const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs }; const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null); async function page(sb, t, s, f) { const o = []; for (let i = 0; ; i += PAGE) { const { data, error } = await f(sb.from(t).select(s)).order('id', { ascending: true }).range(i, i + PAGE - 1); if (error) throw error; if (!data.length) break; o.push(...data); if (data.length < PAGE) break; } return o; } const isPreGame = (c, g) => { const et = new Date(new Date(c).getTime() - 4 * 3600 * 1000); const d = et.toISOString().slice(0, 10); return d < g || (d === g && et.getUTCHours() < 19); }; /** Murphy decomposition over K equal-width bins. */ function decompose(rows, bins = 10) { const base = mean(rows.map((r) => r.won)); const uncertainty = base * (1 - base); let reliability = 0; let resolution = 0; const table = []; for (let k = 0; k < bins; k += 1) { const lo = k / bins; const hi = (k + 1) / bins; const slice = rows.filter((r) => r.p >= lo && (hi >= 1 ? r.p <= 1 : r.p < hi)); if (!slice.length) continue; const w = slice.length / rows.length; const fk = mean(slice.map((r) => r.p)); const ok = mean(slice.map((r) => r.won)); reliability += w * (fk - ok) ** 2; resolution += w * (ok - base) ** 2; table.push({ bin: [round2(lo), round2(hi)], n: slice.length, forecast: round4(fk), realized: round4(ok) }); } return { base_rate: round4(base), reliability: round5(reliability), resolution: round5(resolution), uncertainty: round5(uncertainty), brier_check: round5(reliability - resolution + uncertainty), /** What share of the event's variance the model actually explains. */ resolution_share_of_uncertainty: round4(resolution / uncertainty), bins: table, }; } (async () => { const sb = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY, { auth: { persistSession: false } }); const lines = JSON.parse(fs.readFileSync(BOX, 'utf8')).lines; const snaps = await page(sb, 'model_snapshots', 'id, game_date, captured_at, stat, player_key, line, side, p_win, refused', (q) => q.eq('sport', 'mlb').in('stat', STATS)); const picked = new Map(); for (const r of snaps) { if (!isPreGame(r.captured_at, r.game_date) || r.refused || knownNumber(r.p_win) === null) continue; const k = [r.game_date, r.stat, r.player_key, r.line].join('|'); const prev = picked.get(k); if (!prev || knownNumber(r.p_win) > knownNumber(prev.p_win)) picked.set(k, r); } guards.assertPickedSideDedup([...picked.values()].map((r) => ({ propKey: [r.game_date, r.stat, r.player_key, r.line].join('|'), side: r.side, p: knownNumber(r.p_win) }))); const out = {}; for (const stat of STATS) { const rows = []; for (const r of picked.values()) { if (r.stat !== stat) continue; const b = lines[`${r.game_date}|${r.player_key}`]; const L = knownNumber(r.line); if (!b || L === null || !r.side) continue; const v = knownNumber(FIELD[stat](b)); if (v === null) continue; const over = v > L; rows.push({ date: r.game_date, p: knownNumber(r.p_win), won: (String(r.side).toLowerCase() === 'under' ? !over : over) ? 1 : 0 }); } if (rows.length < 100) continue; const raw = decompose(rows); let served = null; if (DEPLOYED.includes(stat)) { const m = lp.fitPlatt(rows); if (m && !m.refused) { const cal = rows.map((r) => ({ ...r, p: lp.applyPlatt(m, r.p) })).filter((r) => knownNumber(r.p) !== null); served = decompose(cal); } } out[stat] = { n: rows.length, deployed: DEPLOYED.includes(stat), raw, served, resolution_change_from_calibration: served ? round5(served.resolution - raw.resolution) : null, reliability_change_from_calibration: served ? round5(served.reliability - raw.reliability) : null, }; } console.log(JSON.stringify(out, null, 2)); process.exit(0); })().catch((e) => { console.error(e); process.exit(1); }); const round5 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 100000) / 100000); const round4 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 10000) / 10000); const round2 = (v) => Math.round(v * 100) / 100;