vyndr/scripts/resolution-diagnosis.js

#!/usr/bin/env node
'use strict';

/**
 * PHASE 2 — put a number on the resolution ceiling.
 *
 * Murphy's decomposition: Brier = reliability - resolution + uncertainty.
 *
 *   reliability  how far each bin's realized rate sits from its forecast (lower
 *                is better; this is what calibration fixes)
 *   resolution   how far the bins' realized rates spread from the base rate
 *                (HIGHER is better; this is discrimination, and NO amount of
 *                calibration can create it)
 *   uncertainty  the base rate's own variance -- a property of the event
 *
 * Calibration moves reliability and leaves resolution untouched by construction:
 * a monotone map relabels bins without re-sorting the rows inside them. So if
 * resolution is near zero, honest numbers are all calibration can ever deliver.
 */

require('dotenv').config();
const fs = require('fs');
const path = require('path');
const { createClient } = require('@supabase/supabase-js');
const lp = require('../src/services/model/lowParamCalibrator');
const guards = require('../src/services/model/calibrationGuards');
const { knownNumber } = require('../src/utils/known');

const BOX = path.join(process.cwd(), '.seq-cache', 'batting-lines.json');
const STATS = ['hits', 'total_bases', 'rbi', 'runs'];
const DEPLOYED = ['hits', 'total_bases'];
const PAGE = 1000;
const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs };
const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);

async function page(sb, t, s, f) {
  const o = [];
  for (let i = 0; ; i += PAGE) {
    const { data, error } = await f(sb.from(t).select(s)).order('id', { ascending: true }).range(i, i + PAGE - 1);
    if (error) throw error; if (!data.length) break; o.push(...data); if (data.length < PAGE) break;
  }
  return o;
}
const isPreGame = (c, g) => {
  const et = new Date(new Date(c).getTime() - 4 * 3600 * 1000);
  const d = et.toISOString().slice(0, 10);
  return d < g || (d === g && et.getUTCHours() < 19);
};

/** Murphy decomposition over K equal-width bins. */
function decompose(rows, bins = 10) {
  const base = mean(rows.map((r) => r.won));
  const uncertainty = base * (1 - base);
  let reliability = 0; let resolution = 0;
  const table = [];
  for (let k = 0; k < bins; k += 1) {
    const lo = k / bins; const hi = (k + 1) / bins;
    const slice = rows.filter((r) => r.p >= lo && (hi >= 1 ? r.p <= 1 : r.p < hi));
    if (!slice.length) continue;
    const w = slice.length / rows.length;
    const fk = mean(slice.map((r) => r.p));
    const ok = mean(slice.map((r) => r.won));
    reliability += w * (fk - ok) ** 2;
    resolution += w * (ok - base) ** 2;
    table.push({ bin: [round2(lo), round2(hi)], n: slice.length, forecast: round4(fk), realized: round4(ok) });
  }
  return {
    base_rate: round4(base),
    reliability: round5(reliability),
    resolution: round5(resolution),
    uncertainty: round5(uncertainty),
    brier_check: round5(reliability - resolution + uncertainty),
    /** What share of the event's variance the model actually explains. */
    resolution_share_of_uncertainty: round4(resolution / uncertainty),
    bins: table,
  };
}

(async () => {
  const sb = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY, { auth: { persistSession: false } });
  const lines = JSON.parse(fs.readFileSync(BOX, 'utf8')).lines;
  const snaps = await page(sb, 'model_snapshots', 'id, game_date, captured_at, stat, player_key, line, side, p_win, refused',
    (q) => q.eq('sport', 'mlb').in('stat', STATS));

  const picked = new Map();
  for (const r of snaps) {
    if (!isPreGame(r.captured_at, r.game_date) || r.refused || knownNumber(r.p_win) === null) continue;
    const k = [r.game_date, r.stat, r.player_key, r.line].join('|');
    const prev = picked.get(k);
    if (!prev || knownNumber(r.p_win) > knownNumber(prev.p_win)) picked.set(k, r);
  }
  guards.assertPickedSideDedup([...picked.values()].map((r) => ({
    propKey: [r.game_date, r.stat, r.player_key, r.line].join('|'), side: r.side, p: knownNumber(r.p_win) })));

  const out = {};
  for (const stat of STATS) {
    const rows = [];
    for (const r of picked.values()) {
      if (r.stat !== stat) continue;
      const b = lines[`${r.game_date}|${r.player_key}`]; const L = knownNumber(r.line);
      if (!b || L === null || !r.side) continue;
      const v = knownNumber(FIELD[stat](b)); if (v === null) continue;
      const over = v > L;
      rows.push({ date: r.game_date, p: knownNumber(r.p_win), won: (String(r.side).toLowerCase() === 'under' ? !over : over) ? 1 : 0 });
    }
    if (rows.length < 100) continue;

    const raw = decompose(rows);
    let served = null;
    if (DEPLOYED.includes(stat)) {
      const m = lp.fitPlatt(rows);
      if (m && !m.refused) {
        const cal = rows.map((r) => ({ ...r, p: lp.applyPlatt(m, r.p) })).filter((r) => knownNumber(r.p) !== null);
        served = decompose(cal);
      }
    }
    out[stat] = {
      n: rows.length,
      deployed: DEPLOYED.includes(stat),
      raw,
      served,
      resolution_change_from_calibration: served ? round5(served.resolution - raw.resolution) : null,
      reliability_change_from_calibration: served ? round5(served.reliability - raw.reliability) : null,
    };
  }
  console.log(JSON.stringify(out, null, 2));
  process.exit(0);
})().catch((e) => { console.error(e); process.exit(1); });

const round5 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 100000) / 100000);
const round4 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 10000) / 10000);
const round2 = (v) => Math.round(v * 100) / 100;