vyndr/scripts/rbi-resolution-decomposition.js

#!/usr/bin/env node
'use strict';

/**
 * PHASES 0-1 — is rbi's 14.51% real, and where does it come from?
 *
 * PHASE 0 is a Beck gate. The 14.51% came from the same decomposition harness
 * whose paging helper produced a false null three times tonight (composite PKs,
 * order-by-id, error swallowed). So the row count is asserted against an
 * independent exact count, every page is error-checked, and 20 raw rows are
 * printed so the decile arithmetic can be audited by hand.
 *
 * PHASE 1 asks what the resolution IS. Resolution rewards a forecast for
 * separating outcomes — but a forecast can separate outcomes by knowing WHO is
 * batting rather than anything about tonight. Three nested forecasts:
 *
 *   PLAYER BASE RATE   leave-one-out frequency for that hitter, nothing else.
 *                      Its resolution is pure across-player spread.
 *   LINEUP SLOT        mean rate for that batting-order position. Real
 *                      predictive signal, but ROLE, not skill.
 *   THE MODEL          served p_win.
 *
 * The part that behaves like our doctrine's "skill" is what the model resolves
 * WITHIN a stratum of similar players — measured directly by stratifying on the
 * player's own base rate and pooling the within-stratum resolutions.
 */

require('dotenv').config();
const fs = require('fs');
const path = require('path');
const { createClient } = require('@supabase/supabase-js');
const guards = require('../src/services/model/calibrationGuards');
const { knownNumber } = require('../src/utils/known');
const { nameKey } = require('../src/utils/playerName');

const BOX = path.join(process.cwd(), '.seq-cache', 'batting-lines.json');
const SEQ = path.join(process.cwd(), '.seq-cache', 'sequences.json');
const STATS = ['rbi', 'hits', 'total_bases', 'runs'];
const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs };
const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);

/** Error-checked pager with an explicit order column. */
async function page(sb, t, sel, orderBy, apply) {
  const out = [];
  for (let i = 0; ; i += 1000) {
    const q = apply ? apply(sb.from(t).select(sel)) : sb.from(t).select(sel);
    const { data, error } = await q.order(orderBy, { ascending: true }).range(i, i + 999);
    if (error) throw new Error(`${t}: ${error.message}`);
    if (!data || !data.length) break;
    out.push(...data);
    if (data.length < 1000) break;
  }
  return out;
}
const isPreGame = (c, g) => {
  const et = new Date(new Date(c).getTime() - 4 * 3600 * 1000);
  const d = et.toISOString().slice(0, 10);
  return d < g || (d === g && et.getUTCHours() < 19);
};

/** Resolution alone: weighted spread of bin realized rates about the base rate. */
function resolutionOf(rows, bins = 10) {
  const base = mean(rows.map((r) => r.won));
  let res = 0;
  const table = [];
  for (let k = 0; k < bins; k += 1) {
    const lo = k / bins; const hi = (k + 1) / bins;
    const sl = rows.filter((r) => r.p >= lo && (hi >= 1 ? r.p <= 1 : r.p < hi));
    if (!sl.length) continue;
    const w = sl.length / rows.length;
    const ok = mean(sl.map((r) => r.won));
    res += w * (ok - base) ** 2;
    table.push({ bin: `${lo.toFixed(1)}-${hi.toFixed(1)}`, n: sl.length, forecast: r4(mean(sl.map((r) => r.p))), realized: r4(ok) });
  }
  return { base_rate: r4(base), resolution: r5(res), uncertainty: r5(base * (1 - base)), share: r4(res / (base * (1 - base))), table };
}

(async () => {
  const sb = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY, { auth: { persistSession: false } });
  const lines = JSON.parse(fs.readFileSync(BOX, 'utf8')).lines;

  // Batting-order slot, reconstructed point-in-time from play-by-play.
  const { games } = JSON.parse(fs.readFileSync(SEQ, 'utf8'));
  const slotOf = new Map();
  for (const g of games) {
    for (const half of ['top', 'bottom']) {
      const seen = []; const set = new Set();
      for (const p of g.pas.filter((x) => x.half === half)) {
        if (!set.has(p.batter)) { set.add(p.batter); seen.push(p); }
        if (seen.length >= 9) break;
      }
      seen.forEach((p, i) => slotOf.set(`${g.date}|${nameKey(p.batter_name || '')}`, i + 1));
    }
  }

  const out = { PHASE_0: {}, PHASE_1: {} };

  for (const stat of STATS) {
    // PHASE 0 — exact count first, then the paged pull must match it.
    const { count: exact, error: cErr } = await sb.from('model_snapshots')
      .select('*', { count: 'exact', head: true }).eq('sport', 'mlb').eq('stat', stat);
    if (cErr) throw new Error(`count ${stat}: ${cErr.message}`);
    const snaps = await page(sb, 'model_snapshots',
      'id, game_date, captured_at, stat, player_key, player_name, line, side, p_win, refused', 'id',
      (q) => q.eq('sport', 'mlb').eq('stat', stat));
    if (snaps.length !== exact) throw new Error(`PHASE 0 FAIL ${stat}: paged ${snaps.length} != exact ${exact}`);

    const picked = new Map();
    for (const r of snaps) {
      if (!isPreGame(r.captured_at, r.game_date) || r.refused || knownNumber(r.p_win) === null) continue;
      const k = [r.game_date, r.player_key, r.line].join('|');
      const prev = picked.get(k);
      if (!prev || knownNumber(r.p_win) > knownNumber(prev.p_win)) picked.set(k, r);
    }
    guards.assertPickedSideDedup([...picked.values()].map((r) => ({ propKey: [r.game_date, r.player_key, r.line].join('|'), side: r.side, p: knownNumber(r.p_win) })));

    const rows = [];
    for (const r of picked.values()) {
      const b = lines[`${r.game_date}|${r.player_key}`]; const L = knownNumber(r.line);
      if (!b || L === null || !r.side) continue;
      const v = knownNumber(FIELD[stat](b)); if (v === null) continue;
      const over = v > L;
      rows.push({
        date: r.game_date, key: r.player_key, name: r.player_name, line: L, side: r.side,
        actual: v, p: knownNumber(r.p_win),
        won: (String(r.side).toLowerCase() === 'under' ? !over : over) ? 1 : 0,
        slot: slotOf.get(`${r.game_date}|${r.player_key}`) ?? null,
      });
    }

    const model = resolutionOf(rows);
    out.PHASE_0[stat] = { exact_rows: exact, paged_rows: snaps.length, scorable: rows.length, model_resolution: model.resolution, model_share: model.share, deciles: model.table };
    if (stat === 'rbi') {
      out.PHASE_0.rbi_raw_sample = rows.slice(0, 20).map((r) => ({ name: r.name, date: r.date, line: r.line, side: r.side, p_win: r.p, actual_rbi: r.actual, won: r.won }));
    }

    // ── (a) PLAYER BASE RATE, leave-one-out ──
    const byPlayer = new Map();
    for (const r of rows) { const c = byPlayer.get(r.key) || { n: 0, w: 0 }; c.n += 1; c.w += r.won; byPlayer.set(r.key, c); }
    const baseRows = rows.filter((r) => byPlayer.get(r.key).n >= 3)
      .map((r) => { const c = byPlayer.get(r.key); return { ...r, p: (c.w - r.won) / (c.n - 1) }; });
    const baseOnly = resolutionOf(baseRows);

    // ── (b) LINEUP SLOT, leave-one-out ──
    const bySlot = new Map();
    for (const r of rows) { if (r.slot == null) continue; const c = bySlot.get(r.slot) || { n: 0, w: 0 }; c.n += 1; c.w += r.won; bySlot.set(r.slot, c); }
    const slotRows = rows.filter((r) => r.slot != null && bySlot.get(r.slot).n >= 10)
      .map((r) => { const c = bySlot.get(r.slot); return { ...r, p: (c.w - r.won) / (c.n - 1) }; });
    const slotOnly = slotRows.length ? resolutionOf(slotRows) : null;

    // ── (c) WITHIN-STRATUM: does the model still separate similar players? ──
    // Stratify on the player's own base rate, then pool the model's resolution
    // computed INSIDE each stratum. Across-player spread is held constant, so
    // what survives is discrimination between comparable hitters.
    const strata = [[0, 0.45], [0.45, 0.6], [0.6, 0.75], [0.75, 1.01]];
    let within = 0; let wTot = 0; const strataDetail = [];
    for (const [lo, hi] of strata) {
      const sl = rows.filter((r) => { const c = byPlayer.get(r.key); if (!c || c.n < 3) return false; const b = c.w / c.n; return b >= lo && b < hi; });
      if (sl.length < 40) continue;
      const rr = resolutionOf(sl);
      within += sl.length * rr.resolution; wTot += sl.length;
      strataDetail.push({ stratum: `${lo}-${hi}`, n: sl.length, base: rr.base_rate, resolution: rr.resolution });
    }
    const withinRes = wTot ? within / wTot : null;

    out.PHASE_1[stat] = {
      n: rows.length,
      model_resolution: model.resolution,
      model_share_of_variance: model.share,
      a_player_base_rate_only: { n: baseRows.length, resolution: baseOnly.resolution, share: baseOnly.share },
      b_lineup_slot_only: slotOnly ? { n: slotRows.length, resolution: slotOnly.resolution, share: slotOnly.share } : null,
      c_within_stratum_resolution: r5(withinRes),
      strata: strataDetail,
      base_rate_explains_pct: baseOnly.resolution ? r4(Math.min(1, baseOnly.resolution / model.resolution)) : null,
      within_stratum_share_of_model: withinRes != null && model.resolution ? r4(withinRes / model.resolution) : null,
    };
  }

  console.log(JSON.stringify(out, null, 2));
  process.exit(0);
})().catch((e) => { console.error('FAILED:', e.message); process.exit(1); });

const r5 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 100000) / 100000);
const r4 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 10000) / 10000);