vyndr/scripts/lodo-calibration.js

#!/usr/bin/env node
'use strict';

/**
 * lodo-calibration — Phases 2 and 3.
 *
 * The ≥40 date-cluster floor was factorGate's interval bar for a CAUSAL claim,
 * mis-applied to a monotone shrink-to-observed layer. Calibration makes no causal
 * claim, consumes no Bonferroni slot, and has a bounded failure mode (it can only
 * over- or under-shrink). Its real risk is that the correction is DATE-DRIVEN —
 * that one unusual day's offensive environment is doing all the work.
 *
 * Leave-one-date-out tests exactly that, and it is a harder bar than a cluster
 * count: a single date whose removal reverses the improvement, or flips the
 * favourite-longshot sign, fails the stat outright.
 *
 * ── WHAT LODO IS AND IS NOT ──────────────────────────────────────────────
 * Refitting on all-but-one date uses dates that follow the held-out one, so this
 * is a STABILITY test, not a point-in-time backtest. The point-in-time result is
 * separate and already established (fit-past / apply-forward, CI excluding zero
 * on hits / TB / RBI). Both are required; neither substitutes for the other.
 *
 *   SUPABASE_URL=... node scripts/lodo-calibration.js
 */

require('dotenv').config();
const fs = require('fs');
const path = require('path');
const { createClient } = require('@supabase/supabase-js');
const cal = require('../src/services/model/calibration');
const guards = require('../src/services/model/calibrationGuards');
const { knownNumber } = require('../src/utils/known');

const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
const BOX = path.join(process.cwd(), '.seq-cache', 'batting-lines.json');
const STATS = ['hits', 'total_bases', 'rbi', 'runs'];
const PAGE = 1000;
/** The favourite bucket where the over-prediction concentrates. */
const FAVOURITE_FLOOR = 0.9;
/**
 * Minimum rows on a held-out date for that drop to be informative.
 *
 * POWER-DERIVED AND PRE-COMMITTED (n* = 70). Not chosen here, and not tunable
 * from here -- it is imported so the value that decides the verdicts cannot be
 * edited alongside them.
 */
const { LODO_TEST, LODO_POWER_FLOOR } = require('../src/services/model/calibrationRegistry');

const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs };
const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);

async function page(sb, table, select, apply) {
  const out = [];
  for (let from = 0; ; from += PAGE) {
    const { data, error } = await apply(sb.from(table).select(select))
      .order('id', { ascending: true }).range(from, from + PAGE - 1);
    if (error) throw error;
    if (!data || data.length === 0) break;
    out.push(...data);
    if (data.length < PAGE) break;
  }
  return out;
}

const isPreGame = (capturedAt, gameDate) => {
  const et = new Date(new Date(capturedAt).getTime() - 4 * 3600 * 1000);
  const d = et.toISOString().slice(0, 10);
  return d < gameDate || (d === gameDate && et.getUTCHours() < 19);
};

async function main() {
  const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
  const lines = JSON.parse(fs.readFileSync(BOX, 'utf8')).lines;

  const snaps = await page(sb, 'model_snapshots',
    'id, game_date, captured_at, stat, player_key, line, side, p_win, refused',
    (q) => q.eq('sport', 'mlb').in('stat', STATS));

  // Build the RAW population first so the guard has something to catch.
  const raw = [];
  const picked = new Map();
  for (const r of snaps) {
    if (!isPreGame(r.captured_at, r.game_date)) continue;
    if (r.refused || knownNumber(r.p_win) === null) continue;
    const propKey = [r.game_date, r.stat, r.player_key, r.line].join('|');
    raw.push({ propKey, side: r.side, p: knownNumber(r.p_win) });
    const prev = picked.get(propKey);
    if (!prev || knownNumber(r.p_win) > knownNumber(prev.p_win)) picked.set(propKey, r);
  }

  // GUARD 1 — prove the raw population would have lied, then prove dedup fixes it.
  const rawCheck = guards.checkPickedSideDedup(raw);
  const pickedRows = [...picked.values()].map((r) => ({
    propKey: [r.game_date, r.stat, r.player_key, r.line].join('|'),
    side: r.side, p: knownNumber(r.p_win),
  }));
  guards.assertPickedSideDedup(pickedRows);          // throws if dedup failed

  const out = {
    guard_1_raw_population: { violated: rawCheck.violated, mean_p: rawCheck.mean_p, both_sides_share: rawCheck.both_sides_share },
    guard_1_after_dedup: guards.checkPickedSideDedup(pickedRows),
    per_stat: {},
  };

  for (const stat of STATS) {
    const rows = [];
    for (const r of picked.values()) {
      if (r.stat !== stat) continue;
      const b = lines[`${r.game_date}|${r.player_key}`];
      const L = knownNumber(r.line);
      if (!b || L === null || !r.side) continue;
      const v = knownNumber(FIELD[stat](b));
      if (v === null) continue;
      const over = v > L;
      rows.push({
        date: r.game_date,
        p: knownNumber(r.p_win),
        won: (String(r.side).toLowerCase() === 'under' ? !over : over) ? 1 : 0,
      });
    }
    const dates = [...new Set(rows.map((r) => r.date))].sort();

    const full = cal.fitIsotonic(rows.map((r) => ({ p: r.p, won: r.won })));
    if (!full) {
      out.per_stat[stat] = {
        n: rows.length, dates: dates.length,
        lodo: 'NOT RUN', decision: 'REFUSE',
        reason: `no isotonic map is fittable at n=${rows.length} (needs ${cal.MIN_TOTAL || 200})`,
      };
      continue;
    }

    // ── LEAVE ONE DATE OUT, at THIS stat's own informative bar ──
    const spec = LODO_TEST[stat];
    const MIN_HELD_ROWS = spec ? spec.n_star : Infinity;
    const table = [];
    for (const d of dates) {
      const fit = rows.filter((r) => r.date !== d);
      const held = rows.filter((r) => r.date === d);
      if (held.length < MIN_HELD_ROWS) {
        table.push({ dropped: d, held_n: held.length, verdict: 'UNINFORMATIVE', reason: 'too few rows on this date' });
        continue;
      }
      const map = cal.fitIsotonic(fit.map((r) => ({ p: r.p, won: r.won })));
      const applied = guards.applyOrRefuse(map, held, cal.applyIsotonic);
      if (!applied.ok) {
        table.push({ dropped: d, held_n: held.length, verdict: 'UNINFORMATIVE', reason: applied.reason });
        continue;
      }
      const ys = applied.rows.map((r) => r.won);
      const bRaw = guards.safeBrier(applied.rows.map((r) => r.p), ys);
      const bCal = guards.safeBrier(applied.rows.map((r) => r.pc), ys);
      if (bRaw === null || bCal === null) {
        table.push({ dropped: d, held_n: held.length, verdict: 'UNINFORMATIVE', reason: 'a null reached the metric' });
        continue;
      }
      const fav = applied.rows.filter((r) => r.p >= FAVOURITE_FLOOR);
      const favBias = fav.length >= 5 ? mean(fav.map((r) => r.p)) - mean(fav.map((r) => r.won)) : null;
      table.push({
        dropped: d,
        held_n: held.length,
        brier_delta: round4(bCal - bRaw),
        improves: bCal < bRaw,
        favourite_n: fav.length,
        favourite_bias: favBias === null ? null : round4(favBias),
        favourite_sign_holds: favBias === null ? null : favBias > 0,
        verdict: bCal < bRaw ? 'holds' : 'REVERSES',
      });
    }

    const informative = table.filter((t) => t.verdict !== 'UNINFORMATIVE');
    const reversals = informative.filter((t) => t.verdict === 'REVERSES');

    // THE DECISION RULE IS BINOMIAL, not zero-tolerance. Under stability each
    // informative drop reverses with prob Phi(-k), so demanding zero reversals
    // failed stable stats roughly half the time.
    const cutoff = spec ? spec.cutoff : 0;
    const exceedsCutoff = reversals.length > cutoff;

    // AND THE TEST MUST BE ABLE TO FAIL. Below the power floor it cannot, so it
    // cannot pass either -- "could not test" must never read as "passed".
    const underpowered = !spec || spec.power < LODO_POWER_FLOOR;
    const verdict = underpowered ? 'UNTESTABLE_BY_LODO' : (exceedsCutoff ? 'FAIL' : 'PASS');
    const passes = verdict === 'PASS';

    out.per_stat[stat] = {
      n: rows.length,
      dates: dates.length,
      lodo_table: table,
      informative_drops: informative.length,
      brier_reversals: informative.filter((t) => t.verdict === 'REVERSES').length,
      favourite_sign_flips: informative.filter((t) => t.favourite_sign_holds === false).length,
      favourite_sign_untested: informative.filter((t) => t.favourite_sign_holds === null).length,
      n_star: spec ? spec.n_star : null,
      cutoff,
      reversal_count: reversals.length,
      reversing_dates: reversals.map((t) => ({ date: t.dropped, held_n: t.held_n, delta: t.brier_delta })),
      test_power: spec ? spec.power : null,
      lodo: verdict,
      reason: underpowered
        ? `power ${spec ? spec.power : 0} < floor ${LODO_POWER_FLOOR} — this test would miss a real date-driven failure more than nine times in ten, so it can neither pass nor fail the stat`
        : (exceedsCutoff
          ? `${reversals.length} reversals among ${informative.length} informative drops exceeds the cutoff of ${cutoff}`
          : `${reversals.length} reversals among ${informative.length} informative drops is within the cutoff of ${cutoff}`),
    };
  }

  console.log(JSON.stringify(out, null, 2));
  process.exit(0);
}

const round4 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 10000) / 10000);

main().catch((e) => { console.error(e); process.exit(1); });