vyndr/scripts/derive-lodo-threshold.js

#!/usr/bin/env node
'use strict';

/**
 * PHASE 0 — derive the LODO held-row threshold from POWER, blind to outcomes.
 *
 * ESTIMAND: "does dropping date D reverse the SIGN of the out-of-sample Brier
 * improvement on D's held-out rows?"
 *
 * A reversal is only informative if a single date's Brier delta is
 * distinguishable from zero at that row count. Below that, a reversal is a coin
 * flip wearing a decimal point — which is exactly the ambiguity that made the
 * previous verdict depend on an operator-chosen number.
 *
 * ── THE DERIVATION ───────────────────────────────────────────────────────
 * The per-row Brier difference is
 *
 *     d_i = (pc_i - y_i)^2 - (p_i - y_i)^2
 *
 * and a date's Brier delta is the MEAN of d over that date's rows. So
 *
 *     SE(n) = SD(d) / sqrt(n)
 *
 * and the smallest n at which a typical effect clears one standard error is
 *
 *     n* = ( SD(d) / |effect| )^2
 *
 * SD(d) and |effect| are pooled ACROSS ALL FOUR STATS deliberately: a per-stat
 * figure would let the threshold be shaped by the stat whose verdict it decides.
 *
 * THIS SCRIPT PRINTS NO STAT VERDICT AND NO DATE. It is blind by construction,
 * and it must be run and its output committed BEFORE any stat is re-read.
 *
 *   SUPABASE_URL=... node scripts/derive-lodo-threshold.js
 */

require('dotenv').config();
const fs = require('fs');
const path = require('path');
const { createClient } = require('@supabase/supabase-js');
const cal = require('../src/services/model/calibration');
const guards = require('../src/services/model/calibrationGuards');
const { knownNumber } = require('../src/utils/known');

const BOX = path.join(process.cwd(), '.seq-cache', 'batting-lines.json');
const STATS = ['hits', 'total_bases', 'rbi', 'runs'];
const PAGE = 1000;

const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs };
const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);

async function page(sb, t, s, f) {
  const o = [];
  for (let i = 0; ; i += PAGE) {
    const { data, error } = await f(sb.from(t).select(s)).order('id', { ascending: true }).range(i, i + PAGE - 1);
    if (error) throw error;
    if (!data.length) break;
    o.push(...data);
    if (data.length < PAGE) break;
  }
  return o;
}

const isPreGame = (c, g) => {
  const et = new Date(new Date(c).getTime() - 4 * 3600 * 1000);
  const d = et.toISOString().slice(0, 10);
  return d < g || (d === g && et.getUTCHours() < 19);
};

(async () => {
  const sb = createClient(process.env.SUPABASE_URL,
    process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY, { auth: { persistSession: false } });
  const lines = JSON.parse(fs.readFileSync(BOX, 'utf8')).lines;

  const snaps = await page(sb, 'model_snapshots',
    'id, game_date, captured_at, stat, player_key, line, side, p_win, refused',
    (q) => q.eq('sport', 'mlb').in('stat', STATS));

  const picked = new Map();
  for (const r of snaps) {
    if (!isPreGame(r.captured_at, r.game_date) || r.refused || knownNumber(r.p_win) === null) continue;
    const k = [r.game_date, r.stat, r.player_key, r.line].join('|');
    const prev = picked.get(k);
    if (!prev || knownNumber(r.p_win) > knownNumber(prev.p_win)) picked.set(k, r);
  }
  guards.assertPickedSideDedup([...picked.values()].map((r) => ({
    propKey: [r.game_date, r.stat, r.player_key, r.line].join('|'), side: r.side, p: knownNumber(r.p_win),
  })));

  // Pooled per-row Brier differences, across all four stats.
  const diffs = [];
  for (const stat of STATS) {
    const rows = [];
    for (const r of picked.values()) {
      if (r.stat !== stat) continue;
      const b = lines[`${r.game_date}|${r.player_key}`];
      const L = knownNumber(r.line);
      if (!b || L === null || !r.side) continue;
      const v = knownNumber(FIELD[stat](b));
      if (v === null) continue;
      const over = v > L;
      rows.push({ p: knownNumber(r.p_win), won: (String(r.side).toLowerCase() === 'under' ? !over : over) ? 1 : 0 });
    }
    const map = cal.fitIsotonic(rows.map((r) => ({ p: r.p, won: r.won })));
    if (!map) continue;
    for (const r of rows) {
      const pc = cal.applyIsotonic(map, r.p);
      if (knownNumber(pc) === null) continue;
      diffs.push((pc - r.won) ** 2 - (r.p - r.won) ** 2);
    }
  }

  const m = mean(diffs);
  const sd = Math.sqrt(diffs.reduce((s, d) => s + (d - m) ** 2, 0) / (diffs.length - 1));
  const effect = Math.abs(m);
  const nStar = Math.ceil((sd / effect) ** 2);

  const curve = [10, 20, 25, 30, 50, 75, 100, 150, 200, 300, 500].map((n) => ({
    n,
    se: round5(sd / Math.sqrt(n)),
    effect_over_se: round3(effect / (sd / Math.sqrt(n))),
    informative: effect >= sd / Math.sqrt(n),
  }));

  console.log(JSON.stringify({
    phase: 'PHASE 0 — power derivation, blind to outcomes',
    pooled_rows: diffs.length,
    per_row_brier_diff_sd: round5(sd),
    pooled_effect_abs_mean: round5(effect),
    n_star: nStar,
    rule: 'n* = (SD(d) / |effect|)^2 — the smallest held-row count at which a typical Brier delta clears one standard error',
    se_vs_n: curve,
    blind: 'no stat verdict, no date, and no reversal is referenced anywhere in this output',
  }, null, 2));
  process.exit(0);
})().catch((e) => { console.error(e); process.exit(1); });

const round5 = (v) => Math.round(v * 100000) / 100000;
const round3 = (v) => Math.round(v * 1000) / 1000;