vyndr/scripts/validate-lowparam.js

#!/usr/bin/env node
'use strict';

/**
 * PHASE 3 — validate the low-parameter correction as a NEW estimator.
 *
 * No grandfathering: it must beat RAW out-of-sample with a DATE-BLOCK bootstrap
 * interval excluding zero. It is also scored against the retired isotonic map on
 * the identical held-out rows, so the swap is a measured comparison rather than
 * a preference.
 */

require('dotenv').config();
const fs = require('fs');
const path = require('path');
const { createClient } = require('@supabase/supabase-js');
const cal = require('../src/services/model/calibration');
const lp = require('../src/services/model/lowParamCalibrator');
const guards = require('../src/services/model/calibrationGuards');
const { knownNumber } = require('../src/utils/known');

const BOX = path.join(process.cwd(), '.seq-cache', 'batting-lines.json');
const STATS = ['hits', 'total_bases', 'rbi', 'runs'];
const PAGE = 1000; const ITERS = 4000;
const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs };
const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);

async function page(sb, t, s, f) {
  const o = [];
  for (let i = 0; ; i += PAGE) {
    const { data, error } = await f(sb.from(t).select(s)).order('id', { ascending: true }).range(i, i + PAGE - 1);
    if (error) throw error; if (!data.length) break; o.push(...data); if (data.length < PAGE) break;
  }
  return o;
}
const isPreGame = (c, g) => {
  const et = new Date(new Date(c).getTime() - 4 * 3600 * 1000);
  const d = et.toISOString().slice(0, 10);
  return d < g || (d === g && et.getUTCHours() < 19);
};
function makeRnd(seed) { let s = seed >>> 0; return () => { s ^= s << 13; s >>>= 0; s ^= s >>> 17; s ^= s << 5; s >>>= 0; return s / 4294967296; }; }

/** Paired date-block bootstrap on a Brier difference. */
function dateBlockCI(rows, keyA, keyB, seed) {
  const byDate = new Map();
  for (const r of rows) { if (!byDate.has(r.date)) byDate.set(r.date, []); byDate.get(r.date).push(r); }
  const keys = [...byDate.keys()]; const rnd = makeRnd(seed); const diffs = [];
  for (let it = 0; it < ITERS; it += 1) {
    const s = [];
    for (let i = 0; i < keys.length; i += 1) s.push(...byDate.get(keys[Math.floor(rnd() * keys.length)]));
    const a = guards.safeBrier(s.map((r) => r[keyA]), s.map((r) => r.won));
    const b = guards.safeBrier(s.map((r) => r[keyB]), s.map((r) => r.won));
    if (a === null || b === null) continue;
    diffs.push(a - b);
  }
  diffs.sort((x, y) => x - y);
  return diffs.length
    ? { ci: [round4(diffs[Math.floor(diffs.length * 0.025)]), round4(diffs[Math.floor(diffs.length * 0.975)])], date_blocks: keys.length }
    : { ci: null, date_blocks: keys.length };
}

(async () => {
  const sb = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY, { auth: { persistSession: false } });
  const lines = JSON.parse(fs.readFileSync(BOX, 'utf8')).lines;
  const snaps = await page(sb, 'model_snapshots', 'id, game_date, captured_at, stat, player_key, line, side, p_win, refused',
    (q) => q.eq('sport', 'mlb').in('stat', STATS));

  const picked = new Map();
  for (const r of snaps) {
    if (!isPreGame(r.captured_at, r.game_date) || r.refused || knownNumber(r.p_win) === null) continue;
    const k = [r.game_date, r.stat, r.player_key, r.line].join('|');
    const prev = picked.get(k);
    if (!prev || knownNumber(r.p_win) > knownNumber(prev.p_win)) picked.set(k, r);
  }
  guards.assertPickedSideDedup([...picked.values()].map((r) => ({
    propKey: [r.game_date, r.stat, r.player_key, r.line].join('|'), side: r.side, p: knownNumber(r.p_win) })));

  const out = {};
  for (const stat of STATS) {
    const rows = [];
    for (const r of picked.values()) {
      if (r.stat !== stat) continue;
      const b = lines[`${r.game_date}|${r.player_key}`]; const L = knownNumber(r.line);
      if (!b || L === null || !r.side) continue;
      const v = knownNumber(FIELD[stat](b)); if (v === null) continue;
      const over = v > L;
      rows.push({ date: r.game_date, p: knownNumber(r.p_win), won: (String(r.side).toLowerCase() === 'under' ? !over : over) ? 1 : 0 });
    }
    rows.sort((a, b) => String(a.date).localeCompare(String(b.date)));
    const dates = [...new Set(rows.map((r) => r.date))].sort();
    const perDate = new Map(); for (const r of rows) perDate.set(r.date, (perDate.get(r.date) || 0) + 1);
    let acc = 0; let cut = dates[dates.length - 1];
    for (const d of dates) { acc += perDate.get(d); if (acc >= rows.length * 0.45) { cut = d; break; } }

    const fit = rows.filter((r) => r.date < cut);
    const ev = rows.filter((r) => r.date >= cut);
    const platt = lp.fitPlatt(fit);
    const iso = cal.fitIsotonic(fit.map((r) => ({ p: r.p, won: r.won })));

    if (!platt || ev.length < 50) {
      out[stat] = { n: rows.length, fit_n: fit.length, eval_n: ev.length, decision: 'REFUSE', reason: 'no low-parameter fit or too little held out' };
      continue;
    }
    const scored = ev.map((r) => ({ ...r, plat: lp.applyPlatt(platt, r.p), isoP: iso ? cal.applyIsotonic(iso, r.p) : null }))
      .filter((r) => knownNumber(r.plat) !== null);
    const ys = scored.map((r) => r.won);
    const bRaw = guards.safeBrier(scored.map((r) => r.p), ys);
    const bPlat = guards.safeBrier(scored.map((r) => r.plat), ys);
    const isoRows = scored.filter((r) => knownNumber(r.isoP) !== null);
    const bIso = isoRows.length ? guards.safeBrier(isoRows.map((r) => r.isoP), isoRows.map((r) => r.won)) : null;

    const vsRaw = dateBlockCI(scored, 'plat', 'p', 20260808);
    const vsIso = isoRows.length ? dateBlockCI(isoRows, 'plat', 'isoP', 20260808) : { ci: null };

    const beatsRaw = bPlat < bRaw && vsRaw.ci && vsRaw.ci[1] < 0;
    out[stat] = {
      n: rows.length, dates: dates.length, split_at: cut, fit_n: fit.length, eval_n: scored.length,
      platt: { a: platt.a, b: platt.b, flattens: platt.flattens, fit_dates: platt.fit_dates, shrinkage: platt.shrinkage },
      brier_raw: round4(bRaw), brier_lowparam: round4(bPlat), brier_isotonic: bIso === null ? null : round4(bIso),
      delta_vs_raw: round4(bPlat - bRaw), ci_vs_raw: vsRaw.ci, eval_date_blocks: vsRaw.date_blocks,
      delta_vs_isotonic: bIso === null ? null : round4(bPlat - bIso), ci_vs_isotonic: vsIso.ci,
      decision: beatsRaw ? 'DEPLOY-PROVISIONAL' : 'REFUSE',
      reason: beatsRaw ? 'beats raw out-of-sample with a date-block interval excluding zero'
        : 'does not beat raw at a date-block interval excluding zero',
    };
  }
  console.log(JSON.stringify(out, null, 2));
  process.exit(0);
})().catch((e) => { console.error(e); process.exit(1); });

const round4 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 10000) / 10000);