vyndr/scripts/test-favourite-bias-robust.js

#!/usr/bin/env node
'use strict';

/**
 * PHASE 1 — is the favourite-longshot bias real WITHOUT a calibration map?
 *
 * Four orders have refined a stability gate on 19 dates. LODO turned out to be
 * structurally underpowered (0.014–0.093) and the deploy intervals rest on 2–4
 * date clusters. So we stop certifying the stability of a specific MAP, and ask
 * the one question this sample might actually answer:
 *
 *     does the model over-predict its own favourites, robustly?
 *
 * That claim is MODEL-FREE and MAP-FREE — it is a property of (p_win, outcome)
 * pairs, needs no isotonic fit, and can therefore be tested without any of the
 * machinery whose stability we cannot certify.
 *
 * ── DATE-BLOCK BOOTSTRAP ─────────────────────────────────────────────────
 * Resampling ROWS would treat 200 props from one night as 200 readings of that
 * night's offensive environment. Whole DATES are resampled instead, which is the
 * honest unit and a far harsher one at 5–17 dates.
 *
 * VERDICT is pre-stated: ROBUST iff the >0.9 over-prediction sign survives in
 * >=95% of pooled date-block resamples AND replicates in >=3 of 4 stats on the
 * same criterion. Anything else is NOT-ROBUST, and NOT-ROBUST means we serve raw.
 *
 *   SUPABASE_URL=... node scripts/test-favourite-bias-robust.js
 */

require('dotenv').config();
const fs = require('fs');
const path = require('path');
const { createClient } = require('@supabase/supabase-js');
const guards = require('../src/services/model/calibrationGuards');
const { knownNumber } = require('../src/utils/known');

const BOX = path.join(process.cwd(), '.seq-cache', 'batting-lines.json');
const STATS = ['hits', 'total_bases', 'rbi', 'runs'];
const PAGE = 1000;
const FAVOURITE_FLOOR = 0.9;
const ITERS = 5000;
/** Pre-stated pass marks. */
const SIGN_STABILITY_REQUIRED = 0.95;
const STATS_MUST_REPLICATE = 3;

const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs };
const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);

async function page(sb, t, s, f) {
  const o = [];
  for (let i = 0; ; i += PAGE) {
    const { data, error } = await f(sb.from(t).select(s)).order('id', { ascending: true }).range(i, i + PAGE - 1);
    if (error) throw error;
    if (!data.length) break;
    o.push(...data);
    if (data.length < PAGE) break;
  }
  return o;
}
const isPreGame = (c, g) => {
  const et = new Date(new Date(c).getTime() - 4 * 3600 * 1000);
  const d = et.toISOString().slice(0, 10);
  return d < g || (d === g && et.getUTCHours() < 19);
};
function makeRnd(seed) {
  let s = seed >>> 0;
  return () => { s ^= s << 13; s >>>= 0; s ^= s >>> 17; s ^= s << 5; s >>>= 0; return s / 4294967296; };
}

/** Over-prediction in the favourite bin: predicted − realized. Positive = over. */
function favouriteBias(rows) {
  const fav = rows.filter((r) => r.p >= FAVOURITE_FLOOR);
  if (fav.length < 5) return null;
  return mean(fav.map((r) => r.p)) - mean(fav.map((r) => r.won));
}

/** Resample whole DATES with replacement; report how often the sign survives. */
function dateBlockSignStability(rows, seed) {
  const byDate = new Map();
  for (const r of rows) {
    if (!byDate.has(r.date)) byDate.set(r.date, []);
    byDate.get(r.date).push(r);
  }
  const keys = [...byDate.keys()];
  const rnd = makeRnd(seed);
  let positive = 0; let indeterminate = 0; const draws = [];
  for (let it = 0; it < ITERS; it += 1) {
    const sample = [];
    for (let i = 0; i < keys.length; i += 1) sample.push(...byDate.get(keys[Math.floor(rnd() * keys.length)]));
    const b = favouriteBias(sample);
    // A resample with too few favourites cannot speak — counted, never guessed.
    if (b === null) { indeterminate += 1; continue; }
    draws.push(b);
    if (b > 0) positive += 1;
  }
  const usable = ITERS - indeterminate;
  draws.sort((a, b) => a - b);
  return {
    date_blocks: keys.length,
    usable_resamples: usable,
    indeterminate_resamples: indeterminate,
    sign_stability: usable ? round4(positive / usable) : null,
    ci_90: draws.length ? [round4(draws[Math.floor(draws.length * 0.05)]), round4(draws[Math.floor(draws.length * 0.95)])] : null,
  };
}

function deciles(rows) {
  const out = [];
  for (let lo = 0.3; lo < 1.0; lo += 0.1) {
    const hi = lo + 0.1;
    const slice = rows.filter((r) => r.p >= lo && (hi >= 1 ? r.p <= 1 : r.p < hi));
    if (slice.length < 15) continue;
    const pred = mean(slice.map((r) => r.p));
    const real = mean(slice.map((r) => r.won));
    out.push({ bin: [round2(lo), round2(hi)], n: slice.length, predicted: round4(pred), realized: round4(real), over_prediction: round4(pred - real) });
  }
  return out;
}

(async () => {
  const sb = createClient(process.env.SUPABASE_URL,
    process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY, { auth: { persistSession: false } });
  const lines = JSON.parse(fs.readFileSync(BOX, 'utf8')).lines;

  const snaps = await page(sb, 'model_snapshots',
    'id, game_date, captured_at, stat, player_key, line, side, p_win, refused',
    (q) => q.eq('sport', 'mlb').in('stat', STATS));

  const picked = new Map();
  for (const r of snaps) {
    if (!isPreGame(r.captured_at, r.game_date) || r.refused || knownNumber(r.p_win) === null) continue;
    const k = [r.game_date, r.stat, r.player_key, r.line].join('|');
    const prev = picked.get(k);
    if (!prev || knownNumber(r.p_win) > knownNumber(prev.p_win)) picked.set(k, r);
  }
  guards.assertPickedSideDedup([...picked.values()].map((r) => ({
    propKey: [r.game_date, r.stat, r.player_key, r.line].join('|'), side: r.side, p: knownNumber(r.p_win),
  })));

  const byStat = {}; const pooled = [];
  for (const stat of STATS) byStat[stat] = [];
  for (const r of picked.values()) {
    const b = lines[`${r.game_date}|${r.player_key}`];
    const L = knownNumber(r.line);
    if (!b || L === null || !r.side) continue;
    const v = knownNumber(FIELD[r.stat](b));
    if (v === null) continue;
    const over = v > L;
    const row = { date: r.game_date, p: knownNumber(r.p_win), won: (String(r.side).toLowerCase() === 'under' ? !over : over) ? 1 : 0 };
    byStat[r.stat].push(row); pooled.push(row);
  }

  const pooledResult = {
    n: pooled.length,
    deciles: deciles(pooled),
    favourite_bias: round4(favouriteBias(pooled)),
    ...dateBlockSignStability(pooled, 20260808),
  };

  const perStat = {};
  let replicated = 0;
  for (const stat of STATS) {
    const rows = byStat[stat];
    const fb = favouriteBias(rows);
    const stab = dateBlockSignStability(rows, 20260808);
    const ok = fb !== null && fb > 0 && stab.sign_stability !== null && stab.sign_stability >= SIGN_STABILITY_REQUIRED;
    if (ok) replicated += 1;
    perStat[stat] = { n: rows.length, deciles: deciles(rows), favourite_bias: fb === null ? null : round4(fb), ...stab, replicates: ok };
  }

  const pooledOk = pooledResult.favourite_bias > 0 && pooledResult.sign_stability >= SIGN_STABILITY_REQUIRED;
  const verdict = pooledOk && replicated >= STATS_MUST_REPLICATE ? 'ROBUST' : 'NOT-ROBUST';

  console.log(JSON.stringify({
    phase: 'PHASE 1 — model-free, map-free favourite-longshot bias test',
    criteria: { sign_stability_required: SIGN_STABILITY_REQUIRED, stats_must_replicate: STATS_MUST_REPLICATE, favourite_floor: FAVOURITE_FLOOR },
    pooled: pooledResult,
    per_stat: perStat,
    stats_replicating: replicated,
    VERDICT: verdict,
    consequence: verdict === 'ROBUST'
      ? 'proceed to a low-parameter correction, validated as a NEW estimator'
      : 'serve raw; the bias is not certifiable on this sample',
  }, null, 2));
  process.exit(0);
})().catch((e) => { console.error(e); process.exit(1); });

const round4 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 10000) / 10000);
const round2 = (v) => Math.round(v * 100) / 100;