vyndr/scripts/hits-v1-holdout.js

#!/usr/bin/env node
'use strict';

/**
 * hits-v1-holdout — STEP 3. POINT-IN-TIME REPLAY, HITS ROWS ONLY.
 *
 * THREE guards this script exists to enforce, all of which have burned a
 * measurement in this codebase before:
 *
 *   1. HITS ROWS ONLY. Averaging hits into the other stats would hide the
 *      effect entirely — hits is one stat among nine and the ladder's failure is
 *      specific to it.
 *
 *   2. DIRECTION-ALIGNED. `p_win` is P(GRADED SIDE); `proj_p_over_line` and
 *      `proj_hits_p_over` are P(OVER). 26% of matched hits rows are
 *      under-graded, and comparing a raw P(over) against an under-side outcome
 *      measures the model BACKWARDS. That artifact alone accounted for 41% of
 *      the ladder's apparent loss when it was first measured.
 *
 *   3. NO LOOKAHEAD. This is the guard specific to a replay. For each settled
 *      row, the player's game log is rebuilt STRICTLY BEFORE that row's
 *      game_date, and the multiplier is the REAL `combined_multiplier` recorded
 *      on the row at grade time. A replay that used today's full log would be
 *      scoring a prediction with the answer in hand — a fabricated result, and
 *      a worse lie than no measurement.
 *
 * CONTAMINATION EXCLUSION (mandatory). Rows whose price/book were stamped from a
 * NON-TAKEABLE book between 2026-08-01 and the write-path fix are tagged
 * `quarantine_reason LIKE 'nontakeable_book%'` and are EXCLUDED: their locked
 * price describes a market you could not have bet.
 *
 * WHAT THIS IS AND IS NOT. It is a backtest, and it is labelled one. The verdict
 * of record is the FORWARD ledger accrual, which starts at the next snapshot.
 * Stated limits: statsapi is read as it stands today (retroactive stat
 * corrections are invisible), and LEAGUE_HIT_RATE / PRIOR_AB are constants set
 * today — at 20 at-bats against a regular's 200–400 the prior moves a settled
 * hitter by thousandths, but it is not zero.
 *
 *   node scripts/hits-v1-holdout.js
 */

require('dotenv').config();
const { createClient } = require('@supabase/supabase-js');
const binomialHits = require('../src/services/projection/binomialHits');
const mlb = require('../src/services/adapters/mlbStatsAdapter');
const { knownNumber } = require('../src/utils/known');
const { normalizeName } = require('../src/utils/playerName');

const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;

/** Pearson correlation — the resolution measure the ladder is judged on. */
function corr(xs, ys) {
  const n = xs.length;
  if (n < 3) return null;
  const mx = xs.reduce((a, b) => a + b, 0) / n;
  const my = ys.reduce((a, b) => a + b, 0) / n;
  let sxy = 0; let sxx = 0; let syy = 0;
  for (let i = 0; i < n; i += 1) {
    const dx = xs[i] - mx; const dy = ys[i] - my;
    sxy += dx * dy; sxx += dx * dx; syy += dy * dy;
  }
  if (sxx <= 0 || syy <= 0) return null;
  return Math.round((sxy / Math.sqrt(sxx * syy)) * 10000) / 10000;
}
const mean = (a) => (a.length ? Math.round((a.reduce((x, y) => x + y, 0) / a.length) * 10000) / 10000 : null);
const sd = (a) => {
  if (a.length < 2) return null;
  const m = a.reduce((x, y) => x + y, 0) / a.length;
  return Math.round(Math.sqrt(a.reduce((s, v) => s + (v - m) ** 2, 0) / (a.length - 1)) * 10000) / 10000;
};
/** Brier score — lower is better. Reported beside resolution as a check. */
const brier = (ps, ys) => (ps.length
  ? Math.round((ps.reduce((s, p, i) => s + (p - ys[i]) ** 2, 0) / ps.length) * 10000) / 10000
  : null);

/**
 * Paired bootstrap CI on a DIFFERENCE of resolutions.
 *
 * Both models score the SAME rows, so their errors are correlated and comparing
 * two independent standard errors would overstate the uncertainty. Resampling
 * rows as pairs preserves that dependence. Deterministic seed — a measurement
 * that changes between runs is not a measurement.
 */
function bootstrapDiff(rowsIn, keyA, keyB, iters = 4000, seed = 20260802) {
  if (rowsIn.length < 20) return null;
  let s = seed >>> 0;
  const rnd = () => { // xorshift32 — deterministic, no Math.random
    s ^= s << 13; s >>>= 0; s ^= s >>> 17; s ^= s << 5; s >>>= 0;
    return s / 4294967296;
  };
  const n = rowsIn.length;
  const diffs = [];
  for (let it = 0; it < iters; it += 1) {
    const ys = []; const a = []; const bArr = [];
    for (let i = 0; i < n; i += 1) {
      const r = rowsIn[Math.floor(rnd() * n)];
      ys.push(r.won); a.push(r[keyA]); bArr.push(r[keyB]);
    }
    const ca = corr(a, ys); const cb = corr(bArr, ys);
    if (ca == null || cb == null) continue;
    diffs.push(ca - cb);
  }
  if (diffs.length < 100) return null;
  diffs.sort((x, y) => x - y);
  const q = (p) => Math.round(diffs[Math.floor(p * (diffs.length - 1))] * 10000) / 10000;
  return {
    point: Math.round(((corr(rowsIn.map((r) => r[keyA]), rowsIn.map((r) => r.won)) || 0)
      - (corr(rowsIn.map((r) => r[keyB]), rowsIn.map((r) => r.won)) || 0)) * 10000) / 10000,
    ci95: [q(0.025), q(0.975)],
    // The question the promotion gate actually asks.
    p_improves: Math.round((diffs.filter((d) => d > 0).length / diffs.length) * 1000) / 1000,
  };
}

async function main() {
  if (!SB_URL || !SB_KEY) throw new Error('SUPABASE_URL / service key required');
  const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });

  const { data, error } = await sb
    .from('ledger_entries')
    .select('id, player_name, player_key, stat, line, side, outcome, game_date, p_win, proj_p_over_line, proj_factors, quarantine_reason')
    .eq('sport', 'mlb')
    .is('user_id', null)
    .eq('stat', 'hits')
    .in('outcome', ['hit', 'miss'])
    .not('p_win', 'is', null)
    .not('proj_p_over_line', 'is', null);
  if (error) throw error;

  // Contamination exclusion, applied in JS so the filter is visible here.
  const rows = (data || []).filter((r) => !(r.quarantine_reason || '').startsWith('nontakeable_book'));

  // Resolve each distinct player ONCE, and cache the full game log.
  const logCache = new Map();
  const names = [...new Set(rows.map((r) => r.player_name).filter(Boolean))];
  let resolved = 0;
  for (const name of names) {
    try {
      const found = await mlb.searchPlayer(name);
      if (!found || !found.id) { logCache.set(name, null); continue; }
      const log = await mlb.getPlayerGameLog(found.id);
      logCache.set(name, Array.isArray(log) ? log : null);
      if (log && log.length) resolved += 1;
    } catch { logCache.set(name, null); }
  }

  const out = [];
  const reasons = {};
  const bump = (k) => { reasons[k] = (reasons[k] || 0) + 1; };

  for (const r of rows) {
    const full = logCache.get(r.player_name);
    if (!full) { bump('no_game_log'); continue; }
    const gameDate = String(r.game_date || '').slice(0, 10);
    if (!gameDate) { bump('no_game_date'); continue; }

    // ── NO LOOKAHEAD ──────────────────────────────────────────────────────
    // Strictly BEFORE the graded game. A row dated the same day is the game
    // being predicted; including it would hand the model the answer.
    const priorLog = full.filter((g) => g && g.date && String(g.date).slice(0, 10) < gameDate);
    if (priorLog.length < binomialHits.HITS_MIN_GAMES) { bump('thin_prior_log'); continue; }

    // The REAL grade-time multiplier, recorded on the row at lock.
    const m = knownNumber(r.proj_factors && r.proj_factors.combined_multiplier);
    const proj = binomialHits.projectHits({
      rows: priorLog, line: Number(r.line), multiplier: m == null ? 1 : m,
    });
    if (!proj) { bump('inputs_underivable'); continue; }

    // ── DIRECTION-ALIGN to the graded side ────────────────────────────────
    const under = String(r.side || '').toLowerCase() === 'under';
    const won = r.outcome === 'hit' ? 1 : 0;
    out.push({
      won,
      under,
      champ: Number(r.p_win),
      ladder: under ? 1 - Number(r.proj_p_over_line) : Number(r.proj_p_over_line),
      hitsv1: under ? 1 - proj.p_over_line : proj.p_over_line,
      line: Number(r.line),
      games_prior: priorLog.length,
    });
  }

  const slice = (rowsIn, label) => {
    const ys = rowsIn.map((x) => x.won);
    const c = rowsIn.map((x) => x.champ);
    const l = rowsIn.map((x) => x.ladder);
    const h = rowsIn.map((x) => x.hitsv1);
    return {
      slice: label,
      n: rowsIn.length,
      under_rows: rowsIn.filter((x) => x.under).length,
      base_rate: mean(ys),
      resolution: { champion: corr(c, ys), current_ladder: corr(l, ys), hits_v1: corr(h, ys) },
      brier: { champion: brier(c, ys), current_ladder: brier(l, ys), hits_v1: brier(h, ys) },
      mean_p: { champion: mean(c), current_ladder: mean(l), hits_v1: mean(h) },
      sd_p: { champion: sd(c), current_ladder: sd(l), hits_v1: sd(h) },
    };
  };

  console.log(JSON.stringify({
    measurement: 'POINT-IN-TIME REPLAY (backtest) — verdict of record is the forward ledger accrual',
    guards: {
      hits_rows_only: true,
      direction_aligned: true,
      no_lookahead: 'game log truncated strictly before each row game_date',
      grade_time_multiplier: 'real combined_multiplier from the row',
      contamination_excluded: 'nontakeable_book*',
    },
    candidate_rows: rows.length,
    players_resolved: `${resolved}/${names.length}`,
    matched_rows: out.length,
    dropped: reasons,
    overall: slice(out, 'all hits rows'),
    // Is the comparison a RESULT or a noise reading? Paired bootstrap, so the
    // shared rows are not double-counted as independent evidence.
    paired_bootstrap: {
      note: 'difference in resolution, 4000 paired resamples, deterministic seed',
      hits_v1_minus_ladder: bootstrapDiff(out, 'hitsv1', 'ladder'),
      champion_minus_ladder: bootstrapDiff(out, 'champ', 'ladder'),
      champion_minus_hits_v1: bootstrapDiff(out, 'champ', 'hitsv1'),
      at_line_0_5: {
        hits_v1_minus_ladder: bootstrapDiff(out.filter((x) => x.line === 0.5), 'hitsv1', 'ladder'),
        champion_minus_ladder: bootstrapDiff(out.filter((x) => x.line === 0.5), 'champ', 'ladder'),
      },
    },
    by_line: [0.5, 1.5, 2.5].map((ln) => slice(out.filter((x) => x.line === ln), `line ${ln}`))
      .filter((s) => s.n > 0),
  }, null, 2));

  process.exit(0);
}

main().catch((e) => { console.error(e); process.exit(1); });