Backtest harness — the validator, built refusal-first

Phase 0 gate PASSED: the join is clean. No FK exists; the natural key (sport, player_key, stat, line, side, game_date) yields 283 clean 1:1 joins with ZERO ambiguity. game_id is NOT usable — 400/550 snapshot rows carry UNK@UNK because home/away names weren't threaded into the grader until Order 1.6. Non-joining rows are EXPECTED, not errors: retention stores both sides plus refusals; the ledger keeps only the graded side. Outcomes are NOT denormalized — ledger_entries stays the source of truth. BUILT TEST-FIRST, and the first property proven is the REFUSAL, not the math. Below threshold the harness emits INSUFFICIENT with n and the shortfall and NO rate anywhere in the payload, so a downstream renderer cannot surface one by accident. A test asserts the payload contains no hit_rate number at all. - Wilson intervals (correct at the n we actually have, unlike the normal approximation which emits negative lower bounds). - Strata NEVER mix sport or model_version. - Denominator excludes quarantined, void, unrecoverable, pending, push — asserted by test. - Monotonicity refuses to RANK buckets whose intervals overlap; it reports "not distinguishable on this sample". - Probability calibration (Brier + reliability) also respects the threshold: a thin sample returns status INSUFFICIENT and a NULL score. - Replay seam reads the STORED feature vector only. A row whose input was never retained is UN-BACKTESTABLE, never scored with substituted current data. Identity replay reproduces the live prediction exactly. The tests caught a real bug in my own code: `Number(null) === 0` let a null p_win through as a confident 0% forecast — this codebase's signature fabrication bug, inside the harness whose entire purpose is refusing invented numbers. Fixed with a strict null guard. FIRST LIVE RUN — the correct, passing output: VERDICT: INSUFFICIENT_HISTORY (can_validate=false) 283 joined -> 35 scored (120 quarantined, 124 pending, 4 terminal) C n=18 (short by 2), B n=17 (short by 3) strata: mlb 7, wnba 28 — never mixed migration 028 adds harness_results (append-only trend log; INSUFFICIENT rows are expected and correct) and opsWatch.harnessStaleAlarm pages if the harness stops running — a validator that isn't running looks exactly like one that keeps passing. Suite 283/3403 green, build exit 0. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SmNjJAwEnqHPtXbvSZR8kA
2026-07-20 11:31:48 -04:00
parent f73fb64a43
commit e809a0eb3c
5 changed files with 530 additions and 0 deletions
@@ -0,0 +1,277 @@
+'use strict';
+
+/**
+ * BACKTEST HARNESS (Session 64) — the validator every metric family is gated on.
+ *
+ * ITS FIRST PROPERTY IS THE REFUSAL. A harness that emits a confident number on
+ * a handful of rows is worse than no harness: it launders noise into a claim,
+ * and claims become marketing. Below the sample threshold this returns
+ * INSUFFICIENT with n and the shortfall — and NO rate anywhere in the payload,
+ * so a downstream renderer cannot accidentally surface one.
+ *
+ * Design rules, each enforced by a test:
+ *  - NEVER mix sports or model_versions in a bucket. A backtest across model
+ *    eras is meaningless, and `ledger_entries` is already contaminated
+ *    pre-2026-07-19 with no per-row version.
+ *  - The denominator excludes quarantined, void, unrecoverable, pending AND
+ *    push — the same discipline the public record already uses.
+ *  - Overlapping confidence intervals are reported as NOT DISTINGUISHABLE
+ *    rather than ranked. Two buckets 4 points apart on n=25 are noise.
+ *  - The replay seam reads the STORED feature vector only. A backtest that
+ *    refetches today's data to score a past night is invalid; a row whose
+ *    input was never retained is UN-BACKTESTABLE, and says so.
+ *
+ * Pure functions over rows — no I/O, so it is unit-testable and cannot
+ * accidentally reach the network mid-backtest.
+ */
+
+const DEFAULT_MIN_SAMPLE = Number(process.env.BACKTEST_MIN_SAMPLE || 20);
+const TERMINAL_NON_RESULTS = new Set(['void', 'unrecoverable']);
+
+/**
+ * Wilson score interval — correct for small n, unlike the normal approximation
+ * (which happily emits negative lower bounds on the sample sizes we actually
+ * have). z=1.96 → 95%.
+ */
+function wilsonInterval(hits, n, z = 1.96) {
+  if (!n || n <= 0) return { low: null, high: null };
+  const p = hits / n;
+  const z2 = z * z;
+  const denom = 1 + z2 / n;
+  const centre = p + z2 / (2 * n);
+  const margin = z * Math.sqrt((p * (1 - p) + z2 / (4 * n)) / n);
+  return {
+    low: Math.max(0, (centre - margin) / denom),
+    high: Math.min(1, (centre + margin) / denom),
+  };
+}
+
+/** Split rows into scored vs each excluded reason. The counts are reported so
+ *  a shrinking denominator is visible rather than silent. */
+function partition(rows) {
+  const scored = [];
+  const counts = {
+    total: rows.length, scored: 0,
+    excluded_quarantine: 0, excluded_terminal: 0,
+    excluded_pending: 0, excluded_push: 0,
+  };
+  for (const r of rows || []) {
+    if (!r) continue;
+    if (r.quarantine_reason || r.snap_quarantine) { counts.excluded_quarantine += 1; continue; }
+    const o = r.outcome;
+    if (o == null) { counts.excluded_pending += 1; continue; }
+    if (o === 'push') { counts.excluded_push += 1; continue; }
+    if (TERMINAL_NON_RESULTS.has(o)) { counts.excluded_terminal += 1; continue; }
+    if (o !== 'hit' && o !== 'miss') { counts.excluded_terminal += 1; continue; }
+    scored.push(r);
+  }
+  counts.scored = scored.length;
+  return { scored, counts };
+}
+
+/** One bucket's calibration. Below `minSample` it reports the refusal and NO rate. */
+function bucketStats(bucket, rows, minSample) {
+  const n = rows.length;
+  const hits = rows.filter((r) => r.outcome === 'hit').length;
+  if (n < minSample) {
+    return { bucket, n, hits, status: 'INSUFFICIENT', need: minSample, short_by: minSample - n, hit_rate: null, ci_low: null, ci_high: null };
+  }
+  const ci = wilsonInterval(hits, n);
+  return {
+    bucket, n, hits, status: 'OK',
+    hit_rate: hits / n,
+    ci_low: ci.low, ci_high: ci.high,
+  };
+}
+
+function groupBy(rows, keyFn) {
+  const m = new Map();
+  for (const r of rows) {
+    const k = keyFn(r);
+    if (!m.has(k)) m.set(k, []);
+    m.get(k).push(r);
+  }
+  return m;
+}
+
+/**
+ * Monotonicity: does a higher grade actually hit more often? Only buckets that
+ * BOTH cleared the sample gate are compared, and a comparison whose intervals
+ * overlap is reported as not distinguishable rather than ranked.
+ */
+function monotonicity(buckets, order) {
+  const ok = buckets.filter((b) => b.status === 'OK');
+  const comparisons = [];
+  for (let i = 0; i < ok.length; i += 1) {
+    for (let j = i + 1; j < ok.length; j += 1) {
+      const a = ok[i];
+      const b = ok[j];
+      const ra = order.indexOf(a.bucket);
+      const rb = order.indexOf(b.bucket);
+      if (ra === -1 || rb === -1) continue;
+      const better = ra < rb ? a : b;      // earlier in order = higher grade
+      const worse = ra < rb ? b : a;
+      const overlap = !(better.ci_low > worse.ci_high || worse.ci_low > better.ci_high);
+      comparisons.push({
+        pair: `${a.bucket}>${b.bucket}`,
+        higher: better.bucket, lower: worse.bucket,
+        higher_rate: better.hit_rate, lower_rate: worse.hit_rate,
+        distinguishable: !overlap,
+        holds: overlap ? null : better.hit_rate > worse.hit_rate,
+      });
+    }
+  }
+  let verdict;
+  if (comparisons.length === 0) verdict = 'INSUFFICIENT';
+  else if (comparisons.every((c) => !c.distinguishable)) verdict = 'NOT_DISTINGUISHABLE';
+  else if (comparisons.filter((c) => c.distinguishable).every((c) => c.holds)) verdict = 'MONOTONIC';
+  else verdict = 'BROKEN';
+  return { verdict, comparisons };
+}
+
+/** Brier score + reliability curve. Rows without a p_win are EXCLUDED — never
+ *  defaulted to 0.5, which would invent a prediction we did not make. */
+function probabilityCalibration(rows, bins = 10, minSample = null) {
+  // STRICT null guard. `Number(null) === 0` is this codebase's signature
+  // fabrication bug, and a null p_win slipping through would be scored as a
+  // confident 0% prediction — inventing a forecast we never made, inside the
+  // harness whose whole purpose is to refuse invented numbers.
+  const usable = rows.filter((r) => r && r.p_win != null && r.p_win !== '' && Number.isFinite(Number(r.p_win)));
+  if (usable.length === 0) return { n: 0, status: 'INSUFFICIENT', brier: null, reliability: [] };
+  let sum = 0;
+  const buckets = Array.from({ length: bins }, () => ({ n: 0, hits: 0, p_sum: 0 }));
+  for (const r of usable) {
+    const p = Number(r.p_win);
+    const y = r.outcome === 'hit' ? 1 : 0;
+    sum += (p - y) ** 2;
+    const idx = Math.min(bins - 1, Math.max(0, Math.floor(p * bins)));
+    buckets[idx].n += 1;
+    buckets[idx].hits += y;
+    buckets[idx].p_sum += p;
+  }
+  // The refusal applies here too: a Brier score on a handful of rows is noise
+  // wearing a decimal point. Below threshold we report the count and NO score,
+  // so a reader can never mistake it for a validated number.
+  if (minSample != null && usable.length < minSample) {
+    return { n: usable.length, status: 'INSUFFICIENT', need: minSample, brier: null, reliability: [] };
+  }
+  return {
+    n: usable.length,
+    status: 'OK',
+    brier: sum / usable.length,
+    reliability: buckets
+      .map((b, i) => (b.n ? {
+        bin: `${(i / bins).toFixed(1)}-${((i + 1) / bins).toFixed(1)}`,
+        n: b.n, predicted: b.p_sum / b.n, realized: b.hits / b.n,
+      } : null))
+      .filter(Boolean),
+  };
+}
+
+const FOUR_ORDER = ['A+', 'A', 'B', 'C', 'D', 'F'];
+const ELEVEN_ORDER = ['A+', 'A', 'A-', 'B+', 'B', 'B-', 'C+', 'C', 'C-', 'D', 'F'];
+
+/**
+ * Run the backtest.
+ * @param {Array} rows joined snapshot+outcome rows
+ * @param {Object} opts { minSample }
+ */
+function runBacktest(rows, opts = {}) {
+  const minSample = Number.isFinite(opts.minSample) ? opts.minSample : DEFAULT_MIN_SAMPLE;
+  const { scored, counts } = partition(rows || []);
+
+  // Strata: NEVER mix sport or model_version.
+  const strata = [];
+  for (const [key, group] of groupBy(scored, (r) => `${r.sport}|${r.model_version}`)) {
+    const [sport, model_version] = key.split('|');
+    const g4 = [...groupBy(group, (r) => r.grade || '?')]
+      .map(([b, rs]) => bucketStats(b, rs, minSample));
+    const g11 = [...groupBy(group, (r) => r.grade_11 || '?')]
+      .map(([b, rs]) => bucketStats(b, rs, minSample));
+    strata.push({
+      sport, model_version, n: group.length,
+      grade_buckets: g4,
+      grade_11_buckets: g11,
+      monotonicity: monotonicity(g4, FOUR_ORDER),
+      probability: probabilityCalibration(group, 10, minSample),
+    });
+  }
+
+  // Flat views (used by the tests + the log) — computed over ALL scored rows.
+  const grade_buckets = [...groupBy(scored, (r) => r.grade || '?')]
+    .map(([b, rs]) => bucketStats(b, rs, minSample));
+  const grade_11_buckets = [...groupBy(scored, (r) => r.grade_11 || '?')]
+    .map(([b, rs]) => bucketStats(b, rs, minSample));
+
+  const anyOk = grade_buckets.some((b) => b.status === 'OK');
+  return {
+    generated_at: opts.now ? opts.now() : new Date().toISOString(),
+    min_sample: minSample,
+    counts,
+    // THE REFUSAL: with nothing above threshold there is no validation to give.
+    can_validate: anyOk,
+    verdict: anyOk ? 'PARTIAL' : 'INSUFFICIENT_HISTORY',
+    grade_buckets,
+    grade_11_buckets,
+    monotonicity: monotonicity(grade_buckets, FOUR_ORDER),
+    probability: probabilityCalibration(scored, 10, minSample),
+    strata,
+  };
+}
+
+/**
+ * ALTERNATIVE-MODEL REPLAY SEAM.
+ *
+ * `scoreFn(row)` sees ONLY the stored row (including its retained feature
+ * vector) and returns `{ p_win }` — or `{ unavailable: '<reason>' }` when an
+ * input it needs was never retained. Those rows are counted as un-backtestable
+ * instead of being scored with substituted current data, which would make the
+ * result a fiction.
+ */
+function replayAlternative(rows, scoreFn, opts = {}) {
+  const minSample = Number.isFinite(opts.minSample) ? opts.minSample : DEFAULT_MIN_SAMPLE;
+  const { scored } = partition(rows || []);
+  const liveRows = [];
+  const candRows = [];
+  let un_backtestable = 0;
+
+  for (const r of scored) {
+    let out;
+    try { out = scoreFn(r); } catch { out = { unavailable: 'scorer_threw' }; }
+    if (!out || out.unavailable || out.p_win == null || !Number.isFinite(Number(out.p_win))) {
+      un_backtestable += 1;
+      continue;
+    }
+    liveRows.push(r);
+    candRows.push({ ...r, p_win: Number(out.p_win) });
+  }
+
+  const live = probabilityCalibration(liveRows);
+  const candidate = probabilityCalibration(candRows);
+  const identical = live.n === candidate.n
+    && live.n > 0
+    && Math.abs((live.brier ?? 0) - (candidate.brier ?? 0)) < 1e-9;
+
+  return {
+    n_considered: scored.length,
+    un_backtestable,
+    live,
+    candidate,
+    identical,
+    // Same refusal rule as the main report.
+    verdict: candidate.n >= minSample ? 'COMPARABLE' : 'INSUFFICIENT_HISTORY',
+    improvement: (candidate.n >= minSample && live.brier != null && candidate.brier != null)
+      ? live.brier - candidate.brier    // positive = candidate is better
+      : null,
+  };
+}
+
+module.exports = {
+  runBacktest,
+  replayAlternative,
+  wilsonInterval,
+  probabilityCalibration,
+  monotonicity,
+  partition,
+  DEFAULT_MIN_SAMPLE,
+};