Power-derive the LODO threshold: hits restored through the gate, rbi/runs

routed as date-driven PHASE 0 — threshold derived BLIND, before any stat was re-read. A reversal is informative only if that date's Brier delta is distinguishable from zero at its row count. Per-row Brier difference d_i = (pc-y)^2 - (p-y)^2, so SE(n) = SD(d)/sqrt(n) and n* = (SD(d)/|effect|)^2. Pooled across all four stats so no single stat's verdict could shape the threshold deciding it: pooled rows 3,417 | SD(d) 0.09816 | |effect| 0.01175 n* = (0.09816/0.01175)^2 = 69.8 -> 70 The hand-chosen 20 sat at 0.54 SE -- a coin flip. That is the defect this removes, and why the previous verdict moved with the number. Committed as calibrationRegistry.LODO_MIN_HELD_ROWS = 70 with LODO_THRESHOLD_BASIS; a test recomputes (SD/effect)^2 and asserts it equals the constant, so it cannot drift from its own justification. The derivation script prints no stat verdict, no date and no reversal. PHASE 1 — LODO at n*, applied cold: hits 5 informative drops, 0 reversals PASS total_bases 4 informative drops, 0 reversals PASS rbi reverses 2026-08-01 (n=99) FAIL runs reverses 08-01 (n=86), 08-05 (244) FAIL hits held-out deltas -0.0041/-0.0080/-0.0192/-0.0140/-0.0139 across 123-272 row dates, favourite sign holding on every testable drop. THIS IS THE INSTRUMENT FINALLY POWERED, NOT VINDICATION OF A PREDICTION -- the withdrawal at 6ae11f1 was correct on the instrument available then, which admitted 20- and 25-row dates as evidence. Nothing about hits changed; the threshold stopped being chosen. PHASE 2 — both failures are DATE-DRIVEN, not underpowered. Every reversal sits above n*=70 (99, 86, 244), so no threshold and no further accrual rescues either: isotonic is fitting day-structure. Routed to the low-parameter calibrator queue (Platt/beta), not built here. PHASE 3 — CALIBRATION_DEPLOYED is now ['hits','total_bases'], frozen and tested, both PROVISIONAL with auto-demotion armed and the >=40 date-cluster promotion bar unchanged. hits stackability for chain.chainAcross is RESTORED, and the record shows it returned through the powered gate rather than by fiat. hits bands rebuilt on p_win_calibrated (765 eval rows): every archetype still one band, still base_rate -- calibrated YES, proven-per-archetype NO. PHASE 4 logged: the deploy set is now set by a power-derived, pre-committed, tested constant rather than an operator-chosen number. At 6ae11f1 that rule moved the live path AGAINST the operator; it has now moved it back on the same evidence because the instrument changed. Both directions are the rule working. And calibrated p_win separates within archetype no better than raw across 13 archetype slots on two deployed stats -- per-archetype separation will come from proven factors or not at all. p_win never mutated; no Bonferroni slot consumed; counter and frozen clusters verified byte-identical file by file. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W1sivYNqY2TS5ftykmHBU9
2026-08-06 20:15:32 -04:00
parent 6ae11f1193
commit 1f40014256
7 changed files with 397 additions and 11 deletions
@@ -0,0 +1,139 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * PHASE 0 — derive the LODO held-row threshold from POWER, blind to outcomes.
+ *
+ * ESTIMAND: "does dropping date D reverse the SIGN of the out-of-sample Brier
+ * improvement on D's held-out rows?"
+ *
+ * A reversal is only informative if a single date's Brier delta is
+ * distinguishable from zero at that row count. Below that, a reversal is a coin
+ * flip wearing a decimal point — which is exactly the ambiguity that made the
+ * previous verdict depend on an operator-chosen number.
+ *
+ * ── THE DERIVATION ───────────────────────────────────────────────────────
+ * The per-row Brier difference is
+ *
+ *     d_i = (pc_i - y_i)^2 - (p_i - y_i)^2
+ *
+ * and a date's Brier delta is the MEAN of d over that date's rows. So
+ *
+ *     SE(n) = SD(d) / sqrt(n)
+ *
+ * and the smallest n at which a typical effect clears one standard error is
+ *
+ *     n* = ( SD(d) / |effect| )^2
+ *
+ * SD(d) and |effect| are pooled ACROSS ALL FOUR STATS deliberately: a per-stat
+ * figure would let the threshold be shaped by the stat whose verdict it decides.
+ *
+ * THIS SCRIPT PRINTS NO STAT VERDICT AND NO DATE. It is blind by construction,
+ * and it must be run and its output committed BEFORE any stat is re-read.
+ *
+ *   SUPABASE_URL=... node scripts/derive-lodo-threshold.js
+ */
+
+require('dotenv').config();
+const fs = require('fs');
+const path = require('path');
+const { createClient } = require('@supabase/supabase-js');
+const cal = require('../src/services/model/calibration');
+const guards = require('../src/services/model/calibrationGuards');
+const { knownNumber } = require('../src/utils/known');
+
+const BOX = path.join(process.cwd(), '.seq-cache', 'batting-lines.json');
+const STATS = ['hits', 'total_bases', 'rbi', 'runs'];
+const PAGE = 1000;
+
+const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs };
+const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);
+
+async function page(sb, t, s, f) {
+  const o = [];
+  for (let i = 0; ; i += PAGE) {
+    const { data, error } = await f(sb.from(t).select(s)).order('id', { ascending: true }).range(i, i + PAGE - 1);
+    if (error) throw error;
+    if (!data.length) break;
+    o.push(...data);
+    if (data.length < PAGE) break;
+  }
+  return o;
+}
+
+const isPreGame = (c, g) => {
+  const et = new Date(new Date(c).getTime() - 4 * 3600 * 1000);
+  const d = et.toISOString().slice(0, 10);
+  return d < g || (d === g && et.getUTCHours() < 19);
+};
+
+(async () => {
+  const sb = createClient(process.env.SUPABASE_URL,
+    process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY, { auth: { persistSession: false } });
+  const lines = JSON.parse(fs.readFileSync(BOX, 'utf8')).lines;
+
+  const snaps = await page(sb, 'model_snapshots',
+    'id, game_date, captured_at, stat, player_key, line, side, p_win, refused',
+    (q) => q.eq('sport', 'mlb').in('stat', STATS));
+
+  const picked = new Map();
+  for (const r of snaps) {
+    if (!isPreGame(r.captured_at, r.game_date) || r.refused || knownNumber(r.p_win) === null) continue;
+    const k = [r.game_date, r.stat, r.player_key, r.line].join('|');
+    const prev = picked.get(k);
+    if (!prev || knownNumber(r.p_win) > knownNumber(prev.p_win)) picked.set(k, r);
+  }
+  guards.assertPickedSideDedup([...picked.values()].map((r) => ({
+    propKey: [r.game_date, r.stat, r.player_key, r.line].join('|'), side: r.side, p: knownNumber(r.p_win),
+  })));
+
+  // Pooled per-row Brier differences, across all four stats.
+  const diffs = [];
+  for (const stat of STATS) {
+    const rows = [];
+    for (const r of picked.values()) {
+      if (r.stat !== stat) continue;
+      const b = lines[`${r.game_date}|${r.player_key}`];
+      const L = knownNumber(r.line);
+      if (!b || L === null || !r.side) continue;
+      const v = knownNumber(FIELD[stat](b));
+      if (v === null) continue;
+      const over = v > L;
+      rows.push({ p: knownNumber(r.p_win), won: (String(r.side).toLowerCase() === 'under' ? !over : over) ? 1 : 0 });
+    }
+    const map = cal.fitIsotonic(rows.map((r) => ({ p: r.p, won: r.won })));
+    if (!map) continue;
+    for (const r of rows) {
+      const pc = cal.applyIsotonic(map, r.p);
+      if (knownNumber(pc) === null) continue;
+      diffs.push((pc - r.won) ** 2 - (r.p - r.won) ** 2);
+    }
+  }
+
+  const m = mean(diffs);
+  const sd = Math.sqrt(diffs.reduce((s, d) => s + (d - m) ** 2, 0) / (diffs.length - 1));
+  const effect = Math.abs(m);
+  const nStar = Math.ceil((sd / effect) ** 2);
+
+  const curve = [10, 20, 25, 30, 50, 75, 100, 150, 200, 300, 500].map((n) => ({
+    n,
+    se: round5(sd / Math.sqrt(n)),
+    effect_over_se: round3(effect / (sd / Math.sqrt(n))),
+    informative: effect >= sd / Math.sqrt(n),
+  }));
+
+  console.log(JSON.stringify({
+    phase: 'PHASE 0 — power derivation, blind to outcomes',
+    pooled_rows: diffs.length,
+    per_row_brier_diff_sd: round5(sd),
+    pooled_effect_abs_mean: round5(effect),
+    n_star: nStar,
+    rule: 'n* = (SD(d) / |effect|)^2 — the smallest held-row count at which a typical Brier delta clears one standard error',
+    se_vs_n: curve,
+    blind: 'no stat verdict, no date, and no reversal is referenced anywhere in this output',
+  }, null, 2));
+  process.exit(0);
+})().catch((e) => { console.error(e); process.exit(1); });
+
+const round5 = (v) => Math.round(v * 100000) / 100000;
+const round3 = (v) => Math.round(v * 1000) / 1000;
@@ -38,8 +38,14 @@ const STATS = ['hits', 'total_bases', 'rbi', 'runs'];
 const PAGE = 1000;
 /** The favourite bucket where the over-prediction concentrates. */
 const FAVOURITE_FLOOR = 0.9;
-/** Minimum rows on a held-out date for that drop to be informative. */
-const MIN_HELD_ROWS = 20;
+/**
+ * Minimum rows on a held-out date for that drop to be informative.
+ *
+ * POWER-DERIVED AND PRE-COMMITTED (n* = 70). Not chosen here, and not tunable
+ * from here -- it is imported so the value that decides the verdicts cannot be
+ * edited alongside them.
+ */
+const { LODO_MIN_HELD_ROWS: MIN_HELD_ROWS } = require('../src/services/model/calibrationRegistry');

 const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs };
 const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);