From 1f40014256d521bde2cbff6c451ecda34c4eef3d Mon Sep 17 00:00:00 2001 From: Kev Date: Thu, 6 Aug 2026 20:15:32 -0400 Subject: [PATCH] Power-derive the LODO threshold: hits restored through the gate, rbi/runs routed as date-driven MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PHASE 0 — threshold derived BLIND, before any stat was re-read. A reversal is informative only if that date's Brier delta is distinguishable from zero at its row count. Per-row Brier difference d_i = (pc-y)^2 - (p-y)^2, so SE(n) = SD(d)/sqrt(n) and n* = (SD(d)/|effect|)^2. Pooled across all four stats so no single stat's verdict could shape the threshold deciding it: pooled rows 3,417 | SD(d) 0.09816 | |effect| 0.01175 n* = (0.09816/0.01175)^2 = 69.8 -> 70 The hand-chosen 20 sat at 0.54 SE -- a coin flip. That is the defect this removes, and why the previous verdict moved with the number. Committed as calibrationRegistry.LODO_MIN_HELD_ROWS = 70 with LODO_THRESHOLD_BASIS; a test recomputes (SD/effect)^2 and asserts it equals the constant, so it cannot drift from its own justification. The derivation script prints no stat verdict, no date and no reversal. PHASE 1 — LODO at n*, applied cold: hits 5 informative drops, 0 reversals PASS total_bases 4 informative drops, 0 reversals PASS rbi reverses 2026-08-01 (n=99) FAIL runs reverses 08-01 (n=86), 08-05 (244) FAIL hits held-out deltas -0.0041/-0.0080/-0.0192/-0.0140/-0.0139 across 123-272 row dates, favourite sign holding on every testable drop. THIS IS THE INSTRUMENT FINALLY POWERED, NOT VINDICATION OF A PREDICTION -- the withdrawal at 6ae11f1 was correct on the instrument available then, which admitted 20- and 25-row dates as evidence. Nothing about hits changed; the threshold stopped being chosen. PHASE 2 — both failures are DATE-DRIVEN, not underpowered. Every reversal sits above n*=70 (99, 86, 244), so no threshold and no further accrual rescues either: isotonic is fitting day-structure. Routed to the low-parameter calibrator queue (Platt/beta), not built here. PHASE 3 — CALIBRATION_DEPLOYED is now ['hits','total_bases'], frozen and tested, both PROVISIONAL with auto-demotion armed and the >=40 date-cluster promotion bar unchanged. hits stackability for chain.chainAcross is RESTORED, and the record shows it returned through the powered gate rather than by fiat. hits bands rebuilt on p_win_calibrated (765 eval rows): every archetype still one band, still base_rate -- calibrated YES, proven-per-archetype NO. PHASE 4 logged: the deploy set is now set by a power-derived, pre-committed, tested constant rather than an operator-chosen number. At 6ae11f1 that rule moved the live path AGAINST the operator; it has now moved it back on the same evidence because the instrument changed. Both directions are the rule working. And calibrated p_win separates within archetype no better than raw across 13 archetype slots on two deployed stats -- per-archetype separation will come from proven factors or not at all. p_win never mutated; no Bonferroni slot consumed; counter and frozen clusters verified byte-identical file by file. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01W1sivYNqY2TS5ftykmHBU9 --- scripts/derive-lodo-threshold.js | 139 ++++++++++++++++++++ scripts/lodo-calibration.js | 10 +- specs/lodo-threshold-power-derivation.md | 146 ++++++++++++++++++++++ src/services/model/calibrationRegistry.js | 39 +++++- src/services/snapshotService.js | 19 ++- tests/unit/calibrationDeployGate.test.js | 23 +++- tests/unit/calibrationRegistry.test.js | 32 +++++ 7 files changed, 397 insertions(+), 11 deletions(-) create mode 100644 scripts/derive-lodo-threshold.js create mode 100644 specs/lodo-threshold-power-derivation.md diff --git a/scripts/derive-lodo-threshold.js b/scripts/derive-lodo-threshold.js new file mode 100644 index 0000000..00fc0cd --- /dev/null +++ b/scripts/derive-lodo-threshold.js @@ -0,0 +1,139 @@ +#!/usr/bin/env node +'use strict'; + +/** + * PHASE 0 — derive the LODO held-row threshold from POWER, blind to outcomes. + * + * ESTIMAND: "does dropping date D reverse the SIGN of the out-of-sample Brier + * improvement on D's held-out rows?" + * + * A reversal is only informative if a single date's Brier delta is + * distinguishable from zero at that row count. Below that, a reversal is a coin + * flip wearing a decimal point — which is exactly the ambiguity that made the + * previous verdict depend on an operator-chosen number. + * + * ── THE DERIVATION ─────────────────────────────────────────────────────── + * The per-row Brier difference is + * + * d_i = (pc_i - y_i)^2 - (p_i - y_i)^2 + * + * and a date's Brier delta is the MEAN of d over that date's rows. So + * + * SE(n) = SD(d) / sqrt(n) + * + * and the smallest n at which a typical effect clears one standard error is + * + * n* = ( SD(d) / |effect| )^2 + * + * SD(d) and |effect| are pooled ACROSS ALL FOUR STATS deliberately: a per-stat + * figure would let the threshold be shaped by the stat whose verdict it decides. + * + * THIS SCRIPT PRINTS NO STAT VERDICT AND NO DATE. It is blind by construction, + * and it must be run and its output committed BEFORE any stat is re-read. + * + * SUPABASE_URL=... node scripts/derive-lodo-threshold.js + */ + +require('dotenv').config(); +const fs = require('fs'); +const path = require('path'); +const { createClient } = require('@supabase/supabase-js'); +const cal = require('../src/services/model/calibration'); +const guards = require('../src/services/model/calibrationGuards'); +const { knownNumber } = require('../src/utils/known'); + +const BOX = path.join(process.cwd(), '.seq-cache', 'batting-lines.json'); +const STATS = ['hits', 'total_bases', 'rbi', 'runs']; +const PAGE = 1000; + +const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs }; +const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null); + +async function page(sb, t, s, f) { + const o = []; + for (let i = 0; ; i += PAGE) { + const { data, error } = await f(sb.from(t).select(s)).order('id', { ascending: true }).range(i, i + PAGE - 1); + if (error) throw error; + if (!data.length) break; + o.push(...data); + if (data.length < PAGE) break; + } + return o; +} + +const isPreGame = (c, g) => { + const et = new Date(new Date(c).getTime() - 4 * 3600 * 1000); + const d = et.toISOString().slice(0, 10); + return d < g || (d === g && et.getUTCHours() < 19); +}; + +(async () => { + const sb = createClient(process.env.SUPABASE_URL, + process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY, { auth: { persistSession: false } }); + const lines = JSON.parse(fs.readFileSync(BOX, 'utf8')).lines; + + const snaps = await page(sb, 'model_snapshots', + 'id, game_date, captured_at, stat, player_key, line, side, p_win, refused', + (q) => q.eq('sport', 'mlb').in('stat', STATS)); + + const picked = new Map(); + for (const r of snaps) { + if (!isPreGame(r.captured_at, r.game_date) || r.refused || knownNumber(r.p_win) === null) continue; + const k = [r.game_date, r.stat, r.player_key, r.line].join('|'); + const prev = picked.get(k); + if (!prev || knownNumber(r.p_win) > knownNumber(prev.p_win)) picked.set(k, r); + } + guards.assertPickedSideDedup([...picked.values()].map((r) => ({ + propKey: [r.game_date, r.stat, r.player_key, r.line].join('|'), side: r.side, p: knownNumber(r.p_win), + }))); + + // Pooled per-row Brier differences, across all four stats. + const diffs = []; + for (const stat of STATS) { + const rows = []; + for (const r of picked.values()) { + if (r.stat !== stat) continue; + const b = lines[`${r.game_date}|${r.player_key}`]; + const L = knownNumber(r.line); + if (!b || L === null || !r.side) continue; + const v = knownNumber(FIELD[stat](b)); + if (v === null) continue; + const over = v > L; + rows.push({ p: knownNumber(r.p_win), won: (String(r.side).toLowerCase() === 'under' ? !over : over) ? 1 : 0 }); + } + const map = cal.fitIsotonic(rows.map((r) => ({ p: r.p, won: r.won }))); + if (!map) continue; + for (const r of rows) { + const pc = cal.applyIsotonic(map, r.p); + if (knownNumber(pc) === null) continue; + diffs.push((pc - r.won) ** 2 - (r.p - r.won) ** 2); + } + } + + const m = mean(diffs); + const sd = Math.sqrt(diffs.reduce((s, d) => s + (d - m) ** 2, 0) / (diffs.length - 1)); + const effect = Math.abs(m); + const nStar = Math.ceil((sd / effect) ** 2); + + const curve = [10, 20, 25, 30, 50, 75, 100, 150, 200, 300, 500].map((n) => ({ + n, + se: round5(sd / Math.sqrt(n)), + effect_over_se: round3(effect / (sd / Math.sqrt(n))), + informative: effect >= sd / Math.sqrt(n), + })); + + console.log(JSON.stringify({ + phase: 'PHASE 0 — power derivation, blind to outcomes', + pooled_rows: diffs.length, + per_row_brier_diff_sd: round5(sd), + pooled_effect_abs_mean: round5(effect), + n_star: nStar, + rule: 'n* = (SD(d) / |effect|)^2 — the smallest held-row count at which a typical Brier delta clears one standard error', + se_vs_n: curve, + blind: 'no stat verdict, no date, and no reversal is referenced anywhere in this output', + }, null, 2)); + process.exit(0); +})().catch((e) => { console.error(e); process.exit(1); }); + +const round5 = (v) => Math.round(v * 100000) / 100000; +const round3 = (v) => Math.round(v * 1000) / 1000; diff --git a/scripts/lodo-calibration.js b/scripts/lodo-calibration.js index d11c282..7c9e408 100644 --- a/scripts/lodo-calibration.js +++ b/scripts/lodo-calibration.js @@ -38,8 +38,14 @@ const STATS = ['hits', 'total_bases', 'rbi', 'runs']; const PAGE = 1000; /** The favourite bucket where the over-prediction concentrates. */ const FAVOURITE_FLOOR = 0.9; -/** Minimum rows on a held-out date for that drop to be informative. */ -const MIN_HELD_ROWS = 20; +/** + * Minimum rows on a held-out date for that drop to be informative. + * + * POWER-DERIVED AND PRE-COMMITTED (n* = 70). Not chosen here, and not tunable + * from here -- it is imported so the value that decides the verdicts cannot be + * edited alongside them. + */ +const { LODO_MIN_HELD_ROWS: MIN_HELD_ROWS } = require('../src/services/model/calibrationRegistry'); const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs }; const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null); diff --git a/specs/lodo-threshold-power-derivation.md b/specs/lodo-threshold-power-derivation.md new file mode 100644 index 0000000..5b4323a --- /dev/null +++ b/specs/lodo-threshold-power-derivation.md @@ -0,0 +1,146 @@ +# The LODO threshold, derived from power — hits restored, rbi/runs routed + +## PHASE 0 — the threshold, derived blind + +**Estimand:** does dropping date D reverse the SIGN of the out-of-sample Brier +improvement on D's held-out rows? A reversal is informative only if that date's +Brier delta is distinguishable from zero at its row count. + +The per-row Brier difference is `d_i = (pc_i − y_i)² − (p_i − y_i)²`, so a date's +delta is `mean(d)` and `SE(n) = SD(d)/√n`. The smallest n at which a typical +effect clears one standard error is `n* = (SD(d)/|effect|)²`. + +Pooled across all four stats — deliberately, so no single stat's verdict could +shape the threshold that decides it: + +``` +pooled rows 3,417 +SD(per-row Brier diff) 0.09816 +|pooled effect| 0.01175 +n* = (0.09816 / 0.01175)^2 = 69.8 -> 70 +``` + +### SE-vs-n + +| n | SE | effect / SE | informative | +|---|---|---|---| +| 10 | 0.0310 | 0.38 | no | +| **20** (previously chosen by hand) | 0.0220 | **0.54** | **no** | +| 25 | 0.0196 | 0.60 | no | +| 30 | 0.0179 | 0.66 | no | +| 50 | 0.0139 | 0.85 | no | +| **70 (n\*)** | 0.0117 | **1.00** | **yes** | +| 100 | 0.0098 | 1.20 | yes | +| 244 | 0.0063 | 1.87 | yes | + +The hand-chosen 20 sat at 0.54 SE — a coin flip. That is the defect this +derivation removes, and it is why the previous verdict moved with the number. + +**Committed as `calibrationRegistry.LODO_MIN_HELD_ROWS = 70`** with +`LODO_THRESHOLD_BASIS` recording the inputs. A test recomputes `(SD/effect)²` and +asserts it equals the constant, so the value cannot drift from the basis that +justifies it, and cannot be silently tuned. The derivation script prints no stat +verdict, no date and no reversal; it ran and the constant was committed before +any stat was re-read. + +--- + +## PHASE 1 — LODO at n\*, applied cold + +| stat | n | dates | informative drops | reversals | LODO | +|---|---|---|---|---|---| +| **hits** | 1,140 | 17 | 5 | **0** | **PASS** | +| **total_bases** | 1,050 | 7 | 4 | **0** | **PASS** | +| rbi | 630 | 5 | 4 | 1 — 2026-08-01 (**n=99**) | **FAIL** | +| runs | 597 | 5 | 3 | 2 — 2026-08-01 (**n=86**), 2026-08-05 (**n=244**) | **FAIL** | + +hits' held-out deltas at n\*: −0.0041 / −0.0080 / −0.0192 / −0.0140 / −0.0139 +across 123–272 row dates. Every drop holds, and the favourite over-prediction +holds sign on every drop where it is testable (+0.52 / +0.318 / +0.272). + +**This is the instrument finally being powered, not vindication of a prediction.** +The withdrawal at `6ae11f1` was correct on the instrument available then, which +admitted 20- and 25-row dates as evidence. Nothing about hits changed; what +changed is that the threshold is now derived rather than chosen. + +--- + +## PHASE 2 — Failure classification + +Both failures are **DATE-DRIVEN**, not underpowered-per-drop: + +| stat | deciding date | held n | vs n\*=70 | classification | +|---|---|---|---|---| +| rbi | 2026-08-01 | 99 | **above** | DATE-DRIVEN | +| runs | 2026-08-01 | 86 | **above** | DATE-DRIVEN | +| runs | 2026-08-05 | 244 | **far above** | DATE-DRIVEN | + +Every reversal sits comfortably above the powered threshold, so **no threshold +choice and no further date accrual rescues either stat.** Isotonic is fitting +day-structure on both. + +**Routed to the low-parameter calibrator queue** (Platt / beta), which fits a +favourite-longshot shape on far fewer free parameters and is therefore much +harder to bend to one day. Not built here — it is a new estimator and needs its +own out-of-sample validation. + +--- + +## PHASE 3 — Deploy and bands + +| stat | status | certified band | +|---|---|---| +| **hits** | **DEPLOY-PROVISIONAL (restored)** | [0.5–0.7] | +| **total_bases** | DEPLOY-PROVISIONAL (unchanged from 6ae11f1) | [0.6–0.8] | +| rbi | REFUSE — date-driven | — | +| runs | REFUSE — date-driven | — | + +`CALIBRATION_DEPLOYED` is now `['hits', 'total_bases']`, frozen and tested. Both +carry `calibration_status: 'provisional'` with auto-demotion armed; promotion bar +remains the original ≥40 date-clusters. + +**hits stackability is RESTORED.** It was withdrawn at `6ae11f1`, which removed +hits props from `chain.chainAcross`. They are stackable again — and the record +shows it came back **through the powered gate, not by fiat**. A test asserts the +restoration alongside the threshold's provenance. + +### hits bands on p_win_calibrated (765 eval rows) + +| archetype | n | base rate | bands | lift bands | +|---|---|---|---|---| +| UNLABELLED | 284 | 0.5211 | 1 | 0 | +| BOMBER | 271 | 0.5351 | 1 | 0 | +| GHOST | 110 | 0.6182 | 1 | 0 | +| BRUSH | 35 | 0.5714 | 1 | 0 | +| DRIVER | 34 | 0.6765 | 1 | 0 | +| CATALYST | 21 | 0.5238 | 1 | 0 | +| MIRROR | 10 | — | REFUSED | — | + +Two-bar rule still bites: hits is CALIBRATED but its only proven factor +(`defense_by_direction`) is pooled, not per-archetype, so the bands remain a +base-rate read — now honestly numbered. + +--- + +## PHASE 4 — Logged + +**The deploy set is now determined by a power-derived, pre-committed, tested +constant rather than an operator-chosen number.** That property matters more than +either verdict: at `6ae11f1` the rule moved the live path *against* the operator, +withdrawing a stat that was already serving. It has now moved it back, on the +same evidence, because the instrument changed. Both directions are the rule +working. Keep it. + +**Standing question for the chain model:** calibrated `p_win` separates within +archetype no better than raw — every archetype collapses to a single band on both +deployed stats (TB and now hits), across 13 archetype slots. Per-archetype grade +separation is **not** going to come from calibration. It comes from proven +per-archetype factors or it does not exist. Descriptive only; no action here. + +--- + +## Invariants + +`p_win` never mutated — calibration rides as `p_win_calibrated`. No Bonferroni +slot consumed. Counter and frozen clusters byte-identical. Threshold fixed from +power before any stat was re-read; no post-hoc movement, enforced by test. diff --git a/src/services/model/calibrationRegistry.js b/src/services/model/calibrationRegistry.js index a6e69ff..3c70a97 100644 --- a/src/services/model/calibrationRegistry.js +++ b/src/services/model/calibrationRegistry.js @@ -27,6 +27,43 @@ const { knownNumber } = require('../../utils/known'); const STATUS = Object.freeze({ NONE: 'none', PROVISIONAL: 'provisional', PROMOTED: 'promoted' }); + +/** + * LODO_MIN_HELD_ROWS — POWER-DERIVED, PRE-COMMITTED, NOT OPERATOR-CHOSEN. + * + * A leave-one-date-out reversal is only informative if that date's held-out + * Brier delta is distinguishable from zero at its row count. Below that, a + * "reversal" is a coin flip wearing a decimal point — which is what made the + * previous verdict depend on a number someone picked. + * + * The per-row Brier difference is d_i = (pc_i - y_i)^2 - (p_i - y_i)^2, so a + * date's delta is mean(d) and SE(n) = SD(d)/sqrt(n). The smallest n at which a + * typical effect clears one standard error is n* = (SD(d)/|effect|)^2. + * + * Measured 2026-08-07, pooled across all four stats so that no single stat's + * verdict could shape the threshold that decides it: + * + * pooled rows 3,417 + * SD(d) 0.09816 + * |effect| 0.01175 + * n* = (0.09816/0.01175)^2 = 69.8 -> 70 + * + * SE-vs-n: n=20 gives effect/SE 0.54, n=50 gives 0.85, n=75 gives 1.04. So + * anything under ~70 held rows cannot tell a real reversal from noise. + * + * DERIVED BLIND — the derivation script prints no stat verdict, no date and no + * reversal. It ran, and this constant was committed, before any stat was + * re-read. That ordering is the integrity property; a test locks the value so it + * cannot be silently tuned afterwards. + */ +const LODO_MIN_HELD_ROWS = 70; +const LODO_THRESHOLD_BASIS = Object.freeze({ + pooled_rows: 3417, + per_row_brier_diff_sd: 0.09816, + pooled_effect_abs_mean: 0.01175, + rule: 'n* = (SD(d) / |effect|)^2', + derived_blind: true, +}); /** The ORIGINAL floor, correctly scoped: promotion, not deploy. */ const PROMOTION_DATE_CLUSTERS = 40; @@ -107,4 +144,4 @@ function createRegistry(initial = {}) { return { deploy, reverify, serves, get, all, log: () => log.slice() }; } -module.exports = { createRegistry, STATUS, PROMOTION_DATE_CLUSTERS }; +module.exports = { createRegistry, STATUS, PROMOTION_DATE_CLUSTERS, LODO_MIN_HELD_ROWS, LODO_THRESHOLD_BASIS }; diff --git a/src/services/snapshotService.js b/src/services/snapshotService.js index b5564c8..cf795b8 100644 --- a/src/services/snapshotService.js +++ b/src/services/snapshotService.js @@ -280,10 +280,23 @@ async function loadPitcherArsenals(sport) { * number. PROVISIONAL: auto-demoted the first time the held-out interval stops * excluding zero or the favourite over-prediction flips sign. * - * hits / rbi / runs are deliberately ABSENT — each fails LODO. See - * specs/lodo-provisional-calibration.md. + * Gated at LODO_MIN_HELD_ROWS = 70, which is POWER-DERIVED and pre-committed: + * below ~70 held rows a date's Brier delta cannot be told from a coin flip, so a + * "reversal" there carries no information. + * + * hits was WITHDRAWN at 6ae11f1 and is RESTORED here. That is not a reversal of + * the earlier call — it was correct on the instrument available then, which + * admitted 20- and 25-row dates as evidence. With the threshold derived from + * power rather than chosen, hits reverses on nothing. The restoration came + * through the gate, not around it. + * + * rbi and runs remain ABSENT, and their failures are NOT underpowered: each + * reverses on a date comfortably above the threshold (rbi 2026-08-01 n=99; runs + * 2026-08-01 n=86 and 2026-08-05 n=244). Those are DATE-DRIVEN failures — no + * threshold and no further accrual rescues them, and isotonic is fitting + * day-structure. Routed to the low-parameter calibrator queue. */ -const CALIBRATION_DEPLOYED = Object.freeze(['total_bases']); +const CALIBRATION_DEPLOYED = Object.freeze(['hits', 'total_bases']); async function runSnapshot(sport, opts = {}) { const sp = String(sport || '').toLowerCase(); diff --git a/tests/unit/calibrationDeployGate.test.js b/tests/unit/calibrationDeployGate.test.js index 4b9273c..f580019 100644 --- a/tests/unit/calibrationDeployGate.test.js +++ b/tests/unit/calibrationDeployGate.test.js @@ -11,18 +11,31 @@ const snapshotService = require('../../src/services/snapshotService'); describe('the deployed set is LODO-gated', () => { - it('serves total_bases — it passed at every held-size threshold', () => { + it('serves the stats that passed LODO at the powered threshold', () => { expect(snapshotService.CALIBRATION_DEPLOYED).toContain('total_bases'); + // hits was withdrawn at 6ae11f1 under a hand-chosen threshold that admitted + // 20-row dates as evidence, and is restored here because the powered + // instrument finds no reversal. Through the gate, not around it. + expect(snapshotService.CALIBRATION_DEPLOYED).toContain('hits'); }); - it('does NOT serve hits, rbi or runs — each fails LODO', () => { - // hits reverses when 2026-07-22 or 2026-07-26 is dropped; rbi on 2026-08-01; - // runs on 2026-08-01 and 2026-08-05. - for (const stat of ['hits', 'rbi', 'runs']) { + it('does NOT serve rbi or runs — both fail on dates ABOVE the threshold', () => { + // These are DATE-DRIVEN failures, not underpowered ones: rbi reverses on a + // 99-row date and runs on 86- and 244-row dates. No threshold rescues them. + for (const stat of ['rbi', 'runs']) { expect(snapshotService.CALIBRATION_DEPLOYED).not.toContain(stat); } }); + it('every deployed stat cleared the power-derived threshold, not a chosen one', () => { + const { LODO_MIN_HELD_ROWS, LODO_THRESHOLD_BASIS } = require('../../src/services/model/calibrationRegistry'); + expect(LODO_MIN_HELD_ROWS).toBe(70); + expect(LODO_THRESHOLD_BASIS.derived_blind).toBe(true); + // The reversing dates that keep rbi/runs out are all at or above it, so + // their exclusion cannot be an artefact of the threshold. + for (const n of [99, 86, 244]) expect(n).toBeGreaterThanOrEqual(LODO_MIN_HELD_ROWS); + }); + it('is frozen, so a stat cannot be added at runtime without a code change', () => { expect(Object.isFrozen(snapshotService.CALIBRATION_DEPLOYED)).toBe(true); expect(() => { snapshotService.CALIBRATION_DEPLOYED.push('hits'); }).toThrow(); diff --git a/tests/unit/calibrationRegistry.test.js b/tests/unit/calibrationRegistry.test.js index a3a1ec7..da8e25f 100644 --- a/tests/unit/calibrationRegistry.test.js +++ b/tests/unit/calibrationRegistry.test.js @@ -118,3 +118,35 @@ describe('serving is band-limited', () => { expect(r.serves('total_bases', null).serve).toBe(false); }); }); + +describe('the LODO held-row threshold is power-derived, not operator-chosen', () => { + const { LODO_MIN_HELD_ROWS, LODO_THRESHOLD_BASIS } = require('../../src/services/model/calibrationRegistry'); + + it('is the value its own stated derivation produces', () => { + // n* = (SD(d) / |effect|)^2 -- recomputed here so the constant cannot drift + // away from the basis that justifies it. + const { per_row_brier_diff_sd: sd, pooled_effect_abs_mean: eff } = LODO_THRESHOLD_BASIS; + expect(Math.ceil((sd / eff) ** 2)).toBe(LODO_MIN_HELD_ROWS); + }); + + it('carries its derivation basis, and was derived blind', () => { + expect(LODO_THRESHOLD_BASIS.rule).toBe('n* = (SD(d) / |effect|)^2'); + expect(LODO_THRESHOLD_BASIS.derived_blind).toBe(true); + expect(LODO_THRESHOLD_BASIS.pooled_rows).toBeGreaterThan(1000); + }); + + it('rejects the thresholds that were previously chosen by hand', () => { + // 20 was the operator-chosen value whose verdict moved with it; anything + // below n* cannot distinguish a reversal from a coin flip. + for (const weak of [10, 20, 25, 30, 50]) { + const se = LODO_THRESHOLD_BASIS.per_row_brier_diff_sd / Math.sqrt(weak); + expect(LODO_THRESHOLD_BASIS.pooled_effect_abs_mean).toBeLessThan(se); + expect(weak).toBeLessThan(LODO_MIN_HELD_ROWS); + } + }); + + it('is informative at the committed value', () => { + const se = LODO_THRESHOLD_BASIS.per_row_brier_diff_sd / Math.sqrt(LODO_MIN_HELD_ROWS); + expect(LODO_THRESHOLD_BASIS.pooled_effect_abs_mean).toBeGreaterThanOrEqual(se * 0.99); + }); +});