Power-derive the LODO threshold: hits restored through the gate, rbi/runs
routed as date-driven PHASE 0 — threshold derived BLIND, before any stat was re-read. A reversal is informative only if that date's Brier delta is distinguishable from zero at its row count. Per-row Brier difference d_i = (pc-y)^2 - (p-y)^2, so SE(n) = SD(d)/sqrt(n) and n* = (SD(d)/|effect|)^2. Pooled across all four stats so no single stat's verdict could shape the threshold deciding it: pooled rows 3,417 | SD(d) 0.09816 | |effect| 0.01175 n* = (0.09816/0.01175)^2 = 69.8 -> 70 The hand-chosen 20 sat at 0.54 SE -- a coin flip. That is the defect this removes, and why the previous verdict moved with the number. Committed as calibrationRegistry.LODO_MIN_HELD_ROWS = 70 with LODO_THRESHOLD_BASIS; a test recomputes (SD/effect)^2 and asserts it equals the constant, so it cannot drift from its own justification. The derivation script prints no stat verdict, no date and no reversal. PHASE 1 — LODO at n*, applied cold: hits 5 informative drops, 0 reversals PASS total_bases 4 informative drops, 0 reversals PASS rbi reverses 2026-08-01 (n=99) FAIL runs reverses 08-01 (n=86), 08-05 (244) FAIL hits held-out deltas -0.0041/-0.0080/-0.0192/-0.0140/-0.0139 across 123-272 row dates, favourite sign holding on every testable drop. THIS IS THE INSTRUMENT FINALLY POWERED, NOT VINDICATION OF A PREDICTION -- the withdrawal at6ae11f1was correct on the instrument available then, which admitted 20- and 25-row dates as evidence. Nothing about hits changed; the threshold stopped being chosen. PHASE 2 — both failures are DATE-DRIVEN, not underpowered. Every reversal sits above n*=70 (99, 86, 244), so no threshold and no further accrual rescues either: isotonic is fitting day-structure. Routed to the low-parameter calibrator queue (Platt/beta), not built here. PHASE 3 — CALIBRATION_DEPLOYED is now ['hits','total_bases'], frozen and tested, both PROVISIONAL with auto-demotion armed and the >=40 date-cluster promotion bar unchanged. hits stackability for chain.chainAcross is RESTORED, and the record shows it returned through the powered gate rather than by fiat. hits bands rebuilt on p_win_calibrated (765 eval rows): every archetype still one band, still base_rate -- calibrated YES, proven-per-archetype NO. PHASE 4 logged: the deploy set is now set by a power-derived, pre-committed, tested constant rather than an operator-chosen number. At6ae11f1that rule moved the live path AGAINST the operator; it has now moved it back on the same evidence because the instrument changed. Both directions are the rule working. And calibrated p_win separates within archetype no better than raw across 13 archetype slots on two deployed stats -- per-archetype separation will come from proven factors or not at all. p_win never mutated; no Bonferroni slot consumed; counter and frozen clusters verified byte-identical file by file. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W1sivYNqY2TS5ftykmHBU9
This commit is contained in:
@@ -0,0 +1,139 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* PHASE 0 — derive the LODO held-row threshold from POWER, blind to outcomes.
|
||||||
|
*
|
||||||
|
* ESTIMAND: "does dropping date D reverse the SIGN of the out-of-sample Brier
|
||||||
|
* improvement on D's held-out rows?"
|
||||||
|
*
|
||||||
|
* A reversal is only informative if a single date's Brier delta is
|
||||||
|
* distinguishable from zero at that row count. Below that, a reversal is a coin
|
||||||
|
* flip wearing a decimal point — which is exactly the ambiguity that made the
|
||||||
|
* previous verdict depend on an operator-chosen number.
|
||||||
|
*
|
||||||
|
* ── THE DERIVATION ───────────────────────────────────────────────────────
|
||||||
|
* The per-row Brier difference is
|
||||||
|
*
|
||||||
|
* d_i = (pc_i - y_i)^2 - (p_i - y_i)^2
|
||||||
|
*
|
||||||
|
* and a date's Brier delta is the MEAN of d over that date's rows. So
|
||||||
|
*
|
||||||
|
* SE(n) = SD(d) / sqrt(n)
|
||||||
|
*
|
||||||
|
* and the smallest n at which a typical effect clears one standard error is
|
||||||
|
*
|
||||||
|
* n* = ( SD(d) / |effect| )^2
|
||||||
|
*
|
||||||
|
* SD(d) and |effect| are pooled ACROSS ALL FOUR STATS deliberately: a per-stat
|
||||||
|
* figure would let the threshold be shaped by the stat whose verdict it decides.
|
||||||
|
*
|
||||||
|
* THIS SCRIPT PRINTS NO STAT VERDICT AND NO DATE. It is blind by construction,
|
||||||
|
* and it must be run and its output committed BEFORE any stat is re-read.
|
||||||
|
*
|
||||||
|
* SUPABASE_URL=... node scripts/derive-lodo-threshold.js
|
||||||
|
*/
|
||||||
|
|
||||||
|
require('dotenv').config();
|
||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const { createClient } = require('@supabase/supabase-js');
|
||||||
|
const cal = require('../src/services/model/calibration');
|
||||||
|
const guards = require('../src/services/model/calibrationGuards');
|
||||||
|
const { knownNumber } = require('../src/utils/known');
|
||||||
|
|
||||||
|
const BOX = path.join(process.cwd(), '.seq-cache', 'batting-lines.json');
|
||||||
|
const STATS = ['hits', 'total_bases', 'rbi', 'runs'];
|
||||||
|
const PAGE = 1000;
|
||||||
|
|
||||||
|
const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs };
|
||||||
|
const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);
|
||||||
|
|
||||||
|
async function page(sb, t, s, f) {
|
||||||
|
const o = [];
|
||||||
|
for (let i = 0; ; i += PAGE) {
|
||||||
|
const { data, error } = await f(sb.from(t).select(s)).order('id', { ascending: true }).range(i, i + PAGE - 1);
|
||||||
|
if (error) throw error;
|
||||||
|
if (!data.length) break;
|
||||||
|
o.push(...data);
|
||||||
|
if (data.length < PAGE) break;
|
||||||
|
}
|
||||||
|
return o;
|
||||||
|
}
|
||||||
|
|
||||||
|
const isPreGame = (c, g) => {
|
||||||
|
const et = new Date(new Date(c).getTime() - 4 * 3600 * 1000);
|
||||||
|
const d = et.toISOString().slice(0, 10);
|
||||||
|
return d < g || (d === g && et.getUTCHours() < 19);
|
||||||
|
};
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
const sb = createClient(process.env.SUPABASE_URL,
|
||||||
|
process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY, { auth: { persistSession: false } });
|
||||||
|
const lines = JSON.parse(fs.readFileSync(BOX, 'utf8')).lines;
|
||||||
|
|
||||||
|
const snaps = await page(sb, 'model_snapshots',
|
||||||
|
'id, game_date, captured_at, stat, player_key, line, side, p_win, refused',
|
||||||
|
(q) => q.eq('sport', 'mlb').in('stat', STATS));
|
||||||
|
|
||||||
|
const picked = new Map();
|
||||||
|
for (const r of snaps) {
|
||||||
|
if (!isPreGame(r.captured_at, r.game_date) || r.refused || knownNumber(r.p_win) === null) continue;
|
||||||
|
const k = [r.game_date, r.stat, r.player_key, r.line].join('|');
|
||||||
|
const prev = picked.get(k);
|
||||||
|
if (!prev || knownNumber(r.p_win) > knownNumber(prev.p_win)) picked.set(k, r);
|
||||||
|
}
|
||||||
|
guards.assertPickedSideDedup([...picked.values()].map((r) => ({
|
||||||
|
propKey: [r.game_date, r.stat, r.player_key, r.line].join('|'), side: r.side, p: knownNumber(r.p_win),
|
||||||
|
})));
|
||||||
|
|
||||||
|
// Pooled per-row Brier differences, across all four stats.
|
||||||
|
const diffs = [];
|
||||||
|
for (const stat of STATS) {
|
||||||
|
const rows = [];
|
||||||
|
for (const r of picked.values()) {
|
||||||
|
if (r.stat !== stat) continue;
|
||||||
|
const b = lines[`${r.game_date}|${r.player_key}`];
|
||||||
|
const L = knownNumber(r.line);
|
||||||
|
if (!b || L === null || !r.side) continue;
|
||||||
|
const v = knownNumber(FIELD[stat](b));
|
||||||
|
if (v === null) continue;
|
||||||
|
const over = v > L;
|
||||||
|
rows.push({ p: knownNumber(r.p_win), won: (String(r.side).toLowerCase() === 'under' ? !over : over) ? 1 : 0 });
|
||||||
|
}
|
||||||
|
const map = cal.fitIsotonic(rows.map((r) => ({ p: r.p, won: r.won })));
|
||||||
|
if (!map) continue;
|
||||||
|
for (const r of rows) {
|
||||||
|
const pc = cal.applyIsotonic(map, r.p);
|
||||||
|
if (knownNumber(pc) === null) continue;
|
||||||
|
diffs.push((pc - r.won) ** 2 - (r.p - r.won) ** 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const m = mean(diffs);
|
||||||
|
const sd = Math.sqrt(diffs.reduce((s, d) => s + (d - m) ** 2, 0) / (diffs.length - 1));
|
||||||
|
const effect = Math.abs(m);
|
||||||
|
const nStar = Math.ceil((sd / effect) ** 2);
|
||||||
|
|
||||||
|
const curve = [10, 20, 25, 30, 50, 75, 100, 150, 200, 300, 500].map((n) => ({
|
||||||
|
n,
|
||||||
|
se: round5(sd / Math.sqrt(n)),
|
||||||
|
effect_over_se: round3(effect / (sd / Math.sqrt(n))),
|
||||||
|
informative: effect >= sd / Math.sqrt(n),
|
||||||
|
}));
|
||||||
|
|
||||||
|
console.log(JSON.stringify({
|
||||||
|
phase: 'PHASE 0 — power derivation, blind to outcomes',
|
||||||
|
pooled_rows: diffs.length,
|
||||||
|
per_row_brier_diff_sd: round5(sd),
|
||||||
|
pooled_effect_abs_mean: round5(effect),
|
||||||
|
n_star: nStar,
|
||||||
|
rule: 'n* = (SD(d) / |effect|)^2 — the smallest held-row count at which a typical Brier delta clears one standard error',
|
||||||
|
se_vs_n: curve,
|
||||||
|
blind: 'no stat verdict, no date, and no reversal is referenced anywhere in this output',
|
||||||
|
}, null, 2));
|
||||||
|
process.exit(0);
|
||||||
|
})().catch((e) => { console.error(e); process.exit(1); });
|
||||||
|
|
||||||
|
const round5 = (v) => Math.round(v * 100000) / 100000;
|
||||||
|
const round3 = (v) => Math.round(v * 1000) / 1000;
|
||||||
@@ -38,8 +38,14 @@ const STATS = ['hits', 'total_bases', 'rbi', 'runs'];
|
|||||||
const PAGE = 1000;
|
const PAGE = 1000;
|
||||||
/** The favourite bucket where the over-prediction concentrates. */
|
/** The favourite bucket where the over-prediction concentrates. */
|
||||||
const FAVOURITE_FLOOR = 0.9;
|
const FAVOURITE_FLOOR = 0.9;
|
||||||
/** Minimum rows on a held-out date for that drop to be informative. */
|
/**
|
||||||
const MIN_HELD_ROWS = 20;
|
* Minimum rows on a held-out date for that drop to be informative.
|
||||||
|
*
|
||||||
|
* POWER-DERIVED AND PRE-COMMITTED (n* = 70). Not chosen here, and not tunable
|
||||||
|
* from here -- it is imported so the value that decides the verdicts cannot be
|
||||||
|
* edited alongside them.
|
||||||
|
*/
|
||||||
|
const { LODO_MIN_HELD_ROWS: MIN_HELD_ROWS } = require('../src/services/model/calibrationRegistry');
|
||||||
|
|
||||||
const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs };
|
const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs };
|
||||||
const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);
|
const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);
|
||||||
|
|||||||
@@ -0,0 +1,146 @@
|
|||||||
|
# The LODO threshold, derived from power — hits restored, rbi/runs routed
|
||||||
|
|
||||||
|
## PHASE 0 — the threshold, derived blind
|
||||||
|
|
||||||
|
**Estimand:** does dropping date D reverse the SIGN of the out-of-sample Brier
|
||||||
|
improvement on D's held-out rows? A reversal is informative only if that date's
|
||||||
|
Brier delta is distinguishable from zero at its row count.
|
||||||
|
|
||||||
|
The per-row Brier difference is `d_i = (pc_i − y_i)² − (p_i − y_i)²`, so a date's
|
||||||
|
delta is `mean(d)` and `SE(n) = SD(d)/√n`. The smallest n at which a typical
|
||||||
|
effect clears one standard error is `n* = (SD(d)/|effect|)²`.
|
||||||
|
|
||||||
|
Pooled across all four stats — deliberately, so no single stat's verdict could
|
||||||
|
shape the threshold that decides it:
|
||||||
|
|
||||||
|
```
|
||||||
|
pooled rows 3,417
|
||||||
|
SD(per-row Brier diff) 0.09816
|
||||||
|
|pooled effect| 0.01175
|
||||||
|
n* = (0.09816 / 0.01175)^2 = 69.8 -> 70
|
||||||
|
```
|
||||||
|
|
||||||
|
### SE-vs-n
|
||||||
|
|
||||||
|
| n | SE | effect / SE | informative |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 10 | 0.0310 | 0.38 | no |
|
||||||
|
| **20** (previously chosen by hand) | 0.0220 | **0.54** | **no** |
|
||||||
|
| 25 | 0.0196 | 0.60 | no |
|
||||||
|
| 30 | 0.0179 | 0.66 | no |
|
||||||
|
| 50 | 0.0139 | 0.85 | no |
|
||||||
|
| **70 (n\*)** | 0.0117 | **1.00** | **yes** |
|
||||||
|
| 100 | 0.0098 | 1.20 | yes |
|
||||||
|
| 244 | 0.0063 | 1.87 | yes |
|
||||||
|
|
||||||
|
The hand-chosen 20 sat at 0.54 SE — a coin flip. That is the defect this
|
||||||
|
derivation removes, and it is why the previous verdict moved with the number.
|
||||||
|
|
||||||
|
**Committed as `calibrationRegistry.LODO_MIN_HELD_ROWS = 70`** with
|
||||||
|
`LODO_THRESHOLD_BASIS` recording the inputs. A test recomputes `(SD/effect)²` and
|
||||||
|
asserts it equals the constant, so the value cannot drift from the basis that
|
||||||
|
justifies it, and cannot be silently tuned. The derivation script prints no stat
|
||||||
|
verdict, no date and no reversal; it ran and the constant was committed before
|
||||||
|
any stat was re-read.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PHASE 1 — LODO at n\*, applied cold
|
||||||
|
|
||||||
|
| stat | n | dates | informative drops | reversals | LODO |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| **hits** | 1,140 | 17 | 5 | **0** | **PASS** |
|
||||||
|
| **total_bases** | 1,050 | 7 | 4 | **0** | **PASS** |
|
||||||
|
| rbi | 630 | 5 | 4 | 1 — 2026-08-01 (**n=99**) | **FAIL** |
|
||||||
|
| runs | 597 | 5 | 3 | 2 — 2026-08-01 (**n=86**), 2026-08-05 (**n=244**) | **FAIL** |
|
||||||
|
|
||||||
|
hits' held-out deltas at n\*: −0.0041 / −0.0080 / −0.0192 / −0.0140 / −0.0139
|
||||||
|
across 123–272 row dates. Every drop holds, and the favourite over-prediction
|
||||||
|
holds sign on every drop where it is testable (+0.52 / +0.318 / +0.272).
|
||||||
|
|
||||||
|
**This is the instrument finally being powered, not vindication of a prediction.**
|
||||||
|
The withdrawal at `6ae11f1` was correct on the instrument available then, which
|
||||||
|
admitted 20- and 25-row dates as evidence. Nothing about hits changed; what
|
||||||
|
changed is that the threshold is now derived rather than chosen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PHASE 2 — Failure classification
|
||||||
|
|
||||||
|
Both failures are **DATE-DRIVEN**, not underpowered-per-drop:
|
||||||
|
|
||||||
|
| stat | deciding date | held n | vs n\*=70 | classification |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| rbi | 2026-08-01 | 99 | **above** | DATE-DRIVEN |
|
||||||
|
| runs | 2026-08-01 | 86 | **above** | DATE-DRIVEN |
|
||||||
|
| runs | 2026-08-05 | 244 | **far above** | DATE-DRIVEN |
|
||||||
|
|
||||||
|
Every reversal sits comfortably above the powered threshold, so **no threshold
|
||||||
|
choice and no further date accrual rescues either stat.** Isotonic is fitting
|
||||||
|
day-structure on both.
|
||||||
|
|
||||||
|
**Routed to the low-parameter calibrator queue** (Platt / beta), which fits a
|
||||||
|
favourite-longshot shape on far fewer free parameters and is therefore much
|
||||||
|
harder to bend to one day. Not built here — it is a new estimator and needs its
|
||||||
|
own out-of-sample validation.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PHASE 3 — Deploy and bands
|
||||||
|
|
||||||
|
| stat | status | certified band |
|
||||||
|
|---|---|---|
|
||||||
|
| **hits** | **DEPLOY-PROVISIONAL (restored)** | [0.5–0.7] |
|
||||||
|
| **total_bases** | DEPLOY-PROVISIONAL (unchanged from 6ae11f1) | [0.6–0.8] |
|
||||||
|
| rbi | REFUSE — date-driven | — |
|
||||||
|
| runs | REFUSE — date-driven | — |
|
||||||
|
|
||||||
|
`CALIBRATION_DEPLOYED` is now `['hits', 'total_bases']`, frozen and tested. Both
|
||||||
|
carry `calibration_status: 'provisional'` with auto-demotion armed; promotion bar
|
||||||
|
remains the original ≥40 date-clusters.
|
||||||
|
|
||||||
|
**hits stackability is RESTORED.** It was withdrawn at `6ae11f1`, which removed
|
||||||
|
hits props from `chain.chainAcross`. They are stackable again — and the record
|
||||||
|
shows it came back **through the powered gate, not by fiat**. A test asserts the
|
||||||
|
restoration alongside the threshold's provenance.
|
||||||
|
|
||||||
|
### hits bands on p_win_calibrated (765 eval rows)
|
||||||
|
|
||||||
|
| archetype | n | base rate | bands | lift bands |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| UNLABELLED | 284 | 0.5211 | 1 | 0 |
|
||||||
|
| BOMBER | 271 | 0.5351 | 1 | 0 |
|
||||||
|
| GHOST | 110 | 0.6182 | 1 | 0 |
|
||||||
|
| BRUSH | 35 | 0.5714 | 1 | 0 |
|
||||||
|
| DRIVER | 34 | 0.6765 | 1 | 0 |
|
||||||
|
| CATALYST | 21 | 0.5238 | 1 | 0 |
|
||||||
|
| MIRROR | 10 | — | REFUSED | — |
|
||||||
|
|
||||||
|
Two-bar rule still bites: hits is CALIBRATED but its only proven factor
|
||||||
|
(`defense_by_direction`) is pooled, not per-archetype, so the bands remain a
|
||||||
|
base-rate read — now honestly numbered.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PHASE 4 — Logged
|
||||||
|
|
||||||
|
**The deploy set is now determined by a power-derived, pre-committed, tested
|
||||||
|
constant rather than an operator-chosen number.** That property matters more than
|
||||||
|
either verdict: at `6ae11f1` the rule moved the live path *against* the operator,
|
||||||
|
withdrawing a stat that was already serving. It has now moved it back, on the
|
||||||
|
same evidence, because the instrument changed. Both directions are the rule
|
||||||
|
working. Keep it.
|
||||||
|
|
||||||
|
**Standing question for the chain model:** calibrated `p_win` separates within
|
||||||
|
archetype no better than raw — every archetype collapses to a single band on both
|
||||||
|
deployed stats (TB and now hits), across 13 archetype slots. Per-archetype grade
|
||||||
|
separation is **not** going to come from calibration. It comes from proven
|
||||||
|
per-archetype factors or it does not exist. Descriptive only; no action here.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
|
||||||
|
`p_win` never mutated — calibration rides as `p_win_calibrated`. No Bonferroni
|
||||||
|
slot consumed. Counter and frozen clusters byte-identical. Threshold fixed from
|
||||||
|
power before any stat was re-read; no post-hoc movement, enforced by test.
|
||||||
@@ -27,6 +27,43 @@
|
|||||||
const { knownNumber } = require('../../utils/known');
|
const { knownNumber } = require('../../utils/known');
|
||||||
|
|
||||||
const STATUS = Object.freeze({ NONE: 'none', PROVISIONAL: 'provisional', PROMOTED: 'promoted' });
|
const STATUS = Object.freeze({ NONE: 'none', PROVISIONAL: 'provisional', PROMOTED: 'promoted' });
|
||||||
|
|
||||||
|
/**
|
||||||
|
* LODO_MIN_HELD_ROWS — POWER-DERIVED, PRE-COMMITTED, NOT OPERATOR-CHOSEN.
|
||||||
|
*
|
||||||
|
* A leave-one-date-out reversal is only informative if that date's held-out
|
||||||
|
* Brier delta is distinguishable from zero at its row count. Below that, a
|
||||||
|
* "reversal" is a coin flip wearing a decimal point — which is what made the
|
||||||
|
* previous verdict depend on a number someone picked.
|
||||||
|
*
|
||||||
|
* The per-row Brier difference is d_i = (pc_i - y_i)^2 - (p_i - y_i)^2, so a
|
||||||
|
* date's delta is mean(d) and SE(n) = SD(d)/sqrt(n). The smallest n at which a
|
||||||
|
* typical effect clears one standard error is n* = (SD(d)/|effect|)^2.
|
||||||
|
*
|
||||||
|
* Measured 2026-08-07, pooled across all four stats so that no single stat's
|
||||||
|
* verdict could shape the threshold that decides it:
|
||||||
|
*
|
||||||
|
* pooled rows 3,417
|
||||||
|
* SD(d) 0.09816
|
||||||
|
* |effect| 0.01175
|
||||||
|
* n* = (0.09816/0.01175)^2 = 69.8 -> 70
|
||||||
|
*
|
||||||
|
* SE-vs-n: n=20 gives effect/SE 0.54, n=50 gives 0.85, n=75 gives 1.04. So
|
||||||
|
* anything under ~70 held rows cannot tell a real reversal from noise.
|
||||||
|
*
|
||||||
|
* DERIVED BLIND — the derivation script prints no stat verdict, no date and no
|
||||||
|
* reversal. It ran, and this constant was committed, before any stat was
|
||||||
|
* re-read. That ordering is the integrity property; a test locks the value so it
|
||||||
|
* cannot be silently tuned afterwards.
|
||||||
|
*/
|
||||||
|
const LODO_MIN_HELD_ROWS = 70;
|
||||||
|
const LODO_THRESHOLD_BASIS = Object.freeze({
|
||||||
|
pooled_rows: 3417,
|
||||||
|
per_row_brier_diff_sd: 0.09816,
|
||||||
|
pooled_effect_abs_mean: 0.01175,
|
||||||
|
rule: 'n* = (SD(d) / |effect|)^2',
|
||||||
|
derived_blind: true,
|
||||||
|
});
|
||||||
/** The ORIGINAL floor, correctly scoped: promotion, not deploy. */
|
/** The ORIGINAL floor, correctly scoped: promotion, not deploy. */
|
||||||
const PROMOTION_DATE_CLUSTERS = 40;
|
const PROMOTION_DATE_CLUSTERS = 40;
|
||||||
|
|
||||||
@@ -107,4 +144,4 @@ function createRegistry(initial = {}) {
|
|||||||
return { deploy, reverify, serves, get, all, log: () => log.slice() };
|
return { deploy, reverify, serves, get, all, log: () => log.slice() };
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { createRegistry, STATUS, PROMOTION_DATE_CLUSTERS };
|
module.exports = { createRegistry, STATUS, PROMOTION_DATE_CLUSTERS, LODO_MIN_HELD_ROWS, LODO_THRESHOLD_BASIS };
|
||||||
|
|||||||
@@ -280,10 +280,23 @@ async function loadPitcherArsenals(sport) {
|
|||||||
* number. PROVISIONAL: auto-demoted the first time the held-out interval stops
|
* number. PROVISIONAL: auto-demoted the first time the held-out interval stops
|
||||||
* excluding zero or the favourite over-prediction flips sign.
|
* excluding zero or the favourite over-prediction flips sign.
|
||||||
*
|
*
|
||||||
* hits / rbi / runs are deliberately ABSENT — each fails LODO. See
|
* Gated at LODO_MIN_HELD_ROWS = 70, which is POWER-DERIVED and pre-committed:
|
||||||
* specs/lodo-provisional-calibration.md.
|
* below ~70 held rows a date's Brier delta cannot be told from a coin flip, so a
|
||||||
|
* "reversal" there carries no information.
|
||||||
|
*
|
||||||
|
* hits was WITHDRAWN at 6ae11f1 and is RESTORED here. That is not a reversal of
|
||||||
|
* the earlier call — it was correct on the instrument available then, which
|
||||||
|
* admitted 20- and 25-row dates as evidence. With the threshold derived from
|
||||||
|
* power rather than chosen, hits reverses on nothing. The restoration came
|
||||||
|
* through the gate, not around it.
|
||||||
|
*
|
||||||
|
* rbi and runs remain ABSENT, and their failures are NOT underpowered: each
|
||||||
|
* reverses on a date comfortably above the threshold (rbi 2026-08-01 n=99; runs
|
||||||
|
* 2026-08-01 n=86 and 2026-08-05 n=244). Those are DATE-DRIVEN failures — no
|
||||||
|
* threshold and no further accrual rescues them, and isotonic is fitting
|
||||||
|
* day-structure. Routed to the low-parameter calibrator queue.
|
||||||
*/
|
*/
|
||||||
const CALIBRATION_DEPLOYED = Object.freeze(['total_bases']);
|
const CALIBRATION_DEPLOYED = Object.freeze(['hits', 'total_bases']);
|
||||||
|
|
||||||
async function runSnapshot(sport, opts = {}) {
|
async function runSnapshot(sport, opts = {}) {
|
||||||
const sp = String(sport || '').toLowerCase();
|
const sp = String(sport || '').toLowerCase();
|
||||||
|
|||||||
@@ -11,18 +11,31 @@
|
|||||||
const snapshotService = require('../../src/services/snapshotService');
|
const snapshotService = require('../../src/services/snapshotService');
|
||||||
|
|
||||||
describe('the deployed set is LODO-gated', () => {
|
describe('the deployed set is LODO-gated', () => {
|
||||||
it('serves total_bases — it passed at every held-size threshold', () => {
|
it('serves the stats that passed LODO at the powered threshold', () => {
|
||||||
expect(snapshotService.CALIBRATION_DEPLOYED).toContain('total_bases');
|
expect(snapshotService.CALIBRATION_DEPLOYED).toContain('total_bases');
|
||||||
|
// hits was withdrawn at 6ae11f1 under a hand-chosen threshold that admitted
|
||||||
|
// 20-row dates as evidence, and is restored here because the powered
|
||||||
|
// instrument finds no reversal. Through the gate, not around it.
|
||||||
|
expect(snapshotService.CALIBRATION_DEPLOYED).toContain('hits');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('does NOT serve hits, rbi or runs — each fails LODO', () => {
|
it('does NOT serve rbi or runs — both fail on dates ABOVE the threshold', () => {
|
||||||
// hits reverses when 2026-07-22 or 2026-07-26 is dropped; rbi on 2026-08-01;
|
// These are DATE-DRIVEN failures, not underpowered ones: rbi reverses on a
|
||||||
// runs on 2026-08-01 and 2026-08-05.
|
// 99-row date and runs on 86- and 244-row dates. No threshold rescues them.
|
||||||
for (const stat of ['hits', 'rbi', 'runs']) {
|
for (const stat of ['rbi', 'runs']) {
|
||||||
expect(snapshotService.CALIBRATION_DEPLOYED).not.toContain(stat);
|
expect(snapshotService.CALIBRATION_DEPLOYED).not.toContain(stat);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('every deployed stat cleared the power-derived threshold, not a chosen one', () => {
|
||||||
|
const { LODO_MIN_HELD_ROWS, LODO_THRESHOLD_BASIS } = require('../../src/services/model/calibrationRegistry');
|
||||||
|
expect(LODO_MIN_HELD_ROWS).toBe(70);
|
||||||
|
expect(LODO_THRESHOLD_BASIS.derived_blind).toBe(true);
|
||||||
|
// The reversing dates that keep rbi/runs out are all at or above it, so
|
||||||
|
// their exclusion cannot be an artefact of the threshold.
|
||||||
|
for (const n of [99, 86, 244]) expect(n).toBeGreaterThanOrEqual(LODO_MIN_HELD_ROWS);
|
||||||
|
});
|
||||||
|
|
||||||
it('is frozen, so a stat cannot be added at runtime without a code change', () => {
|
it('is frozen, so a stat cannot be added at runtime without a code change', () => {
|
||||||
expect(Object.isFrozen(snapshotService.CALIBRATION_DEPLOYED)).toBe(true);
|
expect(Object.isFrozen(snapshotService.CALIBRATION_DEPLOYED)).toBe(true);
|
||||||
expect(() => { snapshotService.CALIBRATION_DEPLOYED.push('hits'); }).toThrow();
|
expect(() => { snapshotService.CALIBRATION_DEPLOYED.push('hits'); }).toThrow();
|
||||||
|
|||||||
@@ -118,3 +118,35 @@ describe('serving is band-limited', () => {
|
|||||||
expect(r.serves('total_bases', null).serve).toBe(false);
|
expect(r.serves('total_bases', null).serve).toBe(false);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('the LODO held-row threshold is power-derived, not operator-chosen', () => {
|
||||||
|
const { LODO_MIN_HELD_ROWS, LODO_THRESHOLD_BASIS } = require('../../src/services/model/calibrationRegistry');
|
||||||
|
|
||||||
|
it('is the value its own stated derivation produces', () => {
|
||||||
|
// n* = (SD(d) / |effect|)^2 -- recomputed here so the constant cannot drift
|
||||||
|
// away from the basis that justifies it.
|
||||||
|
const { per_row_brier_diff_sd: sd, pooled_effect_abs_mean: eff } = LODO_THRESHOLD_BASIS;
|
||||||
|
expect(Math.ceil((sd / eff) ** 2)).toBe(LODO_MIN_HELD_ROWS);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('carries its derivation basis, and was derived blind', () => {
|
||||||
|
expect(LODO_THRESHOLD_BASIS.rule).toBe('n* = (SD(d) / |effect|)^2');
|
||||||
|
expect(LODO_THRESHOLD_BASIS.derived_blind).toBe(true);
|
||||||
|
expect(LODO_THRESHOLD_BASIS.pooled_rows).toBeGreaterThan(1000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects the thresholds that were previously chosen by hand', () => {
|
||||||
|
// 20 was the operator-chosen value whose verdict moved with it; anything
|
||||||
|
// below n* cannot distinguish a reversal from a coin flip.
|
||||||
|
for (const weak of [10, 20, 25, 30, 50]) {
|
||||||
|
const se = LODO_THRESHOLD_BASIS.per_row_brier_diff_sd / Math.sqrt(weak);
|
||||||
|
expect(LODO_THRESHOLD_BASIS.pooled_effect_abs_mean).toBeLessThan(se);
|
||||||
|
expect(weak).toBeLessThan(LODO_MIN_HELD_ROWS);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('is informative at the committed value', () => {
|
||||||
|
const se = LODO_THRESHOLD_BASIS.per_row_brier_diff_sd / Math.sqrt(LODO_MIN_HELD_ROWS);
|
||||||
|
expect(LODO_THRESHOLD_BASIS.pooled_effect_abs_mean).toBeGreaterThanOrEqual(se * 0.99);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user