diff --git a/scripts/test-favourite-bias-robust.js b/scripts/test-favourite-bias-robust.js new file mode 100644 index 0000000..8d6b8c5 --- /dev/null +++ b/scripts/test-favourite-bias-robust.js @@ -0,0 +1,189 @@ +#!/usr/bin/env node +'use strict'; + +/** + * PHASE 1 — is the favourite-longshot bias real WITHOUT a calibration map? + * + * Four orders have refined a stability gate on 19 dates. LODO turned out to be + * structurally underpowered (0.014–0.093) and the deploy intervals rest on 2–4 + * date clusters. So we stop certifying the stability of a specific MAP, and ask + * the one question this sample might actually answer: + * + * does the model over-predict its own favourites, robustly? + * + * That claim is MODEL-FREE and MAP-FREE — it is a property of (p_win, outcome) + * pairs, needs no isotonic fit, and can therefore be tested without any of the + * machinery whose stability we cannot certify. + * + * ── DATE-BLOCK BOOTSTRAP ───────────────────────────────────────────────── + * Resampling ROWS would treat 200 props from one night as 200 readings of that + * night's offensive environment. Whole DATES are resampled instead, which is the + * honest unit and a far harsher one at 5–17 dates. + * + * VERDICT is pre-stated: ROBUST iff the >0.9 over-prediction sign survives in + * >=95% of pooled date-block resamples AND replicates in >=3 of 4 stats on the + * same criterion. Anything else is NOT-ROBUST, and NOT-ROBUST means we serve raw. + * + * SUPABASE_URL=... node scripts/test-favourite-bias-robust.js + */ + +require('dotenv').config(); +const fs = require('fs'); +const path = require('path'); +const { createClient } = require('@supabase/supabase-js'); +const guards = require('../src/services/model/calibrationGuards'); +const { knownNumber } = require('../src/utils/known'); + +const BOX = path.join(process.cwd(), '.seq-cache', 'batting-lines.json'); +const STATS = ['hits', 'total_bases', 'rbi', 'runs']; +const PAGE = 1000; +const FAVOURITE_FLOOR = 0.9; +const ITERS = 5000; +/** Pre-stated pass marks. */ +const SIGN_STABILITY_REQUIRED = 0.95; +const STATS_MUST_REPLICATE = 3; + +const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs }; +const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null); + +async function page(sb, t, s, f) { + const o = []; + for (let i = 0; ; i += PAGE) { + const { data, error } = await f(sb.from(t).select(s)).order('id', { ascending: true }).range(i, i + PAGE - 1); + if (error) throw error; + if (!data.length) break; + o.push(...data); + if (data.length < PAGE) break; + } + return o; +} +const isPreGame = (c, g) => { + const et = new Date(new Date(c).getTime() - 4 * 3600 * 1000); + const d = et.toISOString().slice(0, 10); + return d < g || (d === g && et.getUTCHours() < 19); +}; +function makeRnd(seed) { + let s = seed >>> 0; + return () => { s ^= s << 13; s >>>= 0; s ^= s >>> 17; s ^= s << 5; s >>>= 0; return s / 4294967296; }; +} + +/** Over-prediction in the favourite bin: predicted − realized. Positive = over. */ +function favouriteBias(rows) { + const fav = rows.filter((r) => r.p >= FAVOURITE_FLOOR); + if (fav.length < 5) return null; + return mean(fav.map((r) => r.p)) - mean(fav.map((r) => r.won)); +} + +/** Resample whole DATES with replacement; report how often the sign survives. */ +function dateBlockSignStability(rows, seed) { + const byDate = new Map(); + for (const r of rows) { + if (!byDate.has(r.date)) byDate.set(r.date, []); + byDate.get(r.date).push(r); + } + const keys = [...byDate.keys()]; + const rnd = makeRnd(seed); + let positive = 0; let indeterminate = 0; const draws = []; + for (let it = 0; it < ITERS; it += 1) { + const sample = []; + for (let i = 0; i < keys.length; i += 1) sample.push(...byDate.get(keys[Math.floor(rnd() * keys.length)])); + const b = favouriteBias(sample); + // A resample with too few favourites cannot speak — counted, never guessed. + if (b === null) { indeterminate += 1; continue; } + draws.push(b); + if (b > 0) positive += 1; + } + const usable = ITERS - indeterminate; + draws.sort((a, b) => a - b); + return { + date_blocks: keys.length, + usable_resamples: usable, + indeterminate_resamples: indeterminate, + sign_stability: usable ? round4(positive / usable) : null, + ci_90: draws.length ? [round4(draws[Math.floor(draws.length * 0.05)]), round4(draws[Math.floor(draws.length * 0.95)])] : null, + }; +} + +function deciles(rows) { + const out = []; + for (let lo = 0.3; lo < 1.0; lo += 0.1) { + const hi = lo + 0.1; + const slice = rows.filter((r) => r.p >= lo && (hi >= 1 ? r.p <= 1 : r.p < hi)); + if (slice.length < 15) continue; + const pred = mean(slice.map((r) => r.p)); + const real = mean(slice.map((r) => r.won)); + out.push({ bin: [round2(lo), round2(hi)], n: slice.length, predicted: round4(pred), realized: round4(real), over_prediction: round4(pred - real) }); + } + return out; +} + +(async () => { + const sb = createClient(process.env.SUPABASE_URL, + process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY, { auth: { persistSession: false } }); + const lines = JSON.parse(fs.readFileSync(BOX, 'utf8')).lines; + + const snaps = await page(sb, 'model_snapshots', + 'id, game_date, captured_at, stat, player_key, line, side, p_win, refused', + (q) => q.eq('sport', 'mlb').in('stat', STATS)); + + const picked = new Map(); + for (const r of snaps) { + if (!isPreGame(r.captured_at, r.game_date) || r.refused || knownNumber(r.p_win) === null) continue; + const k = [r.game_date, r.stat, r.player_key, r.line].join('|'); + const prev = picked.get(k); + if (!prev || knownNumber(r.p_win) > knownNumber(prev.p_win)) picked.set(k, r); + } + guards.assertPickedSideDedup([...picked.values()].map((r) => ({ + propKey: [r.game_date, r.stat, r.player_key, r.line].join('|'), side: r.side, p: knownNumber(r.p_win), + }))); + + const byStat = {}; const pooled = []; + for (const stat of STATS) byStat[stat] = []; + for (const r of picked.values()) { + const b = lines[`${r.game_date}|${r.player_key}`]; + const L = knownNumber(r.line); + if (!b || L === null || !r.side) continue; + const v = knownNumber(FIELD[r.stat](b)); + if (v === null) continue; + const over = v > L; + const row = { date: r.game_date, p: knownNumber(r.p_win), won: (String(r.side).toLowerCase() === 'under' ? !over : over) ? 1 : 0 }; + byStat[r.stat].push(row); pooled.push(row); + } + + const pooledResult = { + n: pooled.length, + deciles: deciles(pooled), + favourite_bias: round4(favouriteBias(pooled)), + ...dateBlockSignStability(pooled, 20260808), + }; + + const perStat = {}; + let replicated = 0; + for (const stat of STATS) { + const rows = byStat[stat]; + const fb = favouriteBias(rows); + const stab = dateBlockSignStability(rows, 20260808); + const ok = fb !== null && fb > 0 && stab.sign_stability !== null && stab.sign_stability >= SIGN_STABILITY_REQUIRED; + if (ok) replicated += 1; + perStat[stat] = { n: rows.length, deciles: deciles(rows), favourite_bias: fb === null ? null : round4(fb), ...stab, replicates: ok }; + } + + const pooledOk = pooledResult.favourite_bias > 0 && pooledResult.sign_stability >= SIGN_STABILITY_REQUIRED; + const verdict = pooledOk && replicated >= STATS_MUST_REPLICATE ? 'ROBUST' : 'NOT-ROBUST'; + + console.log(JSON.stringify({ + phase: 'PHASE 1 — model-free, map-free favourite-longshot bias test', + criteria: { sign_stability_required: SIGN_STABILITY_REQUIRED, stats_must_replicate: STATS_MUST_REPLICATE, favourite_floor: FAVOURITE_FLOOR }, + pooled: pooledResult, + per_stat: perStat, + stats_replicating: replicated, + VERDICT: verdict, + consequence: verdict === 'ROBUST' + ? 'proceed to a low-parameter correction, validated as a NEW estimator' + : 'serve raw; the bias is not certifiable on this sample', + }, null, 2)); + process.exit(0); +})().catch((e) => { console.error(e); process.exit(1); }); + +const round4 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 10000) / 10000); +const round2 = (v) => Math.round(v * 100) / 100; diff --git a/scripts/validate-lowparam.js b/scripts/validate-lowparam.js new file mode 100644 index 0000000..9eee2df --- /dev/null +++ b/scripts/validate-lowparam.js @@ -0,0 +1,131 @@ +#!/usr/bin/env node +'use strict'; + +/** + * PHASE 3 — validate the low-parameter correction as a NEW estimator. + * + * No grandfathering: it must beat RAW out-of-sample with a DATE-BLOCK bootstrap + * interval excluding zero. It is also scored against the retired isotonic map on + * the identical held-out rows, so the swap is a measured comparison rather than + * a preference. + */ + +require('dotenv').config(); +const fs = require('fs'); +const path = require('path'); +const { createClient } = require('@supabase/supabase-js'); +const cal = require('../src/services/model/calibration'); +const lp = require('../src/services/model/lowParamCalibrator'); +const guards = require('../src/services/model/calibrationGuards'); +const { knownNumber } = require('../src/utils/known'); + +const BOX = path.join(process.cwd(), '.seq-cache', 'batting-lines.json'); +const STATS = ['hits', 'total_bases', 'rbi', 'runs']; +const PAGE = 1000; const ITERS = 4000; +const FIELD = { hits: (b) => b.hits, total_bases: (b) => b.totalBases, rbi: (b) => b.rbi, runs: (b) => b.runs }; +const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null); + +async function page(sb, t, s, f) { + const o = []; + for (let i = 0; ; i += PAGE) { + const { data, error } = await f(sb.from(t).select(s)).order('id', { ascending: true }).range(i, i + PAGE - 1); + if (error) throw error; if (!data.length) break; o.push(...data); if (data.length < PAGE) break; + } + return o; +} +const isPreGame = (c, g) => { + const et = new Date(new Date(c).getTime() - 4 * 3600 * 1000); + const d = et.toISOString().slice(0, 10); + return d < g || (d === g && et.getUTCHours() < 19); +}; +function makeRnd(seed) { let s = seed >>> 0; return () => { s ^= s << 13; s >>>= 0; s ^= s >>> 17; s ^= s << 5; s >>>= 0; return s / 4294967296; }; } + +/** Paired date-block bootstrap on a Brier difference. */ +function dateBlockCI(rows, keyA, keyB, seed) { + const byDate = new Map(); + for (const r of rows) { if (!byDate.has(r.date)) byDate.set(r.date, []); byDate.get(r.date).push(r); } + const keys = [...byDate.keys()]; const rnd = makeRnd(seed); const diffs = []; + for (let it = 0; it < ITERS; it += 1) { + const s = []; + for (let i = 0; i < keys.length; i += 1) s.push(...byDate.get(keys[Math.floor(rnd() * keys.length)])); + const a = guards.safeBrier(s.map((r) => r[keyA]), s.map((r) => r.won)); + const b = guards.safeBrier(s.map((r) => r[keyB]), s.map((r) => r.won)); + if (a === null || b === null) continue; + diffs.push(a - b); + } + diffs.sort((x, y) => x - y); + return diffs.length + ? { ci: [round4(diffs[Math.floor(diffs.length * 0.025)]), round4(diffs[Math.floor(diffs.length * 0.975)])], date_blocks: keys.length } + : { ci: null, date_blocks: keys.length }; +} + +(async () => { + const sb = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY, { auth: { persistSession: false } }); + const lines = JSON.parse(fs.readFileSync(BOX, 'utf8')).lines; + const snaps = await page(sb, 'model_snapshots', 'id, game_date, captured_at, stat, player_key, line, side, p_win, refused', + (q) => q.eq('sport', 'mlb').in('stat', STATS)); + + const picked = new Map(); + for (const r of snaps) { + if (!isPreGame(r.captured_at, r.game_date) || r.refused || knownNumber(r.p_win) === null) continue; + const k = [r.game_date, r.stat, r.player_key, r.line].join('|'); + const prev = picked.get(k); + if (!prev || knownNumber(r.p_win) > knownNumber(prev.p_win)) picked.set(k, r); + } + guards.assertPickedSideDedup([...picked.values()].map((r) => ({ + propKey: [r.game_date, r.stat, r.player_key, r.line].join('|'), side: r.side, p: knownNumber(r.p_win) }))); + + const out = {}; + for (const stat of STATS) { + const rows = []; + for (const r of picked.values()) { + if (r.stat !== stat) continue; + const b = lines[`${r.game_date}|${r.player_key}`]; const L = knownNumber(r.line); + if (!b || L === null || !r.side) continue; + const v = knownNumber(FIELD[stat](b)); if (v === null) continue; + const over = v > L; + rows.push({ date: r.game_date, p: knownNumber(r.p_win), won: (String(r.side).toLowerCase() === 'under' ? !over : over) ? 1 : 0 }); + } + rows.sort((a, b) => String(a.date).localeCompare(String(b.date))); + const dates = [...new Set(rows.map((r) => r.date))].sort(); + const perDate = new Map(); for (const r of rows) perDate.set(r.date, (perDate.get(r.date) || 0) + 1); + let acc = 0; let cut = dates[dates.length - 1]; + for (const d of dates) { acc += perDate.get(d); if (acc >= rows.length * 0.45) { cut = d; break; } } + + const fit = rows.filter((r) => r.date < cut); + const ev = rows.filter((r) => r.date >= cut); + const platt = lp.fitPlatt(fit); + const iso = cal.fitIsotonic(fit.map((r) => ({ p: r.p, won: r.won }))); + + if (!platt || ev.length < 50) { + out[stat] = { n: rows.length, fit_n: fit.length, eval_n: ev.length, decision: 'REFUSE', reason: 'no low-parameter fit or too little held out' }; + continue; + } + const scored = ev.map((r) => ({ ...r, plat: lp.applyPlatt(platt, r.p), isoP: iso ? cal.applyIsotonic(iso, r.p) : null })) + .filter((r) => knownNumber(r.plat) !== null); + const ys = scored.map((r) => r.won); + const bRaw = guards.safeBrier(scored.map((r) => r.p), ys); + const bPlat = guards.safeBrier(scored.map((r) => r.plat), ys); + const isoRows = scored.filter((r) => knownNumber(r.isoP) !== null); + const bIso = isoRows.length ? guards.safeBrier(isoRows.map((r) => r.isoP), isoRows.map((r) => r.won)) : null; + + const vsRaw = dateBlockCI(scored, 'plat', 'p', 20260808); + const vsIso = isoRows.length ? dateBlockCI(isoRows, 'plat', 'isoP', 20260808) : { ci: null }; + + const beatsRaw = bPlat < bRaw && vsRaw.ci && vsRaw.ci[1] < 0; + out[stat] = { + n: rows.length, dates: dates.length, split_at: cut, fit_n: fit.length, eval_n: scored.length, + platt: { a: platt.a, b: platt.b, flattens: platt.flattens, fit_dates: platt.fit_dates, shrinkage: platt.shrinkage }, + brier_raw: round4(bRaw), brier_lowparam: round4(bPlat), brier_isotonic: bIso === null ? null : round4(bIso), + delta_vs_raw: round4(bPlat - bRaw), ci_vs_raw: vsRaw.ci, eval_date_blocks: vsRaw.date_blocks, + delta_vs_isotonic: bIso === null ? null : round4(bPlat - bIso), ci_vs_isotonic: vsIso.ci, + decision: beatsRaw ? 'DEPLOY-PROVISIONAL' : 'REFUSE', + reason: beatsRaw ? 'beats raw out-of-sample with a date-block interval excluding zero' + : 'does not beat raw at a date-block interval excluding zero', + }; + } + console.log(JSON.stringify(out, null, 2)); + process.exit(0); +})().catch((e) => { console.error(e); process.exit(1); }); + +const round4 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 10000) / 10000); diff --git a/specs/robust-bias-lowparam.md b/specs/robust-bias-lowparam.md new file mode 100644 index 0000000..5c1e0a3 --- /dev/null +++ b/specs/robust-bias-lowparam.md @@ -0,0 +1,137 @@ +# The bias is robust; the map was not. Low-parameter correction deployed. + +## PHASE 0 — the sample-limit truth, on record + +**On 19 dates, BOTH stability instruments are underpowered. This is the SAMPLE, +not a fixable instrument.** No future order should re-open the gate-refinement +loop expecting a different answer at this N. + +- **LODO power 0.014–0.093** against a strong date-driven instability. Across + every k from 1.0 to 2.0, the best any stat reaches is 0.337. +- **Deploy CIs rest on 2–4 date clusters.** A cluster-robust interval at 2 + clusters has ~1 degree of freedom and a near-undefined width. + +Neither certifies forward stability of a specific map. Four orders refined a gate +the sample cannot support; that loop stops here. + +**Record correction on runs:** its `1f40014` DATE-DRIVEN classification was an +artefact of the coin-flip ruler — 2 reversals in 3 drops never cleared a cutoff +of 2. runs is an ordinary "no fittable map" refusal. **Not date-driven.** + +--- + +## PHASE 1 — the robust claim: ROBUST + +Model-free, map-free, on the picked-side deduped population. Date-block bootstrap +(whole dates resampled, 5,000 draws). + +### Pooled — the shape is textbook favourite-longshot + +| bin | n | predicted | realized | over-prediction | +|---|---|---|---|---| +| 0.5–0.6 | 1,021 | 0.5477 | 0.5553 | −0.0076 | +| 0.6–0.7 | 961 | 0.6432 | 0.6004 | +0.0428 | +| 0.7–0.8 | 745 | 0.7420 | 0.6456 | +0.0963 | +| 0.8–0.9 | 459 | 0.8430 | 0.6841 | +0.1589 | +| **0.9–1.0** | 157 | 0.9075 | **0.6624** | **+0.2451** | + +Pooled sign stability **0.9946** over 17 date blocks, 90% CI [+0.136, +0.300], +zero indeterminate resamples. + +### Per stat — 4 of 4 replicate + +| stat | n | >0.9 bias | date blocks | sign stability | 90% CI | replicates | +|---|---|---|---|---|---|---| +| hits | 1,140 | +0.2435 | 17 | 0.994 | [0.120, 0.321] | yes | +| total_bases | 1,050 | +0.2816 | 7 | 1.000 | [0.154, 0.394] | yes | +| rbi | 630 | +0.2107 | 5 | 1.000 | [0.156, 0.245] | yes | +| runs | 597 | +0.2367 | 5 | 0.998 | [0.082, 0.314] | yes | + +**VERDICT: ROBUST** — pooled ≥95% and 4/4 stats (bar was 3/4). + +Worth noting alongside it: **realized rate plateaus at ~0.65–0.68 from p=0.7 +upward.** The 0.9+ bucket (0.6624) performs no better than the 0.8–0.9 bucket +(0.6841). The model has no genuinely high-confidence reads, only high-confidence +*numbers*. + +--- + +## PHASE 3 — low-parameter correction, validated as a new estimator + +Platt: `p_cal = sigmoid(a·logit(p) + b)`. Two parameters over the whole curve, so +it **cannot** encode "this Tuesday was odd" — which is precisely the failure mode +we cannot rule out for isotonic on this sample. + +Shrunk toward identity by fit-date count: `w = D/(D+10)`, applied as +`w·p_platt + (1−w)·p_raw`. A thin fit is therefore applied at reduced strength. + +| stat | a | shrink | eval n | blocks | Brier raw | low-param | Δ vs raw | CI (date-block) | decision | +|---|---|---|---|---|---|---|---|---|---| +| **hits** | 0.406 | 0.565 | 765 | 4 | 0.2626 | 0.2540 | **−0.0086** | [−0.0112, −0.0069] | **DEPLOY** | +| **total_bases** | 0.472 | 0.333 | 625 | 2 | 0.2490 | 0.2429 | **−0.0061** | [−0.0062, −0.0059] | **DEPLOY** | +| rbi | 0.775 | 0.231 | 425 | 2 | 0.2011 | 0.2007 | −0.0004 | [−0.0007, **0**] | REFUSE | +| runs | −0.032 | — | — | — | — | — | — | — | **REFUSE (slope)** | + +### A guard the first run needed + +runs fitted **a = −0.032**. A non-positive slope does not flatten an +over-confident forecaster — it **inverts** it, and near zero the curve collapses +to a constant, predicting the base rate for everything. That *lowers* Brier +(shrinking a miscalibrated forecaster toward its base rate always does) while +destroying all resolution, so it would have **scored as a win while making the +product worthless**. `MIN_SLOPE` now refuses it by name, with a test. + +### Stated plainly: isotonic scored better, and we are not using it + +On the identical held-out rows, isotonic beat the low-parameter fit on hits +(+0.0028, CI [0.0013, 0.0045]) and rbi (+0.0042, CI [0.0003, 0.0092]), and tied +on total_bases (−0.0009, CI spanning zero). + +**The swap is a capacity judgement, not a measurement.** The evaluation window +spans 2–4 date blocks, so "isotonic wins OOS" there is weak evidence, and it is +exactly what a flexible map would produce if it captured structure shared by the +fit and evaluation periods. That reasoning is a judgement and is labelled as one. + +--- + +## PHASE 4 — deploy and labelling + +| stat | served | basis | +|---|---|---| +| hits | low-parameter correction | `direction_robust_magnitude_provisional` | +| total_bases | low-parameter correction | `direction_robust_magnitude_provisional` | +| **rbi** | **RAW — withdrawn** | deployed on isotonic at `ced4042`; low-param does not beat raw | +| runs | RAW | slope refused | + +The **direction** is bootstrap-robust; the **magnitude** is thin-sample and +conservatively shrunk (0.565 hits, 0.333 TB). Customer-facing letter unchanged. + +Auto-demotion remains armed via `calibrationRegistry.reverify`: a sign flip in +the >0.9 bucket or a CI crossing zero demotes to raw and logs the breaking date. +Promotion to non-provisional stays at the original ≥40 date-cluster bar. + +--- + +## PHASE 5 — the standing finding, stated hard + +**Across 18 archetype slots on three stats, calibrated `p_win` separates within +archetype NO BETTER than raw. Every slot collapses to one band, indistinguishable +from its own base rate. Zero slots show lift.** + +Per-archetype grade separation is **not coming from calibration**. It comes from +**proven factors or it does not exist.** + +This reframes the roadmap. Calibration has now been pursued through five orders +and has delivered exactly what it can deliver — honest numbers on two stats — and +nothing at all on the question the grade product actually turns on. The next real +lever is factors on the stats that lack them. + +--- + +## Invariants + +`p_win` never mutated — the correction rides as `p_win_calibrated`. Calibration +consumed no Bonferroni slot. The robust-claim test ran before any calibrator was +built and could have terminated the session at Phase 2. Counter and frozen +clusters verified file-by-file (14 modules, including `calibration.js` and +`calibrationService.js`, both untouched and simply no longer on the serving path). diff --git a/src/services/model/lowParamCalibrator.js b/src/services/model/lowParamCalibrator.js new file mode 100644 index 0000000..5d28cbf --- /dev/null +++ b/src/services/model/lowParamCalibrator.js @@ -0,0 +1,127 @@ +'use strict'; + +/** + * lowParamCalibrator — a two-parameter favourite-longshot correction. + * + * Isotonic has one free parameter per distinct prediction level, which on 19 + * dates is far more freedom than the sample can discipline — it can and does + * chase a single night's structure. Platt scaling has exactly TWO parameters + * over the whole curve: + * + * p_cal = sigmoid(a * logit(p) + b) + * + * `a < 1` flattens an over-confident forecaster toward the base rate, which is + * precisely the favourite-longshot shape measured here (over-prediction rising + * monotonically from -0.008 at p~0.55 to +0.245 above 0.9). Two parameters + * cannot represent "this Tuesday was odd", which is the entire point. + * + * ── SHRINKAGE TOWARD IDENTITY ──────────────────────────────────────────── + * Even two parameters are fitted on few dates, so the correction is blended + * back toward the raw forecast by a weight tied to how many dates were seen: + * + * w = D / (D + D0) + * p_final = w * p_platt + (1 - w) * p_raw + * + * At 5 fit dates w = 0.33 — the correction is applied at a third of its fitted + * strength. At 40 dates it is 0.80. A thin-sample fit therefore cannot + * over-correct, and the blend is monotone because both inputs are. + * + * The DIRECTION of this correction is bootstrap-robust; its MAGNITUDE is + * thin-sample. Shrinkage is how that distinction is expressed in the number + * rather than only in a label. + */ + +const { knownNumber } = require('../../utils/known'); + +/** Dates at which the fit earns half its weight. */ +const SHRINK_HALF_DATES = 10; +/** Probabilities are clamped off 0/1 before the logit. */ +const EPS = 1e-6; +/** Below this many rows there is nothing to fit. */ +const MIN_FIT_ROWS = 100; +/** + * A slope at or below this is not a correction. Ordering must be preserved and + * the curve must not collapse to a constant. + */ +const MIN_SLOPE = 0.05; + +const clamp01 = (p) => Math.min(1 - EPS, Math.max(EPS, p)); +const logit = (p) => Math.log(clamp01(p) / (1 - clamp01(p))); +const sigmoid = (z) => 1 / (1 + Math.exp(-z)); + +/** + * Fit `a` and `b` by Newton–Raphson on the log-likelihood. Two parameters, so + * this converges in a handful of steps and has no tuning of its own. + */ +function fitPlatt(rows, opts = {}) { + const pts = (rows || []) + .map((r) => ({ x: logit(knownNumber(r.p)), y: knownNumber(r.won) })) + .filter((r) => Number.isFinite(r.x) && (r.y === 0 || r.y === 1)); + if (pts.length < (opts.minRows ?? MIN_FIT_ROWS)) return null; + + let a = 1; let b = 0; + for (let it = 0; it < 100; it += 1) { + let g0 = 0; let g1 = 0; let h00 = 0; let h01 = 0; let h11 = 0; + for (const { x, y } of pts) { + const p = sigmoid(a * x + b); + const e = p - y; + const w = p * (1 - p); + g0 += e * x; g1 += e; + h00 += w * x * x; h01 += w * x; h11 += w; + } + const det = h00 * h11 - h01 * h01; + if (!Number.isFinite(det) || Math.abs(det) < 1e-12) break; + const da = (g0 * h11 - g1 * h01) / det; + const db = (g1 * h00 - g0 * h01) / det; + a -= da; b -= db; + if (Math.abs(da) < 1e-10 && Math.abs(db) < 1e-10) break; + } + if (!Number.isFinite(a) || !Number.isFinite(b)) return null; + + // ── THE SLOPE MUST CORRECT, NOT ABANDON ──────────────────────────────── + // `a` in (0, 1] is a flattening: ordering preserved, over-confidence pulled + // in. `a <= 0` INVERTS the forecast, and `a` near zero collapses the curve to + // a constant — the fit has decided p_win carries nothing and is predicting the + // base rate for everything. That lowers Brier (shrinking a miscalibrated + // forecaster toward its base rate always does) while destroying resolution, + // so it would score as a win while making the product worthless. + // + // Measured: runs fitted a = -0.032. Refused here rather than deployed. + if (a <= (opts.minSlope ?? MIN_SLOPE)) { + return { refused: true, a: round5(a), b: round5(b), reason: a <= 0 + ? 'fitted slope is not positive — the correction would invert the forecast' + : 'fitted slope is near zero — the fit collapses to a constant and abandons the forecast' }; + } + + const dates = new Set((rows || []).map((r) => r.date).filter(Boolean)).size; + const half = opts.shrinkHalfDates ?? SHRINK_HALF_DATES; + const shrink = dates > 0 ? dates / (dates + half) : 0; + + return { + a: round5(a), + b: round5(b), + fit_rows: pts.length, + fit_dates: dates, + shrinkage: round4(shrink), + /** Flattening a forecaster means a < 1; reported so the shape is checkable. */ + flattens: a < 1, + }; +} + +/** + * Apply the fitted correction, shrunk toward the raw forecast. + * Returns null when unreadable — never a silently uncorrected number. + */ +function applyPlatt(model, p) { + const x = knownNumber(p); + if (!model || model.refused || x === null) return null; + const raw = clamp01(x); + const corrected = sigmoid(model.a * logit(raw) + model.b); + const w = model.shrinkage; + return round5(w * corrected + (1 - w) * raw); +} + +const round5 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 100000) / 100000); +const round4 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 10000) / 10000); + +module.exports = { fitPlatt, applyPlatt, SHRINK_HALF_DATES, MIN_FIT_ROWS, MIN_SLOPE, logit, sigmoid }; diff --git a/src/services/model/lowParamService.js b/src/services/model/lowParamService.js new file mode 100644 index 0000000..7c55d88 --- /dev/null +++ b/src/services/model/lowParamService.js @@ -0,0 +1,102 @@ +'use strict'; + +/** + * lowParamService — the production side of the two-parameter correction. + * + * Mirrors calibrationService's interface so the serving path swaps cleanly, but + * fits a Platt curve instead of an isotonic map. The reason for the swap is + * capacity, not score: on 19 dates we cannot certify the stability of a map with + * one free parameter per prediction level, and LODO turned out to have 1.4-9.3% + * power to tell us otherwise. Two parameters cannot encode "this Tuesday was + * odd", which is exactly the failure we cannot rule out for isotonic. + * + * Stated plainly because it is a judgement rather than a measurement: on the + * held-out window isotonic scored BETTER than this on hits (+0.0028) and rbi + * (+0.0042) and tied on total_bases. That window spans 2-4 date blocks, so it is + * weak evidence either way, and it is consistent with a flexible map having + * captured structure shared by fit and evaluation periods. + * + * Same point-in-time cut as before: fitted ONLY on games that are already over. + */ + +const lp = require('./lowParamCalibrator'); +const cal = require('./calibration'); +const { knownNumber } = require('../../utils/known'); + +const MIN_FIT = 200; +const HOLDOUT_FRACTION = 0.35; + +/** Build from settled rows: fit on the older part, certify bands on the newer. */ +function build(rows, opts = {}) { + const clean = (rows || []) + .map((r) => ({ p: knownNumber(r.p), won: knownNumber(r.won), date: String(r.date || '') })) + .filter((r) => r.p !== null && (r.won === 0 || r.won === 1)) + .sort((a, b) => a.date.localeCompare(b.date)); + if (clean.length < (opts.minFit ?? MIN_FIT)) return null; + + const cut = Math.floor(clean.length * (1 - (opts.holdout ?? HOLDOUT_FRACTION))); + const fitRows = clean.slice(0, cut); + const certRows = clean.slice(cut); + if (fitRows.length < (opts.minFit ?? MIN_FIT) || certRows.length < 50) return null; + + const model = lp.fitPlatt(fitRows, opts); + // A refused fit (inverting or collapsed slope) yields no calibrator at all. + if (!model || model.refused) return null; + + const corrected = certRows + .map((r) => ({ ...r, p: lp.applyPlatt(model, r.p) })) + .filter((r) => knownNumber(r.p) !== null); + const bands = cal.certifyBands(corrected, { + tolerance: opts.tolerance ?? 0.05, + minBin: opts.minBin ?? 40, + }); + + return { + model, + bands, + fit_n: fitRows.length, + certify_n: certRows.length, + fitted_through: fitRows[fitRows.length - 1].date, + shrinkage: model.shrinkage, + calibrate(p) { + const raw = knownNumber(p); + if (raw === null) return { p_raw: null, p_calibrated: null, calibrated: false, reason: 'absent' }; + const c = lp.applyPlatt(model, raw); + if (c === null) return { p_raw: raw, p_calibrated: null, calibrated: false, reason: 'no_model_value' }; + const inBand = cal.inCertifiedBand(bands, c); + return { + p_raw: raw, + p_calibrated: Math.round(c * 1000) / 1000, + calibrated: inBand, + reason: inBand ? null : 'outside_certified_band', + }; + }, + }; +} + +/** Load settled history and build, POINT-IN-TIME (strictly before today). */ +async function fromLedger(sb, { sport = 'mlb', stat = 'hits', before = null, ...opts } = {}) { + if (!sb) return null; + const cutoff = before || new Intl.DateTimeFormat('en-CA', { + timeZone: 'America/New_York', year: 'numeric', month: '2-digit', day: '2-digit', + }).format(new Date()); + const rows = []; + for (let from = 0; ; from += 1000) { + const { data, error } = await sb.from('ledger_entries') + .select('p_win, outcome, game_date, quarantine_reason') + .eq('sport', sport).is('user_id', null).eq('stat', stat) + .in('outcome', ['hit', 'miss']).not('p_win', 'is', null) + .lt('game_date', cutoff) + .range(from, from + 999); + if (error || !data || data.length === 0) break; + rows.push(...data); + if (data.length < 1000) break; + } + const clean = rows + .filter((r) => !(r.quarantine_reason || '').startsWith('nontakeable_book')) + .map((r) => ({ p: Number(r.p_win), won: r.outcome === 'hit' ? 1 : 0, date: String(r.game_date) })); + const built = build(clean, opts); + return built ? { ...built, cutoff } : null; +} + +module.exports = { build, fromLedger, MIN_FIT, HOLDOUT_FRACTION }; diff --git a/src/services/snapshotService.js b/src/services/snapshotService.js index da6ce49..67ea865 100644 --- a/src/services/snapshotService.js +++ b/src/services/snapshotService.js @@ -276,36 +276,37 @@ async function loadPitcherArsenals(sport) { } /** - * Stats that may serve a calibrated number, and ON WHAT BASIS. + * Stats that may serve a calibrated number, and on what basis. * - * ── LODO CANNOT EVALUATE ANY OF THEM ───────────────────────────────────── - * The LODO gate was audited and rebuilt as a coherent pair (per-stat - * informativeness bar + binomial reversal cutoff at FP <= 0.05). At this date - * count its POWER against a strong date-driven instability is 0.093 / 0.093 / - * 0.045 / 0.014 — it would miss a real failure more than nine times in ten. A - * gate that cannot fail cannot pass, so every stat is UNTESTABLE-BY-LODO and - * none of these deploys claim LODO stability. + * ── THE BIAS IS ROBUST; THE MAP WAS NOT CERTIFIABLE ────────────────────── + * Tested model-free and map-free on the picked-side population: the model + * over-predicts its own favourites, and the sign survives 99.5% of date-block + * resamples pooled and replicates in 4 of 4 stats. Over-prediction rises + * monotonically from -0.008 near p=0.55 to +0.245 above 0.9. * - * The previous zero-reversal rule was incoherent: at a 1-SE bar a stable stat - * reverses on ~16% of drops, so demanding zero failed stable stats ~50% of the - * time. Re-read under the binomial cutoff, NEITHER rbi (1 reversal) NOR runs - * (2) exceeds its cutoff of 2 — both prior FAILs were false. + * What could NOT be certified on 19 dates is the stability of a specific + * isotonic MAP — LODO has 1.4-9.3% power there. So isotonic is retired and the + * correction is a TWO-PARAMETER Platt curve, which has no capacity to encode a + * single odd day, shrunk toward the raw forecast by fit-date count. * - * ── SO THE DEPLOY BASIS IS THE DATE-CLUSTERED CI ALONE ─────────────────── - * hits, total_bases and rbi each have a point-in-time held-out interval - * excluding zero. That is the ONLY support they have, and it is thin — the - * interval rests on 4, 2 and 2 date clusters respectively. Auto-demotion is - * therefore the sole stability guard, not a backstop to a passed test. + * Validated as a NEW estimator against RAW, date-block bootstrap: + * hits a=0.406 shrink 0.565 0.2626 -> 0.2540 CI [-0.0112,-0.0069] + * total_bases a=0.472 shrink 0.333 0.2490 -> 0.2429 CI [-0.0062,-0.0059] * - * runs is absent: no isotonic map was fittable at its point-in-time split, so it - * has no CI support to stand on either. + * rbi is WITHDRAWN (deployed last order on isotonic): the low-parameter fit does + * not beat raw, CI [-0.0007, 0] touching zero. runs is refused by the slope + * guard — it fitted a = -0.032, which would invert the forecast rather than + * flatten it. Both now serve RAW. + */ +const CALIBRATION_DEPLOYED = Object.freeze(['hits', 'total_bases']); +/** + * The DIRECTION of the correction is bootstrap-robust; its MAGNITUDE is fitted + * on few dates and deliberately shrunk toward identity. The customer-facing + * letter is unchanged; this is what the internal record says. */ -const CALIBRATION_DEPLOYED = Object.freeze(['hits', 'total_bases', 'rbi']); -/** Why each deployed stat is allowed to serve. Not one of them is LODO-stable. */ const CALIBRATION_BASIS = Object.freeze({ - hits: 'ci_only_lodo_untestable', - total_bases: 'ci_only_lodo_untestable', - rbi: 'ci_only_lodo_untestable', + hits: 'direction_robust_magnitude_provisional', + total_bases: 'direction_robust_magnitude_provisional', }); async function runSnapshot(sport, opts = {}) { @@ -757,7 +758,7 @@ async function runSnapshot(sport, opts = {}) { if (sp === 'mlb') { for (const stat of CALIBRATION_DEPLOYED) { try { - const calSvc = deps.calibrationService || require('./model/calibrationService'); + const calSvc = deps.calibrationService || require('./model/lowParamService'); const sbc = require('../utils/supabase').getSupabaseServiceClient(); const calibrator = sbc ? await calSvc.fromLedger(sbc, { sport: 'mlb', stat }) : null; if (calibrator) { @@ -772,7 +773,7 @@ async function runSnapshot(sport, opts = {}) { g.calibration_basis = CALIBRATION_BASIS[stat] || null; if (out.calibrated) marked += 1; } - console.log(`[calibration] ${sp} ${stat} (PROVISIONAL) — ${marked} stackable; fit n=${calibrator.fit_n} through ${calibrator.fitted_through}`); + console.log(`[calibration] ${sp} ${stat} (low-param, PROVISIONAL) — ${marked} stackable; fit n=${calibrator.fit_n} through ${calibrator.fitted_through}, a=${calibrator.model.a} shrink=${calibrator.shrinkage}`); } else { console.log(`[calibration] ${sp} ${stat} — no calibrator (thin history); nothing is stackable`); } diff --git a/tests/unit/calibrationDeployGate.test.js b/tests/unit/calibrationDeployGate.test.js index df9e015..4b19a25 100644 --- a/tests/unit/calibrationDeployGate.test.js +++ b/tests/unit/calibrationDeployGate.test.js @@ -3,43 +3,43 @@ /** * Which stats serve a calibrated number, and on what basis. * - * The thing these lock is that "could not test" never reads as "passed". LODO - * has 1.4%-9.3% power at this date count, so no deployed stat claims stability - * from it — each rides a thin date-clustered interval and auto-demotion alone. + * The claim that survived on this sample is the bias DIRECTION, tested model-free + * and map-free. What did not survive is any certification of a specific map's + * stability. These lock that distinction into the serving path. */ const snapshotService = require('../../src/services/snapshotService'); -const { LODO_TEST, LODO_POWER_FLOOR } = require('../../src/services/model/calibrationRegistry'); +const lp = require('../../src/services/model/lowParamCalibrator'); -describe('the deploy set rests on the CI, not on a passed LODO', () => { - it('serves the three stats with a point-in-time interval excluding zero', () => { - expect(snapshotService.CALIBRATION_DEPLOYED).toEqual(['hits', 'total_bases', 'rbi']); +describe('the deploy set rides a low-parameter correction, not isotonic', () => { + it('serves the two stats whose correction beat RAW out-of-sample', () => { + expect(snapshotService.CALIBRATION_DEPLOYED).toEqual(['hits', 'total_bases']); }); - it('does NOT serve runs — no fittable map, so no CI to stand on', () => { + it('WITHDRAWS rbi — the low-parameter fit does not beat raw', () => { + // rbi was deployed at ced4042 on isotonic. Its CI vs raw is [-0.0007, 0], + // which touches zero, so it serves raw again. + expect(snapshotService.CALIBRATION_DEPLOYED).not.toContain('rbi'); + }); + + it('does NOT serve runs — its fitted slope would invert the forecast', () => { expect(snapshotService.CALIBRATION_DEPLOYED).not.toContain('runs'); }); - it('labels every deployed stat as LODO-untestable rather than LODO-stable', () => { + it('labels the basis honestly: direction robust, magnitude provisional', () => { for (const stat of snapshotService.CALIBRATION_DEPLOYED) { - expect(snapshotService.CALIBRATION_BASIS[stat]).toBe('ci_only_lodo_untestable'); + expect(snapshotService.CALIBRATION_BASIS[stat]).toBe('direction_robust_magnitude_provisional'); } }); - it('no stat may claim LODO stability, because the test cannot fail', () => { - for (const t of Object.values(LODO_TEST)) expect(t.power).toBeLessThan(LODO_POWER_FLOOR); - }); - - it('rbi and runs were FALSE FAILS under the old zero-reversal rule', () => { - // Re-read under the binomial cutoff: 1 and 2 reversals, cutoff 2 for both. - expect(LODO_TEST.rbi.cutoff).toBe(2); - expect(LODO_TEST.runs.cutoff).toBe(2); - // rbi returns to the deploy set on CI support; runs still has none. - expect(snapshotService.CALIBRATION_DEPLOYED).toContain('rbi'); + it('the served correction cannot encode a single odd day', () => { + // Two parameters over the whole curve is the entire reason for the swap. + expect(typeof lp.fitPlatt).toBe('function'); + expect(lp.MIN_SLOPE).toBeGreaterThan(0); }); it('is frozen, so a stat cannot be added at runtime', () => { expect(Object.isFrozen(snapshotService.CALIBRATION_DEPLOYED)).toBe(true); - expect(() => { snapshotService.CALIBRATION_DEPLOYED.push('runs'); }).toThrow(); + expect(() => { snapshotService.CALIBRATION_DEPLOYED.push('rbi'); }).toThrow(); }); }); diff --git a/tests/unit/lowParamCalibrator.test.js b/tests/unit/lowParamCalibrator.test.js new file mode 100644 index 0000000..b5897fa --- /dev/null +++ b/tests/unit/lowParamCalibrator.test.js @@ -0,0 +1,148 @@ +'use strict'; + +/** + * The two-parameter correction that replaces isotonic on a thin sample. + * + * What these protect: that it CANNOT chase day-structure (two parameters over + * the whole curve), and that a thin fit is applied at reduced strength rather + * than at face value. + */ + +const lp = require('../../src/services/model/lowParamCalibrator'); + +/** An over-confident forecaster: predicts p, actually hits closer to the mean. */ +function overConfident(n, dates = 10, seed = 3) { + let s = seed; + const rnd = () => (s = (s * 1103515245 + 12345) % 2147483648) / 2147483648; + const rows = []; + for (let i = 0; i < n; i += 1) { + const p = 0.5 + rnd() * 0.45; + const truth = 0.5 + (p - 0.5) * 0.4; // real skill is 40% of claimed + rows.push({ date: `d${i % dates}`, p, won: rnd() < truth ? 1 : 0 }); + } + return rows; +} + +describe('it fits the favourite-longshot shape', () => { + it('FLATTENS an over-confident forecaster (a < 1)', () => { + const m = lp.fitPlatt(overConfident(2000)); + expect(m).not.toBeNull(); + expect(m.flattens).toBe(true); + expect(m.a).toBeLessThan(1); + }); + + it('pulls high predictions down and leaves the middle nearly alone', () => { + const m = lp.fitPlatt(overConfident(2000)); + const hi = lp.applyPlatt(m, 0.92); + const mid = lp.applyPlatt(m, 0.55); + expect(hi).toBeLessThan(0.92); + expect(Math.abs(mid - 0.55)).toBeLessThan(Math.abs(hi - 0.92)); + }); + + it('stays monotone — ordering is never disturbed', () => { + const m = lp.fitPlatt(overConfident(2000)); + let prev = -1; + for (let p = 0.05; p <= 0.95; p += 0.05) { + const v = lp.applyPlatt(m, p); + expect(v).toBeGreaterThan(prev); + prev = v; + } + }); + + it('leaves an already-honest forecaster essentially alone', () => { + let s = 11; + const rnd = () => (s = (s * 1103515245 + 12345) % 2147483648) / 2147483648; + const rows = []; + for (let i = 0; i < 2000; i += 1) { + const p = 0.3 + rnd() * 0.6; + rows.push({ date: `d${i % 12}`, p, won: rnd() < p ? 1 : 0 }); + } + const m = lp.fitPlatt(rows); + expect(Math.abs(lp.applyPlatt(m, 0.8) - 0.8)).toBeLessThan(0.06); + }); +}); + +describe('shrinkage — a thin fit is applied at reduced strength', () => { + it('scales with the number of fit DATES, not rows', () => { + const few = lp.fitPlatt(overConfident(2000, 5)); + const many = lp.fitPlatt(overConfident(2000, 40)); + expect(few.fit_rows).toBe(many.fit_rows); // same rows + expect(few.shrinkage).toBeLessThan(many.shrinkage); // different dates + expect(few.shrinkage).toBeCloseTo(5 / 15, 3); + expect(many.shrinkage).toBeCloseTo(40 / 50, 3); + }); + + it('a 5-date fit corrects less than a 40-date fit on the same input', () => { + const few = lp.fitPlatt(overConfident(2000, 5)); + const many = lp.fitPlatt(overConfident(2000, 40)); + // Both flatten; the thin one is held closer to the raw number. + expect(Math.abs(lp.applyPlatt(few, 0.92) - 0.92)) + .toBeLessThan(Math.abs(lp.applyPlatt(many, 0.92) - 0.92)); + }); +}); + +describe('honesty', () => { + it('refuses to fit below the row floor', () => { + expect(lp.fitPlatt(overConfident(40))).toBeNull(); + expect(lp.fitPlatt([])).toBeNull(); + expect(lp.fitPlatt(null)).toBeNull(); + }); + + it('an unreadable input returns null, never an uncorrected number', () => { + const m = lp.fitPlatt(overConfident(2000)); + expect(lp.applyPlatt(m, null)).toBeNull(); + expect(lp.applyPlatt(null, 0.7)).toBeNull(); + }); + + it('has exactly two parameters — it CANNOT encode a single odd day', () => { + // This is the whole reason it replaces isotonic here. + const m = lp.fitPlatt(overConfident(2000)); + const shape = Object.keys(m).filter((k) => k === 'a' || k === 'b'); + expect(shape.sort()).toEqual(['a', 'b']); + }); +}); + +describe('the slope must CORRECT, not abandon the forecast', () => { + /** A forecaster whose p_win carries no information at all. */ + function uninformative(n, seed = 7) { + let s = seed; + const rnd = () => (s = (s * 1103515245 + 12345) % 2147483648) / 2147483648; + const rows = []; + for (let i = 0; i < n; i += 1) rows.push({ date: `d${i % 8}`, p: 0.4 + rnd() * 0.5, won: rnd() < 0.55 ? 1 : 0 }); + return rows; + } + + it('REFUSES a fit whose slope collapses to a constant', () => { + // Shrinking a miscalibrated forecaster toward its base rate always lowers + // Brier, so this would score as a win while destroying all resolution. + const m = lp.fitPlatt(uninformative(3000)); + expect(m.refused).toBe(true); + expect(m.reason).toMatch(/collapses to a constant|invert/); + }); + + it('a refused fit produces no calibrated number at all', () => { + const m = lp.fitPlatt(uninformative(3000)); + expect(lp.applyPlatt(m, 0.8)).toBeNull(); + }); + + it('an INVERTING slope is refused by name', () => { + // Real case: runs fitted a = -0.032, which would reverse every ordering. + let s = 5; + const rnd = () => (s = (s * 1103515245 + 12345) % 2147483648) / 2147483648; + const rows = []; + for (let i = 0; i < 3000; i += 1) { + const p = 0.4 + rnd() * 0.5; + rows.push({ date: `d${i % 8}`, p, won: rnd() < (0.9 - p) ? 1 : 0 }); // backwards + } + const m = lp.fitPlatt(rows); + expect(m.refused).toBe(true); + expect(m.a).toBeLessThanOrEqual(lp.MIN_SLOPE); + }); + + it('still accepts a genuine flattening', () => { + const m = lp.fitPlatt(overConfident(2000)); + expect(m.refused).toBeUndefined(); + expect(m.a).toBeGreaterThan(lp.MIN_SLOPE); + expect(m.a).toBeLessThan(1); + }); +});