diff --git a/BUILD-STATE.md b/BUILD-STATE.md index 5a4f94e..af9cfed 100755 --- a/BUILD-STATE.md +++ b/BUILD-STATE.md @@ -3,6 +3,28 @@ ## Last Updated 2026-08-03 +## Session 88 (2026-08-04) — Re-adjudication: nothing to demote, hole closed ✅ +Spec: `specs/re-adjudication.md`. 4,238 tests / 337 suites green, build exit 0. +Counter byte-identical. Nothing recalibrated — nothing needed to be. +- **PROVEN SET IS EMPTY, verified 3 ways** (proven-status, featureRegistry + summary, validatedSkills). Zero conditioning entries ever reached PROVEN, so + STEP 3 (demote) and STEP 4 (recalibrate) are vacuous — correctly. +- **Correction: the cumulative gate did NOT catch a false positive last session.** + It caught nothing; it tightened α 0.0026 → 0.0013, demonstrating the mechanism. +- **THE REAL HOLE, CLOSED:** `promote()` could bypass cumulative correction. + `isSufficient` now requires `bonferroni_tests`, refuses anything below the + cumulative count, and refuses a p that doesn't clear 0.05/tests. Same guard on + `recordConditioning(PROVEN)`. Verified: no-correction / per-session-8-vs-38 / + weak-p all refused; cumulative-38 with p=0.0005 accepted. +- **Cumulative correction now NATIVE on all analysis paths** — pitcher-prove-k + and tb-solo-and-interactions migrated off per-session counts. +- **`reAblation.js` built** (standing second line): pure/injectable, records both + p-values + both test counts per verdict, `PENDING_RETEST` when there is no + fresh measurement (absence is not evidence). +- **Net effect on the proven set: ZERO.** No demotions, no recalibrations, no + ledger event — announcing a recalibration that changed nothing would itself be + a false signal of rigour. + ## Session 87 (2026-08-03) — Defence ingested; cumulative correction locked ✅ Spec: `specs/defense-ingest-and-cumulative-correction.md`. 4,228 tests / 336 suites green, build exit 0. Counter + clusters byte-identical. diff --git a/CLAUDE.md b/CLAUDE.md index a7b799f..6405136 100755 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1529,6 +1529,34 @@ phased plan in the Session-57 conversation / BUILD-STATE Next section). - `team_defense` keys on Savant's DISPLAY NAME (a nickname, "Cubs") while game logs give full names ("Chicago Cubs") — match on both. +## Re-adjudication + the promotion bar (Session 88 — non-obvious) +- **NOTHING HAS EVER BEEN PROVEN.** `proven-status.js` = EMPTY; `validatedSkills()` + = {} for all archetypes; 0 conditioning entries. The only PROVEN *feature* is + `recent_frequency_prior` — the incumbent COUNTER itself (S78 ablation showed it + is ~100% of the champion's resolution). It is the baseline, not a conditioning + interaction; demoting it would leave nothing to grade from. +- **The cumulative correction did NOT catch a false positive.** It caught nothing + (empty proven set). It tightened α 0.0026 → 0.0013 in one session — the + mechanism working, not a demotion. Don't restate that as a catch. +- **THE REAL HOLE (now closed): `promote()` could bypass the cumulative + correction.** `isSufficient` now REQUIRES `evidence.bonferroni_tests`, refuses + it if lower than `opts.cumulativeTests`, and refuses a `p_value` that doesn't + clear `0.05 / bonferroni_tests`. Same rule guards + `recordConditioning(status:PROVEN)`. This is what makes a retroactive + re-adjudication pass unnecessary — the bar is applied at promotion time. +- **Cumulative correction is now NATIVE on every analysis path** — `cluster-prove`, + `pitcher-prove-k` and `tb-solo-and-interactions` all use `testLedger`. If you + add a new analysis script, wire it or it silently corrects per-session. +- **`src/services/model/reAblation.js` is the standing second line.** Pure + + injectable (no DB, no measurement) so the decision rule can't drift from the + gate's. Records BOTH p-values and BOTH test counts per verdict so a demotion is + re-derivable. **No fresh measurement = `PENDING_RETEST`, never DEMOTE** — + absence of a re-test is not evidence, and demoting on it would punish whichever + stat is off-season. A feature promoted at α=0.05/20 CAN demote on the same + p-value once the bar is 0.05/60; that is correct, not unfair. +- **Don't emit a public "recalibrated after re-adjudication" ledger event when + nothing changed** — announcing rigour that did no work is itself a false signal. + ## Active Skills - vyndr-voice (all user-facing output) - prop-analysis (grading methodology) diff --git a/scripts/pitcher-prove-k.js b/scripts/pitcher-prove-k.js index a7f7231..75d444e 100644 --- a/scripts/pitcher-prove-k.js +++ b/scripts/pitcher-prove-k.js @@ -233,7 +233,18 @@ async function main() { }); } - const TESTS = SOLO.length + INTERACTIONS.length; + // ── CUMULATIVE BONFERRONI ───────────────────────────────────────────── + // The denominator is every DISTINCT hypothesis this programme has tested, not + // this run's. A per-session count gives each new order a fresh, generous alpha + // and lets the false-positive rate compound silently. + const tl = require('../src/services/model/testLedger'); + const mcStore = tl.supabaseStore(sb); + const mc = await tl.recordAndCount(mcStore, [ + ...SOLO.map((f) => ({ sport: 'mlb', stat: 'strikeouts', archetype: null, interaction: `solo:${f}`, target: 'counter_residual' })), + ...INTERACTIONS.map((x) => ({ sport: 'mlb', stat: 'strikeouts', archetype: null, interaction: x.key, target: 'counter_residual' })), + ]); + const TESTS = mc.cumulative_tests; + const complete = (keys) => rows.filter((r) => keys.every((k) => knownNumber(r[k]) !== null)); const solo = {}; @@ -322,6 +333,7 @@ async function main() { archetype_mix: rows.reduce((a, r) => { const k = r.archetype || 'unclassified'; a[k] = (a[k] || 0) + 1; return a; }, {}), gate_spec: cv.VALIDATION_REQUIREMENTS, bonferroni_tests: TESTS, + multiple_comparisons: { ...mc, note: 'cumulative across the programme lifetime, not this session' }, step1_solo_baseline: solo, step3_interactions: interactions, step3b_within_archetype_carrier: strata, diff --git a/scripts/tb-solo-and-interactions.js b/scripts/tb-solo-and-interactions.js index 9851e4e..392bf6e 100644 --- a/scripts/tb-solo-and-interactions.js +++ b/scripts/tb-solo-and-interactions.js @@ -331,7 +331,18 @@ async function main() { } // Bonferroni denominator = every test in this family (solo + interaction). - const TESTS = SOLO.length + INTERACTIONS.length; + // ── CUMULATIVE BONFERRONI ───────────────────────────────────────────── + // The denominator is every DISTINCT hypothesis this programme has tested, not + // this run's. A per-session count gives each new order a fresh, generous alpha + // and lets the false-positive rate compound silently. + const tl = require('../src/services/model/testLedger'); + const mcStore = tl.supabaseStore(sb); + const mc = await tl.recordAndCount(mcStore, [ + ...SOLO.map((f) => ({ sport: 'mlb', stat: 'total_bases', archetype: null, interaction: `solo:${f}`, target: 'counter_residual' })), + ...INTERACTIONS.map((x) => ({ sport: 'mlb', stat: 'total_bases', archetype: null, interaction: x.key, target: 'counter_residual' })), + ]); + const TESTS = mc.cumulative_tests; + // ── STEP 1 — SOLO PASS (the control) ──────────────────────────────────── const solo = {}; @@ -397,6 +408,7 @@ async function main() { rows_scored: rows.length, gate_spec: cv.VALIDATION_REQUIREMENTS, bonferroni_tests: TESTS, + multiple_comparisons: { ...mc, note: 'cumulative across the programme lifetime, not this session' }, n_gap_note: `the gate needs ${cv.VALIDATION_REQUIREMENTS.min_historical_instances} rows; this run has ${rows.length}`, archetype_coverage: { labelled: rows.filter((r) => r.archetype).length, diff --git a/specs/re-adjudication.md b/specs/re-adjudication.md new file mode 100644 index 0000000..ebd9769 --- /dev/null +++ b/specs/re-adjudication.md @@ -0,0 +1,123 @@ +# RE-ADJUDICATION UNDER CUMULATIVE CORRECTION + +**2026-08-04.** Counter byte-identical. Nothing recalibrated, because nothing +needed to be. + +> **There is nothing to re-adjudicate: the proven set is empty, and has always +> been empty.** Verified three ways below. +> +> **And the cumulative correction did not catch a false positive last session.** +> It caught nothing — there was nothing in the proven set to catch. What it did +> was tighten α from 0.0026 to 0.0013 within one session, which demonstrated the +> mechanism working, not a demotion. +> +> **The real exposure was elsewhere, and it was real: a promotion could bypass +> the cumulative correction entirely.** That hole is now closed structurally, so +> the retroactive pass this order describes can never become necessary. + +--- + +## 1. The registry state, verified (STEP 1 + 2) + +``` +scripts/proven-status.js → PROVEN_SET: EMPTY + hits LOSES · total_bases INCONCLUSIVE · strikeouts INCONCLUSIVE + +featureRegistry → PROVEN features: ["recent_frequency_prior"] + validatedSkills(BOMBER|GHOST|TORCH|FLAME|SCALPEL|SINKER) = {} + conditioning entries recorded: 0 +``` + +The single PROVEN *feature* is `recent_frequency_prior` — **the incumbent counter +itself**, proven by the S78 ablation showing it is ~100% of the champion's +resolution. It is the baseline every challenger is measured against, not a +conditioning interaction, and demoting it would leave the model with nothing to +grade from. **No conditioning entry has ever reached PROVEN**, so there is no +archetype-slot to re-run and no grade that depends on a demoted feature. + +**STEP 3 (demote) and STEP 4 (recalibrate) are therefore vacuous — correctly.** +`readjudicateAll([], …)` returns `NOTHING TO RE-ADJUDICATE — the proven set is +empty`, which is the honest output rather than a no-op glossed over. + +## 2. The hole that WAS open, and is now closed + +The order's underlying worry — *make sure the standard applied from the start, so +no retroactive correction is ever needed* — was well founded. Two audits: + +**(a) `promote()` did not require the cumulative denominator.** It checked n, +lift and CI, but nothing stopped a future session from testing eight hypotheses, +correcting by eight, and promoting on a p-value that would not survive the +programme's real denominator. **That is exactly the hole that makes a +"re-adjudicate everything" pass necessary later.** + +Now structural — `isSufficient` refuses evidence that: +- carries **no** `bonferroni_tests` at all; +- was corrected against **fewer** tests than the cumulative count; +- has a `p_value` that does not clear `0.05 / bonferroni_tests`. + +``` +no correction recorded → refused +per-session (8) when cumulative is 38 → refused +cumulative (38) but p = 0.01 > 0.0013 → refused +cumulative (38) and p = 0.0005 → accepted +``` + +The same rule guards `recordConditioning(status: PROVEN)`. Locked by test. + +**(b) Two of four analysis scripts still corrected per-session.** +`pitcher-prove-k.js` and `tb-solo-and-interactions.js` used +`SOLO.length + INTERACTIONS.length`; both now use the cumulative ledger, as +`cluster-prove.js` already did. **Cumulative correction is native across every +analysis path.** + +## 3. Standing re-ablation (STEP 5, second line) — built + +`src/services/model/reAblation.js`. Pure and injectable: it takes evidence and +returns auditable verdicts, performs no measurement, and reaches no database, so +the decision rule cannot quietly drift from the gate's. + +Every verdict records **both p-values and both test counts**, so a demotion can +be re-derived by anyone: + +``` +original_p_value · original_bonferroni_tests · original_corrected_alpha +current_p_value · cumulative_bonferroni_tests · cumulative_corrected_alpha +verdict + reason +``` + +**A feature promoted at α=0.05/20 and re-tested against α=0.05/60 can demote on +the same p-value.** That is not unfair — the bar rose because the programme has +since had more chances to get lucky, and holding old results to the old bar is +how a lucky draw becomes permanent. + +**No fresh measurement is `PENDING_RETEST`, never a demotion.** Absence of a +re-test is not evidence a feature stopped working, and demoting on it would +punish whichever stat happens to be off-season. + +Three ways a proven feature can stop being true are documented in the module: it +was never true; the game changed; or the bar rose. + +## 4. Net effect on the proven set + +**Zero.** Nothing survived because nothing was there; nothing demoted; nothing +recalibrated; no ledger event was emitted, because emitting "recalibrated after +re-adjudication" when nothing changed would itself be a false signal of rigour. + +Frozen clusters were **not exempted** — they were examined and contain no proven +conditioning. The counter is byte-identical. + +## 5. Going forward + +- **Cumulative correction is native** on every analysis path and **required** for + every promotion. +- **Standing re-ablation is available** and should run whenever a proven entry + exists and fresh sample arrives. +- **The bar only rises.** Current cumulative count 38, α = 0.0013. Every new + hypothesis tightens it for everything after — so re-testing standing candidates + is now mathematically the disciplined choice over inventing new ones. +- **The first promotion this programme ever makes will already have been held to + the cumulative bar**, which is the outcome this order was actually asking for. + +**Not recommended:** treating the empty proven set as a problem to solve by +lowering the bar, or emitting a public "re-adjudication" event for a pass that +changed nothing. diff --git a/src/services/model/featureRegistry.js b/src/services/model/featureRegistry.js index 640e7cb..a1f450d 100644 --- a/src/services/model/featureRegistry.js +++ b/src/services/model/featureRegistry.js @@ -210,11 +210,27 @@ function isLive(sport, key) { return statusOf(sport, key) === STATUS.PROVEN; } +const BASE_ALPHA = 0.05; + /** - * Evidence sufficient to promote: a real sample, positive lift, and a CI that - * excludes zero on the good side. Anything less is a story about a number. + * Evidence sufficient to promote: a real sample, positive lift, a CI that + * excludes zero on the good side, AND a p-value that clears the CUMULATIVELY + * corrected alpha. + * + * THE CUMULATIVE REQUIREMENT IS STRUCTURAL, NOT ADVISORY. Before it, the + * correction existed but nothing forced a promotion to use it: a session could + * test eight hypotheses, correct by eight, and promote on a p-value that would + * not survive the programme's real denominator. That is precisely the hole that + * makes a retroactive "re-adjudicate everything" pass necessary later — so it is + * closed here rather than audited for afterwards. + * + * `opts.cumulativeTests` is the programme-lifetime count (from testLedger). When + * supplied, evidence corrected against FEWER tests than that is refused: you may + * not promote on a laxer bar than the programme has earned. It is injectable so + * the unit suite never needs a database, and omitted only when no cumulative + * count is available at all. */ -function isSufficient(evidence) { +function isSufficient(evidence, opts = {}) { if (!evidence || typeof evidence !== 'object') return false; const n = Number(evidence.n); const lift = Number(evidence.lift); @@ -224,17 +240,40 @@ function isSufficient(evidence) { if (!Array.isArray(ci) || ci.length !== 2) return false; const [lo, hi] = ci.map(Number); if (!Number.isFinite(lo) || !Number.isFinite(hi)) return false; - return lo > 0; // the whole interval above zero — improvement, not a coin flip + if (!(lo > 0)) return false; // the whole interval above zero + + // The correction must be present, and must be the cumulative one. + const tests = Number(evidence.bonferroni_tests); + if (!Number.isFinite(tests) || tests < 1) return false; + const cum = Number(opts.cumulativeTests); + if (Number.isFinite(cum) && tests < cum) return false; + + // If a p-value is supplied it must clear the corrected alpha. (It is optional + // only because some evidence is a bootstrap CI with no p — the CI check above + // already carries that case.) + const p = Number(evidence.p_value); + if (Number.isFinite(p) && !(p < BASE_ALPHA / tests)) return false; + return true; } /** * Promote a CANDIDATE to PROVEN. Refuses without sufficient evidence, and there * is no override parameter on purpose. */ -function promote(sport, key, evidence, at = null) { +function promote(sport, key, evidence, at = null, opts = {}) { const f = byKey.get(idOf(sport, key)); if (!f) return { ok: false, reason: 'unknown_feature' }; - if (!isSufficient(evidence)) return { ok: false, reason: 'insufficient_evidence', required: { min_n: MIN_PROMOTION_N, lift: '>0', ci95_low: '>0' } }; + if (!isSufficient(evidence, opts)) { + return { + ok: false, + reason: 'insufficient_evidence', + required: { + min_n: MIN_PROMOTION_N, lift: '>0', ci95_low: '>0', + bonferroni_tests: 'required, and >= the cumulative programme count', + p_value: 'if present, must be < 0.05 / bonferroni_tests', + }, + }; + } f.history.push({ from: f.status, to: STATUS.PROVEN, evidence, at }); f.status = STATUS.PROVEN; f.evidence = evidence; @@ -302,12 +341,12 @@ const conditioning = []; * interaction validates — an untagged proven entry cannot contribute to a * coherent profile, so it is refused. */ -function recordConditioning({ sport, archetype, stat, interaction, skill, status, lift = null, evidence = null }) { +function recordConditioning({ sport, archetype, stat, interaction, skill, status, lift = null, evidence = null }, opts = {}) { if (!Object.keys(SKILLS).includes(String(skill || '').toUpperCase())) { return { ok: false, reason: 'untagged_or_unknown_skill', known: Object.keys(SKILLS) }; } if (!Object.values(STATUS).includes(status)) return { ok: false, reason: 'bad_status' }; - if (status === STATUS.PROVEN && !isSufficient(evidence)) { + if (status === STATUS.PROVEN && !isSufficient(evidence, opts)) { return { ok: false, reason: 'insufficient_evidence_for_proven' }; } const row = { @@ -349,7 +388,7 @@ function __reset() { } module.exports = { - STATUS, MIN_PROMOTION_N, + STATUS, MIN_PROMOTION_N, BASE_ALPHA, allFeatures, liveFeatures, candidateFeatures, statusOf, isLive, recordStatVerdict, statusForStat, liveFeaturesForStat, candidateFeaturesForStat, SKILLS, recordConditioning, conditioningFor, validatedSkills, diff --git a/src/services/model/reAblation.js b/src/services/model/reAblation.js new file mode 100644 index 0000000..72e5052 --- /dev/null +++ b/src/services/model/reAblation.js @@ -0,0 +1,108 @@ +'use strict'; + +/** + * reAblation — THE SECOND LINE OF DEFENCE. + * + * The gate decides whether a feature earns its place. This decides whether it + * KEEPS it. Those are different questions and only the first has ever been asked + * here. + * + * WHY A PROVEN FEATURE CAN STOP BEING TRUE. Three ways, all of them real and + * none of them a bug: + * + * 1. IT WAS NEVER TRUE. It cleared the bar on a lucky draw. More data is the + * only thing that reveals this, and the cumulative correction makes it + * rarer without making it impossible. + * 2. THE GAME CHANGED. Baseball is not stationary — a league-wide shift in how + * pitchers are used, or a rule change, can retire a real effect. + * 3. THE BAR ROSE. The cumulative denominator only grows, so a feature proved + * at alpha 0.05/20 is being held to 0.05/60 a year later. A feature that + * cleared the old bar and not the new one is not being punished unfairly — + * it is being held to what the programme has since earned the right to ask. + * + * A ledger that tightens its own standard and demotes its own features is more + * credible than one that only ever adds. So the demotion is recorded with BOTH + * p-values and the test count each was corrected against — anyone can see + * exactly why, and re-derive it. + * + * PURE AND INJECTABLE: it takes evidence in and returns verdicts. It performs no + * measurement itself and reaches no database, so the decision rule is testable + * without a network and cannot quietly drift from the rule the gate uses. + */ + +const { BASE_ALPHA } = require('./featureRegistry'); + +/** + * Re-adjudicate ONE proven entry against the current cumulative denominator. + * + * @param {object} entry what was promoted, and on what evidence + * @param {object} current the fresh measurement (may be absent) + * @param {number} cumulativeTests the programme-lifetime distinct test count + * @returns {object} an auditable verdict — never a bare boolean + */ +function readjudicate(entry, current, cumulativeTests) { + const tests = Number(cumulativeTests); + const correctedAlpha = Number.isFinite(tests) && tests >= 1 ? BASE_ALPHA / tests : BASE_ALPHA; + const originalTests = Number(entry && entry.evidence && entry.evidence.bonferroni_tests); + const originalAlpha = Number.isFinite(originalTests) && originalTests >= 1 + ? BASE_ALPHA / originalTests : null; + const originalP = Number(entry && entry.evidence && entry.evidence.p_value); + + const base = { + key: entry && entry.key, + archetype: entry && entry.archetype, + stat: entry && entry.stat, + original_p_value: Number.isFinite(originalP) ? originalP : null, + original_bonferroni_tests: Number.isFinite(originalTests) ? originalTests : null, + original_corrected_alpha: originalAlpha, + cumulative_bonferroni_tests: Number.isFinite(tests) ? tests : null, + cumulative_corrected_alpha: correctedAlpha, + }; + + // NO FRESH MEASUREMENT — deliberately NOT a demotion. Absence of a re-test is + // not evidence a feature stopped working, and demoting on it would punish + // whichever stat happens to be off-season. + if (!current || !Number.isFinite(Number(current.p_value))) { + return { ...base, verdict: 'PENDING_RETEST', reason: 'no fresh measurement available' }; + } + + const p = Number(current.p_value); + const n = Number(current.n); + const survives = p < correctedAlpha; + return { + ...base, + current_p_value: p, + current_n: Number.isFinite(n) ? n : null, + verdict: survives ? 'SURVIVES' : 'DEMOTE', + reason: survives + ? `p ${p} < cumulative alpha ${correctedAlpha}` + : `p ${p} no longer clears the cumulative alpha ${correctedAlpha}` + + (originalAlpha !== null ? ` (it cleared ${originalAlpha} when promoted)` : ''), + }; +} + +/** + * Re-adjudicate a whole registry. Returns the verdicts plus a summary that + * states plainly what happened, including the case that matters most right now: + * an empty proven set has nothing to re-adjudicate, and saying so is the honest + * result rather than a no-op to be glossed. + */ +function readjudicateAll(provenEntries, measurements, cumulativeTests) { + const entries = provenEntries || []; + const verdicts = entries.map((e) => readjudicate(e, (measurements || {})[e.key], cumulativeTests)); + return { + cumulative_bonferroni_tests: cumulativeTests, + cumulative_corrected_alpha: Number.isFinite(Number(cumulativeTests)) && cumulativeTests >= 1 + ? BASE_ALPHA / cumulativeTests : BASE_ALPHA, + proven_entries_examined: entries.length, + survived: verdicts.filter((v) => v.verdict === 'SURVIVES').length, + demoted: verdicts.filter((v) => v.verdict === 'DEMOTE').length, + pending_retest: verdicts.filter((v) => v.verdict === 'PENDING_RETEST').length, + verdicts, + summary: entries.length === 0 + ? 'NOTHING TO RE-ADJUDICATE — the proven set is empty.' + : `${verdicts.filter((v) => v.verdict === 'DEMOTE').length} demoted of ${entries.length}.`, + }; +} + +module.exports = { readjudicate, readjudicateAll }; diff --git a/tests/unit/reAblation.test.js b/tests/unit/reAblation.test.js new file mode 100644 index 0000000..032652c --- /dev/null +++ b/tests/unit/reAblation.test.js @@ -0,0 +1,109 @@ +'use strict'; + +/** + * Standing re-ablation + the structural bar on promotion. + * + * Together these close the hole that makes a retroactive "re-adjudicate + * everything" pass necessary: a promotion cannot use a laxer correction than + * the programme has earned, and a proven feature is re-checked against the + * cumulative bar rather than the one it was promoted under. + */ + +const ra = require('../../src/services/model/reAblation'); +const reg = require('../../src/services/model/featureRegistry'); + +afterEach(() => reg.__reset()); + +const promoted = (p, tests) => ({ + key: 'mlb|hits|BOMBER|barrel_x_park', + archetype: 'BOMBER', stat: 'hits', + evidence: { p_value: p, bonferroni_tests: tests, n: 600 }, +}); + +describe('re-adjudication against the CUMULATIVE bar', () => { + it('demotes a feature that cleared the old bar but not the grown one', () => { + // Promoted at alpha 0.05/20 = 0.0025 with p = 0.002. The programme has since + // tested 60 hypotheses, so the bar is 0.05/60 = 0.00083. + const v = ra.readjudicate(promoted(0.002, 20), { p_value: 0.002, n: 700 }, 60); + expect(v.verdict).toBe('DEMOTE'); + expect(v.original_corrected_alpha).toBeCloseTo(0.0025, 6); + expect(v.cumulative_corrected_alpha).toBeCloseTo(0.05 / 60, 8); + // Auditable: BOTH p-values and BOTH test counts are on the record. + expect(v.original_p_value).toBe(0.002); + expect(v.current_p_value).toBe(0.002); + expect(v.original_bonferroni_tests).toBe(20); + expect(v.cumulative_bonferroni_tests).toBe(60); + expect(v.reason).toMatch(/no longer clears/); + }); + + it('keeps a feature strong enough to clear the grown bar', () => { + const v = ra.readjudicate(promoted(0.00001, 20), { p_value: 0.00002, n: 900 }, 60); + expect(v.verdict).toBe('SURVIVES'); + }); + + it('NO fresh measurement is NOT a demotion — absence is not evidence', () => { + // Demoting here would punish whichever stat happens to be off-season. + const v = ra.readjudicate(promoted(0.001, 20), null, 60); + expect(v.verdict).toBe('PENDING_RETEST'); + const v2 = ra.readjudicate(promoted(0.001, 20), { n: 900 }, 60); + expect(v2.verdict).toBe('PENDING_RETEST'); + }); + + it('an empty proven set re-adjudicates to nothing, and says so plainly', () => { + const out = ra.readjudicateAll([], {}, 38); + expect(out.proven_entries_examined).toBe(0); + expect(out.demoted).toBe(0); + expect(out.summary).toMatch(/NOTHING TO RE-ADJUDICATE/); + expect(out.cumulative_corrected_alpha).toBeCloseTo(0.05 / 38, 8); + }); + + it('reports per-slot, so a feature can survive for one archetype and die for another', () => { + const strong = { ...promoted(0.00001, 20), key: 'k1', archetype: 'GHOST' }; + const weak = { ...promoted(0.002, 20), key: 'k2', archetype: 'BOMBER' }; + const out = ra.readjudicateAll([strong, weak], { + k1: { p_value: 0.00002, n: 800 }, + k2: { p_value: 0.002, n: 800 }, + }, 60); + expect(out.survived).toBe(1); + expect(out.demoted).toBe(1); + expect(out.verdicts.find((v) => v.key === 'k1').archetype).toBe('GHOST'); + expect(out.verdicts.find((v) => v.key === 'k2').verdict).toBe('DEMOTE'); + }); +}); + +describe('promotion cannot use a laxer correction than the programme earned', () => { + const base = { n: 600, lift: 0.04, ci95: [0.01, 0.07] }; + + it('refuses evidence with NO correction recorded at all', () => { + expect(reg.promote('mlb', 'batter_barrel_pct', base).ok).toBe(false); + }); + + it('refuses a PER-SESSION correction when the cumulative count is higher', () => { + const r = reg.promote('mlb', 'batter_barrel_pct', + { ...base, bonferroni_tests: 8 }, null, { cumulativeTests: 38 }); + expect(r.ok).toBe(false); + expect(r.reason).toBe('insufficient_evidence'); + }); + + it('refuses a p-value that does not clear the corrected alpha', () => { + expect(reg.promote('mlb', 'batter_barrel_pct', + { ...base, bonferroni_tests: 38, p_value: 0.01 }, null, { cumulativeTests: 38 }).ok).toBe(false); + }); + + it('accepts evidence corrected cumulatively AND significant under it', () => { + const r = reg.promote('mlb', 'batter_barrel_pct', + { ...base, bonferroni_tests: 38, p_value: 0.0005 }, null, { cumulativeTests: 38 }); + expect(r.ok).toBe(true); + expect(reg.isLive('mlb', 'batter_barrel_pct')).toBe(true); + }); + + it('the same rule guards a PROVEN conditioning entry', () => { + const bad = reg.recordConditioning({ + sport: 'mlb', archetype: 'GHOST', stat: 'hits', interaction: 'defense_x_contact', + skill: 'CONTACT', status: reg.STATUS.PROVEN, + evidence: { ...base, bonferroni_tests: 8 }, + }, { cumulativeTests: 38 }); + expect(bad.ok).toBe(false); + expect(bad.reason).toBe('insufficient_evidence_for_proven'); + }); +}); diff --git a/tests/unit/skillModel.test.js b/tests/unit/skillModel.test.js index e29129c..3feacf5 100644 --- a/tests/unit/skillModel.test.js +++ b/tests/unit/skillModel.test.js @@ -35,8 +35,14 @@ describe('D3 — earn its place or it is out', () => { }); it('promotes on sufficient evidence, and records WHY', () => { - const ev = { n: 600, lift: 0.04, ci95: [0.012, 0.068], measured_at: '2026-09-01' }; - expect(reg.promote('mlb', 'batter_barrel_pct', ev).ok).toBe(true); + // The correction is now part of "sufficient": evidence must carry the + // CUMULATIVE bonferroni_tests it was corrected against, so a promotion can + // never use a laxer bar than the programme has earned. + const ev = { + n: 600, lift: 0.04, ci95: [0.012, 0.068], measured_at: '2026-09-01', + bonferroni_tests: 38, p_value: 0.0005, + }; + expect(reg.promote('mlb', 'batter_barrel_pct', ev, null, { cumulativeTests: 38 }).ok).toBe(true); expect(reg.isLive('mlb', 'batter_barrel_pct')).toBe(true); expect(reg.liveFeatures('mlb').has('batter_barrel_pct')).toBe(true); }); @@ -52,7 +58,7 @@ describe('D3 — earn its place or it is out', () => { }); it('per-sport: a feature proven for MLB says nothing about another sport', () => { - reg.promote('mlb', 'batter_barrel_pct', { n: 600, lift: 0.04, ci95: [0.01, 0.07] }); + reg.promote('mlb', 'batter_barrel_pct', { n: 600, lift: 0.04, ci95: [0.01, 0.07], bonferroni_tests: 38, p_value: 0.0005 }, null, { cumulativeTests: 38 }); expect(reg.isLive('nba', 'batter_barrel_pct')).toBe(false); expect(reg.statusOf('nba', 'batter_barrel_pct')).toBeNull(); });