diff --git a/scripts/teeth-certified-probability.js b/scripts/teeth-certified-probability.js index 8d47ba2..816b347 100644 --- a/scripts/teeth-certified-probability.js +++ b/scripts/teeth-certified-probability.js @@ -72,15 +72,19 @@ logicTooth(1, 'runtime SHA verification waits for a snapshot despite working run // ── 2 — the estimator must carry a reconstructable identity ────────────── injectionTooth(2, 'runtime estimator lacks a reconstructable artifact identity', 'src/services/model/probabilityContractService.js', - ` knot_digest: digest(fitted.map),`, - ` knot_digest: 'static-placeholder',`, + ` knot_count: Array.isArray(fitted.map) ? fitted.map.length : null, + knot_digest: digest(fitted.map),`, + ` knot_count: null, + knot_digest: 'static-placeholder',`, 'tests/unit/probabilityContract.test.js'); // ── 3 — the artifact must be tied to the certified model era ───────────── injectionTooth(3, 'runtime artifact differs from the certified artifact', 'src/services/model/probabilityContractService.js', - ` model_version: contract.model_version,`, - ` model_version: 'engine1@some-other-era',`, + ` model_version: contract.model_version, + fit_as_of: fitted.cutoff || null,`, + ` model_version: 'engine1@some-other-era', + fit_as_of: fitted.cutoff || null,`, 'tests/unit/probabilityContractShadow.test.js'); // ── 4 — point-in-time evidence ─────────────────────────────────────────── @@ -128,6 +132,30 @@ logicTooth(25, 'shadow flag parses loosely or defaults ON', () => { detail: `strict '1' compare=${strict}; sport-scoped=${scoped}; stat-scoped=${statScoped}` }; }); +// ── 9 — the fit policy must not drift silently ─────────────────────────── +injectionTooth(9, 'current fit policy silently changed before measurement', + 'src/services/model/fitPolicy.js', + ` model_version: 'engine1@2026-08-07-fullwindow', + data_selection: Object.freeze({`, + ` model_version: 'engine1@2026-07-20', + data_selection: Object.freeze({`, + 'tests/unit/fitPolicy.test.js'); + +// ── 10 — a refit may not become servable just because it ran ───────────── +injectionTooth(10, 'future refit becomes servable without a validity gate', + 'src/services/model/fitPolicy.js', + ` /** A policy-invalid artifact is NEVER servable. There is no override. */ + servable: violations.length === 0,`, + ` servable: true,`, + 'tests/unit/fitPolicy.test.js'); + +// ── 10b — support may not be widened by a refit ────────────────────────── +injectionTooth(26, 'a refit widens the region it is trusted in', + 'src/services/model/fitPolicy.js', + ` violations.push(VIOLATION.SUPPORT_WIDENED);`, + ` { /* widening allowed */ }`, + 'tests/unit/fitPolicy.test.js'); + const unreachable = [ { id: 17, name: 'UI displays raw exact confidence for UNCERTIFIED', why: 'no live-serving UI path exists; nothing consumes served_probability yet' }, { id: 18, name: 'live EV/Kelly/VALUE consumers bypass served_probability', why: 'live consumers are unchanged by design in this tranche — tooth 16 asserts none exist' }, diff --git a/src/services/model/fitPolicy.js b/src/services/model/fitPolicy.js new file mode 100644 index 0000000..a51ba5d --- /dev/null +++ b/src/services/model/fitPolicy.js @@ -0,0 +1,135 @@ +'use strict'; + +/** + * fitPolicy — WHAT IS BEING CERTIFIED: a curve, or the procedure that makes it. + * + * The artifact has an identity (`knot_digest`, `served_curve`), and identity is + * not certification. A digest says WHICH mapping ran; it says nothing about + * whether the procedure that produced it deserves continuous trust. Tomorrow's + * refit gets its own digest and would be equally "identified" while being fitted + * on anything at all. + * + * VYNDR certifies a PROCEDURE, not a frozen curve. A frozen curve would go stale + * against a live model and would have to be replaced by hand on no schedule; the + * whole "fit past, apply forward" discipline is procedural. So the procedure is + * declared here, versioned, and checkable — and every artifact it produces still + * carries its own digest. + * + * ── THE MEASURED STATE (2026-09-03) ────────────────────────────────────── + * The production procedure DOES NOT MATCH this declaration, and that is the + * whole reason live serving is blocked: + * + * `calibrationService.loadSettledRows` applies NO model_version filter, so + * the fit pools every era. Measured at fit_as_of 2026-09-02: 9,361 settled + * hits rows, of which 3,292 came from the superseded engine1@2026-07-20 and + * 6,069 from the current era. 65% of 9,361 = 6,084 = the production fit_n + * exactly, which is all 3,292 superseded rows plus 2,792 current-era rows — + * 54.1% of the served map fitted on a forecaster it was never certified for, + * while the artifact declares the current era. + * + * Out-of-sample the pooling is not measurably harmful (era-filtered Brier + * 0.24065 vs pooled 0.24068 on 1,120 rows, both intervals spanning zero), so + * this is not a performance claim. It is a provenance contradiction: a + * calibrator declared FOR one era and drawn mostly FROM another. Nothing in the + * procedure prevents the next era change from repeating it, and the freshness + * lag (training_cutoff 2026-08-21 vs fit_as_of 2026-09-02) grows with history. + * + * NOTHING HERE CHANGES THE PRODUCTION FIT. This module declares and CHECKS. + * Changing the fit is a separate, certified act. + */ + +/** + * THE DECLARED PROCEDURE. Every field is a decision that would otherwise be + * implicit in whichever function happened to run. + */ +const POLICY_V1 = Object.freeze({ + policy_version: 'mlb-hits-isotonic-refit@v1', + sport: 'mlb', + stat: 'hits', + /** Only the era the contract is certified for. THIS is what production lacks. */ + model_version: 'engine1@2026-08-07-fullwindow', + data_selection: Object.freeze({ + source: 'ledger_entries', + scope: 'public model record (user_id IS NULL)', + settlement: "outcome IN ('hit','miss') AND p_win IS NOT NULL", + /** STRICTLY before the grading day — a Read may never see its own outcome. */ + horizon: 'game_date < fit_as_of', + excludes: "quarantine_reason LIKE 'nontakeable_book%'", + }), + fit_algorithm: 'isotonic-pav', + fit_algorithm_version: 'calibration.fitIsotonic@2026-08', + /** Minimum rows before a fit may be produced at all. */ + min_fit_rows: 200, + /** Support is a property of the ADJUDICATION, not of tonight's fit. A refit + * may never widen the region it is trusted in — that is self-certification. */ + support_contract: Object.freeze({ certified_bands: Object.freeze([Object.freeze([0.50, 0.80])]), + source: 'adjudication@2026-09-02', may_be_widened_by_refit: false }), + refit_cadence: 'per snapshot run', +}); + +const VIOLATION = Object.freeze({ + ERA_NOT_RESTRICTED: 'ERA_NOT_RESTRICTED', + ERA_MISMATCH: 'ERA_MISMATCH', + ESTIMATOR_MISMATCH: 'ESTIMATOR_MISMATCH', + INSUFFICIENT_FIT_ROWS: 'INSUFFICIENT_FIT_ROWS', + SUPPORT_WIDENED: 'SUPPORT_WIDENED', + NO_ARTIFACT_IDENTITY: 'NO_ARTIFACT_IDENTITY', + NO_TRAINING_CUTOFF: 'NO_TRAINING_CUTOFF', +}); + +const sameBands = (a, b) => JSON.stringify(a) === JSON.stringify(b); + +/** + * THE MINIMUM AUTOMATIC GATES (Step 22). + * + * A newly fitted artifact must NOT become servable merely because the algorithm + * ran. These are the checks that can be made automatically, at fit time, with no + * new measurement — they are deliberately cheap and structural. The statistical + * bars (held-out CI, LODO) stay where they already live, in + * `calibrationRegistry`; this does not build a second governance system. + * + * @param {object} artifact as produced by probabilityContractService + * @param {object} evidence {era_counts} — the era composition of the fit, when known + */ +function validate(artifact, evidence = {}, policy = POLICY_V1) { + const violations = []; + // `servable` is declared on EVERY return path. Omitting it here made + // `validate(null).servable` undefined, which is falsy at a call site and so + // would have looked correct while carrying no assertion at all. + if (!artifact) { + return { valid: false, violations: [VIOLATION.NO_ARTIFACT_IDENTITY], + policy_version: policy.policy_version, servable: false }; + } + + if (!artifact.knot_digest || !artifact.served_curve_digest) violations.push(VIOLATION.NO_ARTIFACT_IDENTITY); + if (!artifact.training_cutoff) violations.push(VIOLATION.NO_TRAINING_CUTOFF); + if (artifact.model_version !== policy.model_version) violations.push(VIOLATION.ERA_MISMATCH); + if (artifact.estimator_type !== 'isotonic') violations.push(VIOLATION.ESTIMATOR_MISMATCH); + if (!(Number(artifact.fit_n) >= policy.min_fit_rows)) violations.push(VIOLATION.INSUFFICIENT_FIT_ROWS); + if (artifact.certified_bands && !sameBands(artifact.certified_bands, policy.support_contract.certified_bands)) { + violations.push(VIOLATION.SUPPORT_WIDENED); + } + + // THE ERA RESTRICTION. `era_counts` is the fit's actual composition. Absent, + // we cannot claim the restriction held — and "we did not check" is recorded + // as a violation rather than passed as clean, because an unverified + // restriction is exactly the state production is in today. + const counts = evidence.era_counts; + if (!counts || typeof counts !== 'object') { + violations.push(VIOLATION.ERA_NOT_RESTRICTED); + } else { + const foreign = Object.entries(counts) + .filter(([era, n]) => era !== policy.model_version && Number(n) > 0); + if (foreign.length) violations.push(VIOLATION.ERA_NOT_RESTRICTED); + } + + return { + valid: violations.length === 0, + violations, + policy_version: policy.policy_version, + /** A policy-invalid artifact is NEVER servable. There is no override. */ + servable: violations.length === 0, + }; +} + +module.exports = { POLICY_V1, VIOLATION, validate }; diff --git a/src/services/model/probabilityContractService.js b/src/services/model/probabilityContractService.js index 80b71c0..d2e4a9c 100644 --- a/src/services/model/probabilityContractService.js +++ b/src/services/model/probabilityContractService.js @@ -17,6 +17,7 @@ const crypto = require('crypto'); const cal = require('./calibration'); const pc = require('./probabilityContract'); +const fitPolicy = require('./fitPolicy'); /** Short, stable content digest. Full sha256 truncated — collision risk here is * irrelevant and 16 hex chars keeps the per-row payload small. */ @@ -74,6 +75,26 @@ async function build(sb, { sport = 'mlb', stat = 'hits', before = null, ...opts // function that produced it, and "isotonic" would be a label rather than a // claim. This is the identity. const curve = servedCurve(fitted.map, contract.certified_bands); + // ── THE FIT-POLICY CHECK (Step 22) ───────────────────────────────────── + // An artifact does not become servable because the algorithm ran. The era + // composition is known STRUCTURALLY, not by an extra read: this service + // applies no model_version filter today, so the restriction demonstrably did + // not hold and the artifact records that rather than claiming it did. + // + // `era_restricted` is a fact about the QUERY, and the query is right here. + const eraRestricted = false; // calibrationService.loadSettledRows applies no model filter + const draft = { + estimator_type: contract.estimator_type, + model_version: contract.model_version, + fit_n: fitted.fit_n ?? null, + training_cutoff: fitted.fitted_through || null, + knot_digest: digest(fitted.map), + served_curve_digest: digest(curve), + certified_bands: contract.certified_bands, + }; + const policy = fitPolicy.validate(draft, eraRestricted + ? { era_counts: { [contract.model_version]: fitted.fit_n } } : {}); + const artifact = Object.freeze({ estimator_type: contract.estimator_type, estimator_version: contract.estimator_version, @@ -86,6 +107,15 @@ async function build(sb, { sport = 'mlb', stat = 'hits', before = null, ...opts knot_digest: digest(fitted.map), served_curve: curve, // complete over certified support served_curve_digest: digest(curve), + // WHICH PROCEDURE, AND WHETHER IT HELD. Recorded on every artifact so a + // later reader sees the policy state of the fit that produced the number, + // not just the number's fingerprint. + fit_policy_version: policy.policy_version, + fit_policy_valid: policy.valid, + fit_policy_violations: Object.freeze(policy.violations), + // A policy-invalid artifact is never servable. Nothing serves today, so + // this is a declaration; it becomes load-bearing the moment serving exists. + servable: policy.servable, }); return { diff --git a/tests/unit/fitPolicy.test.js b/tests/unit/fitPolicy.test.js new file mode 100644 index 0000000..b6e98d7 --- /dev/null +++ b/tests/unit/fitPolicy.test.js @@ -0,0 +1,111 @@ +'use strict'; + +/** + * ARTIFACT IDENTITY IS NOT FIT-POLICY CERTIFICATION. + * + * A digest says WHICH mapping ran. It says nothing about whether the procedure + * that produced it should be trusted to produce tomorrow's. + */ +const fp = require('../../src/services/model/fitPolicy'); +const svc = require('../../src/services/model/probabilityContractService'); +const cal = require('../../src/services/model/calibration'); + +const ERA = 'engine1@2026-08-07-fullwindow'; +const artifact = (over = {}) => ({ + estimator_type: 'isotonic', model_version: ERA, fit_n: 6084, + training_cutoff: '2026-08-21', knot_digest: 'd9d571d728ba76de', + served_curve_digest: '9d731cd88ba073d7', + certified_bands: [[0.50, 0.80]], ...over, +}); + +describe('the declared procedure', () => { + it('pins every decision that would otherwise be implicit', () => { + // If any of these change, they change DELIBERATELY and this test says so. + expect(fp.POLICY_V1.policy_version).toBe('mlb-hits-isotonic-refit@v1'); + expect(fp.POLICY_V1.sport).toBe('mlb'); + expect(fp.POLICY_V1.stat).toBe('hits'); + expect(fp.POLICY_V1.model_version).toBe(ERA); + expect(fp.POLICY_V1.fit_algorithm).toBe('isotonic-pav'); + expect(fp.POLICY_V1.min_fit_rows).toBe(200); + expect(fp.POLICY_V1.data_selection.horizon).toBe('game_date < fit_as_of'); + expect(fp.POLICY_V1.support_contract.certified_bands).toEqual([[0.50, 0.80]]); + expect(fp.POLICY_V1.support_contract.may_be_widened_by_refit).toBe(false); + }); + + it('a refit may never widen the region it is trusted in', () => { + const r = fp.validate(artifact({ certified_bands: [[0.50, 0.95]] }), + { era_counts: { [ERA]: 6084 } }); + expect(r.valid).toBe(false); + expect(r.violations).toContain(fp.VIOLATION.SUPPORT_WIDENED); + expect(r.servable).toBe(false); + }); +}); + +describe('the minimum automatic gates', () => { + it('passes only an era-restricted fit', () => { + const r = fp.validate(artifact(), { era_counts: { [ERA]: 6084 } }); + expect(r.valid).toBe(true); + expect(r.servable).toBe(true); + }); + + it('REFUSES the production era mix — the measured state today', () => { + const r = fp.validate(artifact(), { era_counts: { [ERA]: 2792, 'engine1@2026-07-20': 3292 } }); + expect(r.valid).toBe(false); + expect(r.violations).toContain(fp.VIOLATION.ERA_NOT_RESTRICTED); + expect(r.servable).toBe(false); + }); + + it('"we did not check" is a violation, not a pass', () => { + // An unverified restriction is exactly the state production is in. + const r = fp.validate(artifact(), {}); + expect(r.valid).toBe(false); + expect(r.violations).toContain(fp.VIOLATION.ERA_NOT_RESTRICTED); + }); + + it('catches a missing identity, a wrong era, a wrong estimator and a thin fit', () => { + const ev = { era_counts: { [ERA]: 6084 } }; + expect(fp.validate(artifact({ knot_digest: null }), ev).violations).toContain(fp.VIOLATION.NO_ARTIFACT_IDENTITY); + expect(fp.validate(artifact({ training_cutoff: null }), ev).violations).toContain(fp.VIOLATION.NO_TRAINING_CUTOFF); + expect(fp.validate(artifact({ model_version: 'engine1@2026-07-20' }), ev).violations).toContain(fp.VIOLATION.ERA_MISMATCH); + expect(fp.validate(artifact({ estimator_type: 'low_param' }), ev).violations).toContain(fp.VIOLATION.ESTIMATOR_MISMATCH); + expect(fp.validate(artifact({ fit_n: 12 }), ev).violations).toContain(fp.VIOLATION.INSUFFICIENT_FIT_ROWS); + }); + + it('a null artifact is refused, never treated as vacuously valid', () => { + expect(fp.validate(null).valid).toBe(false); + expect(fp.validate(null).servable).toBe(false); + }); + + it('servable is never true while any violation stands — there is no override', () => { + for (const bad of [{ fit_n: 1 }, { model_version: 'x' }, { estimator_type: 'y' }, { knot_digest: null }]) { + expect(fp.validate(artifact(bad), { era_counts: { [ERA]: 6084 } }).servable).toBe(false); + } + expect(fp.validate.length).toBeLessThanOrEqual(3); // no force/override argument + }); +}); + +describe('the built artifact carries its policy state', () => { + const map = cal.fitIsotonic(Array.from({ length: 900 }, (_, i) => { + const p = Math.round((0.35 + (i % 60) / 100) * 1000) / 1000; + return { p, won: ((i * 2654435761) % 1000) / 1000 < (0.5 + 0.45 * (p - 0.5)) ? 1 : 0, date: `d${i % 14}` }; + }), { minTotal: 200 }); + const build = () => svc.build({}, { calibrationService: { fromLedger: async () => ({ + map, fit_n: 900, fitted_through: '2026-08-21', cutoff: '2026-09-02' }) } }); + + it('records the policy version, the violation and servable:false', async () => { + const b = await build(); + expect(b.artifact.fit_policy_version).toBe('mlb-hits-isotonic-refit@v1'); + expect(b.artifact.fit_policy_valid).toBe(false); + expect(b.artifact.fit_policy_violations).toContain(fp.VIOLATION.ERA_NOT_RESTRICTED); + expect(b.artifact.servable).toBe(false); + }); + + it('the policy violation does NOT distort what the shadow measures', async () => { + // The shadow's job is to measure the certified contract on real rows. If a + // policy violation flipped every row to UNCERTIFIED the shadow would + // measure the violation instead, and the receipt would be worthless. + const b = await build(); + expect(b.resolve({ model_version: ERA, p_win: 0.65 }).probability_state).toBe('CERTIFIED_CALIBRATED'); + expect(b.resolve({ model_version: ERA, p_win: 0.91 }).probability_state).toBe('UNCERTIFIED'); + }); +});