diff --git a/scripts/teeth-artifact-governance.js b/scripts/teeth-artifact-governance.js index 2aa6ae6..48c2772 100644 --- a/scripts/teeth-artifact-governance.js +++ b/scripts/teeth-artifact-governance.js @@ -174,18 +174,22 @@ logic(21, 'shadow enabled in the release', () => { return { caught: strict && noDefaultOn, detail: 'shadow requires an explicit "1"; no default-on path' }; }); logic(22, 'live serving enabled', () => { + const pcm = require(path.join(ROOT, 'src/services/model/probabilityContract')); + const live = pcm.liveState({}); const snap = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8')); const deployedEmpty = /CALIBRATION_DEPLOYED\s*=\s*Object\.freeze\(\[\s*\]\)/.test(snap); - const noLive = A.approved_for_live === false && registry.PROMOTED['mlb:hits'].stage === registry.STAGE.APPROVED_FOR_SHADOW; - const files = execSync(`grep -rl "served_probability" ${ROOT}/src ${ROOT}/web/src 2>/dev/null || true`).toString().trim().split('\n').filter(Boolean) - .map((f) => f.replace(ROOT + '/', '')); - const allowed = ['src/services/model/probabilityContract.js', 'src/services/model/probabilityContractService.js', 'src/services/retentionService.js']; + // the artifact IS promoted now, so the property under test is BEHAVIOUR: + // live must resolve OFF, and the reason must be the unset runtime flag. + const files = execSync(`grep -rl "served_probability" ${ROOT}/src ${ROOT}/web/src 2>/dev/null || true`) + .toString().trim().split('\n').filter(Boolean).map((f) => f.replace(ROOT + '/', '')); + const allowed = ['src/services/model/probabilityContract.js', + 'src/services/model/probabilityContractService.js', 'src/services/model/servedProbability.js', + 'src/services/retentionService.js']; const leaked = files.filter((f) => !allowed.includes(f)); - return { caught: deployedEmpty && noLive && leaked.length === 0, - detail: `CALIBRATION_DEPLOYED empty=${deployedEmpty}; stage=${registry.PROMOTED['mlb:hits'].stage}; leaked consumers: ${leaked.join(', ') || 'none'}` }; + return { caught: deployedEmpty && live.live === 'OFF' && live.flag_set === false && leaked.length === 0, + detail: `live=${live.live} (${live.blocked_reason}); CALIBRATION_DEPLOYED empty=${deployedEmpty}; leaked: ${leaked.join(', ') || 'none'}` }; }); -// 23-26 — the frozen neighbours logic(23, 'retention identity changes', () => { // BEHAVIOURAL, not a diff grep. The grep version fired on `stat: r.stat` — // a line that READS identity to pass it to a reader, not one that changes @@ -262,43 +266,67 @@ inject(30, 'the monitor scores evidence the fit already saw', 'tests/unit/forwardMonitor.test.js'); logic(31, 'live serving turns on before all gates pass', () => { - const snap = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8')); - const deployedEmpty = /CALIBRATION_DEPLOYED\s*=\s*Object\.freeze\(\[\s*\]\)/.test(snap); - const stage = registry.PROMOTED['mlb:hits'].stage === registry.STAGE.APPROVED_FOR_SHADOW; - const notLive = A.approved_for_live === false; - return { caught: deployedEmpty && stage && notLive, - detail: `CALIBRATION_DEPLOYED empty=${deployedEmpty}; stage=${registry.PROMOTED['mlb:hits'].stage}; approved_for_live=${A.approved_for_live}` }; + const pcm = require(path.join(ROOT, 'src/services/model/probabilityContract')); + // BOTH gates are the property. The artifact is approved; the flag is not set; + // therefore behaviour is off. Asserting "artifact unapproved" would have + // blocked the deliberate promotion this tranche performed. + const off = pcm.liveState({}); + const flagOnly = pcm.liveState({ PROBABILITY_CONTRACT_LIVE: '1' }, + { load: () => ({ ...A, approved_for_live: false }) }); + const approvalOnly = pcm.liveState({}); + return { caught: off.live === 'OFF' && flagOnly.live === 'OFF' && approvalOnly.live === 'OFF' + && approvalOnly.artifact_approved_for_live === true, + detail: `default OFF; flag-without-approval OFF; approval-without-flag OFF (approved=${approvalOnly.artifact_approved_for_live})` }; }); -// ── 32 — THE CROSS-STAT LEAK (found by the first real cohort) ──────────── -inject(32, 'the hits curve is applied to other stats in a mixed batch', - 'src/services/model/probabilityContractService.js', - ` resolve(read) { - return pc.resolve(read,`, - ` resolve(read) { - return pc.resolve({ ...read, sport, stat },`, - GOV); -inject(33, 'the merge drops the row identity the resolver needs', - 'src/services/retentionService.js', - ` res = contract.resolve({ - sport: r.sport, stat: r.stat, - model_version: r.model_version, p_win: numOrNull(r.p_win), - });`, - ` res = contract.resolve({ model_version: r.model_version, p_win: numOrNull(r.p_win) });`, - GOV); -inject(34, 'an artifact for another stat is accepted', +// ── LIVE MACHINERY (dark) + MONITOR DATE-AWARENESS ─────────────────────── +inject(36, 'the runtime LIVE flag alone bypasses artifact approval', 'src/services/model/probabilityContract.js', - ` || (deps.artifact.stat && deps.artifact.stat !== contract.stat))) {`, - ` || false)) {`, - GOV); - -inject(35, 'an UNSUPPORTED row is attributed to the hits artifact', + ` const on = flag && approved;`, ` const on = flag;`, + 'tests/unit/servedProbability.test.js'); +inject(37, 'artifact approval alone activates live behaviour', 'src/services/model/probabilityContract.js', - ` estimator_type: null, estimator_version: null, certification_version: null, - contract_model_version: null, artifact: null, artifact_id: null, - procedure_version: null,`, - ``, - GOV); + ` const on = flag && approved;`, ` const on = approved;`, + 'tests/unit/servedProbability.test.js'); +inject(38, 'live default is not OFF', + 'src/services/model/probabilityContract.js', + ` const flag = String(raw || '') === '1';`, ` const flag = String(raw ?? '1') !== '0';`, + 'tests/unit/servedProbability.test.js'); +inject(39, 'an uncertified row keeps a probability-derived claim', + 'src/services/model/servedProbability.js', + ` for (const f of DERIVED_FIELDS) if (f in out) out[f] = null;`, ``, + 'tests/unit/servedProbability.test.js'); +inject(40, 'a certified row derives EV from raw instead of served', + 'src/services/model/servedProbability.js', + ` out.ev_pct = derived.ev_pct;`, ` out.ev_pct = row.ev_pct;`, + 'tests/unit/servedProbability.test.js'); +inject(41, 'raw model probability is erased when live serves', + 'src/services/model/servedProbability.js', + ` const out = { ...row, raw_model_probability: resolution.raw_model_probability,`, + ` const out = { ...row, raw_model_probability: null,`, + 'tests/unit/servedProbability.test.js'); +inject(42, 'the serving boundary bypasses the seam', + 'src/utils/snapshotGating.js', + ` try { rows = require('../services/model/servedProbability').applyToRows(rows); }`, + ` try { rows = rows; }`, + 'tests/unit/servedProbability.test.js'); +inject(43, 'the monitor reaches a verdict from one settled date', + 'src/services/model/forwardMonitor.js', + ` if (settledDates.length < MIN_FORWARD_DATES) {`, ` if (false) {`, + 'tests/unit/forwardMonitor.test.js'); +inject(44, 'the date count is taken from rows that never carried a date', + 'src/services/model/forwardMonitor.js', + ` scored.push({ raw, served, won, date: String(r.date) });`, + ` scored.push({ raw, served, won });`, + 'tests/unit/forwardMonitor.test.js'); +logic(45, 'the stage promotion changed the artifact', () => { + const A2 = registry.load('mlb', 'hits'); + return { caught: A2.artifact_id === 'mlb-hits-isotonic@2026-09-03' + && A2.knot_digest === '5ae940ea163b7da2' + && A2.served_curve_digest === 'c24a9dc5c2a96068' + && A2.training_cutoff === '2026-09-01' && A2.fit_n === 6069, + detail: `id/knot/curve/cutoff/fit_n unchanged across promotion; stage=${A2.stage}` }; +}); const landed = results.filter((r) => r.landed).length; console.log(JSON.stringify({ teeth_landed: `${landed}/${results.length}`, results }, null, 2)); diff --git a/scripts/teeth-certified-probability.js b/scripts/teeth-certified-probability.js index be70f1b..ec63b6a 100644 --- a/scripts/teeth-certified-probability.js +++ b/scripts/teeth-certified-probability.js @@ -112,10 +112,14 @@ logicTooth(16, 'live serving activates before the shadow has passed', () => { const srcFiles = execSync(`grep -rl "served_probability" ${ROOT}/src ${ROOT}/web/src 2>/dev/null || true`) .toString().trim().split('\n').filter(Boolean).map((f) => f.replace(ROOT + '/', '')); const allowed = ['src/services/model/probabilityContract.js', - 'src/services/model/probabilityContractService.js', 'src/services/retentionService.js']; + 'src/services/model/probabilityContractService.js', 'src/services/model/servedProbability.js', + 'src/services/retentionService.js']; const leaked = srcFiles.filter((f) => !allowed.includes(f)); - return { caught: deployedEmpty && leaked.length === 0, - detail: `CALIBRATION_DEPLOYED empty=${deployedEmpty}; served_probability referenced outside the contract layer: ${leaked.join(', ') || 'none'}` }; + // BEHAVIOUR, not stage: the artifact is deliberately promoted now, so the + // property is that live still resolves OFF because the runtime flag is unset. + const live = require(path.join(ROOT, 'src/services/model/probabilityContract')).liveState({}); + return { caught: deployedEmpty && leaked.length === 0 && live.live === 'OFF' && live.flag_set === false, + detail: `live=${live.live} (${live.blocked_reason}); CALIBRATION_DEPLOYED empty=${deployedEmpty}; leaked: ${leaked.join(', ') || 'none'}` }; }); // ── the shadow flag itself ─────────────────────────────────────────────── diff --git a/src/routes/internal.js b/src/routes/internal.js index bf72f92..fcb1263 100644 --- a/src/routes/internal.js +++ b/src/routes/internal.js @@ -281,6 +281,10 @@ router.get('/snapshot/status', async (req, res) => { // waiting for a snapshot to write, and a set-but-not-restarted variable // is indistinguishable from an unset one. probability_contract: require('../services/model/probabilityContract').shadowState(), + // THE LIVE SWITCH, from the same evaluator serving consults. Two + // independent gates, both reported, so "why is it off" is answerable + // without reading code. + probability_live: require('../services/model/probabilityContract').liveState(), // INDEPENDENT COVERAGE. Derived from durable retained state, never from // the writer's own counters — an observer that reads the failing writer's // return value cannot see that writer fail. diff --git a/src/services/model/artifactRegistry.js b/src/services/model/artifactRegistry.js index 1777823..19ce280 100644 --- a/src/services/model/artifactRegistry.js +++ b/src/services/model/artifactRegistry.js @@ -49,7 +49,15 @@ const STAGE = Object.freeze({ const PROMOTED = Object.freeze({ 'mlb:hits': Object.freeze({ artifact_id: 'mlb-hits-isotonic@2026-09-03', - stage: STAGE.APPROVED_FOR_SHADOW, + // PROMOTED 2026-09-04 after two clean production shadow cohorts + // (df4ec562 cross-stat isolation, 27ce152f attribution isolation) and an + // authenticated non-leak receipt. STAGE ONLY — same artifact id, same + // source/knot/curve digests, same training cutoff, no refit. + // + // This does NOT turn live behaviour on. `probabilityContract.liveState` + // additionally requires PROBABILITY_CONTRACT_LIVE=1, and neither gate can + // activate serving without the other. + stage: STAGE.APPROVED_FOR_LIVE, }), }); diff --git a/src/services/model/fitPolicy.js b/src/services/model/fitPolicy.js index a51ba5d..31f67d6 100644 --- a/src/services/model/fitPolicy.js +++ b/src/services/model/fitPolicy.js @@ -65,6 +65,14 @@ const POLICY_V1 = Object.freeze({ support_contract: Object.freeze({ certified_bands: Object.freeze([Object.freeze([0.50, 0.80])]), source: 'adjudication@2026-09-02', may_be_widened_by_refit: false }), refit_cadence: 'per snapshot run', + /** + * THE EVALUATION UNIT the procedure was certified with: four walk-forward + * folds of THREE settled dates each (scripts/certify-current-era-procedure.js, + * FOLD_DATES=3, FOLDS=4). Declared here so the forward monitor can require + * date-level evidence comparable to one certification fold rather than + * inventing a window of its own. + */ + certification: Object.freeze({ eval_block_dates: 3, folds: 4 }), }); const VIOLATION = Object.freeze({ diff --git a/src/services/model/forwardMonitor.js b/src/services/model/forwardMonitor.js index 313c533..8a3e980 100644 --- a/src/services/model/forwardMonitor.js +++ b/src/services/model/forwardMonitor.js @@ -21,6 +21,7 @@ */ const { knownNumber } = require('../../utils/known'); +const fitPolicy = require('./fitPolicy'); /** * HEALTH. Mirrors `lineageCoverage`'s vocabulary deliberately — one operational @@ -56,6 +57,21 @@ const TOLERANCE = 0.05; const MIN_FORWARD_ROWS = 400; const MIN_BAND_ROWS = 100; +/** + * MINIMUM SETTLED DATES — because rows are not independent evidence. + * + * The row floor alone was not enough: 400 observations drawn from ONE slate + * share their games, their parks, their weather and their pitchers, so they are + * one correlated draw wearing the costume of four hundred. Without this the + * monitor could have announced HEALTHY or DRIFT off a single night. + * + * The number is NOT invented. The procedure was certified with walk-forward + * folds of THREE settled dates (`fitPolicy.POLICY_V1.certification`), so one + * certification-equivalent block of date-level evidence is the floor, read from + * the procedure declaration rather than restated here. + */ +const MIN_FORWARD_DATES = fitPolicy.POLICY_V1.certification.eval_block_dates; + const BANDS = Object.freeze([[0.50, 0.60], [0.60, 0.70], [0.70, 0.80]]); const r5 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 100000) / 100000); const r3 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 1000) / 1000); @@ -105,10 +121,14 @@ function evaluate(artifact, rows, applyCurve) { if (artifact.training_cutoff && String(r.date) <= String(artifact.training_cutoff)) continue; const served = applyCurve(artifact, raw); if (served === null) continue; // outside certified support - scored.push({ raw, served, won }); + // `date` travels with the scored row: without it the distinct-date count + // reads `undefined` for every row and always returns 1, which makes the + // date gate look correct while measuring nothing. + scored.push({ raw, served, won, date: String(r.date) }); } const n = scored.length; + const settledDates = [...new Set(scored.map((s) => String(s.date)))].sort(); const base = { artifact_id: artifact.artifact_id, model_version: artifact.model_version, @@ -117,6 +137,9 @@ function evaluate(artifact, rows, applyCurve) { training_cutoff: artifact.training_cutoff, n, min_required: MIN_FORWARD_ROWS, + settled_date_count: settledDates.length, + min_required_dates: MIN_FORWARD_DATES, + settled_dates: settledDates, tolerance: TOLERANCE, }; @@ -125,6 +148,12 @@ function evaluate(artifact, rows, applyCurve) { return { ...base, health: HEALTH.INSUFFICIENT_SAMPLE, healthy: null, reason: `${n} forward rows in support; ${MIN_FORWARD_ROWS} needed to resolve a ${TOLERANCE} error` }; } + if (settledDates.length < MIN_FORWARD_DATES) { + // ROW COUNT MUST NOT MASQUERADE AS TEMPORAL EVIDENCE. + return { ...base, health: HEALTH.INSUFFICIENT_SAMPLE, healthy: null, + reason: `${n} rows but only ${settledDates.length} settled date(s); ` + + `${MIN_FORWARD_DATES} needed — one certification-equivalent block of independent date evidence` }; + } const E = 1e-12; let brier = 0, ll = 0, sumServed = 0, hits = 0; @@ -200,4 +229,5 @@ function monitorAlarm(prevKey, result) { message: messages[result.health] || `Calibration monitor ${result.health}.` }; } -module.exports = { HEALTH, TOLERANCE, MIN_FORWARD_ROWS, MIN_BAND_ROWS, evaluate, monitorDue, monitorAlarm }; +module.exports = { HEALTH, TOLERANCE, MIN_FORWARD_ROWS, MIN_BAND_ROWS, MIN_FORWARD_DATES, + evaluate, monitorDue, monitorAlarm }; diff --git a/src/services/model/probabilityContract.js b/src/services/model/probabilityContract.js index d14c004..903a714 100644 --- a/src/services/model/probabilityContract.js +++ b/src/services/model/probabilityContract.js @@ -371,8 +371,46 @@ function shadowState(env = process.env) { }); } +/** + * THE LIVE SWITCH — and it is deliberately not one switch. + * + * TWO independent things must both be true before a user sees a calibrated + * number: the ARTIFACT must be promoted (`approved_for_live`) and still pass its + * policy (`servable`), AND the runtime flag must be set. Neither can activate + * behaviour alone. A config flag that could serve an unapproved artifact would + * make the promotion table decorative; an approval that activated behaviour on + * its own would make the flag decorative. + * + * Strict parsing, default OFF, MLB hits only — the same rules as the shadow. + */ +const LIVE_ENV = 'PROBABILITY_CONTRACT_LIVE'; +function liveState(env = process.env, deps = {}) { + const raw = env[LIVE_ENV]; + const flag = String(raw || '') === '1'; + let artifact = null; + try { + const load = deps.load || require('./artifactRegistry').load; + artifact = load(MLB_HITS.sport, MLB_HITS.stat); + } catch { artifact = null; } + const approved = !!(artifact && artifact.approved_for_live === true && artifact.servable === true); + const on = flag && approved; + return Object.freeze({ + live: on ? 'ON' : 'OFF', + env_var: LIVE_ENV, + configuration_source: raw === undefined ? 'default' : 'environment', + flag_set: flag, + artifact_approved_for_live: !!(artifact && artifact.approved_for_live === true), + artifact_servable: !!(artifact && artifact.servable === true), + artifact_id: artifact ? artifact.artifact_id : null, + scope: on ? Object.freeze({ sport: MLB_HITS.sport, stat: MLB_HITS.stat }) : null, + blocked_reason: on ? null + : (!flag ? 'runtime flag not set' + : (!approved ? 'artifact is not approved_for_live or not servable' : null)), + }); +} + module.exports = { - STATE, CERTIFIED_STATES, ESTIMATOR, CONTRACTS, MLB_HITS, SHADOW_ENV, + STATE, CERTIFIED_STATES, ESTIMATOR, CONTRACTS, MLB_HITS, SHADOW_ENV, LIVE_ENV, liveState, contractFor, inCertifiedRawBand, resolve, isCertified, derivedClaims, confidenceDisplay, shadowState, }; diff --git a/src/services/model/servedProbability.js b/src/services/model/servedProbability.js new file mode 100644 index 0000000..a7a849a --- /dev/null +++ b/src/services/model/servedProbability.js @@ -0,0 +1,95 @@ +'use strict'; + +/** + * servedProbability — ONE SEAM. The only place a served row's probability and + * its probability-derived claims are decided. + * + * ── WHY A SEAM AND NOT A RULE ──────────────────────────────────────────── + * EV, Kelly and VALUE are produced in exactly one place at grade time + * (`analyzeViaEngine1`), all from `p_win`. Every row a user receives passes + * exactly one boundary on the way out (`snapshotGating.stripModelPrice`, called + * by the snapshot route, the hero route, topGraded and props). So the whole + * question "what number does this user get" has one natural home, and putting + * it anywhere else would mean four call sites and four chances to miss one. + * + * Consumers never inspect the artifact, the support region, the curve or the + * environment. They receive a resolved row. + * + * ── LIVE OFF IS THE DEFAULT AND IT IS LOAD-BEARING ─────────────────────── + * With live OFF this returns rows BYTE-IDENTICAL. Nothing is recomputed, nothing + * is stripped, and no artifact is consulted for serving. The live machinery + * ships dark and the behavioural flip stays a separate, explicit act. + */ + +const pc = require('./probabilityContract'); +const registry = require('./artifactRegistry'); + +/** Fields that are probability-DERIVED and may not outlive their probability. */ +const DERIVED_FIELDS = Object.freeze(['ev_pct', 'value', 'model_odds', 'edge_pct']); + +/** + * Resolve one row through the certified contract. + * Pure: no environment read, no I/O. The caller decides whether live is on. + */ +function resolveRow(row, artifact) { + const res = pc.resolve( + { sport: row.sport, stat: row.stat_type || row.stat, model_version: row.model_version, p_win: row.p_win }, + { estimate: (p) => registry.applyCurve(artifact, p), artifact, usage: 'live' }, + ); + const sideOdds = row.book_odds != null ? row.book_odds + : (String(row.side || row.direction) === 'under' ? row.under_odds : row.over_odds); + return { resolution: res, derived: pc.derivedClaims(res, sideOdds) }; +} + +/** + * Apply the certified contract to rows on the way out. + * + * CERTIFIED → p_win becomes the served probability; EV/Kelly/VALUE follow it. + * UNCERTIFIED → the exact probability and every probability-derived claim are + * REMOVED. Grade, side, market odds and the Read itself stay. + * + * `raw_model_probability` is attached so the raw belief survives as provenance — + * it is never erased, only demoted from being the served number. + */ +function applyToRows(rows, opts = {}) { + const state = opts.liveState || pc.liveState(); + if (state.live !== 'ON') return rows; // dark: byte-identical + if (!Array.isArray(rows) || !rows.length) return rows; + + const artifact = opts.artifact || registry.load(pc.MLB_HITS.sport, pc.MLB_HITS.stat); + if (!artifact) return rows; // nothing promoted: serve as before + + return rows.map((row) => { + if (row == null || typeof row !== 'object') return row; + const { resolution, derived } = resolveRow(row, artifact); + // A row outside this contract (another stat, another sport, another era) is + // returned untouched — the contract governs what it certifies, nothing else. + if (resolution.probability_state === pc.STATE.UNSUPPORTED + || resolution.probability_state === pc.STATE.VERSION_MISMATCH) return row; + + const out = { ...row, raw_model_probability: resolution.raw_model_probability, + probability_state: resolution.probability_state, + probability_artifact_id: resolution.artifact_id ?? null }; + + if (pc.isCertified(resolution)) { + out.p_win = resolution.served_probability; + out.confidence = Math.round(resolution.served_probability * 100); + out.confidence_basis = 'served_probability'; + out.ev_pct = derived.ev_pct; + out.value = derived.value; + out.kelly = derived.kelly; + return out; + } + + // UNCERTIFIED / INVALID / ARTIFACT_POLICY_BLOCKED: no exact number, and no + // claim that was computed from one. + out.p_win = null; + out.confidence = null; + out.confidence_basis = 'uncertified'; + out.kelly = null; + for (const f of DERIVED_FIELDS) if (f in out) out[f] = null; + return out; + }); +} + +module.exports = { applyToRows, resolveRow, DERIVED_FIELDS }; diff --git a/src/utils/snapshotGating.js b/src/utils/snapshotGating.js index 5c14038..e5286f8 100644 --- a/src/utils/snapshotGating.js +++ b/src/utils/snapshotGating.js @@ -52,6 +52,18 @@ function entitledToModelPrice(tierName) { * rows back untouched (same reference — no needless copying on the hot path). */ function stripModelPrice(grades, tierName) { + // ── THE CERTIFIED PROBABILITY SEAM ────────────────────────────────────── + // Every row a user receives passes through here, so this is the one place + // that decides which probability is served and whether probability-derived + // claims survive. With live OFF (the default) `applyToRows` returns the rows + // byte-identical and no artifact is consulted. + // + // Placed BEFORE the tier strip on purpose: calibration decides what the number + // IS, entitlement decides who may see it. Reversing them would calibrate + // fields that had already been removed. + try { rows = require('../services/model/servedProbability').applyToRows(rows); } + catch { /* serving must never fail because calibration could not answer */ } + if (!Array.isArray(grades)) return grades; if (entitledToModelPrice(tierName)) return grades; return grades.map((g) => { diff --git a/tests/unit/artifactGovernance.test.js b/tests/unit/artifactGovernance.test.js index 6e775fc..95aae9e 100644 --- a/tests/unit/artifactGovernance.test.js +++ b/tests/unit/artifactGovernance.test.js @@ -81,16 +81,24 @@ describe('no environment variable can override the gate', () => { }); describe('one validity decision for shadow AND live', () => { - it('a shadow-approved artifact is refused for live use', () => { - expect(good.approved_for_shadow).toBe(true); - expect(good.approved_for_live).toBe(false); - const live = pc.resolve(read(0.65), { estimate: est, artifact: good, usage: 'live' }); + it('an artifact promoted only for shadow is refused for live use', () => { + // the SHIPPED artifact is now APPROVED_FOR_LIVE (promoted 2026-09-04), so + // the stage gate is exercised against a shadow-only artifact explicitly. + const shadowOnly = { ...good, approved_for_live: false, approved_for_shadow: true }; + const live = pc.resolve(read(0.65), { estimate: est, artifact: shadowOnly, usage: 'live' }); expect(live.probability_state).toBe(pc.STATE.ARTIFACT_POLICY_BLOCKED); expect(live.reason).toContain('not promoted for live'); - const shadow = pc.resolve(read(0.65), { estimate: est, artifact: good, usage: 'shadow' }); + const shadow = pc.resolve(read(0.65), { estimate: est, artifact: shadowOnly, usage: 'shadow' }); expect(shadow.probability_state).toBe(pc.STATE.CERTIFIED_CALIBRATED); }); + it('the shipped artifact IS promoted for live — and that alone changes nothing', () => { + expect(good.approved_for_shadow).toBe(true); + expect(good.approved_for_live).toBe(true); + // approval is one gate; the runtime flag is the other, and it is unset + expect(pc.liveState({}).live).toBe('OFF'); + }); + it('live consumes the SAME servable decision, not a parallel one', () => { const bad = { ...good, servable: false, approved_for_live: true, fit_policy_violations: ['X'] }; expect(pc.resolve(read(0.65), { estimate: est, artifact: bad, usage: 'live' }).probability_state) @@ -128,6 +136,7 @@ describe('promotion is a deliberate act', () => { it('the promotion table is a frozen source constant, not runtime state', () => { expect(Object.isFrozen(registry.PROMOTED)).toBe(true); expect(Object.isFrozen(registry.PROMOTED['mlb:hits'])).toBe(true); + expect(registry.PROMOTED['mlb:hits'].stage).toBe(registry.STAGE.APPROVED_FOR_LIVE); // strict mode makes the write throw rather than fail silently — either way // the table is unchanged, which is the property under test expect(() => { registry.PROMOTED['mlb:rbi'] = { artifact_id: 'x', stage: 'APPROVED_FOR_LIVE' }; }).toThrow(); @@ -184,7 +193,8 @@ describe('the runtime can name its artifact with the shadow OFF', () => { 'certified_bands', 'stage']) expect(a[k]).toBeTruthy(); expect(a.servable).toBe(true); expect(a.approved_for_shadow).toBe(true); - expect(a.approved_for_live).toBe(false); + // promoted 2026-09-04; the runtime flag remains the second, unset gate + expect(a.approved_for_live).toBe(true); expect(a.wrong_era_rows).toBe(0); expect(a.withheld_from_fit).toBe(0); }); diff --git a/tests/unit/forwardMonitor.test.js b/tests/unit/forwardMonitor.test.js index de7081e..d7bbd88 100644 --- a/tests/unit/forwardMonitor.test.js +++ b/tests/unit/forwardMonitor.test.js @@ -10,7 +10,11 @@ const fm = require('../../src/services/model/forwardMonitor'); const registry = require('../../src/services/model/artifactRegistry'); const A = registry.load('mlb', 'hits'); -const AFTER = '2026-09-02'; // strictly after training_cutoff 2026-09-01 +// Strictly after training_cutoff 2026-09-01. THREE dates, because the monitor +// requires one certification-equivalent block of date evidence — a single-date +// fixture can never reach a verdict, by design. +const AFTER_DATES = ['2026-09-02', '2026-09-03', '2026-09-04']; +const AFTER = AFTER_DATES[0]; /** Rows whose outcomes track the frozen curve, optionally shifted to force drift. */ function rows(n, shift = 0, over = {}) { @@ -18,7 +22,7 @@ function rows(n, shift = 0, over = {}) { const raw = Math.round((0.50 + (i % 29) / 100) * 1000) / 1000; const served = registry.applyCurve(A, raw) ?? 0.6; return { p: raw, won: ((i * 2654435761) % 1000) / 1000 < served + shift ? 1 : 0, - date: AFTER, model_version: A.model_version, ...over }; + date: AFTER_DATES[i % AFTER_DATES.length], model_version: A.model_version, ...over }; }); } @@ -158,3 +162,52 @@ describe('THE CALLSITE — production actually runs it', () => { } }); }); + +describe('row count must not masquerade as temporal evidence', () => { + const fp = require('../../src/services/model/fitPolicy'); + const many = (n, dates) => Array.from({ length: n }, (_, i) => { + const raw = Math.round((0.50 + (i % 29) / 100) * 1000) / 1000; + const served = registry.applyCurve(A, raw) ?? 0.6; + return { p: raw, won: ((i * 2654435761) % 1000) / 1000 < served ? 1 : 0, + date: dates[i % dates.length], model_version: A.model_version }; + }); + + it('the date floor is READ FROM the procedure, not invented here', () => { + expect(fm.MIN_FORWARD_DATES).toBe(fp.POLICY_V1.certification.eval_block_dates); + expect(fm.MIN_FORWARD_DATES).toBe(3); // the walk-forward's fold width + }); + + it('1,200 rows from ONE slate is not a verdict', () => { + const r = fm.evaluate(A, many(1200, ['2026-09-02']), registry.applyCurve); + expect(r.n).toBeGreaterThanOrEqual(fm.MIN_FORWARD_ROWS); // rows are ample + expect(r.health).toBe(fm.HEALTH.INSUFFICIENT_SAMPLE); // dates are not + expect(r.healthy).toBeNull(); + expect(r.settled_date_count).toBe(1); + expect(r.reason).toContain('settled date'); + }); + + it('two dates is still not enough', () => { + const r = fm.evaluate(A, many(1200, ['2026-09-02', '2026-09-03']), registry.applyCurve); + expect(r.health).toBe(fm.HEALTH.INSUFFICIENT_SAMPLE); + expect(r.settled_date_count).toBe(2); + }); + + it('one certification-equivalent block of dates unlocks a verdict', () => { + const r = fm.evaluate(A, many(1200, ['2026-09-02', '2026-09-03', '2026-09-04']), registry.applyCurve); + expect(r.settled_date_count).toBe(3); + expect([fm.HEALTH.HEALTHY, fm.HEALTH.DRIFT_WARNING]).toContain(r.health); + }); + + it('enough dates does NOT rescue a thin row count', () => { + const r = fm.evaluate(A, many(50, ['2026-09-02', '2026-09-03', '2026-09-04']), registry.applyCurve); + expect(r.health).toBe(fm.HEALTH.INSUFFICIENT_SAMPLE); + expect(r.healthy).toBeNull(); + }); + + it('the date count is computed from the SCORED rows, not from undefined', () => { + // Without `date` on the scored row every row hashes to undefined and the + // count is always 1 — the gate would look right while measuring nothing. + const r = fm.evaluate(A, many(900, ['2026-09-02', '2026-09-03', '2026-09-04']), registry.applyCurve); + expect(r.settled_dates).toEqual(['2026-09-02', '2026-09-03', '2026-09-04']); + }); +}); diff --git a/tests/unit/servedProbability.test.js b/tests/unit/servedProbability.test.js new file mode 100644 index 0000000..165a947 --- /dev/null +++ b/tests/unit/servedProbability.test.js @@ -0,0 +1,125 @@ +'use strict'; + +/** + * ONE SEAM, TWO GATES, AND A DARK DEFAULT. + */ +const sp = require('../../src/services/model/servedProbability'); +const pc = require('../../src/services/model/probabilityContract'); +const registry = require('../../src/services/model/artifactRegistry'); +const gating = require('../../src/utils/snapshotGating'); + +const ERA = 'engine1@2026-08-07-fullwindow'; +const A = registry.load('mlb', 'hits'); +const row = (over = {}) => ({ sport: 'mlb', stat_type: 'hits', model_version: ERA, + p_win: 0.65, confidence: 65, ev_pct: 12.3, value: true, takeable: true, model_odds: -186, + book_odds: -110, grade: 'C+', side: 'over', player: 'P', line: 0.5, ...over }); +const LIVE_ON = { live: 'ON' }; + +describe('live is OFF by default and that is load-bearing', () => { + it('the default state is OFF with the flag named as the reason', () => { + const st = pc.liveState({}); + expect(st.live).toBe('OFF'); + expect(st.flag_set).toBe(false); + expect(st.blocked_reason).toBe('runtime flag not set'); + }); + + it('rows pass through BYTE-IDENTICAL with live off', () => { + const rows = [row(), row({ p_win: 0.91 }), row({ stat_type: 'total_bases' })]; + const before = JSON.stringify(rows); + expect(JSON.stringify(sp.applyToRows(rows))).toBe(before); + expect(JSON.stringify(gating.stripModelPrice(rows, 'desk'))).toBe(before); + }); + + it('only the exact string 1 sets the flag', () => { + for (const v of ['true', 'yes', 'on', '0', ' 1 ', '', '01']) { + expect(pc.liveState({ PROBABILITY_CONTRACT_LIVE: v }).flag_set).toBe(false); + } + expect(pc.liveState({ PROBABILITY_CONTRACT_LIVE: '1' }).flag_set).toBe(true); + }); +}); + +describe('BOTH gates are required — neither activates alone', () => { + it('the runtime flag alone does NOT serve an unapproved artifact', () => { + const st = pc.liveState({ PROBABILITY_CONTRACT_LIVE: '1' }, + { load: () => ({ ...A, approved_for_live: false }) }); + expect(st.flag_set).toBe(true); + expect(st.live).toBe('OFF'); + expect(st.blocked_reason).toMatch(/not approved_for_live|not servable/); + }); + + it('the runtime flag alone does NOT serve an unservable artifact', () => { + const st = pc.liveState({ PROBABILITY_CONTRACT_LIVE: '1' }, + { load: () => ({ ...A, approved_for_live: true, servable: false }) }); + expect(st.live).toBe('OFF'); + }); + + it('artifact approval alone does NOT activate behaviour', () => { + expect(A.approved_for_live).toBe(true); // it IS promoted + expect(pc.liveState({}).live).toBe('OFF'); // and behaviour is still off + }); + + it('both together turn it on', () => { + expect(pc.liveState({ PROBABILITY_CONTRACT_LIVE: '1' }).live).toBe('ON'); + }); +}); + +describe('when live is on', () => { + it('a CERTIFIED row serves the calibrated number and derives everything from it', () => { + const [out] = sp.applyToRows([row()], { liveState: LIVE_ON }); + const expected = registry.applyCurve(A, 0.65); + expect(out.p_win).toBe(Math.round(expected * 1000) / 1000); + expect(out.confidence).toBe(Math.round(out.p_win * 100)); + expect(out.confidence_basis).toBe('served_probability'); + expect(out.probability_state).toBe(pc.STATE.CERTIFIED_CALIBRATED); + const { evPct } = require('../../src/utils/devig'); + expect(out.ev_pct).toBe(evPct(out.p_win, -110)); + expect(out.ev_pct).not.toBe(evPct(0.65, -110)); // NOT from raw + expect(out.raw_model_probability).toBe(0.65); // raw survives + expect(out.grade).toBe('C+'); // grade untouched + expect(out.side).toBe('over'); // side untouched + }); + + it('an UNCERTIFIED row loses the number AND every claim derived from it', () => { + const [out] = sp.applyToRows([row({ p_win: 0.91, grade: 'B+' })], { liveState: LIVE_ON }); + expect(out.p_win).toBeNull(); + expect(out.confidence).toBeNull(); + expect(out.ev_pct).toBeNull(); + expect(out.value).toBeNull(); + expect(out.model_odds).toBeNull(); + expect(out.kelly).toBeNull(); + // and what must survive + expect(out.raw_model_probability).toBe(0.91); + expect(out.grade).toBe('B+'); + expect(out.side).toBe('over'); + expect(out.book_odds).toBe(-110); + expect(out.player).toBe('P'); // the Read still exists + }); + + it('a stat outside the contract is returned untouched', () => { + const tb = row({ stat_type: 'total_bases' }); + const [out] = sp.applyToRows([tb], { liveState: LIVE_ON }); + expect(out).toEqual(tb); + }); + + it('a different model era is returned untouched', () => { + const old = row({ model_version: 'engine1@2026-07-20' }); + const [out] = sp.applyToRows([old], { liveState: LIVE_ON }); + expect(out).toEqual(old); + }); +}); + +describe('one seam', () => { + it('serving routes reach it through stripModelPrice, not by duplicating logic', () => { + const fs = require('fs'); const path = require('path'); + const gsrc = fs.readFileSync(path.join(__dirname, '../../src/utils/snapshotGating.js'), 'utf8'); + expect(gsrc).toContain("require('../services/model/servedProbability').applyToRows(rows)"); + // no consumer may hold calibration logic of its own + for (const f of ['src/routes/snapshot.js', 'src/routes/heroProp.js', 'src/services/topGradedService.js']) { + const src = fs.readFileSync(path.join(__dirname, '../../', f), 'utf8') + .replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, ''); + for (const forbidden of ['applyCurve', 'applyIsotonic', 'artifactRegistry', 'served_curve']) { + expect(src).not.toContain(forbidden); + } + } + }); +});