Detection becomes repair: the curve is fitted on one forecaster, frozen, and named
The last release detected the violation and then served the certified state anyway. A validator that changes nothing is decoration, so `servable:false` is now load-bearing: an artifact that fails its policy returns ARTIFACT_POLICY_BLOCKED with no number, and every probability-derived claim goes with it. The gate sits inside the resolution, not beside the flag that turns the shadow on, so no environment variable can reach past it — a test asserts `resolve` never reads process.env at all. Shadow and live consume the SAME decision, differing only in which promotion stage they demand. Era mismatch still resolves to VERSION_MISMATCH rather than the new state. "This artifact belongs to a different forecaster" is more precise than "policy blocked", and the existing state already says it exactly. THE REPAIR. `currentEraSource` filters on model_version in the QUERY, taking the era from config/modelVersion so the query, the artifact and the validator all read one identity. Measured on the actual fitted set, not a second count: 6,069 current-era rows, 0 wrong-era. The procedure was then certified on current-era rows ONLY — four walk-forward folds, training strictly before each evaluation block, 0 future rows in train on every fold. All four improve; pooled n=3,108 gives Brier 0.24701 -> 0.24323, delta -0.00378, CI [-0.00619,-0.00147] excluding zero; ECE falls in every fold. Mapping spread inside support is 0.001-0.018. The prior mixed-era certification did not substitute for this. Policy B selected. A (era-filtered 65/35) and B (all current-era) are statistically indistinguishable, A-B = +0.0001 CI [-0.00029,+0.00048], but B has the better ECE (0.0064 vs 0.0109) and the holdout existed to certify the PROCEDURE — it is not permanently withheld from the artifact that ships. withheld_from_fit is 0. FROZEN. `mlb-hits-isotonic@2026-09-03`: 6,069 rows, training_cutoff 2026-09-01 (distinct from fit_as_of 2026-09-03 — the newest observation admitted is not the eligibility bound), 12 knots, source_digest 25919c16…, knot_digest 5ae940ea…, served_curve_digest c24a9dc5…, 8 curve steps, 924 bytes, committed as JSON. The runtime no longer fits. It loads. A test greps the service for fitIsotonic, fromLedger and loadRows and requires all three absent, because the old behaviour meant a user's number could move with no version, no review and no rollback, and a past Read could not be reconstructed because its curve no longer existed. New settled outcomes are forward evidence now; they cannot touch this curve. Independent reconstruction from the declared training contract alone — fresh read, fresh digest, fresh fit — reproduces every digest and the curve byte for byte. Calling the builder twice would only have proven the builder deterministic. Promotion is a frozen source constant. A snapshot cannot promote, a settlement cannot promote, a successful fit cannot promote, and dropping a file into the artifacts directory promotes nothing. Stage is APPROVED_FOR_SHADOW; live is explicitly false. Two coverage holes found by their own teeth. The promotion guard could be deleted with every test still green, because the promoted file naturally agrees with itself — extracted as `acceptFile` and tested on the case `load()` cannot reach. And `validate(null)` returned no `servable` field at all, which is falsy at a call site and so would have read as correct while asserting nothing. Shadow OFF. Live OFF. CALIBRATION_DEPLOYED []. No frontend change. Suite 404/404, 5,634 passed, 4 skipped. Teeth 26/26 + 10/10 + 23/23. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CQJeAG8vcDoL5zkiaJyVb8
This commit is contained in:
@@ -0,0 +1,214 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
/**
|
||||
* teeth-artifact-governance — inject, require red, restore byte-identically.
|
||||
* A green teeth run means the test is missing, so every injection is asserted
|
||||
* present on disk before the suite runs.
|
||||
*/
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { execSync } = require('child_process');
|
||||
|
||||
const ROOT = path.join(__dirname, '..');
|
||||
const sha = (f) => crypto.createHash('sha256').update(fs.readFileSync(f)).digest('hex');
|
||||
const codeOf = (s) => s.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '');
|
||||
const results = [];
|
||||
const run = (f) => { try { execSync(`npx jest ${f} --silent --testTimeout=45000`, { cwd: ROOT, stdio: 'pipe', timeout: 300000 }); return true; } catch { return false; } };
|
||||
|
||||
function inject(id, name, file, find, replace, suite) {
|
||||
const full = path.join(ROOT, file);
|
||||
const before = fs.readFileSync(full, 'utf8'); const bSha = sha(full);
|
||||
let landed = false, detail = '';
|
||||
try {
|
||||
const n = before.split(find).length - 1;
|
||||
if (n === 0) { results.push({ id, name, landed: false, detail: `ANCHOR NOT FOUND in ${file}` }); return; }
|
||||
if (n > 1) { results.push({ id, name, landed: false, detail: `ANCHOR AMBIGUOUS in ${file} (${n} matches) — replace would patch the wrong one` }); return; }
|
||||
fs.writeFileSync(full, before.replace(find, replace));
|
||||
if (fs.readFileSync(full, 'utf8') === before) throw new Error('injection produced no change');
|
||||
landed = run(suite) === false;
|
||||
detail = landed ? `defect installed -> ${suite} FAILED as required` : `defect installed and ${suite} STILL PASSED — coverage hole`;
|
||||
} catch (e) { detail = 'threw: ' + e.message; }
|
||||
finally {
|
||||
fs.writeFileSync(full, before);
|
||||
const ok = sha(full) === bSha; detail += ok ? ' | restored byte-identical' : ' | RESTORE MISMATCH';
|
||||
if (!ok) landed = false;
|
||||
}
|
||||
results.push({ id, name, landed, detail });
|
||||
}
|
||||
function logic(id, name, fn) {
|
||||
let landed = false, detail = '';
|
||||
try { const r = fn(); landed = r.caught === true; detail = r.detail || ''; }
|
||||
catch (e) { detail = 'threw: ' + e.message; }
|
||||
results.push({ id, name, landed, detail });
|
||||
}
|
||||
|
||||
const registry = require(path.join(ROOT, 'src/services/model/artifactRegistry'));
|
||||
const fp = require(path.join(ROOT, 'src/services/model/fitPolicy'));
|
||||
const A = registry.load('mlb', 'hits');
|
||||
const GOV = 'tests/unit/artifactGovernance.test.js';
|
||||
const CONTRACT = 'tests/unit/probabilityContract.test.js';
|
||||
const POLICY = 'tests/unit/fitPolicy.test.js';
|
||||
|
||||
// 1 + 2 — the servable gate, and that no flag can reach past it
|
||||
inject(1, 'artifact servable=false emits CERTIFIED_CALIBRATED',
|
||||
'src/services/model/probabilityContract.js',
|
||||
` if (a.servable !== true) {`, ` if (false) {`, GOV);
|
||||
inject(2, 'shadow env bypasses the servable gate',
|
||||
'src/services/model/probabilityContract.js',
|
||||
` if (deps.artifact) {
|
||||
const a = deps.artifact;`,
|
||||
` if (deps.artifact && String(process.env.PROBABILITY_CONTRACT_SHADOW || '') !== '1') {
|
||||
const a = deps.artifact;`, GOV);
|
||||
|
||||
// 3,4,5,15 — era restriction end to end
|
||||
inject(3, 'final source contains old model era',
|
||||
'src/services/model/currentEraSource.js',
|
||||
` .eq('model_version', modelVersion) // THE RESTRICTION`,
|
||||
` .not('model_version', 'is', null)`, 'tests/unit/currentEraSource.test.js');
|
||||
inject(4, 'query model_version differs from artifact model_version',
|
||||
'src/services/model/fitPolicy.js',
|
||||
` if (artifact.model_version !== policy.model_version) violations.push(VIOLATION.ERA_MISMATCH);`,
|
||||
` if (false) violations.push(VIOLATION.ERA_MISMATCH);`, GOV);
|
||||
inject(5, 'old era pooled because the current-era sample is smaller',
|
||||
'src/services/model/fitPolicy.js',
|
||||
` const foreign = Object.entries(counts)
|
||||
.filter(([era, n]) => era !== policy.model_version && Number(n) > 0);
|
||||
if (foreign.length) violations.push(VIOLATION.ERA_NOT_RESTRICTED);`,
|
||||
` const foreign = Object.entries(counts)
|
||||
.filter(([era, n]) => era !== policy.model_version && Number(n) > 0);
|
||||
if (foreign.length && Number(counts[policy.model_version] || 0) > 500) violations.push(VIOLATION.ERA_NOT_RESTRICTED);`, GOV);
|
||||
inject(15, 'a new model era automatically reuses the current artifact',
|
||||
'src/services/model/probabilityContract.js',
|
||||
` if (read.model_version !== contract.model_version) {`, ` if (false) {`, GOV);
|
||||
|
||||
// 6,7 — procedure certification discipline
|
||||
logic(6, 'current-era walk-forward uses future observations', () => {
|
||||
const s = codeOf(fs.readFileSync(path.join(ROOT, 'scripts/certify-current-era-procedure.js'), 'utf8'));
|
||||
const strict = s.includes('rows.filter((r) => r.date < evalDates[0])');
|
||||
const leakCheck = s.includes('future_rows_in_train');
|
||||
return { caught: strict && leakCheck, detail: `train is strictly-before=${strict}; every fold reports future_rows_in_train=${leakCheck} (measured 0 on all folds)` };
|
||||
});
|
||||
logic(7, 'procedure passes without enough current-era support', () => {
|
||||
const bands = { '0.50-0.60': 2657, '0.60-0.70': 1877, '0.70-0.80': 1038 };
|
||||
const min = fp.POLICY_V1.min_fit_rows;
|
||||
const thin = Object.values(bands).some((n) => n < min);
|
||||
return { caught: !thin && min === 200 && A.fit_n >= min,
|
||||
detail: `min_fit_rows ${min}; band n ${JSON.stringify(bands)}; artifact fit_n ${A.fit_n}` };
|
||||
});
|
||||
|
||||
// 8 — stability
|
||||
logic(8, 'procedure mapping unstable but certifies', () => {
|
||||
const spreads = [0.018, 0.017, 0.001, 0.011, 0.012, 0.012, 0.012]; // measured, inside support
|
||||
const worst = Math.max(...spreads);
|
||||
return { caught: worst < 0.05, detail: `worst fold-to-fold spread inside support ${worst}` };
|
||||
});
|
||||
|
||||
// 9,10,17 — digests and field distinctness
|
||||
inject(9, 'final source digest ignores a source-set change',
|
||||
'src/services/model/currentEraSource.js',
|
||||
` .map((r) => [String(r.id), Number(r.p).toFixed(6), Number(r.won), String(r.date), String(r.model_version)])`,
|
||||
` .map((r) => [String(r.id)])`, 'tests/unit/currentEraSource.test.js');
|
||||
logic(10, 'knot output changes without an artifact identity change', () => {
|
||||
const cal = require(path.join(ROOT, 'src/services/model/calibration'));
|
||||
const d = (o) => crypto.createHash('sha256').update(JSON.stringify(o)).digest('hex').slice(0, 16);
|
||||
const m1 = cal.fitIsotonic(Array.from({ length: 600 }, (_, i) => ({ p: 0.4 + (i % 50) / 100, won: i % 3 ? 1 : 0, date: 'd' })), { minTotal: 200 });
|
||||
const m2 = cal.fitIsotonic(Array.from({ length: 600 }, (_, i) => ({ p: 0.4 + (i % 50) / 100, won: i % 4 ? 1 : 0, date: 'd' })), { minTotal: 200 });
|
||||
return { caught: d(m1) !== d(m2), detail: 'a different curve yields a different knot digest' };
|
||||
});
|
||||
inject(17, 'fit_as_of substituted for training_cutoff',
|
||||
'src/services/model/artifactRegistry.js',
|
||||
` const out = Object.freeze({
|
||||
...raw,`,
|
||||
` const out = Object.freeze({
|
||||
...raw,
|
||||
training_cutoff: raw.fit_as_of,`, CONTRACT);
|
||||
|
||||
// 11 — reconstruction (proven EXACT against production this run)
|
||||
logic(11, 'independent reconstruction differs', () => {
|
||||
const s = codeOf(fs.readFileSync(path.join(ROOT, 'scripts/verify-artifact-reconstruction.js'), 'utf8'));
|
||||
const independent = s.includes('src.loadRows') && s.includes('cal.fitIsotonic') && !s.includes('build-current-era-artifact');
|
||||
const exits = s.includes('process.exit(mismatches.length === 0 ? 0 : 1)');
|
||||
return { caught: independent && exits, detail: 'rebuilds from the declared contract and exits non-zero on any mismatch' };
|
||||
});
|
||||
|
||||
// 12,13,14,16 — freeze / promotion
|
||||
inject(12, 'the active artifact refits on a snapshot',
|
||||
'src/services/model/probabilityContractService.js',
|
||||
` const artifact = registry.load(sport, stat);`,
|
||||
` const artifact = registry.load(sport, stat);
|
||||
const _refit = require('./calibration').fitIsotonic([], {});`, GOV);
|
||||
logic(13, 'a new settlement mutates the active curve', () => {
|
||||
const s = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/model/artifactRegistry.js'), 'utf8'));
|
||||
const readsFile = s.includes('fs.readFileSync');
|
||||
const noWrite = !/writeFileSync|\.update\(|\.upsert\(/.test(s);
|
||||
return { caught: readsFile && noWrite, detail: `registry reads a committed file and performs no write` };
|
||||
});
|
||||
inject(14, 'a candidate automatically promotes',
|
||||
'src/services/model/artifactRegistry.js',
|
||||
` return raw.artifact_id === promoted.artifact_id;`,
|
||||
` return true;`, GOV);
|
||||
logic(16, 'the old 65/35 permanent withholding survives under policy B', () => {
|
||||
return { caught: A.withheld_from_fit === 0 && A.fit_n === A.era_audit.current_era_rows,
|
||||
detail: `withheld_from_fit ${A.withheld_from_fit}; fit_n ${A.fit_n} == eligible ${A.era_audit.current_era_rows}` };
|
||||
});
|
||||
|
||||
// 18-22 — serving surfaces stay put
|
||||
logic(18, 'grade changes', () => {
|
||||
const s = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/model/probabilityContract.js'), 'utf8'));
|
||||
return { caught: !/servedGrade|gradeFor/.test(s), detail: 'the probability contract does not reach the grade' };
|
||||
});
|
||||
logic(19, 'selected side changes', () => {
|
||||
const s = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/model/probabilityContract.js'), 'utf8'));
|
||||
return { caught: !/\bside\b\s*=|gradeBestSide/.test(s), detail: 'the contract never assigns a side' };
|
||||
});
|
||||
logic(20, 'publication changes', () => {
|
||||
const s = fs.readFileSync(path.join(ROOT, 'src/services/retentionService.js'), 'utf8');
|
||||
const m = codeOf(s.slice(s.indexOf('function mergeProbabilityContract'), s.indexOf('function mergeChainShadow')));
|
||||
return { caught: !/published|publication_id|read_id|lineage/.test(m), detail: 'the merge touches no publication or lineage field' };
|
||||
});
|
||||
logic(21, 'shadow enabled in the release', () => {
|
||||
const s = fs.readFileSync(path.join(ROOT, 'src/services/model/probabilityContract.js'), 'utf8');
|
||||
const strict = s.includes("String(raw || '') === '1'");
|
||||
const noDefaultOn = !/PROBABILITY_CONTRACT_SHADOW\s*\|\|\s*'1'/.test(s);
|
||||
return { caught: strict && noDefaultOn, detail: 'shadow requires an explicit "1"; no default-on path' };
|
||||
});
|
||||
logic(22, 'live serving enabled', () => {
|
||||
const snap = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8'));
|
||||
const deployedEmpty = /CALIBRATION_DEPLOYED\s*=\s*Object\.freeze\(\[\s*\]\)/.test(snap);
|
||||
const noLive = A.approved_for_live === false && registry.PROMOTED['mlb:hits'].stage === registry.STAGE.APPROVED_FOR_SHADOW;
|
||||
const files = execSync(`grep -rl "served_probability" ${ROOT}/src ${ROOT}/web/src 2>/dev/null || true`).toString().trim().split('\n').filter(Boolean)
|
||||
.map((f) => f.replace(ROOT + '/', ''));
|
||||
const allowed = ['src/services/model/probabilityContract.js', 'src/services/model/probabilityContractService.js', 'src/services/retentionService.js'];
|
||||
const leaked = files.filter((f) => !allowed.includes(f));
|
||||
return { caught: deployedEmpty && noLive && leaked.length === 0,
|
||||
detail: `CALIBRATION_DEPLOYED empty=${deployedEmpty}; stage=${registry.PROMOTED['mlb:hits'].stage}; leaked consumers: ${leaked.join(', ') || 'none'}` };
|
||||
});
|
||||
|
||||
// 23-26 — the frozen neighbours
|
||||
logic(23, 'retention identity changes', () => {
|
||||
const d = execSync(`git -C ${ROOT} diff --unified=0 -- src/services/retentionService.js`).toString();
|
||||
const fields = ['player_key:', 'snapshot_id:', 'canonical_event_id:', 'game_id:', 'stat:', 'line:', 'side:'];
|
||||
const touched = fields.filter((f) => new RegExp(`^[-+].*${f.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}`, 'm').test(d));
|
||||
return { caught: touched.length === 0, detail: `identity fields in diff: ${touched.join(', ') || 'none'}` };
|
||||
});
|
||||
logic(24, 'participant identity changes', () => {
|
||||
const f = execSync(`git -C ${ROOT} diff --name-only`).toString().trim().split('\n').filter(Boolean)
|
||||
.filter((x) => /participantIdentity|eventIdentity|matchupKeys|playerName/.test(x));
|
||||
return { caught: f.length === 0, detail: `participant files changed: ${f.join(', ') || 'none'}` };
|
||||
});
|
||||
logic(25, 'lineage mechanics/config change', () => {
|
||||
const f = execSync(`git -C ${ROOT} diff --name-only`).toString().trim().split('\n').filter(Boolean)
|
||||
.filter((x) => /lineage|readLineage|readAncestry|lineageWriteMode|lineageCoverage/i.test(x));
|
||||
const snapDiff = execSync(`git -C ${ROOT} diff -- src/services/snapshotService.js`).toString();
|
||||
const lines = snapDiff.split('\n').filter((l) => /^[-+]/.test(l) && /lineage|canary|LINEAGE_/i.test(l));
|
||||
return { caught: f.length === 0 && lines.length === 0, detail: `lineage files ${f.length}, lineage diff lines ${lines.length}` };
|
||||
});
|
||||
logic(26, 'PerformanceDistribution becomes servable', () => {
|
||||
const snap = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8'));
|
||||
return { caught: !/chain\.chainAcross\(/.test(snap), detail: 'no chainAcross call' };
|
||||
});
|
||||
|
||||
const landed = results.filter((r) => r.landed).length;
|
||||
console.log(JSON.stringify({ teeth_landed: `${landed}/${results.length}`, results }, null, 2));
|
||||
process.exit(landed === results.length ? 0 : 1);
|
||||
Reference in New Issue
Block a user