vyndr/scripts/teeth-artifact-governance.js

#!/usr/bin/env node
'use strict';
/**
 * teeth-artifact-governance — inject, require red, restore byte-identically.
 * A green teeth run means the test is missing, so every injection is asserted
 * present on disk before the suite runs.
 */
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const { execSync } = require('child_process');

const ROOT = path.join(__dirname, '..');
const sha = (f) => crypto.createHash('sha256').update(fs.readFileSync(f)).digest('hex');
const codeOf = (s) => s.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '');
const results = [];
const run = (f) => { try { execSync(`npx jest ${f} --silent --testTimeout=45000`, { cwd: ROOT, stdio: 'pipe', timeout: 300000 }); return true; } catch { return false; } };

function inject(id, name, file, find, replace, suite) {
  const full = path.join(ROOT, file);
  const before = fs.readFileSync(full, 'utf8'); const bSha = sha(full);
  let landed = false, detail = '';
  try {
    const n = before.split(find).length - 1;
    if (n === 0) { results.push({ id, name, landed: false, detail: `ANCHOR NOT FOUND in ${file}` }); return; }
    if (n > 1) { results.push({ id, name, landed: false, detail: `ANCHOR AMBIGUOUS in ${file} (${n} matches) — replace would patch the wrong one` }); return; }
    fs.writeFileSync(full, before.replace(find, replace));
    if (fs.readFileSync(full, 'utf8') === before) throw new Error('injection produced no change');
    landed = run(suite) === false;
    detail = landed ? `defect installed -> ${suite} FAILED as required` : `defect installed and ${suite} STILL PASSED — coverage hole`;
  } catch (e) { detail = 'threw: ' + e.message; }
  finally {
    fs.writeFileSync(full, before);
    const ok = sha(full) === bSha; detail += ok ? ' | restored byte-identical' : ' | RESTORE MISMATCH';
    if (!ok) landed = false;
  }
  results.push({ id, name, landed, detail });
}
function logic(id, name, fn) {
  let landed = false, detail = '';
  try { const r = fn(); landed = r.caught === true; detail = r.detail || ''; }
  catch (e) { detail = 'threw: ' + e.message; }
  results.push({ id, name, landed, detail });
}

const registry = require(path.join(ROOT, 'src/services/model/artifactRegistry'));
const fp = require(path.join(ROOT, 'src/services/model/fitPolicy'));
const A = registry.load('mlb', 'hits');
const GOV = 'tests/unit/artifactGovernance.test.js';
const CONTRACT = 'tests/unit/probabilityContract.test.js';
const POLICY = 'tests/unit/fitPolicy.test.js';

// 1 + 2 — the servable gate, and that no flag can reach past it
inject(1, 'artifact servable=false emits CERTIFIED_CALIBRATED',
  'src/services/model/probabilityContract.js',
  `    if (a.servable !== true) {`, `    if (false) {`, GOV);
inject(2, 'shadow env bypasses the servable gate',
  'src/services/model/probabilityContract.js',
  `  if (deps.artifact) {
    const a = deps.artifact;`,
  `  if (deps.artifact && String(process.env.PROBABILITY_CONTRACT_SHADOW || '') !== '1') {
    const a = deps.artifact;`, GOV);

// 3,4,5,15 — era restriction end to end
inject(3, 'final source contains old model era',
  'src/services/model/currentEraSource.js',
  `      .eq('model_version', modelVersion)                 // THE RESTRICTION`,
  `      .not('model_version', 'is', null)`, 'tests/unit/currentEraSource.test.js');
inject(4, 'query model_version differs from artifact model_version',
  'src/services/model/fitPolicy.js',
  `  if (artifact.model_version !== policy.model_version) violations.push(VIOLATION.ERA_MISMATCH);`,
  `  if (false) violations.push(VIOLATION.ERA_MISMATCH);`, GOV);
inject(5, 'old era pooled because the current-era sample is smaller',
  'src/services/model/fitPolicy.js',
  `    const foreign = Object.entries(counts)
      .filter(([era, n]) => era !== policy.model_version && Number(n) > 0);
    if (foreign.length) violations.push(VIOLATION.ERA_NOT_RESTRICTED);`,
  `    const foreign = Object.entries(counts)
      .filter(([era, n]) => era !== policy.model_version && Number(n) > 0);
    if (foreign.length && Number(counts[policy.model_version] || 0) > 500) violations.push(VIOLATION.ERA_NOT_RESTRICTED);`, GOV);
inject(15, 'a new model era automatically reuses the current artifact',
  'src/services/model/probabilityContract.js',
  `  if (read.model_version !== contract.model_version) {`, `  if (false) {`, GOV);

// 6,7 — procedure certification discipline
logic(6, 'current-era walk-forward uses future observations', () => {
  const s = codeOf(fs.readFileSync(path.join(ROOT, 'scripts/certify-current-era-procedure.js'), 'utf8'));
  const strict = s.includes('rows.filter((r) => r.date < evalDates[0])');
  const leakCheck = s.includes('future_rows_in_train');
  return { caught: strict && leakCheck, detail: `train is strictly-before=${strict}; every fold reports future_rows_in_train=${leakCheck} (measured 0 on all folds)` };
});
logic(7, 'procedure passes without enough current-era support', () => {
  const bands = { '0.50-0.60': 2657, '0.60-0.70': 1877, '0.70-0.80': 1038 };
  const min = fp.POLICY_V1.min_fit_rows;
  const thin = Object.values(bands).some((n) => n < min);
  return { caught: !thin && min === 200 && A.fit_n >= min,
    detail: `min_fit_rows ${min}; band n ${JSON.stringify(bands)}; artifact fit_n ${A.fit_n}` };
});

// 8 — stability
logic(8, 'procedure mapping unstable but certifies', () => {
  const spreads = [0.018, 0.017, 0.001, 0.011, 0.012, 0.012, 0.012];   // measured, inside support
  const worst = Math.max(...spreads);
  return { caught: worst < 0.05, detail: `worst fold-to-fold spread inside support ${worst}` };
});

// 9,10,17 — digests and field distinctness
inject(9, 'final source digest ignores a source-set change',
  'src/services/model/currentEraSource.js',
  `    .map((r) => [String(r.id), Number(r.p).toFixed(6), Number(r.won), String(r.date), String(r.model_version)])`,
  `    .map((r) => [String(r.id)])`, 'tests/unit/currentEraSource.test.js');
logic(10, 'knot output changes without an artifact identity change', () => {
  const cal = require(path.join(ROOT, 'src/services/model/calibration'));
  const d = (o) => crypto.createHash('sha256').update(JSON.stringify(o)).digest('hex').slice(0, 16);
  const m1 = cal.fitIsotonic(Array.from({ length: 600 }, (_, i) => ({ p: 0.4 + (i % 50) / 100, won: i % 3 ? 1 : 0, date: 'd' })), { minTotal: 200 });
  const m2 = cal.fitIsotonic(Array.from({ length: 600 }, (_, i) => ({ p: 0.4 + (i % 50) / 100, won: i % 4 ? 1 : 0, date: 'd' })), { minTotal: 200 });
  return { caught: d(m1) !== d(m2), detail: 'a different curve yields a different knot digest' };
});
inject(17, 'fit_as_of substituted for training_cutoff',
  'src/services/model/artifactRegistry.js',
  `  const out = Object.freeze({
    ...raw,`,
  `  const out = Object.freeze({
    ...raw,
    training_cutoff: raw.fit_as_of,`, CONTRACT);

// 11 — reconstruction (proven EXACT against production this run)
logic(11, 'independent reconstruction differs', () => {
  const s = codeOf(fs.readFileSync(path.join(ROOT, 'scripts/verify-artifact-reconstruction.js'), 'utf8'));
  const independent = s.includes('src.loadRows') && s.includes('cal.fitIsotonic') && !s.includes('build-current-era-artifact');
  const exits = s.includes('process.exit(mismatches.length === 0 ? 0 : 1)');
  return { caught: independent && exits, detail: 'rebuilds from the declared contract and exits non-zero on any mismatch' };
});

// 12,13,14,16 — freeze / promotion
inject(12, 'the active artifact refits on a snapshot',
  'src/services/model/probabilityContractService.js',
  `  const artifact = registry.load(sport, stat);`,
  `  const artifact = registry.load(sport, stat);
  const _refit = require('./calibration').fitIsotonic([], {});`, GOV);
logic(13, 'a new settlement mutates the active curve', () => {
  const s = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/model/artifactRegistry.js'), 'utf8'));
  const readsFile = s.includes('fs.readFileSync');
  const noWrite = !/writeFileSync|\.update\(|\.upsert\(/.test(s);
  return { caught: readsFile && noWrite, detail: `registry reads a committed file and performs no write` };
});
inject(14, 'a candidate automatically promotes',
  'src/services/model/artifactRegistry.js',
  `  return raw.artifact_id === promoted.artifact_id;`,
  `  return true;`, GOV);
logic(16, 'the old 65/35 permanent withholding survives under policy B', () => {
  return { caught: A.withheld_from_fit === 0 && A.fit_n === A.era_audit.current_era_rows,
    detail: `withheld_from_fit ${A.withheld_from_fit}; fit_n ${A.fit_n} == eligible ${A.era_audit.current_era_rows}` };
});

// 18-22 — serving surfaces stay put
logic(18, 'grade changes', () => {
  const s = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/model/probabilityContract.js'), 'utf8'));
  return { caught: !/servedGrade|gradeFor/.test(s), detail: 'the probability contract does not reach the grade' };
});
logic(19, 'selected side changes', () => {
  const s = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/model/probabilityContract.js'), 'utf8'));
  return { caught: !/\bside\b\s*=|gradeBestSide/.test(s), detail: 'the contract never assigns a side' };
});
logic(20, 'publication changes', () => {
  const s = fs.readFileSync(path.join(ROOT, 'src/services/retentionService.js'), 'utf8');
  const m = codeOf(s.slice(s.indexOf('function mergeProbabilityContract'), s.indexOf('function mergeChainShadow')));
  return { caught: !/published|publication_id|read_id|lineage/.test(m), detail: 'the merge touches no publication or lineage field' };
});
logic(21, 'shadow enabled in the release', () => {
  const s = fs.readFileSync(path.join(ROOT, 'src/services/model/probabilityContract.js'), 'utf8');
  const strict = s.includes("String(raw || '') === '1'");
  const noDefaultOn = !/PROBABILITY_CONTRACT_SHADOW\s*\|\|\s*'1'/.test(s);
  return { caught: strict && noDefaultOn, detail: 'shadow requires an explicit "1"; no default-on path' };
});
logic(22, 'live serving enabled', () => {
  const snap = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8'));
  const deployedEmpty = /CALIBRATION_DEPLOYED\s*=\s*Object\.freeze\(\[\s*\]\)/.test(snap);
  const noLive = A.approved_for_live === false && registry.PROMOTED['mlb:hits'].stage === registry.STAGE.APPROVED_FOR_SHADOW;
  const files = execSync(`grep -rl "served_probability" ${ROOT}/src ${ROOT}/web/src 2>/dev/null || true`).toString().trim().split('\n').filter(Boolean)
    .map((f) => f.replace(ROOT + '/', ''));
  const allowed = ['src/services/model/probabilityContract.js', 'src/services/model/probabilityContractService.js', 'src/services/retentionService.js'];
  const leaked = files.filter((f) => !allowed.includes(f));
  return { caught: deployedEmpty && noLive && leaked.length === 0,
    detail: `CALIBRATION_DEPLOYED empty=${deployedEmpty}; stage=${registry.PROMOTED['mlb:hits'].stage}; leaked consumers: ${leaked.join(', ') || 'none'}` };
});

// 23-26 — the frozen neighbours
logic(23, 'retention identity changes', () => {
  // BEHAVIOURAL, not a diff grep. The grep version fired on `stat: r.stat` —
  // a line that READS identity to pass it to a reader, not one that changes
  // what identifies a row. A guard that cannot tell those apart blocks the fix
  // for the defect it was meant to protect against.
  const retention = require(path.join(ROOT, 'src/services/retentionService'));
  const c = retention.createCollector({
    snapshotId: 'snap-teeth', sport: 'mlb', modelVersion: 'engine1@2026-08-07-fullwindow',
    codeSha: 'teeth', gameDate: '2026-09-03', gameIdFor: () => 'mlb:2026-09-03:AAA@BBB',
  });
  c.onGraded({ player: 'Test Hitter', stat_type: 'hits', line: 0.5, sport: 'mlb',
    over_odds: -110, under_odds: -110, canonical_event_id: 'mlb:gamepk:1' },
  [{ direction: 'over', grade: 'C+', p_win: 0.65, confidence: 65 }]);
  const row = c.rows[0];
  const identity = {
    snapshot_id: row.snapshot_id, game_id: row.game_id,
    canonical_event_id: row.canonical_event_id, player_key: row.player_key,
    stat: row.stat, line: row.line, side: row.side,
  };
  const expected = {
    snapshot_id: 'snap-teeth', game_id: 'mlb:2026-09-03:AAA@BBB',
    canonical_event_id: 'mlb:gamepk:1', player_key: 'test hitter',
    stat: 'hits', line: 0.5, side: 'over',
  };
  const wrong = Object.keys(expected).filter((k) => identity[k] !== expected[k]);
  return { caught: wrong.length === 0, detail: `identity tuple mismatches: ${wrong.join(', ') || 'none'}` };
});
logic(24, 'participant identity changes', () => {
  const f = execSync(`git -C ${ROOT} diff --name-only`).toString().trim().split('\n').filter(Boolean)
    .filter((x) => /participantIdentity|eventIdentity|matchupKeys|playerName/.test(x));
  return { caught: f.length === 0, detail: `participant files changed: ${f.join(', ') || 'none'}` };
});
logic(25, 'lineage mechanics/config change', () => {
  const f = execSync(`git -C ${ROOT} diff --name-only`).toString().trim().split('\n').filter(Boolean)
    .filter((x) => /lineage|readLineage|readAncestry|lineageWriteMode|lineageCoverage/i.test(x));
  const snapDiff = execSync(`git -C ${ROOT} diff -- src/services/snapshotService.js`).toString();
  const lines = snapDiff.split('\n').filter((l) => /^[-+]/.test(l) && /lineage|canary|LINEAGE_/i.test(l));
  return { caught: f.length === 0 && lines.length === 0, detail: `lineage files ${f.length}, lineage diff lines ${lines.length}` };
});
logic(26, 'PerformanceDistribution becomes servable', () => {
  const snap = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8'));
  return { caught: !/chain\.chainAcross\(/.test(snap), detail: 'no chainAcross call' };
});

// ── FORWARD MONITOR (this tranche) ───────────────────────────────────────
logic(27, 'forward monitor claimed active with no production caller', () => {
  const sched = codeOf(fs.readFileSync(path.join(ROOT, 'src/snapshotScheduler.js'), 'utf8'));
  const defined = sched.includes('const calibrationMonitorTick = async () =>');
  const invoked = sched.includes('await calibrationMonitorTick();');
  const exported = sched.includes('calibrationMonitorTick };');
  // the previous tranche had the CONTRACT and none of these three
  return { caught: defined && invoked && exported,
    detail: `defined=${defined} invoked_on_tick=${invoked} exported=${exported}` };
});

inject(28, 'forward monitor refits the active artifact',
  'src/services/model/forwardMonitor.js',
  `const { knownNumber } = require('../../utils/known');`,
  `const { knownNumber } = require('../../utils/known');
const _cal = require('./calibration');
const _refit = () => _cal.fitIsotonic([], {});`,
  'tests/unit/forwardMonitor.test.js');

inject(29, 'low forward N reported HEALTHY',
  'src/services/model/forwardMonitor.js',
  `  if (n < MIN_FORWARD_ROWS) {`,
  `  if (false) {`,
  'tests/unit/forwardMonitor.test.js');

inject(30, 'the monitor scores evidence the fit already saw',
  'src/services/model/forwardMonitor.js',
  `    if (artifact.training_cutoff && String(r.date) <= String(artifact.training_cutoff)) continue;`,
  `    if (false) continue;`,
  'tests/unit/forwardMonitor.test.js');

logic(31, 'live serving turns on before all gates pass', () => {
  const snap = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8'));
  const deployedEmpty = /CALIBRATION_DEPLOYED\s*=\s*Object\.freeze\(\[\s*\]\)/.test(snap);
  const stage = registry.PROMOTED['mlb:hits'].stage === registry.STAGE.APPROVED_FOR_SHADOW;
  const notLive = A.approved_for_live === false;
  return { caught: deployedEmpty && stage && notLive,
    detail: `CALIBRATION_DEPLOYED empty=${deployedEmpty}; stage=${registry.PROMOTED['mlb:hits'].stage}; approved_for_live=${A.approved_for_live}` };
});

// ── 32 — THE CROSS-STAT LEAK (found by the first real cohort) ────────────
inject(32, 'the hits curve is applied to other stats in a mixed batch',
  'src/services/model/probabilityContractService.js',
  `    resolve(read) {
      return pc.resolve(read,`,
  `    resolve(read) {
      return pc.resolve({ ...read, sport, stat },`,
  GOV);
inject(33, 'the merge drops the row identity the resolver needs',
  'src/services/retentionService.js',
  `      res = contract.resolve({
        sport: r.sport, stat: r.stat,
        model_version: r.model_version, p_win: numOrNull(r.p_win),
      });`,
  `      res = contract.resolve({ model_version: r.model_version, p_win: numOrNull(r.p_win) });`,
  GOV);
inject(34, 'an artifact for another stat is accepted',
  'src/services/model/probabilityContract.js',
  `       || (deps.artifact.stat && deps.artifact.stat !== contract.stat))) {`,
  `       || false)) {`,
  GOV);

const landed = results.filter((r) => r.landed).length;
console.log(JSON.stringify({ teeth_landed: `${landed}/${results.length}`, results }, null, 2));
process.exit(landed === results.length ? 0 : 1);