vyndr/scripts/teeth-certified-probability.js

#!/usr/bin/env node
'use strict';
/**
 * teeth-certified-probability — the NEW ground in this tranche.
 *
 * Teeth 6-15 and 19-23 of the order are already landed independently by
 * scripts/teeth-probability-contract.js (23/23) and are not re-asserted here;
 * this runner covers runtime observability, artifact identity, point-in-time
 * evidence, shadow harmlessness, and the activation ordering.
 *
 * Teeth 17, 18 and 24 target a LIVE SERVING path that does not exist yet.
 * They are recorded UNREACHABLE rather than asserted weakly.
 */
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const { execSync } = require('child_process');

const ROOT = path.join(__dirname, '..');
const sha = (f) => crypto.createHash('sha256').update(fs.readFileSync(f)).digest('hex');
const codeOf = (s) => s.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '');
const results = [];

function runSuite(file) {
  try { execSync(`npx jest ${file} --silent --testTimeout=45000`, { cwd: ROOT, stdio: 'pipe', timeout: 300000 });
    return true; } catch { return false; }
}
function injectionTooth(id, name, file, find, replace, suite) {
  const full = path.join(ROOT, file);
  const before = fs.readFileSync(full, 'utf8');
  const beforeSha = sha(full);
  let landed = false, detail = '';
  try {
    if (!before.includes(find)) {
      results.push({ id, name, landed: false, detail: `ANCHOR NOT FOUND in ${file} — the injection would have silently no-opped` });
      return;
    }
    fs.writeFileSync(full, before.replace(find, replace));
    if (fs.readFileSync(full, 'utf8') === before) throw new Error('injection produced no change');
    landed = runSuite(suite) === false;
    detail = landed ? `defect installed -> ${suite} FAILED as required`
                    : `defect installed and ${suite} STILL PASSED — coverage hole`;
  } catch (e) { detail = 'threw: ' + e.message; }
  finally {
    fs.writeFileSync(full, before);
    const ok = sha(full) === beforeSha;
    detail += ok ? ' | restored byte-identical' : ' | RESTORE MISMATCH';
    if (!ok) landed = false;
  }
  results.push({ id, name, landed, detail });
}
function logicTooth(id, name, fn) {
  let landed = false, detail = '';
  try { const r = fn(); landed = r.caught === true; detail = r.detail || ''; }
  catch (e) { detail = 'threw: ' + e.message; }
  results.push({ id, name, landed, detail });
}

// ── 1 — runtime SHA observability actually exists and is used ─────────────
logicTooth(1, 'runtime SHA verification waits for a snapshot despite working runtime status', () => {
  const src = codeOf(fs.readFileSync(path.join(ROOT, 'src/routes/internal.js'), 'utf8'));
  const block = src.slice(src.indexOf("router.get('/snapshot/status'"), src.indexOf("router.get('/snapshot/status'") + 4000);
  const hasSha = /runtime:\s*\{[\s\S]*?code_sha:\s*codeSha\(\)/.test(block);
  const hasStart = /started_at:\s*PROCESS_STARTED_AT/.test(block);
  // and it must resolve from the SAME provenance source, not git HEAD
  const ret = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/retentionService.js'), 'utf8'));
  const sameResolver = /function codeSha\(\)\s*\{\s*return process\.env\.SOURCE_COMMIT/.test(ret);
  return { caught: hasSha && hasStart && sameResolver,
    detail: `status exposes runtime.code_sha=${hasSha} started_at=${hasStart}; same resolver as provenance=${sameResolver}` };
});

// ── 2 — the estimator must carry a reconstructable identity ──────────────
injectionTooth(2, 'runtime estimator lacks a reconstructable artifact identity',
  'src/services/model/probabilityContractService.js',
  `    knot_digest: digest(fitted.map),`,
  `    knot_digest: 'static-placeholder',`,
  'tests/unit/probabilityContract.test.js');

// ── 3 — the artifact must be tied to the certified model era ─────────────
injectionTooth(3, 'runtime artifact differs from the certified artifact',
  'src/services/model/probabilityContractService.js',
  `    model_version: contract.model_version,`,
  `    model_version: 'engine1@some-other-era',`,
  'tests/unit/probabilityContractShadow.test.js');

// ── 4 — point-in-time evidence ───────────────────────────────────────────
injectionTooth(4, 'dynamic refit uses future settlement evidence for an earlier Read',
  'src/services/model/calibrationService.js',
  `      .lt('game_date', cutoff),                // STRICTLY before — the whole point`,
  `      .lte('game_date', cutoff),`,
  'tests/unit/probabilityContract.test.js');

// ── 5 — the shadow may not move served product ───────────────────────────
injectionTooth(5, 'shadow changes current user-facing output',
  'src/services/retentionService.js',
  `    return {
      ...r,
      probability_contract: {`,
  `    return {
      ...r,
      p_win: res.served_probability != null ? res.served_probability : r.p_win,
      probability_contract: {`,
  'tests/unit/probabilityContractShadow.test.js');

// ── 16 — activation ordering ─────────────────────────────────────────────
logicTooth(16, 'live serving activates before the shadow has passed', () => {
  const snap = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8'));
  const deployedEmpty = /CALIBRATION_DEPLOYED\s*=\s*Object\.freeze\(\[\s*\]\)/.test(snap);
  // no live consumer may read served_probability yet
  const srcFiles = execSync(`grep -rl "served_probability" ${ROOT}/src ${ROOT}/web/src 2>/dev/null || true`)
    .toString().trim().split('\n').filter(Boolean).map((f) => f.replace(ROOT + '/', ''));
  const allowed = ['src/services/model/probabilityContract.js',
    'src/services/model/probabilityContractService.js', 'src/services/retentionService.js'];
  const leaked = srcFiles.filter((f) => !allowed.includes(f));
  return { caught: deployedEmpty && leaked.length === 0,
    detail: `CALIBRATION_DEPLOYED empty=${deployedEmpty}; served_probability referenced outside the contract layer: ${leaked.join(', ') || 'none'}` };
});

// ── the shadow flag itself ───────────────────────────────────────────────
logicTooth(25, 'shadow flag parses loosely or defaults ON', () => {
  const snap = fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8');
  const strict = snap.includes("String(process.env.PROBABILITY_CONTRACT_SHADOW || '') === '1'");
  const scoped = snap.includes("&& sp === 'mlb'");
  const statScoped = snap.includes("{ sport: 'mlb', stat: 'hits' }");
  return { caught: strict && scoped && statScoped,
    detail: `strict '1' compare=${strict}; sport-scoped=${scoped}; stat-scoped=${statScoped}` };
});

const unreachable = [
  { id: 17, name: 'UI displays raw exact confidence for UNCERTIFIED', why: 'no live-serving UI path exists; nothing consumes served_probability yet' },
  { id: 18, name: 'live EV/Kelly/VALUE consumers bypass served_probability', why: 'live consumers are unchanged by design in this tranche — tooth 16 asserts none exist' },
  { id: 24, name: 'rollback rewrites historical probability contracts', why: 'nothing is activated, so there is no activation to roll back' },
];

const landed = results.filter((r) => r.landed).length;
console.log(JSON.stringify({ teeth_landed: `${landed}/${results.length}`, results, unreachable }, null, 2));
process.exit(landed === results.length ? 0 : 1);