The mapping that ran now has a name, and the row carries it
Runtime probes say the fleet is on a8de676, one process generation. But
"isotonic" was a label, not a claim: probabilityContractService refits per
snapshot against `game_date < todayEt()`, so the mapping changes as outcomes
settle, and nothing on a row could say WHICH mapping produced its number.
The artifact now has an identity:
estimator_type / estimator_version / certification_version / model_version
fit_as_of the exact lt(game_date) bound 2026-09-02
training_cutoff last date INSIDE the fit 2026-08-21
fit_n / knot_count 6,084 / 28
knot_digest d9d571d728ba76de
served_curve the COMPLETE served function over [0.50,0.80)
served_curve_digest
The served curve is not a sample. p_win is quantised to three decimals at the
source, so a step table at 0.001 granularity is the mapping itself for every
input that can occur — six steps, ~200 bytes. Storing it makes a Read
reconstructable WITHOUT re-deriving a training set that may since have been
re-settled, and a claim you can only verify when the inputs happen not to have
moved is not a reconstructable claim.
Proven, not asserted: the production construction path run twice gives an
identical digest, and an INDEPENDENT reconstruction — re-walk 9,361 settled
ledger rows at the declared bound, refit from scratch — reproduces
d9d571d728ba76de exactly, 28 knots for 28.
A teeth injection found a real defect behind a coverage hole. `resolve` checked
the CONTRACT's model era and never the ARTIFACT's, so a mapping fitted for a
different era could be recorded beside a served number with every test green.
Both the era and the estimator type are now checked, and a mismatch serves
nothing rather than serving quietly.
OBSERVED AND NOT CHANGED: calibrationService splits 65/35 to certify its own
bands, a step this contract does not consume because support comes from the
frozen artifact. So the served map is fitted through 2026-08-21 while 3,277
more recent settled rows sit unused, and that lag grows with history. Changing
it would change the fitted function, which this tranche froze.
Shadow still defaults OFF. CALIBRATION_DEPLOYED still []. served_probability is
referenced by nothing outside the contract layer — asserted by a tooth.
Suite 401/401, 5,593 passed, 4 skipped. Teeth 23/23 (prior) + 7/7 (new).
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CQJeAG8vcDoL5zkiaJyVb8
This commit is contained in:
@@ -0,0 +1,137 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
'use strict';
|
||||||
|
/**
|
||||||
|
* teeth-certified-probability — the NEW ground in this tranche.
|
||||||
|
*
|
||||||
|
* Teeth 6-15 and 19-23 of the order are already landed independently by
|
||||||
|
* scripts/teeth-probability-contract.js (23/23) and are not re-asserted here;
|
||||||
|
* this runner covers runtime observability, artifact identity, point-in-time
|
||||||
|
* evidence, shadow harmlessness, and the activation ordering.
|
||||||
|
*
|
||||||
|
* Teeth 17, 18 and 24 target a LIVE SERVING path that does not exist yet.
|
||||||
|
* They are recorded UNREACHABLE rather than asserted weakly.
|
||||||
|
*/
|
||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const crypto = require('crypto');
|
||||||
|
const { execSync } = require('child_process');
|
||||||
|
|
||||||
|
const ROOT = path.join(__dirname, '..');
|
||||||
|
const sha = (f) => crypto.createHash('sha256').update(fs.readFileSync(f)).digest('hex');
|
||||||
|
const codeOf = (s) => s.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '');
|
||||||
|
const results = [];
|
||||||
|
|
||||||
|
function runSuite(file) {
|
||||||
|
try { execSync(`npx jest ${file} --silent --testTimeout=45000`, { cwd: ROOT, stdio: 'pipe', timeout: 300000 });
|
||||||
|
return true; } catch { return false; }
|
||||||
|
}
|
||||||
|
function injectionTooth(id, name, file, find, replace, suite) {
|
||||||
|
const full = path.join(ROOT, file);
|
||||||
|
const before = fs.readFileSync(full, 'utf8');
|
||||||
|
const beforeSha = sha(full);
|
||||||
|
let landed = false, detail = '';
|
||||||
|
try {
|
||||||
|
if (!before.includes(find)) {
|
||||||
|
results.push({ id, name, landed: false, detail: `ANCHOR NOT FOUND in ${file} — the injection would have silently no-opped` });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
fs.writeFileSync(full, before.replace(find, replace));
|
||||||
|
if (fs.readFileSync(full, 'utf8') === before) throw new Error('injection produced no change');
|
||||||
|
landed = runSuite(suite) === false;
|
||||||
|
detail = landed ? `defect installed -> ${suite} FAILED as required`
|
||||||
|
: `defect installed and ${suite} STILL PASSED — coverage hole`;
|
||||||
|
} catch (e) { detail = 'threw: ' + e.message; }
|
||||||
|
finally {
|
||||||
|
fs.writeFileSync(full, before);
|
||||||
|
const ok = sha(full) === beforeSha;
|
||||||
|
detail += ok ? ' | restored byte-identical' : ' | RESTORE MISMATCH';
|
||||||
|
if (!ok) landed = false;
|
||||||
|
}
|
||||||
|
results.push({ id, name, landed, detail });
|
||||||
|
}
|
||||||
|
function logicTooth(id, name, fn) {
|
||||||
|
let landed = false, detail = '';
|
||||||
|
try { const r = fn(); landed = r.caught === true; detail = r.detail || ''; }
|
||||||
|
catch (e) { detail = 'threw: ' + e.message; }
|
||||||
|
results.push({ id, name, landed, detail });
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 1 — runtime SHA observability actually exists and is used ─────────────
|
||||||
|
logicTooth(1, 'runtime SHA verification waits for a snapshot despite working runtime status', () => {
|
||||||
|
const src = codeOf(fs.readFileSync(path.join(ROOT, 'src/routes/internal.js'), 'utf8'));
|
||||||
|
const block = src.slice(src.indexOf("router.get('/snapshot/status'"), src.indexOf("router.get('/snapshot/status'") + 4000);
|
||||||
|
const hasSha = /runtime:\s*\{[\s\S]*?code_sha:\s*codeSha\(\)/.test(block);
|
||||||
|
const hasStart = /started_at:\s*PROCESS_STARTED_AT/.test(block);
|
||||||
|
// and it must resolve from the SAME provenance source, not git HEAD
|
||||||
|
const ret = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/retentionService.js'), 'utf8'));
|
||||||
|
const sameResolver = /function codeSha\(\)\s*\{\s*return process\.env\.SOURCE_COMMIT/.test(ret);
|
||||||
|
return { caught: hasSha && hasStart && sameResolver,
|
||||||
|
detail: `status exposes runtime.code_sha=${hasSha} started_at=${hasStart}; same resolver as provenance=${sameResolver}` };
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── 2 — the estimator must carry a reconstructable identity ──────────────
|
||||||
|
injectionTooth(2, 'runtime estimator lacks a reconstructable artifact identity',
|
||||||
|
'src/services/model/probabilityContractService.js',
|
||||||
|
` knot_digest: digest(fitted.map),`,
|
||||||
|
` knot_digest: 'static-placeholder',`,
|
||||||
|
'tests/unit/probabilityContract.test.js');
|
||||||
|
|
||||||
|
// ── 3 — the artifact must be tied to the certified model era ─────────────
|
||||||
|
injectionTooth(3, 'runtime artifact differs from the certified artifact',
|
||||||
|
'src/services/model/probabilityContractService.js',
|
||||||
|
` model_version: contract.model_version,`,
|
||||||
|
` model_version: 'engine1@some-other-era',`,
|
||||||
|
'tests/unit/probabilityContractShadow.test.js');
|
||||||
|
|
||||||
|
// ── 4 — point-in-time evidence ───────────────────────────────────────────
|
||||||
|
injectionTooth(4, 'dynamic refit uses future settlement evidence for an earlier Read',
|
||||||
|
'src/services/model/calibrationService.js',
|
||||||
|
` .lt('game_date', cutoff), // STRICTLY before — the whole point`,
|
||||||
|
` .lte('game_date', cutoff),`,
|
||||||
|
'tests/unit/probabilityContract.test.js');
|
||||||
|
|
||||||
|
// ── 5 — the shadow may not move served product ───────────────────────────
|
||||||
|
injectionTooth(5, 'shadow changes current user-facing output',
|
||||||
|
'src/services/retentionService.js',
|
||||||
|
` return {
|
||||||
|
...r,
|
||||||
|
probability_contract: {`,
|
||||||
|
` return {
|
||||||
|
...r,
|
||||||
|
p_win: res.served_probability != null ? res.served_probability : r.p_win,
|
||||||
|
probability_contract: {`,
|
||||||
|
'tests/unit/probabilityContractShadow.test.js');
|
||||||
|
|
||||||
|
// ── 16 — activation ordering ─────────────────────────────────────────────
|
||||||
|
logicTooth(16, 'live serving activates before the shadow has passed', () => {
|
||||||
|
const snap = codeOf(fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8'));
|
||||||
|
const deployedEmpty = /CALIBRATION_DEPLOYED\s*=\s*Object\.freeze\(\[\s*\]\)/.test(snap);
|
||||||
|
// no live consumer may read served_probability yet
|
||||||
|
const srcFiles = execSync(`grep -rl "served_probability" ${ROOT}/src ${ROOT}/web/src 2>/dev/null || true`)
|
||||||
|
.toString().trim().split('\n').filter(Boolean).map((f) => f.replace(ROOT + '/', ''));
|
||||||
|
const allowed = ['src/services/model/probabilityContract.js',
|
||||||
|
'src/services/model/probabilityContractService.js', 'src/services/retentionService.js'];
|
||||||
|
const leaked = srcFiles.filter((f) => !allowed.includes(f));
|
||||||
|
return { caught: deployedEmpty && leaked.length === 0,
|
||||||
|
detail: `CALIBRATION_DEPLOYED empty=${deployedEmpty}; served_probability referenced outside the contract layer: ${leaked.join(', ') || 'none'}` };
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── the shadow flag itself ───────────────────────────────────────────────
|
||||||
|
logicTooth(25, 'shadow flag parses loosely or defaults ON', () => {
|
||||||
|
const snap = fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8');
|
||||||
|
const strict = snap.includes("String(process.env.PROBABILITY_CONTRACT_SHADOW || '') === '1'");
|
||||||
|
const scoped = snap.includes("&& sp === 'mlb'");
|
||||||
|
const statScoped = snap.includes("{ sport: 'mlb', stat: 'hits' }");
|
||||||
|
return { caught: strict && scoped && statScoped,
|
||||||
|
detail: `strict '1' compare=${strict}; sport-scoped=${scoped}; stat-scoped=${statScoped}` };
|
||||||
|
});
|
||||||
|
|
||||||
|
const unreachable = [
|
||||||
|
{ id: 17, name: 'UI displays raw exact confidence for UNCERTIFIED', why: 'no live-serving UI path exists; nothing consumes served_probability yet' },
|
||||||
|
{ id: 18, name: 'live EV/Kelly/VALUE consumers bypass served_probability', why: 'live consumers are unchanged by design in this tranche — tooth 16 asserts none exist' },
|
||||||
|
{ id: 24, name: 'rollback rewrites historical probability contracts', why: 'nothing is activated, so there is no activation to roll back' },
|
||||||
|
];
|
||||||
|
|
||||||
|
const landed = results.filter((r) => r.landed).length;
|
||||||
|
console.log(JSON.stringify({ teeth_landed: `${landed}/${results.length}`, results, unreachable }, null, 2));
|
||||||
|
process.exit(landed === results.length ? 0 : 1);
|
||||||
@@ -152,6 +152,10 @@ function resolve(read = {}, deps = {}) {
|
|||||||
certification_version: contract ? contract.certification_version : null,
|
certification_version: contract ? contract.certification_version : null,
|
||||||
model_version: read.model_version ?? null,
|
model_version: read.model_version ?? null,
|
||||||
contract_model_version: contract ? contract.model_version : null,
|
contract_model_version: contract ? contract.model_version : null,
|
||||||
|
// WHICH FITTED FUNCTION. `estimator_version` names the CERTIFICATION; this
|
||||||
|
// names the exact mapping that ran. Absent when no artifact was supplied,
|
||||||
|
// never invented.
|
||||||
|
artifact: deps.artifact || null,
|
||||||
reason: null,
|
reason: null,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -163,6 +167,17 @@ function resolve(read = {}, deps = {}) {
|
|||||||
// fitted to. A different model era is a different forecaster.
|
// fitted to. A different model era is a different forecaster.
|
||||||
return { ...base, probability_state: STATE.VERSION_MISMATCH, reason: 'model version differs from the certified era' };
|
return { ...base, probability_state: STATE.VERSION_MISMATCH, reason: 'model version differs from the certified era' };
|
||||||
}
|
}
|
||||||
|
// THE ARTIFACT MUST ALSO AGREE. Checking only the contract lets a mapping
|
||||||
|
// built for another era be recorded beside a served number: the gate would
|
||||||
|
// pass on the contract's era while the function that ran belonged to a
|
||||||
|
// different one. Found by a teeth injection that changed the artifact's era
|
||||||
|
// and left every test green.
|
||||||
|
if (deps.artifact && deps.artifact.model_version !== contract.model_version) {
|
||||||
|
return { ...base, probability_state: STATE.VERSION_MISMATCH, reason: 'estimator artifact was fitted for a different model era' };
|
||||||
|
}
|
||||||
|
if (deps.artifact && deps.artifact.estimator_type !== contract.estimator_type) {
|
||||||
|
return { ...base, probability_state: STATE.VERSION_MISMATCH, reason: 'estimator artifact is not the certified estimator type' };
|
||||||
|
}
|
||||||
if (raw === null || raw < 0 || raw > 1) {
|
if (raw === null || raw < 0 || raw > 1) {
|
||||||
return { ...base, probability_state: STATE.INVALID, reason: 'raw probability absent or out of range' };
|
return { ...base, probability_state: STATE.INVALID, reason: 'raw probability absent or out of range' };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,9 +14,45 @@
|
|||||||
* own support is how an estimator certifies itself.
|
* own support is how an estimator certifies itself.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
const crypto = require('crypto');
|
||||||
const cal = require('./calibration');
|
const cal = require('./calibration');
|
||||||
const pc = require('./probabilityContract');
|
const pc = require('./probabilityContract');
|
||||||
|
|
||||||
|
/** Short, stable content digest. Full sha256 truncated — collision risk here is
|
||||||
|
* irrelevant and 16 hex chars keeps the per-row payload small. */
|
||||||
|
const digest = (obj) => crypto.createHash('sha256')
|
||||||
|
.update(JSON.stringify(obj)).digest('hex').slice(0, 16);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* THE SERVED CURVE — the complete served function inside certified support.
|
||||||
|
*
|
||||||
|
* `p_win` is quantised to three decimals at the source
|
||||||
|
* (`analyzeViaEngine1`: Math.round(pWin * 1000) / 1000), so a step table at
|
||||||
|
* 0.001 granularity is not a sample of the mapping — it IS the mapping, for
|
||||||
|
* every input that can actually occur. Six steps, ~200 bytes.
|
||||||
|
*
|
||||||
|
* Storing it on the row makes a Read reconstructable WITHOUT re-deriving the
|
||||||
|
* training set. That matters because settled rows can be re-settled
|
||||||
|
* (`re_settled_at`), so a later refit at the same cutoff is not guaranteed to
|
||||||
|
* reproduce the same map — and a claim you can only verify when the inputs
|
||||||
|
* happen not to have moved is not a reconstructable claim.
|
||||||
|
*/
|
||||||
|
function servedCurve(map, bands) {
|
||||||
|
const steps = [];
|
||||||
|
let prev = null;
|
||||||
|
for (const [lo, hi] of bands) {
|
||||||
|
for (let x = lo; x < hi - 1e-9; x += 0.001) {
|
||||||
|
const raw = Math.round(x * 1000) / 1000;
|
||||||
|
const v = cal.applyIsotonic(map, raw);
|
||||||
|
if (v === null) continue;
|
||||||
|
const rounded = Math.round(v * 1e6) / 1e6;
|
||||||
|
if (rounded !== prev) { steps.push([raw, rounded]); prev = rounded; }
|
||||||
|
}
|
||||||
|
prev = null; // bands are independent segments
|
||||||
|
}
|
||||||
|
return steps;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @returns {null|{estimate, fit_n, fitted_through, contract}} null when there
|
* @returns {null|{estimate, fit_n, fitted_through, contract}} null when there
|
||||||
* is not enough settled history — and null means NOTHING is served, never
|
* is not enough settled history — and null means NOTHING is served, never
|
||||||
@@ -30,8 +66,31 @@ async function build(sb, { sport = 'mlb', stat = 'hits', before = null, ...opts
|
|||||||
const fitted = await svc.fromLedger(sb, { sport, stat, before, ...opts });
|
const fitted = await svc.fromLedger(sb, { sport, stat, before, ...opts });
|
||||||
if (!fitted || !fitted.map) return null;
|
if (!fitted || !fitted.map) return null;
|
||||||
|
|
||||||
|
// ── ARTIFACT IDENTITY ──────────────────────────────────────────────────
|
||||||
|
// The runtime REFITS PER SNAPSHOT against `game_date < todayEt()`, so the
|
||||||
|
// mapping changes as outcomes settle. That is point-in-time correct going
|
||||||
|
// forward — a Read can only ever have seen settlements strictly before its
|
||||||
|
// own day — but without an identity a served number could not be tied to the
|
||||||
|
// function that produced it, and "isotonic" would be a label rather than a
|
||||||
|
// claim. This is the identity.
|
||||||
|
const curve = servedCurve(fitted.map, contract.certified_bands);
|
||||||
|
const artifact = Object.freeze({
|
||||||
|
estimator_type: contract.estimator_type,
|
||||||
|
estimator_version: contract.estimator_version,
|
||||||
|
certification_version: contract.certification_version,
|
||||||
|
model_version: contract.model_version,
|
||||||
|
fit_as_of: fitted.cutoff || null, // the exact lt(game_date) bound
|
||||||
|
training_cutoff: fitted.fitted_through || null, // last date INSIDE the fit
|
||||||
|
fit_n: fitted.fit_n ?? null,
|
||||||
|
knot_count: Array.isArray(fitted.map) ? fitted.map.length : null,
|
||||||
|
knot_digest: digest(fitted.map),
|
||||||
|
served_curve: curve, // complete over certified support
|
||||||
|
served_curve_digest: digest(curve),
|
||||||
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
contract,
|
contract,
|
||||||
|
artifact,
|
||||||
fit_n: fitted.fit_n,
|
fit_n: fitted.fit_n,
|
||||||
fitted_through: fitted.fitted_through,
|
fitted_through: fitted.fitted_through,
|
||||||
cutoff: fitted.cutoff,
|
cutoff: fitted.cutoff,
|
||||||
@@ -39,7 +98,8 @@ async function build(sb, { sport = 'mlb', stat = 'hits', before = null, ...opts
|
|||||||
estimate: (p) => cal.applyIsotonic(fitted.map, p),
|
estimate: (p) => cal.applyIsotonic(fitted.map, p),
|
||||||
/** Resolve one grade through the full contract. */
|
/** Resolve one grade through the full contract. */
|
||||||
resolve(read) {
|
resolve(read) {
|
||||||
return pc.resolve({ ...read, sport, stat }, { estimate: (p) => cal.applyIsotonic(fitted.map, p) });
|
return pc.resolve({ ...read, sport, stat },
|
||||||
|
{ estimate: (p) => cal.applyIsotonic(fitted.map, p), artifact });
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -871,8 +871,13 @@ function mergeProbabilityContract(rows, contract) {
|
|||||||
certification_version: res.certification_version,
|
certification_version: res.certification_version,
|
||||||
model_version: res.model_version,
|
model_version: res.model_version,
|
||||||
reason: res.reason,
|
reason: res.reason,
|
||||||
fitted_through: contract.fitted_through || null,
|
// WHICH FITTED FUNCTION PRODUCED THIS. The estimator refits per
|
||||||
fit_n: contract.fit_n || null,
|
// snapshot, so the certification version alone cannot identify the
|
||||||
|
// mapping that ran. `served_curve` is the complete served function over
|
||||||
|
// certified support at p_win's own 3dp granularity, so the row is
|
||||||
|
// reconstructable without re-deriving a training set that may since
|
||||||
|
// have been re-settled.
|
||||||
|
artifact: res.artifact || null,
|
||||||
derived: derived ? {
|
derived: derived ? {
|
||||||
available: derived.available,
|
available: derived.available,
|
||||||
ev_pct: derived.ev_pct,
|
ev_pct: derived.ev_pct,
|
||||||
|
|||||||
@@ -219,3 +219,111 @@ describe('probabilityContractService', () => {
|
|||||||
expect(built.resolve({ model_version: ERA, p_win: 0.95 }).probability_state).toBe(pc.STATE.UNCERTIFIED);
|
expect(built.resolve({ model_version: ERA, p_win: 0.95 }).probability_state).toBe(pc.STATE.UNCERTIFIED);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('artifact identity — the mapping that actually ran', () => {
|
||||||
|
const cal = require('../../src/services/model/calibration');
|
||||||
|
const mk = (seed) => cal.fitIsotonic(Array.from({ length: 900 }, (_, i) => {
|
||||||
|
const p = Math.round((0.35 + (i % 60) / 100) * 1000) / 1000;
|
||||||
|
return { p, won: ((i * seed) % 1000) / 1000 < (0.5 + 0.45 * (p - 0.5)) ? 1 : 0, date: `d${i % 14}` };
|
||||||
|
}), { minTotal: 200 });
|
||||||
|
const fitted = (map, over = {}) => ({ calibrationService: { fromLedger: async () => ({
|
||||||
|
map, fit_n: 900, fitted_through: 'd13', cutoff: '2026-09-03', ...over }) } });
|
||||||
|
|
||||||
|
it('the same evidence reconstructs the same artifact, digest for digest', async () => {
|
||||||
|
const map = mk(2654435761);
|
||||||
|
const a = await svc.build({}, fitted(map));
|
||||||
|
const b = await svc.build({}, fitted(map));
|
||||||
|
expect(a.artifact.knot_digest).toBe(b.artifact.knot_digest);
|
||||||
|
expect(a.artifact.served_curve_digest).toBe(b.artifact.served_curve_digest);
|
||||||
|
expect(a.artifact.served_curve).toEqual(b.artifact.served_curve);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('DIFFERENT evidence produces a different identity — the digest is not decorative', async () => {
|
||||||
|
const a = await svc.build({}, fitted(mk(2654435761)));
|
||||||
|
const b = await svc.build({}, fitted(mk(40503)));
|
||||||
|
expect(a.artifact.knot_digest).not.toBe(b.artifact.knot_digest);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('carries the point-in-time bound and the training cutoff, distinctly', async () => {
|
||||||
|
const a = await svc.build({}, fitted(mk(2654435761)));
|
||||||
|
expect(a.artifact.fit_as_of).toBe('2026-09-03'); // the lt(game_date) bound
|
||||||
|
expect(a.artifact.training_cutoff).toBe('d13'); // last date inside the fit
|
||||||
|
expect(a.artifact.fit_n).toBe(900);
|
||||||
|
expect(a.artifact.knot_count).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('the served curve IS the served function over certified support, not a sample', async () => {
|
||||||
|
const a = await svc.build({}, fitted(mk(2654435761)));
|
||||||
|
const lookup = (raw) => {
|
||||||
|
let v = null;
|
||||||
|
for (const [from, val] of a.artifact.served_curve) if (raw >= from) v = val;
|
||||||
|
return v;
|
||||||
|
};
|
||||||
|
for (let x = 0.50; x < 0.80 - 1e-9; x += 0.001) {
|
||||||
|
const raw = Math.round(x * 1000) / 1000;
|
||||||
|
const r = a.resolve({ model_version: ERA, p_win: raw });
|
||||||
|
expect(r.served_probability).toBe(Math.round(lookup(raw) * 1000) / 1000);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('the curve covers ONLY certified support — never the unsupported tail', async () => {
|
||||||
|
const a = await svc.build({}, fitted(mk(2654435761)));
|
||||||
|
for (const [from] of a.artifact.served_curve) {
|
||||||
|
expect(from).toBeGreaterThanOrEqual(0.50);
|
||||||
|
expect(from).toBeLessThan(0.80);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('a resolution with no artifact records null rather than inventing one', () => {
|
||||||
|
const r = pc.resolve(read(0.65), { estimate: iso });
|
||||||
|
expect(r.artifact).toBeNull();
|
||||||
|
expect(r.served_probability).not.toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('point in time — a Read can only see settlements before its own day', () => {
|
||||||
|
const calSvc = require('../../src/services/model/calibrationService');
|
||||||
|
|
||||||
|
/** Records the filters actually applied, so this tests behaviour not source. */
|
||||||
|
function recordingClient(rows) {
|
||||||
|
const applied = [];
|
||||||
|
const q = {
|
||||||
|
select: () => q, eq: (c, v) => { applied.push(['eq', c, v]); return q; },
|
||||||
|
is: (c, v) => { applied.push(['is', c, v]); return q; },
|
||||||
|
in: (c, v) => { applied.push(['in', c, v]); return q; },
|
||||||
|
not: (c, o, v) => { applied.push(['not', c, o, v]); return q; },
|
||||||
|
lt: (c, v) => { applied.push(['lt', c, v]); return q; },
|
||||||
|
lte: (c, v) => { applied.push(['lte', c, v]); return q; },
|
||||||
|
gte: (c, v) => { applied.push(['gte', c, v]); return q; },
|
||||||
|
order: () => q, limit: () => q,
|
||||||
|
range: async () => ({ data: rows, error: null, count: rows.length }),
|
||||||
|
then: (res) => res({ data: rows, error: null }),
|
||||||
|
};
|
||||||
|
return { applied, client: { from: () => q } };
|
||||||
|
}
|
||||||
|
|
||||||
|
it('bounds the training walk STRICTLY BEFORE the cutoff, never at or after it', async () => {
|
||||||
|
const { applied, client } = recordingClient([]);
|
||||||
|
await calSvc.loadSettledRows(client, { sport: 'mlb', stat: 'hits', before: '2026-09-02' });
|
||||||
|
const bound = applied.filter((a) => a[1] === 'game_date');
|
||||||
|
expect(bound.length).toBeGreaterThan(0);
|
||||||
|
// strictly-less is the whole discipline: `lte` would admit same-day games
|
||||||
|
expect(bound.some((a) => a[0] === 'lt' && a[2] === '2026-09-02')).toBe(true);
|
||||||
|
expect(bound.some((a) => a[0] === 'lte')).toBe(false);
|
||||||
|
expect(bound.some((a) => a[0] === 'gte')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('the artifact records the bound it was fitted under, so a Read names its own evidence horizon', async () => {
|
||||||
|
const cal = require('../../src/services/model/calibration');
|
||||||
|
const map = cal.fitIsotonic(Array.from({ length: 900 }, (_, i) => {
|
||||||
|
const p = Math.round((0.35 + (i % 60) / 100) * 1000) / 1000;
|
||||||
|
return { p, won: ((i * 2654435761) % 1000) / 1000 < (0.5 + 0.45 * (p - 0.5)) ? 1 : 0, date: `d${i % 14}` };
|
||||||
|
}), { minTotal: 200 });
|
||||||
|
const built = await svc.build({}, { calibrationService: { fromLedger: async () => ({
|
||||||
|
map, fit_n: 900, fitted_through: '2026-08-21', cutoff: '2026-09-02' }) } });
|
||||||
|
expect(built.artifact.fit_as_of).toBe('2026-09-02');
|
||||||
|
expect(built.artifact.training_cutoff).toBe('2026-08-21');
|
||||||
|
// and the two are DIFFERENT questions — the bound, and the last date inside it
|
||||||
|
expect(built.artifact.fit_as_of).not.toBe(built.artifact.training_cutoff);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -78,12 +78,30 @@ describe('the shadow reaches the row', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe('the shadow changes NOTHING that is served', () => {
|
describe('the shadow changes NOTHING that is served', () => {
|
||||||
const SERVED = ['p_win', 'confidence', 'grade', 'ev_pct', 'value', 'takeable', 'side', 'line'];
|
it('leaves EVERY key byte-identical except probability_contract', () => {
|
||||||
it('leaves every served field byte-identical', () => {
|
// Diffing a NAMED LIST of served fields only proves the fields someone
|
||||||
const before = [row(), row({ p_win: 0.91, grade: 'B+' }), row({ side: 'under', p_win: 0.55 })];
|
// thought to list. Diff every key, so a field added later is covered by
|
||||||
|
// this test on the day it appears.
|
||||||
|
const before = [row(), row({ p_win: 0.91, grade: 'B+' }), row({ side: 'under', p_win: 0.55 }),
|
||||||
|
row({ p_win: null, grade: null }), row({ model_version: 'engine1@2026-07-20' })];
|
||||||
const snapshot = JSON.parse(JSON.stringify(before));
|
const snapshot = JSON.parse(JSON.stringify(before));
|
||||||
const after = retention.mergeProbabilityContract(before, contract);
|
const after = retention.mergeProbabilityContract(before, contract);
|
||||||
after.forEach((r, i) => { for (const k of SERVED) expect(r[k]).toEqual(snapshot[i][k]); });
|
after.forEach((r, i) => {
|
||||||
|
const keys = new Set([...Object.keys(r), ...Object.keys(snapshot[i])]);
|
||||||
|
keys.delete('probability_contract');
|
||||||
|
expect(keys.size).toBeGreaterThan(8); // the diff must be non-vacuous
|
||||||
|
for (const k of keys) expect(r[k]).toEqual(snapshot[i][k]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('adds exactly ONE key and no others', () => {
|
||||||
|
const before = row();
|
||||||
|
const after = retention.mergeProbabilityContract([before], contract)[0];
|
||||||
|
const added = Object.keys(after).filter((k) => !(k in before));
|
||||||
|
const removed = Object.keys(before).filter((k) => !(k in after));
|
||||||
|
expect(added).toEqual([]); // declared null upfront
|
||||||
|
expect(removed).toEqual([]);
|
||||||
|
expect(after.probability_contract).not.toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('does not mutate the input rows in place', () => {
|
it('does not mutate the input rows in place', () => {
|
||||||
@@ -124,3 +142,44 @@ describe('the flag is OFF by default', () => {
|
|||||||
expect(src).toMatch(/CALIBRATION_DEPLOYED\s*=\s*Object\.freeze\(\[\s*\]\)/);
|
expect(src).toMatch(/CALIBRATION_DEPLOYED\s*=\s*Object\.freeze\(\[\s*\]\)/);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('the artifact must agree with the contract, not merely accompany it', () => {
|
||||||
|
const cal = require('../../src/services/model/calibration');
|
||||||
|
const svc = require('../../src/services/model/probabilityContractService');
|
||||||
|
const map = cal.fitIsotonic(Array.from({ length: 900 }, (_, i) => {
|
||||||
|
const p = Math.round((0.35 + (i % 60) / 100) * 1000) / 1000;
|
||||||
|
return { p, won: ((i * 2654435761) % 1000) / 1000 < (0.5 + 0.45 * (p - 0.5)) ? 1 : 0, date: `d${i % 14}` };
|
||||||
|
}), { minTotal: 200 });
|
||||||
|
const build = () => svc.build({}, { calibrationService: { fromLedger: async () => ({
|
||||||
|
map, fit_n: 900, fitted_through: '2026-08-21', cutoff: '2026-09-02' }) } });
|
||||||
|
|
||||||
|
it('the built artifact declares the CERTIFIED era and estimator', async () => {
|
||||||
|
const b = await build();
|
||||||
|
expect(b.artifact.model_version).toBe(pc.MLB_HITS.model_version);
|
||||||
|
expect(b.artifact.estimator_type).toBe(pc.MLB_HITS.estimator_type);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('an artifact fitted for another era serves NOTHING, even on a matching read', () => {
|
||||||
|
const r = pc.resolve({ sport: 'mlb', stat: 'hits', model_version: ERA, p_win: 0.65 },
|
||||||
|
{ estimate: () => 0.6, artifact: { model_version: 'engine1@2026-07-20', estimator_type: 'isotonic' } });
|
||||||
|
expect(r.probability_state).toBe(pc.STATE.VERSION_MISMATCH);
|
||||||
|
expect(r.served_probability).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('an artifact of the wrong estimator type serves NOTHING', () => {
|
||||||
|
const r = pc.resolve({ sport: 'mlb', stat: 'hits', model_version: ERA, p_win: 0.65 },
|
||||||
|
{ estimate: () => 0.6, artifact: { model_version: ERA, estimator_type: 'low_param' } });
|
||||||
|
expect(r.probability_state).toBe(pc.STATE.VERSION_MISMATCH);
|
||||||
|
expect(r.served_probability).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('and the row records the mismatch rather than dropping it', () => {
|
||||||
|
const bad = { fit_n: 1, fitted_through: 'x',
|
||||||
|
resolve: (read) => pc.resolve({ ...read, sport: 'mlb', stat: 'hits' },
|
||||||
|
{ estimate: () => 0.6, artifact: { model_version: 'engine1@2026-07-20', estimator_type: 'isotonic' } }) };
|
||||||
|
const [r] = retention.mergeProbabilityContract([row()], bad);
|
||||||
|
expect(r.probability_contract.probability_state).toBe(pc.STATE.VERSION_MISMATCH);
|
||||||
|
expect(r.probability_contract.served_probability).toBeNull();
|
||||||
|
expect(r.probability_contract.raw_model_probability).toBe(0.65);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user