builtbykev
3184c05d51
The reconstruction verifier re-walked the ledger bounded by `fit_as_of` and reported DIFFERS for a completely healthy artifact. The artifact had not moved — its knot, curve and source digests on disk were byte-identical and the runtime was serving exactly them. The TRAINING SOURCE had moved. Measured: the artifact was built at 05:xx UTC with 6,069 rows through 2026-09-01. By 20:00 UTC a re-walk at the same fit_as_of returned 6,237, because 2026-09-02's 168 rows settled in between. Settlement backfills dates that are already before the cutoff, so `fit_as_of` bounds eligibility but does not pin membership. `training_cutoff` — the newest observation actually admitted — does pin it, because those dates are fully settled. Re-walked under that bound the reconstruction is EXACT: fit_n, training_cutoff, source_digest, knot_count, knot_digest, served_curve_digest all equal and the curve identical. This is why the two fields were kept separate and why source_digest exists. The digest caught a moving source and refused to call it a match, which is the behaviour that made the diagnosis possible. No artifact change. No model change. The verifier now states the bound it used. Suite 405/405. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CQJeAG8vcDoL5zkiaJyVb8
85 lines
3.8 KiB
JavaScript
85 lines
3.8 KiB
JavaScript
#!/usr/bin/env node
|
|
'use strict';
|
|
/**
|
|
* verify-artifact-reconstruction — rebuild the promoted artifact from its
|
|
* DECLARED TRAINING CONTRACT and require exact equality.
|
|
*
|
|
* A fresh read, a fresh source digest, a fresh fit, fresh digests. Calling the
|
|
* same builder twice would prove only that the builder is deterministic; this
|
|
* proves the artifact is derivable from what it says about itself.
|
|
*/
|
|
require('dotenv').config({ quiet: true });
|
|
const crypto = require('crypto');
|
|
const { createClient } = require('@supabase/supabase-js');
|
|
const cal = require('../src/services/model/calibration');
|
|
const src = require('../src/services/model/currentEraSource');
|
|
const registry = require('../src/services/model/artifactRegistry');
|
|
|
|
const digest = (o) => crypto.createHash('sha256').update(JSON.stringify(o)).digest('hex').slice(0, 16);
|
|
|
|
function servedCurve(map, bands) {
|
|
const steps = []; let prev = null;
|
|
for (const [lo, hi] of bands) {
|
|
prev = null;
|
|
for (let x = lo; x < hi - 1e-9; x += 0.001) {
|
|
const raw = Math.round(x * 1000) / 1000;
|
|
const v = cal.applyIsotonic(map, raw);
|
|
if (v === null) continue;
|
|
const r = Math.round(v * 1e6) / 1e6;
|
|
if (r !== prev) { steps.push([raw, r]); prev = r; }
|
|
}
|
|
}
|
|
return steps;
|
|
}
|
|
|
|
(async () => {
|
|
const sb = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_SERVICE_KEY,
|
|
{ auth: { persistSession: false } });
|
|
const a = registry.load('mlb', 'hits');
|
|
if (!a) throw new Error('no promoted artifact');
|
|
|
|
// BOUND BY `training_cutoff`, NOT `fit_as_of`.
|
|
//
|
|
// `fit_as_of` is the eligibility bound the ORIGINAL walk used, but it does not
|
|
// pin the training set: settlement backfills. Measured 2026-09-03 — the
|
|
// artifact was built at 05:xx UTC with 6,069 rows through 2026-09-01, and by
|
|
// 20:00 UTC a re-walk at the same `fit_as_of` returned 6,237, because
|
|
// 2026-09-02's 168 rows had settled in between. Reconstruction against
|
|
// `fit_as_of` therefore reports DIFFERS for a completely healthy artifact.
|
|
//
|
|
// `training_cutoff` is the newest observation actually admitted, and those
|
|
// dates are fully settled, so it is stable. `lt` is exclusive, so the bound is
|
|
// the day AFTER the cutoff.
|
|
const cutoffDay = new Date(`${a.training_cutoff}T00:00:00Z`);
|
|
cutoffDay.setUTCDate(cutoffDay.getUTCDate() + 1);
|
|
const bound = cutoffDay.toISOString().slice(0, 10);
|
|
const rows = await src.loadRows(sb, {
|
|
sport: a.sport, stat: a.stat, modelVersion: a.model_version, before: bound,
|
|
});
|
|
rows.sort((x, y) => (x.date < y.date ? -1 : x.date > y.date ? 1 : (x.id < y.id ? -1 : 1)));
|
|
const audit = src.eraAudit(rows, a.model_version);
|
|
const map = cal.fitIsotonic(rows, { minTotal: 200 });
|
|
const curve = servedCurve(map, a.certified_bands);
|
|
|
|
const got = {
|
|
fit_n: rows.length,
|
|
training_cutoff: rows.length ? rows[rows.length - 1].date : null,
|
|
source_digest: src.sourceDigest(rows),
|
|
knot_count: map ? map.length : null,
|
|
knot_digest: digest(map),
|
|
served_curve_digest: digest(curve),
|
|
wrong_era_rows: audit.wrong_era_rows,
|
|
};
|
|
const want = {
|
|
fit_n: a.fit_n, training_cutoff: a.training_cutoff, source_digest: a.source_digest,
|
|
knot_count: a.knot_count, knot_digest: a.knot_digest,
|
|
served_curve_digest: a.served_curve_digest, wrong_era_rows: 0,
|
|
};
|
|
const mismatches = Object.keys(want).filter((k) => got[k] !== want[k]);
|
|
console.log(JSON.stringify({ artifact_id: a.artifact_id, bound_used: `game_date < ${bound}`, want, got,
|
|
curve_identical: JSON.stringify(curve) === JSON.stringify(a.served_curve),
|
|
mismatches, RECONSTRUCTION: mismatches.length === 0 && JSON.stringify(curve) === JSON.stringify(a.served_curve)
|
|
? 'EXACT' : 'DIFFERS' }, null, 1));
|
|
process.exit(mismatches.length === 0 ? 0 : 1);
|
|
})().catch((e) => { console.error(e.message); process.exit(1); });
|