From 3184c05d51241d341548bc89dca1dfdf8e8ce192 Mon Sep 17 00:00:00 2001 From: Kev Date: Thu, 3 Sep 2026 16:12:27 -0400 Subject: [PATCH] fit_as_of does not pin a training set; training_cutoff does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reconstruction verifier re-walked the ledger bounded by `fit_as_of` and reported DIFFERS for a completely healthy artifact. The artifact had not moved — its knot, curve and source digests on disk were byte-identical and the runtime was serving exactly them. The TRAINING SOURCE had moved. Measured: the artifact was built at 05:xx UTC with 6,069 rows through 2026-09-01. By 20:00 UTC a re-walk at the same fit_as_of returned 6,237, because 2026-09-02's 168 rows settled in between. Settlement backfills dates that are already before the cutoff, so `fit_as_of` bounds eligibility but does not pin membership. `training_cutoff` — the newest observation actually admitted — does pin it, because those dates are fully settled. Re-walked under that bound the reconstruction is EXACT: fit_n, training_cutoff, source_digest, knot_count, knot_digest, served_curve_digest all equal and the curve identical. This is why the two fields were kept separate and why source_digest exists. The digest caught a moving source and refused to call it a match, which is the behaviour that made the diagnosis possible. No artifact change. No model change. The verifier now states the bound it used. Suite 405/405. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CQJeAG8vcDoL5zkiaJyVb8 --- scripts/verify-artifact-reconstruction.js | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/scripts/verify-artifact-reconstruction.js b/scripts/verify-artifact-reconstruction.js index c71ecbb..6bbe6dd 100644 --- a/scripts/verify-artifact-reconstruction.js +++ b/scripts/verify-artifact-reconstruction.js @@ -38,9 +38,23 @@ function servedCurve(map, bands) { const a = registry.load('mlb', 'hits'); if (!a) throw new Error('no promoted artifact'); - // ONLY what the artifact declares about its own training contract. + // BOUND BY `training_cutoff`, NOT `fit_as_of`. + // + // `fit_as_of` is the eligibility bound the ORIGINAL walk used, but it does not + // pin the training set: settlement backfills. Measured 2026-09-03 — the + // artifact was built at 05:xx UTC with 6,069 rows through 2026-09-01, and by + // 20:00 UTC a re-walk at the same `fit_as_of` returned 6,237, because + // 2026-09-02's 168 rows had settled in between. Reconstruction against + // `fit_as_of` therefore reports DIFFERS for a completely healthy artifact. + // + // `training_cutoff` is the newest observation actually admitted, and those + // dates are fully settled, so it is stable. `lt` is exclusive, so the bound is + // the day AFTER the cutoff. + const cutoffDay = new Date(`${a.training_cutoff}T00:00:00Z`); + cutoffDay.setUTCDate(cutoffDay.getUTCDate() + 1); + const bound = cutoffDay.toISOString().slice(0, 10); const rows = await src.loadRows(sb, { - sport: a.sport, stat: a.stat, modelVersion: a.model_version, before: a.fit_as_of, + sport: a.sport, stat: a.stat, modelVersion: a.model_version, before: bound, }); rows.sort((x, y) => (x.date < y.date ? -1 : x.date > y.date ? 1 : (x.id < y.id ? -1 : 1))); const audit = src.eraAudit(rows, a.model_version); @@ -62,7 +76,7 @@ function servedCurve(map, bands) { served_curve_digest: a.served_curve_digest, wrong_era_rows: 0, }; const mismatches = Object.keys(want).filter((k) => got[k] !== want[k]); - console.log(JSON.stringify({ artifact_id: a.artifact_id, want, got, + console.log(JSON.stringify({ artifact_id: a.artifact_id, bound_used: `game_date < ${bound}`, want, got, curve_identical: JSON.stringify(curve) === JSON.stringify(a.served_curve), mismatches, RECONSTRUCTION: mismatches.length === 0 && JSON.stringify(curve) === JSON.stringify(a.served_curve) ? 'EXACT' : 'DIFFERS' }, null, 1));