diff --git a/scripts/verify-artifact-reconstruction.js b/scripts/verify-artifact-reconstruction.js index c71ecbb..6bbe6dd 100644 --- a/scripts/verify-artifact-reconstruction.js +++ b/scripts/verify-artifact-reconstruction.js @@ -38,9 +38,23 @@ function servedCurve(map, bands) { const a = registry.load('mlb', 'hits'); if (!a) throw new Error('no promoted artifact'); - // ONLY what the artifact declares about its own training contract. + // BOUND BY `training_cutoff`, NOT `fit_as_of`. + // + // `fit_as_of` is the eligibility bound the ORIGINAL walk used, but it does not + // pin the training set: settlement backfills. Measured 2026-09-03 — the + // artifact was built at 05:xx UTC with 6,069 rows through 2026-09-01, and by + // 20:00 UTC a re-walk at the same `fit_as_of` returned 6,237, because + // 2026-09-02's 168 rows had settled in between. Reconstruction against + // `fit_as_of` therefore reports DIFFERS for a completely healthy artifact. + // + // `training_cutoff` is the newest observation actually admitted, and those + // dates are fully settled, so it is stable. `lt` is exclusive, so the bound is + // the day AFTER the cutoff. + const cutoffDay = new Date(`${a.training_cutoff}T00:00:00Z`); + cutoffDay.setUTCDate(cutoffDay.getUTCDate() + 1); + const bound = cutoffDay.toISOString().slice(0, 10); const rows = await src.loadRows(sb, { - sport: a.sport, stat: a.stat, modelVersion: a.model_version, before: a.fit_as_of, + sport: a.sport, stat: a.stat, modelVersion: a.model_version, before: bound, }); rows.sort((x, y) => (x.date < y.date ? -1 : x.date > y.date ? 1 : (x.id < y.id ? -1 : 1))); const audit = src.eraAudit(rows, a.model_version); @@ -62,7 +76,7 @@ function servedCurve(map, bands) { served_curve_digest: a.served_curve_digest, wrong_era_rows: 0, }; const mismatches = Object.keys(want).filter((k) => got[k] !== want[k]); - console.log(JSON.stringify({ artifact_id: a.artifact_id, want, got, + console.log(JSON.stringify({ artifact_id: a.artifact_id, bound_used: `game_date < ${bound}`, want, got, curve_identical: JSON.stringify(curve) === JSON.stringify(a.served_curve), mismatches, RECONSTRUCTION: mismatches.length === 0 && JSON.stringify(curve) === JSON.stringify(a.served_curve) ? 'EXACT' : 'DIFFERS' }, null, 1));