Files
vyndr/tests/unit/currentEraSource.test.js
builtbykev 044df7a426 The monitoring contract gets a callsite
Traced 2026-09-03: `calibrationRegistry.reverify` had ZERO production callers,
and the only reference to the artifact machinery outside its own directory was
the shadow builder, behind a flag that is off. The previous tranche declared
that future settled outcomes are forward evaluation evidence and shipped no
evaluator — the contract lived in a comment. That is the same shape as
statModel.js and correlateValidator.js, cited for months and never present.

`forwardMonitor` scores the FROZEN artifact on outcomes settled strictly after
its training_cutoff, and `snapshotScheduler.calibrationMonitorTick` runs it on
the existing per-minute cadence, throttled 6h, every failure swallowed. A test
asserts the tick is defined, invoked AND exported, and drives it with a fake to
prove it passes the promoted artifact and a forward-only window.

It cannot change what it watches: the module imports no fitter and no registry,
and a test greps the stripped source for fitIsotonic, fitPlatt, writeFileSync,
upsert, update, promote( , artifactRegistry and PROMOTED.

LOW N IS ITS OWN ANSWER. Below the floor it reports INSUFFICIENT_SAMPLE with
`healthy: null` — never false, never true. The floor is DERIVED, not chosen:
resolving an error of the certified tolerance at two standard errors needs
n >= 0.25/(0.05/2)^2 = 400, and a test recomputes it from TOLERANCE so the two
cannot drift apart. TOLERANCE is 0.05, the same number certifyBands used, so the
monitor can be neither stricter nor laxer than the thing it watches.

Wrong-era forward rows are INVALID, not scored — scoring them would measure a
different forecaster, which is the defect this line of work removed. An
unreadable read is AUDIT_UNAVAILABLE, which is not a health verdict. A drift
alert says in its own text that the artifact is frozen and unchanged, because
the alert is not a demotion.

`currentEraSource.loadRows` gains an optional `after` bound, strictly greater
so the cutoff date itself can never be scored as forward evidence.

SHADOW REMAINS BLOCKED: the production variable is still absent
(configuration_source "default") after a restart at 05:09:02Z, so no shadow
cohort was obtained and no replay was substituted for one.

Artifact unchanged: mlb-hits-isotonic@2026-09-03, knots 5ae940ea163b7da2.
Live OFF. Suite 405/405, 5,654 passed. Teeth 31/31 + 10/10 + 23/23.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CQJeAG8vcDoL5zkiaJyVb8
2026-09-03 01:29:47 -04:00

101 lines
4.3 KiB
JavaScript

'use strict';
/**
* THE INVARIANT: artifact.model_version = X => distinct(training.model_version) = {X}.
*/
const src = require('../../src/services/model/currentEraSource');
const ERA = 'engine1@2026-08-07-fullwindow';
const OLD = 'engine1@2026-07-20';
/** A fake that HONOURS its filters — a pass-through would prove nothing. */
function client(rows) {
const f = {};
const q = {
select: () => q,
eq: (c, v) => { f[c] = v; return q; },
is: () => q,
in: (c, v) => { f[`in_${c}`] = v; return q; },
not: () => q,
lt: (c, v) => { f[`lt_${c}`] = v; return q; },
gt: (c, v) => { f[`gt_${c}`] = v; return q; },
order: () => q,
range: async () => {
let out = rows;
for (const [k, v] of Object.entries(f)) {
if (k.startsWith('in_') || k.startsWith('lt_') || k.startsWith('gt_')) continue;
out = out.filter((r) => r[k] === v);
}
if (f.lt_game_date) out = out.filter((r) => r.game_date < f.lt_game_date);
if (f.gt_game_date) out = out.filter((r) => r.game_date > f.gt_game_date);
return { data: out, error: null, count: out.length };
},
};
return { client: { from: () => q }, filters: f };
}
const ROWS = [
{ id: '1', p_win: 0.6, outcome: 'hit', game_date: '2026-08-12', model_version: ERA, sport: 'mlb', stat: 'hits' },
{ id: '2', p_win: 0.7, outcome: 'miss', game_date: '2026-08-13', model_version: ERA, sport: 'mlb', stat: 'hits' },
{ id: '3', p_win: 0.55, outcome: 'hit', game_date: '2026-08-01', model_version: OLD, sport: 'mlb', stat: 'hits' },
];
describe('the era restriction is in the QUERY', () => {
it('loads only the requested era', async () => {
const { client: c, filters } = client(ROWS);
const out = await src.loadRows(c, { sport: 'mlb', stat: 'hits', modelVersion: ERA, before: '2026-09-03' });
expect(filters.model_version).toBe(ERA);
expect(out.map((r) => r.id).sort()).toEqual(['1', '2']);
expect(out.every((r) => r.model_version === ERA)).toBe(true);
});
it('the era audit is computed from the ACTUAL rows, not a second query', () => {
const mixed = [{ model_version: ERA }, { model_version: ERA }, { model_version: OLD }];
const a = src.eraAudit(mixed, ERA);
expect(a.current_era_rows).toBe(2);
expect(a.wrong_era_rows).toBe(1);
expect(src.eraAudit([{ model_version: ERA }], ERA).wrong_era_rows).toBe(0);
});
it('the optional forward window is STRICTLY after — the cutoff date is never scored', async () => {
const { client: c, filters } = client(ROWS);
const out = await src.loadRows(c, { sport: 'mlb', stat: 'hits', modelVersion: ERA,
before: '2026-09-03', after: '2026-08-12' });
expect(filters.gt_game_date).toBe('2026-08-12');
expect(out.map((r) => r.id)).toEqual(['2']); // 08-12 itself excluded
});
it('omitting the forward window loads the whole eligible history', async () => {
const { client: c } = client(ROWS);
const out = await src.loadRows(c, { sport: 'mlb', stat: 'hits', modelVersion: ERA, before: '2026-09-03' });
expect(out.map((r) => r.id).sort()).toEqual(['1', '2']);
});
it('refuses to run without an explicit era and horizon', async () => {
const { client: c } = client(ROWS);
await expect(src.loadRows(c, { sport: 'mlb', stat: 'hits', before: '2026-09-03' })).rejects.toThrow();
await expect(src.loadRows(c, { sport: 'mlb', stat: 'hits', modelVersion: ERA })).rejects.toThrow();
});
});
describe('the source digest', () => {
const R = [
{ id: 'b', p: 0.6, won: 1, date: 'd2', model_version: ERA },
{ id: 'a', p: 0.5, won: 0, date: 'd1', model_version: ERA },
];
const base = src.sourceDigest(R);
it('ignores database return order', () => {
expect(src.sourceDigest([...R].reverse())).toBe(base);
});
it('moves when a row is added, removed, or materially changed', () => {
expect(src.sourceDigest([R[0]])).not.toBe(base);
expect(src.sourceDigest([...R, { id: 'c', p: 0.7, won: 1, date: 'd3', model_version: ERA }])).not.toBe(base);
expect(src.sourceDigest([{ ...R[0], won: 0 }, R[1]])).not.toBe(base);
expect(src.sourceDigest([{ ...R[0], p: 0.61 }, R[1]])).not.toBe(base);
expect(src.sourceDigest([{ ...R[0], model_version: OLD }, R[1]])).not.toBe(base);
expect(src.sourceDigest([{ ...R[0], date: 'd9' }, R[1]])).not.toBe(base);
});
});