Files
vyndr/tests/unit/opsWatch.test.js
builtbykev e809a0eb3c Backtest harness — the validator, built refusal-first
Phase 0 gate PASSED: the join is clean. No FK exists; the natural key
(sport, player_key, stat, line, side, game_date) yields 283 clean 1:1
joins with ZERO ambiguity. game_id is NOT usable — 400/550 snapshot rows
carry UNK@UNK because home/away names weren't threaded into the grader
until Order 1.6. Non-joining rows are EXPECTED, not errors: retention
stores both sides plus refusals; the ledger keeps only the graded side.
Outcomes are NOT denormalized — ledger_entries stays the source of truth.

BUILT TEST-FIRST, and the first property proven is the REFUSAL, not the
math. Below threshold the harness emits INSUFFICIENT with n and the
shortfall and NO rate anywhere in the payload, so a downstream renderer
cannot surface one by accident. A test asserts the payload contains no
hit_rate number at all.

- Wilson intervals (correct at the n we actually have, unlike the normal
  approximation which emits negative lower bounds).
- Strata NEVER mix sport or model_version.
- Denominator excludes quarantined, void, unrecoverable, pending, push —
  asserted by test.
- Monotonicity refuses to RANK buckets whose intervals overlap; it reports
  "not distinguishable on this sample".
- Probability calibration (Brier + reliability) also respects the
  threshold: a thin sample returns status INSUFFICIENT and a NULL score.
- Replay seam reads the STORED feature vector only. A row whose input was
  never retained is UN-BACKTESTABLE, never scored with substituted current
  data. Identity replay reproduces the live prediction exactly.

The tests caught a real bug in my own code: `Number(null) === 0` let a
null p_win through as a confident 0% forecast — this codebase's signature
fabrication bug, inside the harness whose entire purpose is refusing
invented numbers. Fixed with a strict null guard.

FIRST LIVE RUN — the correct, passing output:
  VERDICT: INSUFFICIENT_HISTORY (can_validate=false)
  283 joined -> 35 scored (120 quarantined, 124 pending, 4 terminal)
  C n=18 (short by 2), B n=17 (short by 3)
  strata: mlb 7, wnba 28 — never mixed

migration 028 adds harness_results (append-only trend log; INSUFFICIENT
rows are expected and correct) and opsWatch.harnessStaleAlarm pages if the
harness stops running — a validator that isn't running looks exactly like
one that keeps passing.

Suite 283/3403 green, build exit 0.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SmNjJAwEnqHPtXbvSZR8kA
2026-07-20 11:31:48 -04:00

335 lines
13 KiB
JavaScript

'use strict';
// Session 8 (A1 board) — opsWatch: the pure ops-alarm logic.
const {
createFailureTracker,
isBadSnapshotResult,
zeroSettleAlarm,
morningHourUtc,
checkQuotaDaily,
countRecentSettles,
buildPulseMessage,
dateET,
} = require('../../src/services/opsWatch');
function memCache(seed = {}) {
const store = { ...seed };
return {
store,
cacheGet: async (k) => (k in store ? store[k] : null),
cacheSet: async (k, v) => { store[k] = v; return true; },
};
}
describe('isBadSnapshotResult', () => {
test('error and skipped/no-props are bad; ok and skipped/no-grades are not', () => {
expect(isBadSnapshotResult({ status: 'error', reason: 'down' })).toBe(true);
expect(isBadSnapshotResult({ status: 'skipped', reason: 'no props' })).toBe(true);
expect(isBadSnapshotResult({ status: 'ok' })).toBe(false);
expect(isBadSnapshotResult({ status: 'skipped', reason: 'no grades' })).toBe(false);
expect(isBadSnapshotResult(null)).toBe(false);
});
});
describe('createFailureTracker', () => {
test('pages exactly once at the third consecutive failure, stays quiet after', () => {
const t = createFailureTracker(3);
const bad = { status: 'error', reason: 'x' };
expect(t.record('mlb', bad).shouldPage).toBe(false); // 1
expect(t.record('mlb', bad).shouldPage).toBe(false); // 2
const third = t.record('mlb', bad);
expect(third.shouldPage).toBe(true); // 3 -> page once
expect(third.count).toBe(3);
expect(t.record('mlb', bad).shouldPage).toBe(false); // 4 -> no re-page
expect(t.count('mlb')).toBe(4);
});
test('any good outcome resets the counter and re-arms the pager', () => {
const t = createFailureTracker(3);
const bad = { status: 'skipped', reason: 'no props' };
t.record('mlb', bad); t.record('mlb', bad);
t.record('mlb', { status: 'ok' }); // reset
expect(t.count('mlb')).toBe(0);
t.record('mlb', bad); t.record('mlb', bad);
expect(t.record('mlb', bad).shouldPage).toBe(true); // re-armed -> pages again
});
test('counts are independent per sport', () => {
const t = createFailureTracker(3);
const bad = { status: 'error' };
t.record('mlb', bad); t.record('mlb', bad);
expect(t.record('wnba', bad).count).toBe(1);
expect(t.count('mlb')).toBe(2);
});
});
describe('zeroSettleAlarm', () => {
test('alarms when settleable rows existed and none settled', () => {
const z = zeroSettleAlarm([{ sport: 'mlb', settled: 0, pending: 12 }]);
expect(z).toEqual({ alarm: true, settled: 0, pending: 12 });
});
test('never alarms on a genuinely empty yesterday (0 rows fetched)', () => {
expect(zeroSettleAlarm([{ sport: 'mlb', settled: 0, pending: 0 }]).alarm).toBe(false);
expect(zeroSettleAlarm([]).alarm).toBe(false);
expect(zeroSettleAlarm(null).alarm).toBe(false);
});
test('no alarm when anything settled', () => {
expect(zeroSettleAlarm([{ sport: 'mlb', settled: 3, pending: 9 }]).alarm).toBe(false);
});
test('WNBA pendings without a finals signal never page (off-day / stale rows)', () => {
// Wave 1 — wnba IS settleable now, but with no finals-probe signal its
// pendings are excluded so an offseason/off-day cannot false-page.
const z = zeroSettleAlarm([
{ sport: 'mlb', settled: 0, pending: 0 },
{ sport: 'wnba', settled: 0, pending: 40 },
]);
expect(z.alarm).toBe(false);
});
test('Wave 1 — WNBA zero-settle WITH finals present DOES page', () => {
const z = zeroSettleAlarm(
[{ sport: 'wnba', settled: 0, pending: 40 }],
{ finalsBySport: { wnba: true } },
);
expect(z.alarm).toBe(true);
expect(z.pending).toBe(40);
});
test('Wave 1 — NBA finals present but SOMETHING settled → no alarm', () => {
const z = zeroSettleAlarm(
[{ sport: 'nba', settled: 5, pending: 30 }],
{ finalsBySport: { nba: true } },
);
expect(z.alarm).toBe(false);
});
test('Wave 1 — NBA finals present but 0 rows to settle → no false alarm', () => {
const z = zeroSettleAlarm(
[{ sport: 'nba', settled: 0, pending: 0 }],
{ finalsBySport: { nba: true } },
);
expect(z.alarm).toBe(false);
});
test('legacy array 2nd arg (settleable list) is still honored', () => {
const z = zeroSettleAlarm([{ sport: 'mlb', settled: 0, pending: 7 }], ['mlb']);
expect(z).toEqual({ alarm: true, settled: 0, pending: 7 });
});
});
describe('morningHourUtc', () => {
test('first configured hour >= 06 UTC (1,3 are late-night ET of the prior day)', () => {
expect(morningHourUtc([14, 19, 22, 1, 3])).toBe(14);
expect(morningHourUtc([19, 14])).toBe(14);
});
test('falls back to the smallest hour when nothing is daytime', () => {
expect(morningHourUtc([1, 3])).toBe(1);
expect(morningHourUtc([])).toBe(14);
});
});
describe('checkQuotaDaily', () => {
const notifyCollector = () => {
const msgs = [];
return { msgs, notify: async (m, o) => { msgs.push({ m, o }); return { sent: true }; } };
};
test('below 80% -> no alert', async () => {
const { msgs, notify } = notifyCollector();
const cache = memCache();
const r = await checkQuotaDaily({
getStatus: async () => ({ pct: 0.5, used: 250, limit: 500 }),
...cache, notify, now: () => new Date('2026-07-11T15:00:00Z'),
});
expect(r.alerted).toBe(false);
expect(msgs).toHaveLength(0);
});
test('>= 80% alerts once, then dedupes for the rest of the day', async () => {
const { msgs, notify } = notifyCollector();
const cache = memCache();
const deps = {
getStatus: async () => ({ pct: 0.82, used: 410, limit: 500, quotaType: 'monthly' }),
...cache, notify, now: () => new Date('2026-07-11T15:00:00Z'),
};
expect((await checkQuotaDaily(deps)).alerted).toBe(true);
const second = await checkQuotaDaily(deps);
expect(second.alerted).toBe(false);
expect(second.deduped).toBe(true);
expect(msgs).toHaveLength(1);
expect(msgs[0].m).toContain('82%');
expect(msgs[0].m).toContain('410/500');
expect(msgs[0].o.priority).toBe('high');
expect(msgs[0].m).not.toContain('!');
expect(cache.store['ops:quota_day:odds-api:2026-07-11']).toBe('1');
});
test('fires again the next day (date-keyed dedupe)', async () => {
const { msgs, notify } = notifyCollector();
const cache = memCache();
const base = {
getStatus: async () => ({ pct: 0.9, used: 450, limit: 500 }),
...cache, notify,
};
await checkQuotaDaily({ ...base, now: () => new Date('2026-07-11T15:00:00Z') });
await checkQuotaDaily({ ...base, now: () => new Date('2026-07-12T15:00:00Z') });
expect(msgs).toHaveLength(2);
});
test('never throws — a broken status probe returns { alerted: false }', async () => {
const r = await checkQuotaDaily({ getStatus: async () => { throw new Error('redis gone'); } });
expect(r.alerted).toBe(false);
expect(r.error).toBe('redis gone');
});
});
describe('countRecentSettles', () => {
test('counts settledAt inside the window across multiple sport logs', () => {
const nowMs = Date.parse('2026-07-11T13:00:00Z');
const logs = [
[ { settledAt: '2026-07-11T14:05:00Z' }, { settledAt: '2026-07-09T14:00:00Z' } ], // 1 in, 1 out
[ { settledAt: '2026-07-10T14:30:00Z' }, { noSettledAt: true } ], // 1 in
];
expect(countRecentSettles(logs, nowMs)).toBe(2);
expect(countRecentSettles([], nowMs)).toBe(0);
expect(countRecentSettles(null, nowMs)).toBe(0);
});
});
describe('buildPulseMessage', () => {
test('one message with every field, real numbers rendered', () => {
const msg = buildPulseMessage({
dateEt: '2026-07-11',
ledgerRows: 42,
settles24h: 38,
quota: { pct: 0.12, used: 61, limit: 500 },
health: { disk_pct: 41, mem_pct: 63 },
});
expect(msg).toContain('VYNDR pulse — 2026-07-11');
expect(msg).toContain('ledger rows yesterday: 42');
expect(msg).toContain('settles last 24h: 38');
expect(msg).toContain('odds-api quota: 12% (61/500)');
expect(msg).toContain('disk 41% · mem 63%');
expect(msg).toContain('desk pack: see /desk');
});
test('missing data renders n/a — never a fabricated zero', () => {
const msg = buildPulseMessage({ dateEt: '2026-07-11', ledgerRows: null, settles24h: null, quota: null, health: null });
expect(msg).toContain('ledger rows yesterday: n/a');
expect(msg).toContain('settles last 24h: n/a');
expect(msg).toContain('odds-api quota: n/a');
expect(msg).toContain('disk n/a · mem n/a');
});
test('VOICE v1.1 — no exclamation points, ever', () => {
const loud = buildPulseMessage({ ledgerRows: 999, settles24h: 999, quota: { pct: 0.99, used: 495, limit: 500 }, health: { disk_pct: 99, mem_pct: 99 } });
expect(loud).not.toContain('!');
expect(buildPulseMessage({})).not.toContain('!');
});
});
describe('dateET', () => {
test('renders the America/New_York calendar date (late UTC rolls back)', () => {
// 03:00 UTC Jul 11 = 11:00 PM ET Jul 10.
expect(dateET(new Date('2026-07-11T03:00:00Z'))).toBe('2026-07-10');
expect(dateET(new Date('2026-07-11T15:00:00Z'))).toBe('2026-07-11');
});
});
describe('retentionZeroWriteAlarm (Session 64)', () => {
const opsWatch = require('../../src/services/opsWatch');
test('pages when a slot graded props but retention wrote zero rows', () => {
const r = opsWatch.retentionZeroWriteAlarm(
[{ sport: 'mlb', status: 'ok', gradeCount: 7 }], { mlb: 0 });
expect(r.alarm).toBe(true);
expect(r.reason).toMatch(/MLB graded 7 but retention wrote 0/);
});
test('an ABSENT count reads as NOTHING, distinct from a reported 0', () => {
const r = opsWatch.retentionZeroWriteAlarm(
[{ sport: 'mlb', status: 'ok', gradeCount: 7 }], {});
expect(r.alarm).toBe(true);
expect(r.reason).toMatch(/wrote NOTHING/);
});
test('pages when retention wrote FEWER rows than the graded slate', () => {
const r = opsWatch.retentionZeroWriteAlarm(
[{ sport: 'wnba', status: 'ok', gradeCount: 25 }], { wnba: 4 });
expect(r.alarm).toBe(true);
expect(r.reason).toMatch(/wrote 4/);
});
test('quiet when retention wrote at least the slate (both sides + refusals)', () => {
const r = opsWatch.retentionZeroWriteAlarm(
[{ sport: 'mlb', status: 'ok', gradeCount: 7 }], { mlb: 50 });
expect(r.alarm).toBe(false);
});
test('a slot that graded nothing never false-pages', () => {
expect(opsWatch.retentionZeroWriteAlarm(
[{ sport: 'soccer', status: 'skipped', reason: 'no props', gradeCount: 0 }], {}).alarm).toBe(false);
expect(opsWatch.retentionZeroWriteAlarm(
[{ sport: 'mlb', status: 'ok', gradeCount: 0 }], { mlb: 0 }).alarm).toBe(false);
});
test('a missing/undefined count is treated as zero, not as fine', () => {
const r = opsWatch.retentionZeroWriteAlarm([{ sport: 'mlb', status: 'ok', gradeCount: 7 }], {});
expect(r.alarm).toBe(true);
});
});
describe('settlementRateAlarm (Session 64)', () => {
const opsWatch = require('../../src/services/opsWatch');
test('pages when a slate resolves below the floor', () => {
const r = opsWatch.settlementRateAlarm([{ sport: 'mlb', settled: 57, voided: 0, unrecoverable: 0, pending: 29 }]);
expect(r.alarm).toBe(true);
expect(r.reason).toMatch(/MLB resolved 57\/86/);
});
test('VOIDS count as resolved — a void is a legitimate terminal state', () => {
const r = opsWatch.settlementRateAlarm([{ sport: 'mlb', settled: 57, voided: 29, unrecoverable: 0, pending: 0 }]);
expect(r.alarm).toBe(false);
});
test('quiet on a healthy pass', () => {
expect(opsWatch.settlementRateAlarm([{ sport: 'wnba', settled: 70, voided: 0, unrecoverable: 0, pending: 0 }]).alarm).toBe(false);
});
test('nothing due never pages', () => {
expect(opsWatch.settlementRateAlarm([{ sport: 'nba', settled: 0, pending: 0 }]).alarm).toBe(false);
});
test('skipped/errored sports are ignored, not counted as failure', () => {
expect(opsWatch.settlementRateAlarm([{ sport: 'soccer', skipped: 'not configured' }]).alarm).toBe(false);
});
});
describe('harnessStaleAlarm (Session 64)', () => {
const opsWatch = require('../../src/services/opsWatch');
const NOW = new Date('2026-07-21T12:00:00Z');
test('never-run is an alarm — a validator that never ran looks like one that passes', () => {
const r = opsWatch.harnessStaleAlarm(null, NOW);
expect(r.alarm).toBe(true);
expect(r.reason).toMatch(/NEVER/);
});
test('a stale run pages', () => {
expect(opsWatch.harnessStaleAlarm('2026-07-19T00:00:00Z', NOW).alarm).toBe(true);
});
test('a recent run is quiet', () => {
expect(opsWatch.harnessStaleAlarm('2026-07-21T06:00:00Z', NOW).alarm).toBe(false);
});
test('an unparseable timestamp is an alarm, not a silent pass', () => {
expect(opsWatch.harnessStaleAlarm('not-a-date', NOW).alarm).toBe(true);
});
});