5a5e37e32e
PHASE 1 — cron capture needed NO wiring. Verified in code: the scheduler tick calls runAll = snapshotService.runAllSnapshots, which loops runSnapshot per sport, which already carries the onGraded -> retention hook. The scheduled path and the manual path are the SAME function. The reason no cron cycle had been captured is simply that no slot has fired since retention deployed (slots are 14/19/22/1/3 UTC; retention landed ~02:55). Induced proof follows the deploy. PHASE 2 — archetype/team/opponent were permanently null because retention persisted at GRADE time, before enrichment attaches them. Retention still COLLECTS at grade time (the only moment the feature vector exists) but now PERSISTS after enrichment, merging those three fields via retentionService.mergeEnrichment. The merge is pure and fills ONLY those three fields — features and every model output are grade-time values and must never be rewritten by enrichment; a test asserts that. Unmatched rows (refusals not in the enriched slate) keep nulls rather than guesses. The empty-slate early return now persists too: a refusal-only slate is still history worth keeping. PHASE 3 — ZERO-WRITE ALARM. opsWatch.retentionZeroWriteAlarm pages at missed-snapshot severity when a slot GRADED props but retention wrote fewer rows than the slate (or nothing). runSnapshot now returns retentionRows so the scheduler can evaluate it. Retention is best-effort by design so it can never break a snapshot — which means a broken write is silent by construction. This is the counterweight. A slot that graded nothing never false-pages; an absent count reads as NOTHING and still pages, distinct from a reported 0. Suite 280/3349 green, build exit 0. Outcome stamping deliberately NOT implemented (depends on the settlement fix). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SmNjJAwEnqHPtXbvSZR8kA
285 lines
11 KiB
JavaScript
285 lines
11 KiB
JavaScript
'use strict';
|
|
|
|
// Session 8 (A1 board) — opsWatch: the pure ops-alarm logic.
|
|
|
|
const {
|
|
createFailureTracker,
|
|
isBadSnapshotResult,
|
|
zeroSettleAlarm,
|
|
morningHourUtc,
|
|
checkQuotaDaily,
|
|
countRecentSettles,
|
|
buildPulseMessage,
|
|
dateET,
|
|
} = require('../../src/services/opsWatch');
|
|
|
|
function memCache(seed = {}) {
|
|
const store = { ...seed };
|
|
return {
|
|
store,
|
|
cacheGet: async (k) => (k in store ? store[k] : null),
|
|
cacheSet: async (k, v) => { store[k] = v; return true; },
|
|
};
|
|
}
|
|
|
|
describe('isBadSnapshotResult', () => {
|
|
test('error and skipped/no-props are bad; ok and skipped/no-grades are not', () => {
|
|
expect(isBadSnapshotResult({ status: 'error', reason: 'down' })).toBe(true);
|
|
expect(isBadSnapshotResult({ status: 'skipped', reason: 'no props' })).toBe(true);
|
|
expect(isBadSnapshotResult({ status: 'ok' })).toBe(false);
|
|
expect(isBadSnapshotResult({ status: 'skipped', reason: 'no grades' })).toBe(false);
|
|
expect(isBadSnapshotResult(null)).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe('createFailureTracker', () => {
|
|
test('pages exactly once at the third consecutive failure, stays quiet after', () => {
|
|
const t = createFailureTracker(3);
|
|
const bad = { status: 'error', reason: 'x' };
|
|
expect(t.record('mlb', bad).shouldPage).toBe(false); // 1
|
|
expect(t.record('mlb', bad).shouldPage).toBe(false); // 2
|
|
const third = t.record('mlb', bad);
|
|
expect(third.shouldPage).toBe(true); // 3 -> page once
|
|
expect(third.count).toBe(3);
|
|
expect(t.record('mlb', bad).shouldPage).toBe(false); // 4 -> no re-page
|
|
expect(t.count('mlb')).toBe(4);
|
|
});
|
|
|
|
test('any good outcome resets the counter and re-arms the pager', () => {
|
|
const t = createFailureTracker(3);
|
|
const bad = { status: 'skipped', reason: 'no props' };
|
|
t.record('mlb', bad); t.record('mlb', bad);
|
|
t.record('mlb', { status: 'ok' }); // reset
|
|
expect(t.count('mlb')).toBe(0);
|
|
t.record('mlb', bad); t.record('mlb', bad);
|
|
expect(t.record('mlb', bad).shouldPage).toBe(true); // re-armed -> pages again
|
|
});
|
|
|
|
test('counts are independent per sport', () => {
|
|
const t = createFailureTracker(3);
|
|
const bad = { status: 'error' };
|
|
t.record('mlb', bad); t.record('mlb', bad);
|
|
expect(t.record('wnba', bad).count).toBe(1);
|
|
expect(t.count('mlb')).toBe(2);
|
|
});
|
|
});
|
|
|
|
describe('zeroSettleAlarm', () => {
|
|
test('alarms when settleable rows existed and none settled', () => {
|
|
const z = zeroSettleAlarm([{ sport: 'mlb', settled: 0, pending: 12 }]);
|
|
expect(z).toEqual({ alarm: true, settled: 0, pending: 12 });
|
|
});
|
|
|
|
test('never alarms on a genuinely empty yesterday (0 rows fetched)', () => {
|
|
expect(zeroSettleAlarm([{ sport: 'mlb', settled: 0, pending: 0 }]).alarm).toBe(false);
|
|
expect(zeroSettleAlarm([]).alarm).toBe(false);
|
|
expect(zeroSettleAlarm(null).alarm).toBe(false);
|
|
});
|
|
|
|
test('no alarm when anything settled', () => {
|
|
expect(zeroSettleAlarm([{ sport: 'mlb', settled: 3, pending: 9 }]).alarm).toBe(false);
|
|
});
|
|
|
|
test('WNBA pendings without a finals signal never page (off-day / stale rows)', () => {
|
|
// Wave 1 — wnba IS settleable now, but with no finals-probe signal its
|
|
// pendings are excluded so an offseason/off-day cannot false-page.
|
|
const z = zeroSettleAlarm([
|
|
{ sport: 'mlb', settled: 0, pending: 0 },
|
|
{ sport: 'wnba', settled: 0, pending: 40 },
|
|
]);
|
|
expect(z.alarm).toBe(false);
|
|
});
|
|
|
|
test('Wave 1 — WNBA zero-settle WITH finals present DOES page', () => {
|
|
const z = zeroSettleAlarm(
|
|
[{ sport: 'wnba', settled: 0, pending: 40 }],
|
|
{ finalsBySport: { wnba: true } },
|
|
);
|
|
expect(z.alarm).toBe(true);
|
|
expect(z.pending).toBe(40);
|
|
});
|
|
|
|
test('Wave 1 — NBA finals present but SOMETHING settled → no alarm', () => {
|
|
const z = zeroSettleAlarm(
|
|
[{ sport: 'nba', settled: 5, pending: 30 }],
|
|
{ finalsBySport: { nba: true } },
|
|
);
|
|
expect(z.alarm).toBe(false);
|
|
});
|
|
|
|
test('Wave 1 — NBA finals present but 0 rows to settle → no false alarm', () => {
|
|
const z = zeroSettleAlarm(
|
|
[{ sport: 'nba', settled: 0, pending: 0 }],
|
|
{ finalsBySport: { nba: true } },
|
|
);
|
|
expect(z.alarm).toBe(false);
|
|
});
|
|
|
|
test('legacy array 2nd arg (settleable list) is still honored', () => {
|
|
const z = zeroSettleAlarm([{ sport: 'mlb', settled: 0, pending: 7 }], ['mlb']);
|
|
expect(z).toEqual({ alarm: true, settled: 0, pending: 7 });
|
|
});
|
|
});
|
|
|
|
describe('morningHourUtc', () => {
|
|
test('first configured hour >= 06 UTC (1,3 are late-night ET of the prior day)', () => {
|
|
expect(morningHourUtc([14, 19, 22, 1, 3])).toBe(14);
|
|
expect(morningHourUtc([19, 14])).toBe(14);
|
|
});
|
|
test('falls back to the smallest hour when nothing is daytime', () => {
|
|
expect(morningHourUtc([1, 3])).toBe(1);
|
|
expect(morningHourUtc([])).toBe(14);
|
|
});
|
|
});
|
|
|
|
describe('checkQuotaDaily', () => {
|
|
const notifyCollector = () => {
|
|
const msgs = [];
|
|
return { msgs, notify: async (m, o) => { msgs.push({ m, o }); return { sent: true }; } };
|
|
};
|
|
|
|
test('below 80% -> no alert', async () => {
|
|
const { msgs, notify } = notifyCollector();
|
|
const cache = memCache();
|
|
const r = await checkQuotaDaily({
|
|
getStatus: async () => ({ pct: 0.5, used: 250, limit: 500 }),
|
|
...cache, notify, now: () => new Date('2026-07-11T15:00:00Z'),
|
|
});
|
|
expect(r.alerted).toBe(false);
|
|
expect(msgs).toHaveLength(0);
|
|
});
|
|
|
|
test('>= 80% alerts once, then dedupes for the rest of the day', async () => {
|
|
const { msgs, notify } = notifyCollector();
|
|
const cache = memCache();
|
|
const deps = {
|
|
getStatus: async () => ({ pct: 0.82, used: 410, limit: 500, quotaType: 'monthly' }),
|
|
...cache, notify, now: () => new Date('2026-07-11T15:00:00Z'),
|
|
};
|
|
expect((await checkQuotaDaily(deps)).alerted).toBe(true);
|
|
const second = await checkQuotaDaily(deps);
|
|
expect(second.alerted).toBe(false);
|
|
expect(second.deduped).toBe(true);
|
|
expect(msgs).toHaveLength(1);
|
|
expect(msgs[0].m).toContain('82%');
|
|
expect(msgs[0].m).toContain('410/500');
|
|
expect(msgs[0].o.priority).toBe('high');
|
|
expect(msgs[0].m).not.toContain('!');
|
|
expect(cache.store['ops:quota_day:odds-api:2026-07-11']).toBe('1');
|
|
});
|
|
|
|
test('fires again the next day (date-keyed dedupe)', async () => {
|
|
const { msgs, notify } = notifyCollector();
|
|
const cache = memCache();
|
|
const base = {
|
|
getStatus: async () => ({ pct: 0.9, used: 450, limit: 500 }),
|
|
...cache, notify,
|
|
};
|
|
await checkQuotaDaily({ ...base, now: () => new Date('2026-07-11T15:00:00Z') });
|
|
await checkQuotaDaily({ ...base, now: () => new Date('2026-07-12T15:00:00Z') });
|
|
expect(msgs).toHaveLength(2);
|
|
});
|
|
|
|
test('never throws — a broken status probe returns { alerted: false }', async () => {
|
|
const r = await checkQuotaDaily({ getStatus: async () => { throw new Error('redis gone'); } });
|
|
expect(r.alerted).toBe(false);
|
|
expect(r.error).toBe('redis gone');
|
|
});
|
|
});
|
|
|
|
describe('countRecentSettles', () => {
|
|
test('counts settledAt inside the window across multiple sport logs', () => {
|
|
const nowMs = Date.parse('2026-07-11T13:00:00Z');
|
|
const logs = [
|
|
[ { settledAt: '2026-07-11T14:05:00Z' }, { settledAt: '2026-07-09T14:00:00Z' } ], // 1 in, 1 out
|
|
[ { settledAt: '2026-07-10T14:30:00Z' }, { noSettledAt: true } ], // 1 in
|
|
];
|
|
expect(countRecentSettles(logs, nowMs)).toBe(2);
|
|
expect(countRecentSettles([], nowMs)).toBe(0);
|
|
expect(countRecentSettles(null, nowMs)).toBe(0);
|
|
});
|
|
});
|
|
|
|
describe('buildPulseMessage', () => {
|
|
test('one message with every field, real numbers rendered', () => {
|
|
const msg = buildPulseMessage({
|
|
dateEt: '2026-07-11',
|
|
ledgerRows: 42,
|
|
settles24h: 38,
|
|
quota: { pct: 0.12, used: 61, limit: 500 },
|
|
health: { disk_pct: 41, mem_pct: 63 },
|
|
});
|
|
expect(msg).toContain('VYNDR pulse — 2026-07-11');
|
|
expect(msg).toContain('ledger rows yesterday: 42');
|
|
expect(msg).toContain('settles last 24h: 38');
|
|
expect(msg).toContain('odds-api quota: 12% (61/500)');
|
|
expect(msg).toContain('disk 41% · mem 63%');
|
|
expect(msg).toContain('desk pack: see /desk');
|
|
});
|
|
|
|
test('missing data renders n/a — never a fabricated zero', () => {
|
|
const msg = buildPulseMessage({ dateEt: '2026-07-11', ledgerRows: null, settles24h: null, quota: null, health: null });
|
|
expect(msg).toContain('ledger rows yesterday: n/a');
|
|
expect(msg).toContain('settles last 24h: n/a');
|
|
expect(msg).toContain('odds-api quota: n/a');
|
|
expect(msg).toContain('disk n/a · mem n/a');
|
|
});
|
|
|
|
test('VOICE v1.1 — no exclamation points, ever', () => {
|
|
const loud = buildPulseMessage({ ledgerRows: 999, settles24h: 999, quota: { pct: 0.99, used: 495, limit: 500 }, health: { disk_pct: 99, mem_pct: 99 } });
|
|
expect(loud).not.toContain('!');
|
|
expect(buildPulseMessage({})).not.toContain('!');
|
|
});
|
|
});
|
|
|
|
describe('dateET', () => {
|
|
test('renders the America/New_York calendar date (late UTC rolls back)', () => {
|
|
// 03:00 UTC Jul 11 = 11:00 PM ET Jul 10.
|
|
expect(dateET(new Date('2026-07-11T03:00:00Z'))).toBe('2026-07-10');
|
|
expect(dateET(new Date('2026-07-11T15:00:00Z'))).toBe('2026-07-11');
|
|
});
|
|
});
|
|
|
|
describe('retentionZeroWriteAlarm (Session 64)', () => {
|
|
const opsWatch = require('../../src/services/opsWatch');
|
|
|
|
test('pages when a slot graded props but retention wrote zero rows', () => {
|
|
const r = opsWatch.retentionZeroWriteAlarm(
|
|
[{ sport: 'mlb', status: 'ok', gradeCount: 7 }], { mlb: 0 });
|
|
expect(r.alarm).toBe(true);
|
|
expect(r.reason).toMatch(/MLB graded 7 but retention wrote 0/);
|
|
});
|
|
|
|
test('an ABSENT count reads as NOTHING, distinct from a reported 0', () => {
|
|
const r = opsWatch.retentionZeroWriteAlarm(
|
|
[{ sport: 'mlb', status: 'ok', gradeCount: 7 }], {});
|
|
expect(r.alarm).toBe(true);
|
|
expect(r.reason).toMatch(/wrote NOTHING/);
|
|
});
|
|
|
|
test('pages when retention wrote FEWER rows than the graded slate', () => {
|
|
const r = opsWatch.retentionZeroWriteAlarm(
|
|
[{ sport: 'wnba', status: 'ok', gradeCount: 25 }], { wnba: 4 });
|
|
expect(r.alarm).toBe(true);
|
|
expect(r.reason).toMatch(/wrote 4/);
|
|
});
|
|
|
|
test('quiet when retention wrote at least the slate (both sides + refusals)', () => {
|
|
const r = opsWatch.retentionZeroWriteAlarm(
|
|
[{ sport: 'mlb', status: 'ok', gradeCount: 7 }], { mlb: 50 });
|
|
expect(r.alarm).toBe(false);
|
|
});
|
|
|
|
test('a slot that graded nothing never false-pages', () => {
|
|
expect(opsWatch.retentionZeroWriteAlarm(
|
|
[{ sport: 'soccer', status: 'skipped', reason: 'no props', gradeCount: 0 }], {}).alarm).toBe(false);
|
|
expect(opsWatch.retentionZeroWriteAlarm(
|
|
[{ sport: 'mlb', status: 'ok', gradeCount: 0 }], { mlb: 0 }).alarm).toBe(false);
|
|
});
|
|
|
|
test('a missing/undefined count is treated as zero, not as fine', () => {
|
|
const r = opsWatch.retentionZeroWriteAlarm([{ sport: 'mlb', status: 'ok', gradeCount: 7 }], {});
|
|
expect(r.alarm).toBe(true);
|
|
});
|
|
});
|