Files
vyndr/tests/unit/opsScheduler.test.js
T
builtbykev 044941d406 S8 (a1): ops — the product watches itself
Settlement alarm (settle-pass THROW pages high; morning zero-settle alarm
keyed off the Postgres ledger settle results, once per ET date, never on an
empty yesterday), per-sport 3-consecutive-slot failure pager (pure
opsWatch.createFailureTracker, pages once per losing streak), odds-api >=80%
quota alert (once per day, Redis-deduped), systemHealth (statfs + os mem,
disk>85 / mem>90 pages), daily 9 AM ET pulse (ONE notification: ledger rows
yesterday via ledgerService.countRowsForDate, settles 24h, quota, disk/mem,
desk line), docs/OPS-RUNBOOK.md (Uptime Kuma monitors, Coolify
deploy-failure -> ntfy, phone subscription). All copy VOICE v1.1 — deadpan,
numbers, no exclamation points (tests lint for it). 2398 -> 2437 tests.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-11 14:34:45 -04:00

208 lines
8.7 KiB
JavaScript

'use strict';
// Session 8 (A1 board) — scheduler ops wiring: settle alarms, failure pager,
// quota check, daily pulse. All deps injected; zero network, zero redis.
const { startSnapshotScheduler } = require('../../src/snapshotScheduler');
function memCache(seed = {}) {
const store = { ...seed };
return {
store,
cacheGet: async (k) => (k in store ? store[k] : null),
cacheSet: async (k, v) => { store[k] = v; return true; },
};
}
function collector() {
const msgs = [];
return { msgs, notify: async (m, o = {}) => { msgs.push({ m, o }); return { sent: true }; } };
}
// Base deps: quiet pipeline, everything healthy, injectable clock.
function baseDeps(cache, notes, dRef) {
return {
...cache,
notify: notes.notify,
now: () => dRef.d,
runAllSnapshots: async () => [{ sport: 'mlb', status: 'ok', gradeCount: 5 }],
settleAllOutcomes: async () => [{ sport: 'mlb', settled: 3, pending: 0 }],
settleAllLedgers: async () => [{ sport: 'mlb', settled: 3, pending: 1 }],
getQuotaStatus: async () => ({ pct: 0.1, used: 50, limit: 500 }),
getSystemHealth: async () => ({ disk_pct: 40, mem_pct: 50 }),
countLedgerRows: async () => 42,
};
}
function makeSched(overrides = {}, startIso = '2026-07-11T14:00:00Z') {
process.env.SNAPSHOT_CRON = '1';
const cache = memCache();
const notes = collector();
const dRef = { d: new Date(startIso) };
const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {});
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
const sched = startSnapshotScheduler({ ...baseDeps(cache, notes, dRef), ...overrides });
clearInterval(sched.interval);
logSpy.mockRestore();
warnSpy.mockRestore();
return { sched, cache, notes, dRef };
}
afterEach(() => { delete process.env.SNAPSHOT_CRON; });
describe('settlement alarm', () => {
test('outcome settle THROW pages high priority', async () => {
const { sched, notes } = makeSched({
settleAllOutcomes: async () => { throw new Error('mlb stats 500'); },
});
await sched.tick();
const alert = notes.msgs.find((x) => x.m.includes('Outcome settlement THREW'));
expect(alert).toBeTruthy();
expect(alert.o.priority).toBe('high');
expect(alert.m).toContain('mlb stats 500');
expect(alert.m).not.toContain('!');
});
test('ledger settle THROW pages high priority', async () => {
const { sched, notes } = makeSched({
settleAllLedgers: async () => { throw new Error('supabase timeout'); },
});
await sched.tick();
const alert = notes.msgs.find((x) => x.m.includes('Ledger settlement THREW'));
expect(alert).toBeTruthy();
expect(alert.o.priority).toBe('high');
});
test('morning zero-settle alarm fires once per day, only when rows were pending', async () => {
const { sched, notes, cache } = makeSched({
settleAllLedgers: async () => [{ sport: 'mlb', settled: 0, pending: 12 }],
});
await sched.tick(); // 14:00 UTC — the morning slot
const alarms = notes.msgs.filter((x) => x.m.includes('0 settles'));
expect(alarms).toHaveLength(1);
expect(alarms[0].m).toContain('12 ledger rows');
expect(alarms[0].o.priority).toBe('high');
expect(cache.store['ops:settle_zero:2026-07-11']).toBe('1');
expect(alarms[0].m).not.toContain('!');
});
test('redis dedupe suppresses a second zero-settle alarm the same ET day', async () => {
const { sched, notes } = makeSched({
settleAllLedgers: async () => [{ sport: 'mlb', settled: 0, pending: 12 }],
cacheGet: async (k) => (k === 'ops:settle_zero:2026-07-11' ? '1' : null),
});
await sched.tick();
expect(notes.msgs.filter((x) => x.m.includes('0 settles'))).toHaveLength(0);
});
test('no alarm at a non-morning slot even with zero settles', async () => {
const { sched, notes } = makeSched({
settleAllLedgers: async () => [{ sport: 'mlb', settled: 0, pending: 12 }],
}, '2026-07-11T19:00:00Z');
await sched.tick();
expect(notes.msgs.filter((x) => x.m.includes('0 settles'))).toHaveLength(0);
});
test('genuinely empty yesterday (0 fetched rows) never alarms', async () => {
const { sched, notes } = makeSched({
settleAllLedgers: async () => [{ sport: 'mlb', settled: 0, pending: 0 }],
});
await sched.tick();
expect(notes.msgs.filter((x) => x.m.includes('0 settles'))).toHaveLength(0);
});
});
describe('persistent snapshot-failure pager', () => {
test('three consecutive failing slots page exactly once; success re-arms', async () => {
let result = { sport: 'mlb', status: 'skipped', reason: 'no props', gradeCount: 0 };
const { sched, notes, dRef } = makeSched({
runAllSnapshots: async () => [result],
});
const slots = ['2026-07-11T14:00:00Z', '2026-07-11T19:00:00Z', '2026-07-11T22:00:00Z', '2026-07-12T01:00:00Z'];
for (const iso of slots.slice(0, 2)) { dRef.d = new Date(iso); await sched.tick(); }
expect(notes.msgs.filter((x) => x.m.includes('consecutive'))).toHaveLength(0);
dRef.d = new Date(slots[2]); await sched.tick(); // third consecutive
let pages = notes.msgs.filter((x) => x.m.includes('consecutive'));
expect(pages).toHaveLength(1);
expect(pages[0].o.priority).toBe('high');
expect(pages[0].m).toContain('MLB');
expect(pages[0].m).toContain('3 consecutive');
expect(pages[0].m).not.toContain('!');
dRef.d = new Date(slots[3]); await sched.tick(); // fourth — no re-page
expect(notes.msgs.filter((x) => x.m.includes('consecutive'))).toHaveLength(1);
// success resets, then three more failures page again
result = { sport: 'mlb', status: 'ok', gradeCount: 4 };
dRef.d = new Date('2026-07-12T03:00:00Z'); await sched.tick();
result = { sport: 'mlb', status: 'error', reason: 'down', gradeCount: 0 };
for (const iso of ['2026-07-12T14:00:00Z', '2026-07-12T19:00:00Z', '2026-07-12T22:00:00Z']) {
dRef.d = new Date(iso); await sched.tick();
}
expect(notes.msgs.filter((x) => x.m.includes('consecutive'))).toHaveLength(2);
});
});
describe('quota daily check', () => {
test('>= 80% after a snapshot run alerts once per day via redis dedupe', async () => {
const { sched, notes, dRef, cache } = makeSched({
getQuotaStatus: async () => ({ pct: 0.84, used: 420, limit: 500, quotaType: 'monthly' }),
});
await sched.tick();
dRef.d = new Date('2026-07-11T19:00:00Z');
await sched.tick(); // second slot, same day — deduped
const alerts = notes.msgs.filter((x) => x.o.title === 'VYNDR quota');
expect(alerts).toHaveLength(1);
expect(alerts[0].m).toContain('84%');
expect(cache.store['ops:quota_day:odds-api:2026-07-11']).toBe('1');
});
});
describe('daily pulse', () => {
test('fires ONE notification at 13:00 UTC with the assembled fields', async () => {
const { sched, notes } = makeSched({}, '2026-07-11T13:00:00Z');
await sched.tick();
await sched.tick(); // same minute — in-process dedupe
const pulses = notes.msgs.filter((x) => x.o.title === 'VYNDR daily pulse');
expect(pulses).toHaveLength(1);
expect(pulses[0].m).toContain('ledger rows yesterday: 42');
expect(pulses[0].m).toContain('odds-api quota: 10% (50/500)');
expect(pulses[0].m).toContain('disk 40% · mem 50%');
expect(pulses[0].m).toContain('desk pack: see /desk');
expect(pulses[0].m).not.toContain('!');
});
test('redis date key suppresses a duplicate pulse after a restart', async () => {
const { sched, notes } = makeSched({
cacheGet: async (k) => (k === 'ops:pulse:2026-07-11' ? '1' : null),
}, '2026-07-11T13:00:00Z');
await sched.tick();
expect(notes.msgs.filter((x) => x.o.title === 'VYNDR daily pulse')).toHaveLength(0);
});
test('does not fire off the pulse hour', async () => {
const { sched, notes } = makeSched({}, '2026-07-11T12:00:00Z');
await sched.tick();
expect(notes.msgs.filter((x) => x.o.title === 'VYNDR daily pulse')).toHaveLength(0);
});
test('box health over threshold pages separately at high priority', async () => {
const { sched, notes } = makeSched({
getSystemHealth: async () => ({ disk_pct: 92, mem_pct: 50 }),
}, '2026-07-11T13:00:00Z');
await sched.tick();
const box = notes.msgs.filter((x) => x.o.title === 'VYNDR box');
expect(box).toHaveLength(1);
expect(box[0].o.priority).toBe('high');
expect(box[0].m).toContain('disk at 92%');
expect(box[0].m).not.toContain('!');
});
test('supabase unconfigured renders n/a, never a fabricated zero', async () => {
const { sched, notes } = makeSched({
countLedgerRows: async () => null,
}, '2026-07-11T13:00:00Z');
await sched.tick();
const pulse = notes.msgs.find((x) => x.o.title === 'VYNDR daily pulse');
expect(pulse.m).toContain('ledger rows yesterday: n/a');
});
});