044941d406
Settlement alarm (settle-pass THROW pages high; morning zero-settle alarm keyed off the Postgres ledger settle results, once per ET date, never on an empty yesterday), per-sport 3-consecutive-slot failure pager (pure opsWatch.createFailureTracker, pages once per losing streak), odds-api >=80% quota alert (once per day, Redis-deduped), systemHealth (statfs + os mem, disk>85 / mem>90 pages), daily 9 AM ET pulse (ONE notification: ledger rows yesterday via ledgerService.countRowsForDate, settles 24h, quota, disk/mem, desk line), docs/OPS-RUNBOOK.md (Uptime Kuma monitors, Coolify deploy-failure -> ntfy, phone subscription). All copy VOICE v1.1 — deadpan, numbers, no exclamation points (tests lint for it). 2398 -> 2437 tests. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
208 lines
8.7 KiB
JavaScript
208 lines
8.7 KiB
JavaScript
'use strict';
|
|
|
|
// Session 8 (A1 board) — scheduler ops wiring: settle alarms, failure pager,
|
|
// quota check, daily pulse. All deps injected; zero network, zero redis.
|
|
|
|
const { startSnapshotScheduler } = require('../../src/snapshotScheduler');
|
|
|
|
function memCache(seed = {}) {
|
|
const store = { ...seed };
|
|
return {
|
|
store,
|
|
cacheGet: async (k) => (k in store ? store[k] : null),
|
|
cacheSet: async (k, v) => { store[k] = v; return true; },
|
|
};
|
|
}
|
|
|
|
function collector() {
|
|
const msgs = [];
|
|
return { msgs, notify: async (m, o = {}) => { msgs.push({ m, o }); return { sent: true }; } };
|
|
}
|
|
|
|
// Base deps: quiet pipeline, everything healthy, injectable clock.
|
|
function baseDeps(cache, notes, dRef) {
|
|
return {
|
|
...cache,
|
|
notify: notes.notify,
|
|
now: () => dRef.d,
|
|
runAllSnapshots: async () => [{ sport: 'mlb', status: 'ok', gradeCount: 5 }],
|
|
settleAllOutcomes: async () => [{ sport: 'mlb', settled: 3, pending: 0 }],
|
|
settleAllLedgers: async () => [{ sport: 'mlb', settled: 3, pending: 1 }],
|
|
getQuotaStatus: async () => ({ pct: 0.1, used: 50, limit: 500 }),
|
|
getSystemHealth: async () => ({ disk_pct: 40, mem_pct: 50 }),
|
|
countLedgerRows: async () => 42,
|
|
};
|
|
}
|
|
|
|
function makeSched(overrides = {}, startIso = '2026-07-11T14:00:00Z') {
|
|
process.env.SNAPSHOT_CRON = '1';
|
|
const cache = memCache();
|
|
const notes = collector();
|
|
const dRef = { d: new Date(startIso) };
|
|
const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {});
|
|
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const sched = startSnapshotScheduler({ ...baseDeps(cache, notes, dRef), ...overrides });
|
|
clearInterval(sched.interval);
|
|
logSpy.mockRestore();
|
|
warnSpy.mockRestore();
|
|
return { sched, cache, notes, dRef };
|
|
}
|
|
|
|
afterEach(() => { delete process.env.SNAPSHOT_CRON; });
|
|
|
|
describe('settlement alarm', () => {
|
|
test('outcome settle THROW pages high priority', async () => {
|
|
const { sched, notes } = makeSched({
|
|
settleAllOutcomes: async () => { throw new Error('mlb stats 500'); },
|
|
});
|
|
await sched.tick();
|
|
const alert = notes.msgs.find((x) => x.m.includes('Outcome settlement THREW'));
|
|
expect(alert).toBeTruthy();
|
|
expect(alert.o.priority).toBe('high');
|
|
expect(alert.m).toContain('mlb stats 500');
|
|
expect(alert.m).not.toContain('!');
|
|
});
|
|
|
|
test('ledger settle THROW pages high priority', async () => {
|
|
const { sched, notes } = makeSched({
|
|
settleAllLedgers: async () => { throw new Error('supabase timeout'); },
|
|
});
|
|
await sched.tick();
|
|
const alert = notes.msgs.find((x) => x.m.includes('Ledger settlement THREW'));
|
|
expect(alert).toBeTruthy();
|
|
expect(alert.o.priority).toBe('high');
|
|
});
|
|
|
|
test('morning zero-settle alarm fires once per day, only when rows were pending', async () => {
|
|
const { sched, notes, cache } = makeSched({
|
|
settleAllLedgers: async () => [{ sport: 'mlb', settled: 0, pending: 12 }],
|
|
});
|
|
await sched.tick(); // 14:00 UTC — the morning slot
|
|
const alarms = notes.msgs.filter((x) => x.m.includes('0 settles'));
|
|
expect(alarms).toHaveLength(1);
|
|
expect(alarms[0].m).toContain('12 ledger rows');
|
|
expect(alarms[0].o.priority).toBe('high');
|
|
expect(cache.store['ops:settle_zero:2026-07-11']).toBe('1');
|
|
expect(alarms[0].m).not.toContain('!');
|
|
});
|
|
|
|
test('redis dedupe suppresses a second zero-settle alarm the same ET day', async () => {
|
|
const { sched, notes } = makeSched({
|
|
settleAllLedgers: async () => [{ sport: 'mlb', settled: 0, pending: 12 }],
|
|
cacheGet: async (k) => (k === 'ops:settle_zero:2026-07-11' ? '1' : null),
|
|
});
|
|
await sched.tick();
|
|
expect(notes.msgs.filter((x) => x.m.includes('0 settles'))).toHaveLength(0);
|
|
});
|
|
|
|
test('no alarm at a non-morning slot even with zero settles', async () => {
|
|
const { sched, notes } = makeSched({
|
|
settleAllLedgers: async () => [{ sport: 'mlb', settled: 0, pending: 12 }],
|
|
}, '2026-07-11T19:00:00Z');
|
|
await sched.tick();
|
|
expect(notes.msgs.filter((x) => x.m.includes('0 settles'))).toHaveLength(0);
|
|
});
|
|
|
|
test('genuinely empty yesterday (0 fetched rows) never alarms', async () => {
|
|
const { sched, notes } = makeSched({
|
|
settleAllLedgers: async () => [{ sport: 'mlb', settled: 0, pending: 0 }],
|
|
});
|
|
await sched.tick();
|
|
expect(notes.msgs.filter((x) => x.m.includes('0 settles'))).toHaveLength(0);
|
|
});
|
|
});
|
|
|
|
describe('persistent snapshot-failure pager', () => {
|
|
test('three consecutive failing slots page exactly once; success re-arms', async () => {
|
|
let result = { sport: 'mlb', status: 'skipped', reason: 'no props', gradeCount: 0 };
|
|
const { sched, notes, dRef } = makeSched({
|
|
runAllSnapshots: async () => [result],
|
|
});
|
|
const slots = ['2026-07-11T14:00:00Z', '2026-07-11T19:00:00Z', '2026-07-11T22:00:00Z', '2026-07-12T01:00:00Z'];
|
|
for (const iso of slots.slice(0, 2)) { dRef.d = new Date(iso); await sched.tick(); }
|
|
expect(notes.msgs.filter((x) => x.m.includes('consecutive'))).toHaveLength(0);
|
|
dRef.d = new Date(slots[2]); await sched.tick(); // third consecutive
|
|
let pages = notes.msgs.filter((x) => x.m.includes('consecutive'));
|
|
expect(pages).toHaveLength(1);
|
|
expect(pages[0].o.priority).toBe('high');
|
|
expect(pages[0].m).toContain('MLB');
|
|
expect(pages[0].m).toContain('3 consecutive');
|
|
expect(pages[0].m).not.toContain('!');
|
|
dRef.d = new Date(slots[3]); await sched.tick(); // fourth — no re-page
|
|
expect(notes.msgs.filter((x) => x.m.includes('consecutive'))).toHaveLength(1);
|
|
// success resets, then three more failures page again
|
|
result = { sport: 'mlb', status: 'ok', gradeCount: 4 };
|
|
dRef.d = new Date('2026-07-12T03:00:00Z'); await sched.tick();
|
|
result = { sport: 'mlb', status: 'error', reason: 'down', gradeCount: 0 };
|
|
for (const iso of ['2026-07-12T14:00:00Z', '2026-07-12T19:00:00Z', '2026-07-12T22:00:00Z']) {
|
|
dRef.d = new Date(iso); await sched.tick();
|
|
}
|
|
expect(notes.msgs.filter((x) => x.m.includes('consecutive'))).toHaveLength(2);
|
|
});
|
|
});
|
|
|
|
describe('quota daily check', () => {
|
|
test('>= 80% after a snapshot run alerts once per day via redis dedupe', async () => {
|
|
const { sched, notes, dRef, cache } = makeSched({
|
|
getQuotaStatus: async () => ({ pct: 0.84, used: 420, limit: 500, quotaType: 'monthly' }),
|
|
});
|
|
await sched.tick();
|
|
dRef.d = new Date('2026-07-11T19:00:00Z');
|
|
await sched.tick(); // second slot, same day — deduped
|
|
const alerts = notes.msgs.filter((x) => x.o.title === 'VYNDR quota');
|
|
expect(alerts).toHaveLength(1);
|
|
expect(alerts[0].m).toContain('84%');
|
|
expect(cache.store['ops:quota_day:odds-api:2026-07-11']).toBe('1');
|
|
});
|
|
});
|
|
|
|
describe('daily pulse', () => {
|
|
test('fires ONE notification at 13:00 UTC with the assembled fields', async () => {
|
|
const { sched, notes } = makeSched({}, '2026-07-11T13:00:00Z');
|
|
await sched.tick();
|
|
await sched.tick(); // same minute — in-process dedupe
|
|
const pulses = notes.msgs.filter((x) => x.o.title === 'VYNDR daily pulse');
|
|
expect(pulses).toHaveLength(1);
|
|
expect(pulses[0].m).toContain('ledger rows yesterday: 42');
|
|
expect(pulses[0].m).toContain('odds-api quota: 10% (50/500)');
|
|
expect(pulses[0].m).toContain('disk 40% · mem 50%');
|
|
expect(pulses[0].m).toContain('desk pack: see /desk');
|
|
expect(pulses[0].m).not.toContain('!');
|
|
});
|
|
|
|
test('redis date key suppresses a duplicate pulse after a restart', async () => {
|
|
const { sched, notes } = makeSched({
|
|
cacheGet: async (k) => (k === 'ops:pulse:2026-07-11' ? '1' : null),
|
|
}, '2026-07-11T13:00:00Z');
|
|
await sched.tick();
|
|
expect(notes.msgs.filter((x) => x.o.title === 'VYNDR daily pulse')).toHaveLength(0);
|
|
});
|
|
|
|
test('does not fire off the pulse hour', async () => {
|
|
const { sched, notes } = makeSched({}, '2026-07-11T12:00:00Z');
|
|
await sched.tick();
|
|
expect(notes.msgs.filter((x) => x.o.title === 'VYNDR daily pulse')).toHaveLength(0);
|
|
});
|
|
|
|
test('box health over threshold pages separately at high priority', async () => {
|
|
const { sched, notes } = makeSched({
|
|
getSystemHealth: async () => ({ disk_pct: 92, mem_pct: 50 }),
|
|
}, '2026-07-11T13:00:00Z');
|
|
await sched.tick();
|
|
const box = notes.msgs.filter((x) => x.o.title === 'VYNDR box');
|
|
expect(box).toHaveLength(1);
|
|
expect(box[0].o.priority).toBe('high');
|
|
expect(box[0].m).toContain('disk at 92%');
|
|
expect(box[0].m).not.toContain('!');
|
|
});
|
|
|
|
test('supabase unconfigured renders n/a, never a fabricated zero', async () => {
|
|
const { sched, notes } = makeSched({
|
|
countLedgerRows: async () => null,
|
|
}, '2026-07-11T13:00:00Z');
|
|
await sched.tick();
|
|
const pulse = notes.msgs.find((x) => x.o.title === 'VYNDR daily pulse');
|
|
expect(pulse.m).toContain('ledger rows yesterday: n/a');
|
|
});
|
|
});
|