S8 (a1): ops — the product watches itself

Settlement alarm (settle-pass THROW pages high; morning zero-settle alarm
keyed off the Postgres ledger settle results, once per ET date, never on an
empty yesterday), per-sport 3-consecutive-slot failure pager (pure
opsWatch.createFailureTracker, pages once per losing streak), odds-api >=80%
quota alert (once per day, Redis-deduped), systemHealth (statfs + os mem,
disk>85 / mem>90 pages), daily 9 AM ET pulse (ONE notification: ledger rows
yesterday via ledgerService.countRowsForDate, settles 24h, quota, disk/mem,
desk line), docs/OPS-RUNBOOK.md (Uptime Kuma monitors, Coolify
deploy-failure -> ntfy, phone subscription). All copy VOICE v1.1 — deadpan,
numbers, no exclamation points (tests lint for it). 2398 -> 2437 tests.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Kev
2026-07-11 14:34:45 -04:00
parent e4d2e79f95
commit 044941d406
11 changed files with 1083 additions and 4 deletions
+207
View File
@@ -0,0 +1,207 @@
'use strict';
// Session 8 (A1 board) — scheduler ops wiring: settle alarms, failure pager,
// quota check, daily pulse. All deps injected; zero network, zero redis.
const { startSnapshotScheduler } = require('../../src/snapshotScheduler');
function memCache(seed = {}) {
const store = { ...seed };
return {
store,
cacheGet: async (k) => (k in store ? store[k] : null),
cacheSet: async (k, v) => { store[k] = v; return true; },
};
}
function collector() {
const msgs = [];
return { msgs, notify: async (m, o = {}) => { msgs.push({ m, o }); return { sent: true }; } };
}
// Base deps: quiet pipeline, everything healthy, injectable clock.
function baseDeps(cache, notes, dRef) {
return {
...cache,
notify: notes.notify,
now: () => dRef.d,
runAllSnapshots: async () => [{ sport: 'mlb', status: 'ok', gradeCount: 5 }],
settleAllOutcomes: async () => [{ sport: 'mlb', settled: 3, pending: 0 }],
settleAllLedgers: async () => [{ sport: 'mlb', settled: 3, pending: 1 }],
getQuotaStatus: async () => ({ pct: 0.1, used: 50, limit: 500 }),
getSystemHealth: async () => ({ disk_pct: 40, mem_pct: 50 }),
countLedgerRows: async () => 42,
};
}
function makeSched(overrides = {}, startIso = '2026-07-11T14:00:00Z') {
process.env.SNAPSHOT_CRON = '1';
const cache = memCache();
const notes = collector();
const dRef = { d: new Date(startIso) };
const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {});
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
const sched = startSnapshotScheduler({ ...baseDeps(cache, notes, dRef), ...overrides });
clearInterval(sched.interval);
logSpy.mockRestore();
warnSpy.mockRestore();
return { sched, cache, notes, dRef };
}
afterEach(() => { delete process.env.SNAPSHOT_CRON; });
describe('settlement alarm', () => {
test('outcome settle THROW pages high priority', async () => {
const { sched, notes } = makeSched({
settleAllOutcomes: async () => { throw new Error('mlb stats 500'); },
});
await sched.tick();
const alert = notes.msgs.find((x) => x.m.includes('Outcome settlement THREW'));
expect(alert).toBeTruthy();
expect(alert.o.priority).toBe('high');
expect(alert.m).toContain('mlb stats 500');
expect(alert.m).not.toContain('!');
});
test('ledger settle THROW pages high priority', async () => {
const { sched, notes } = makeSched({
settleAllLedgers: async () => { throw new Error('supabase timeout'); },
});
await sched.tick();
const alert = notes.msgs.find((x) => x.m.includes('Ledger settlement THREW'));
expect(alert).toBeTruthy();
expect(alert.o.priority).toBe('high');
});
test('morning zero-settle alarm fires once per day, only when rows were pending', async () => {
const { sched, notes, cache } = makeSched({
settleAllLedgers: async () => [{ sport: 'mlb', settled: 0, pending: 12 }],
});
await sched.tick(); // 14:00 UTC — the morning slot
const alarms = notes.msgs.filter((x) => x.m.includes('0 settles'));
expect(alarms).toHaveLength(1);
expect(alarms[0].m).toContain('12 ledger rows');
expect(alarms[0].o.priority).toBe('high');
expect(cache.store['ops:settle_zero:2026-07-11']).toBe('1');
expect(alarms[0].m).not.toContain('!');
});
test('redis dedupe suppresses a second zero-settle alarm the same ET day', async () => {
const { sched, notes } = makeSched({
settleAllLedgers: async () => [{ sport: 'mlb', settled: 0, pending: 12 }],
cacheGet: async (k) => (k === 'ops:settle_zero:2026-07-11' ? '1' : null),
});
await sched.tick();
expect(notes.msgs.filter((x) => x.m.includes('0 settles'))).toHaveLength(0);
});
test('no alarm at a non-morning slot even with zero settles', async () => {
const { sched, notes } = makeSched({
settleAllLedgers: async () => [{ sport: 'mlb', settled: 0, pending: 12 }],
}, '2026-07-11T19:00:00Z');
await sched.tick();
expect(notes.msgs.filter((x) => x.m.includes('0 settles'))).toHaveLength(0);
});
test('genuinely empty yesterday (0 fetched rows) never alarms', async () => {
const { sched, notes } = makeSched({
settleAllLedgers: async () => [{ sport: 'mlb', settled: 0, pending: 0 }],
});
await sched.tick();
expect(notes.msgs.filter((x) => x.m.includes('0 settles'))).toHaveLength(0);
});
});
describe('persistent snapshot-failure pager', () => {
test('three consecutive failing slots page exactly once; success re-arms', async () => {
let result = { sport: 'mlb', status: 'skipped', reason: 'no props', gradeCount: 0 };
const { sched, notes, dRef } = makeSched({
runAllSnapshots: async () => [result],
});
const slots = ['2026-07-11T14:00:00Z', '2026-07-11T19:00:00Z', '2026-07-11T22:00:00Z', '2026-07-12T01:00:00Z'];
for (const iso of slots.slice(0, 2)) { dRef.d = new Date(iso); await sched.tick(); }
expect(notes.msgs.filter((x) => x.m.includes('consecutive'))).toHaveLength(0);
dRef.d = new Date(slots[2]); await sched.tick(); // third consecutive
let pages = notes.msgs.filter((x) => x.m.includes('consecutive'));
expect(pages).toHaveLength(1);
expect(pages[0].o.priority).toBe('high');
expect(pages[0].m).toContain('MLB');
expect(pages[0].m).toContain('3 consecutive');
expect(pages[0].m).not.toContain('!');
dRef.d = new Date(slots[3]); await sched.tick(); // fourth — no re-page
expect(notes.msgs.filter((x) => x.m.includes('consecutive'))).toHaveLength(1);
// success resets, then three more failures page again
result = { sport: 'mlb', status: 'ok', gradeCount: 4 };
dRef.d = new Date('2026-07-12T03:00:00Z'); await sched.tick();
result = { sport: 'mlb', status: 'error', reason: 'down', gradeCount: 0 };
for (const iso of ['2026-07-12T14:00:00Z', '2026-07-12T19:00:00Z', '2026-07-12T22:00:00Z']) {
dRef.d = new Date(iso); await sched.tick();
}
expect(notes.msgs.filter((x) => x.m.includes('consecutive'))).toHaveLength(2);
});
});
describe('quota daily check', () => {
test('>= 80% after a snapshot run alerts once per day via redis dedupe', async () => {
const { sched, notes, dRef, cache } = makeSched({
getQuotaStatus: async () => ({ pct: 0.84, used: 420, limit: 500, quotaType: 'monthly' }),
});
await sched.tick();
dRef.d = new Date('2026-07-11T19:00:00Z');
await sched.tick(); // second slot, same day — deduped
const alerts = notes.msgs.filter((x) => x.o.title === 'VYNDR quota');
expect(alerts).toHaveLength(1);
expect(alerts[0].m).toContain('84%');
expect(cache.store['ops:quota_day:odds-api:2026-07-11']).toBe('1');
});
});
describe('daily pulse', () => {
test('fires ONE notification at 13:00 UTC with the assembled fields', async () => {
const { sched, notes } = makeSched({}, '2026-07-11T13:00:00Z');
await sched.tick();
await sched.tick(); // same minute — in-process dedupe
const pulses = notes.msgs.filter((x) => x.o.title === 'VYNDR daily pulse');
expect(pulses).toHaveLength(1);
expect(pulses[0].m).toContain('ledger rows yesterday: 42');
expect(pulses[0].m).toContain('odds-api quota: 10% (50/500)');
expect(pulses[0].m).toContain('disk 40% · mem 50%');
expect(pulses[0].m).toContain('desk pack: see /desk');
expect(pulses[0].m).not.toContain('!');
});
test('redis date key suppresses a duplicate pulse after a restart', async () => {
const { sched, notes } = makeSched({
cacheGet: async (k) => (k === 'ops:pulse:2026-07-11' ? '1' : null),
}, '2026-07-11T13:00:00Z');
await sched.tick();
expect(notes.msgs.filter((x) => x.o.title === 'VYNDR daily pulse')).toHaveLength(0);
});
test('does not fire off the pulse hour', async () => {
const { sched, notes } = makeSched({}, '2026-07-11T12:00:00Z');
await sched.tick();
expect(notes.msgs.filter((x) => x.o.title === 'VYNDR daily pulse')).toHaveLength(0);
});
test('box health over threshold pages separately at high priority', async () => {
const { sched, notes } = makeSched({
getSystemHealth: async () => ({ disk_pct: 92, mem_pct: 50 }),
}, '2026-07-11T13:00:00Z');
await sched.tick();
const box = notes.msgs.filter((x) => x.o.title === 'VYNDR box');
expect(box).toHaveLength(1);
expect(box[0].o.priority).toBe('high');
expect(box[0].m).toContain('disk at 92%');
expect(box[0].m).not.toContain('!');
});
test('supabase unconfigured renders n/a, never a fabricated zero', async () => {
const { sched, notes } = makeSched({
countLedgerRows: async () => null,
}, '2026-07-11T13:00:00Z');
await sched.tick();
const pulse = notes.msgs.find((x) => x.o.title === 'VYNDR daily pulse');
expect(pulse.m).toContain('ledger rows yesterday: n/a');
});
});
+209
View File
@@ -0,0 +1,209 @@
'use strict';
// Session 8 (A1 board) — opsWatch: the pure ops-alarm logic.
const {
createFailureTracker,
isBadSnapshotResult,
zeroSettleAlarm,
morningHourUtc,
checkQuotaDaily,
countRecentSettles,
buildPulseMessage,
dateET,
} = require('../../src/services/opsWatch');
function memCache(seed = {}) {
const store = { ...seed };
return {
store,
cacheGet: async (k) => (k in store ? store[k] : null),
cacheSet: async (k, v) => { store[k] = v; return true; },
};
}
describe('isBadSnapshotResult', () => {
test('error and skipped/no-props are bad; ok and skipped/no-grades are not', () => {
expect(isBadSnapshotResult({ status: 'error', reason: 'down' })).toBe(true);
expect(isBadSnapshotResult({ status: 'skipped', reason: 'no props' })).toBe(true);
expect(isBadSnapshotResult({ status: 'ok' })).toBe(false);
expect(isBadSnapshotResult({ status: 'skipped', reason: 'no grades' })).toBe(false);
expect(isBadSnapshotResult(null)).toBe(false);
});
});
describe('createFailureTracker', () => {
test('pages exactly once at the third consecutive failure, stays quiet after', () => {
const t = createFailureTracker(3);
const bad = { status: 'error', reason: 'x' };
expect(t.record('mlb', bad).shouldPage).toBe(false); // 1
expect(t.record('mlb', bad).shouldPage).toBe(false); // 2
const third = t.record('mlb', bad);
expect(third.shouldPage).toBe(true); // 3 -> page once
expect(third.count).toBe(3);
expect(t.record('mlb', bad).shouldPage).toBe(false); // 4 -> no re-page
expect(t.count('mlb')).toBe(4);
});
test('any good outcome resets the counter and re-arms the pager', () => {
const t = createFailureTracker(3);
const bad = { status: 'skipped', reason: 'no props' };
t.record('mlb', bad); t.record('mlb', bad);
t.record('mlb', { status: 'ok' }); // reset
expect(t.count('mlb')).toBe(0);
t.record('mlb', bad); t.record('mlb', bad);
expect(t.record('mlb', bad).shouldPage).toBe(true); // re-armed -> pages again
});
test('counts are independent per sport', () => {
const t = createFailureTracker(3);
const bad = { status: 'error' };
t.record('mlb', bad); t.record('mlb', bad);
expect(t.record('wnba', bad).count).toBe(1);
expect(t.count('mlb')).toBe(2);
});
});
describe('zeroSettleAlarm', () => {
test('alarms when settleable rows existed and none settled', () => {
const z = zeroSettleAlarm([{ sport: 'mlb', settled: 0, pending: 12 }]);
expect(z).toEqual({ alarm: true, settled: 0, pending: 12 });
});
test('never alarms on a genuinely empty yesterday (0 rows fetched)', () => {
expect(zeroSettleAlarm([{ sport: 'mlb', settled: 0, pending: 0 }]).alarm).toBe(false);
expect(zeroSettleAlarm([]).alarm).toBe(false);
expect(zeroSettleAlarm(null).alarm).toBe(false);
});
test('no alarm when anything settled', () => {
expect(zeroSettleAlarm([{ sport: 'mlb', settled: 3, pending: 9 }]).alarm).toBe(false);
});
test('ignores sports without a settled-result feed (WNBA pendings are honest)', () => {
const z = zeroSettleAlarm([
{ sport: 'mlb', settled: 0, pending: 0 },
{ sport: 'wnba', settled: 0, pending: 40 },
]);
expect(z.alarm).toBe(false);
});
});
describe('morningHourUtc', () => {
test('first configured hour >= 06 UTC (1,3 are late-night ET of the prior day)', () => {
expect(morningHourUtc([14, 19, 22, 1, 3])).toBe(14);
expect(morningHourUtc([19, 14])).toBe(14);
});
test('falls back to the smallest hour when nothing is daytime', () => {
expect(morningHourUtc([1, 3])).toBe(1);
expect(morningHourUtc([])).toBe(14);
});
});
describe('checkQuotaDaily', () => {
const notifyCollector = () => {
const msgs = [];
return { msgs, notify: async (m, o) => { msgs.push({ m, o }); return { sent: true }; } };
};
test('below 80% -> no alert', async () => {
const { msgs, notify } = notifyCollector();
const cache = memCache();
const r = await checkQuotaDaily({
getStatus: async () => ({ pct: 0.5, used: 250, limit: 500 }),
...cache, notify, now: () => new Date('2026-07-11T15:00:00Z'),
});
expect(r.alerted).toBe(false);
expect(msgs).toHaveLength(0);
});
test('>= 80% alerts once, then dedupes for the rest of the day', async () => {
const { msgs, notify } = notifyCollector();
const cache = memCache();
const deps = {
getStatus: async () => ({ pct: 0.82, used: 410, limit: 500, quotaType: 'monthly' }),
...cache, notify, now: () => new Date('2026-07-11T15:00:00Z'),
};
expect((await checkQuotaDaily(deps)).alerted).toBe(true);
const second = await checkQuotaDaily(deps);
expect(second.alerted).toBe(false);
expect(second.deduped).toBe(true);
expect(msgs).toHaveLength(1);
expect(msgs[0].m).toContain('82%');
expect(msgs[0].m).toContain('410/500');
expect(msgs[0].o.priority).toBe('high');
expect(msgs[0].m).not.toContain('!');
expect(cache.store['ops:quota_day:odds-api:2026-07-11']).toBe('1');
});
test('fires again the next day (date-keyed dedupe)', async () => {
const { msgs, notify } = notifyCollector();
const cache = memCache();
const base = {
getStatus: async () => ({ pct: 0.9, used: 450, limit: 500 }),
...cache, notify,
};
await checkQuotaDaily({ ...base, now: () => new Date('2026-07-11T15:00:00Z') });
await checkQuotaDaily({ ...base, now: () => new Date('2026-07-12T15:00:00Z') });
expect(msgs).toHaveLength(2);
});
test('never throws — a broken status probe returns { alerted: false }', async () => {
const r = await checkQuotaDaily({ getStatus: async () => { throw new Error('redis gone'); } });
expect(r.alerted).toBe(false);
expect(r.error).toBe('redis gone');
});
});
describe('countRecentSettles', () => {
test('counts settledAt inside the window across multiple sport logs', () => {
const nowMs = Date.parse('2026-07-11T13:00:00Z');
const logs = [
[ { settledAt: '2026-07-11T14:05:00Z' }, { settledAt: '2026-07-09T14:00:00Z' } ], // 1 in, 1 out
[ { settledAt: '2026-07-10T14:30:00Z' }, { noSettledAt: true } ], // 1 in
];
expect(countRecentSettles(logs, nowMs)).toBe(2);
expect(countRecentSettles([], nowMs)).toBe(0);
expect(countRecentSettles(null, nowMs)).toBe(0);
});
});
describe('buildPulseMessage', () => {
test('one message with every field, real numbers rendered', () => {
const msg = buildPulseMessage({
dateEt: '2026-07-11',
ledgerRows: 42,
settles24h: 38,
quota: { pct: 0.12, used: 61, limit: 500 },
health: { disk_pct: 41, mem_pct: 63 },
});
expect(msg).toContain('VYNDR pulse — 2026-07-11');
expect(msg).toContain('ledger rows yesterday: 42');
expect(msg).toContain('settles last 24h: 38');
expect(msg).toContain('odds-api quota: 12% (61/500)');
expect(msg).toContain('disk 41% · mem 63%');
expect(msg).toContain('desk pack: see /desk');
});
test('missing data renders n/a — never a fabricated zero', () => {
const msg = buildPulseMessage({ dateEt: '2026-07-11', ledgerRows: null, settles24h: null, quota: null, health: null });
expect(msg).toContain('ledger rows yesterday: n/a');
expect(msg).toContain('settles last 24h: n/a');
expect(msg).toContain('odds-api quota: n/a');
expect(msg).toContain('disk n/a · mem n/a');
});
test('VOICE v1.1 — no exclamation points, ever', () => {
const loud = buildPulseMessage({ ledgerRows: 999, settles24h: 999, quota: { pct: 0.99, used: 495, limit: 500 }, health: { disk_pct: 99, mem_pct: 99 } });
expect(loud).not.toContain('!');
expect(buildPulseMessage({})).not.toContain('!');
});
});
describe('dateET', () => {
test('renders the America/New_York calendar date (late UTC rolls back)', () => {
// 03:00 UTC Jul 11 = 11:00 PM ET Jul 10.
expect(dateET(new Date('2026-07-11T03:00:00Z'))).toBe('2026-07-10');
expect(dateET(new Date('2026-07-11T15:00:00Z'))).toBe('2026-07-11');
});
});
+58
View File
@@ -0,0 +1,58 @@
'use strict';
// Session 8 (A1 board) — systemHealth: box vitals, pure + injectable.
const { getSystemHealth, healthIssues, THRESHOLDS } = require('../../src/services/systemHealth');
describe('getSystemHealth', () => {
test('computes disk_pct from statfs and mem_pct from os', async () => {
const fsImpl = { statfs: async () => ({ blocks: 1000, bavail: 400 }) }; // 60% used
const osImpl = { totalmem: () => 100, freemem: () => 25 }; // 75% used
const h = await getSystemHealth({ fsImpl, osImpl });
expect(h).toEqual({ disk_pct: 60, mem_pct: 75 });
});
test('a failed statfs degrades disk_pct to null, mem still reports', async () => {
const fsImpl = { statfs: async () => { throw new Error('EACCES'); } };
const osImpl = { totalmem: () => 10, freemem: () => 1 };
const h = await getSystemHealth({ fsImpl, osImpl });
expect(h.disk_pct).toBeNull();
expect(h.mem_pct).toBe(90);
});
test('zero/invalid totals degrade to null rather than dividing by zero', async () => {
const h = await getSystemHealth({
fsImpl: { statfs: async () => ({ blocks: 0, bavail: 0 }) },
osImpl: { totalmem: () => 0, freemem: () => 0 },
});
expect(h).toEqual({ disk_pct: null, mem_pct: null });
});
test('real (uninjected) call returns numbers on this box', async () => {
const h = await getSystemHealth();
expect(h.disk_pct === null || (h.disk_pct >= 0 && h.disk_pct <= 100)).toBe(true);
expect(h.mem_pct).toBeGreaterThanOrEqual(0);
expect(h.mem_pct).toBeLessThanOrEqual(100);
});
});
describe('healthIssues', () => {
test('thresholds: disk > 85, mem > 90 — strictly greater', () => {
expect(healthIssues({ disk_pct: 85, mem_pct: 90 })).toEqual([]);
expect(healthIssues({ disk_pct: 86, mem_pct: 90 })).toEqual(['disk at 86% (threshold 85%)']);
expect(healthIssues({ disk_pct: 40, mem_pct: 91 })).toEqual(['memory at 91% (threshold 90%)']);
expect(healthIssues({ disk_pct: 99, mem_pct: 99 })).toHaveLength(2);
});
test('null probes never flag; copy carries no exclamation points', () => {
expect(healthIssues({ disk_pct: null, mem_pct: null })).toEqual([]);
expect(healthIssues(null)).toEqual([]);
for (const line of healthIssues({ disk_pct: 99, mem_pct: 99 })) {
expect(line).not.toContain('!');
}
});
test('exported thresholds match the spec', () => {
expect(THRESHOLDS).toEqual({ disk_pct: 85, mem_pct: 90 });
});
});