Merge S8 (a1): ops — the product watches itself
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> # Conflicts: # BUILD-STATE.md # CLAUDE.md # src/snapshotScheduler.js
This commit is contained in:
+102
-3
@@ -60,9 +60,19 @@ function startSnapshotScheduler(opts = {}) {
|
||||
const settleLedgers = opts.settleAllLedgers || require('./services/ledgerService').settleAllLedgers;
|
||||
const notify = opts.notify || require('./utils/opsNotify').notify;
|
||||
const cacheGet = opts.cacheGet || require('./utils/redis').cacheGet;
|
||||
const cacheSet = opts.cacheSet || require('./utils/redis').cacheSet;
|
||||
const now = opts.now || (() => new Date());
|
||||
// Session 8 (A1 board) — ops: the product watches itself. Pure logic lives
|
||||
// in services/opsWatch + services/systemHealth; this file only wires it.
|
||||
const opsWatch = require('./services/opsWatch');
|
||||
const getSystemHealth = opts.getSystemHealth || require('./services/systemHealth').getSystemHealth;
|
||||
const healthIssues = opts.healthIssues || require('./services/systemHealth').healthIssues;
|
||||
const getQuotaStatus = opts.getQuotaStatus || require('./services/quotaTracker').getQuotaStatus;
|
||||
const countLedgerRows = opts.countLedgerRows || require('./services/ledgerService').countRowsForDate;
|
||||
const failureTracker = opts.failureTracker || opsWatch.createFailureTracker();
|
||||
let lastFiredSlot = null;
|
||||
let lastOverdueSlot = null;
|
||||
let lastPulseDate = null;
|
||||
|
||||
// Session 56 — missed-cron watchdog. Runs every minute (independent of the
|
||||
// fire schedule): if a scheduled slot came and went without a snapshot, alert
|
||||
@@ -83,8 +93,55 @@ function startSnapshotScheduler(opts = {}) {
|
||||
} catch { /* watchdog must never throw */ }
|
||||
};
|
||||
|
||||
// Session 8 — daily 9 AM ET pulse (13:00 UTC), ONE notification. Dedupe is
|
||||
// belt-and-braces: in-process date + a Redis key that survives a restart
|
||||
// inside the same day. Runs on the per-minute cadence, independent of slots.
|
||||
const PULSE_HOUR_UTC = Number.parseInt(process.env.PULSE_HOUR_UTC || '13', 10);
|
||||
const pulseTick = async () => {
|
||||
try {
|
||||
const d = now();
|
||||
if (d.getUTCHours() !== PULSE_HOUR_UTC || d.getUTCMinutes() !== 0) return;
|
||||
const dayKey = d.toISOString().slice(0, 10);
|
||||
if (dayKey === lastPulseDate) return;
|
||||
lastPulseDate = dayKey;
|
||||
if (await cacheGet(`ops:pulse:${dayKey}`)) return; // already sent today (pre-restart)
|
||||
await cacheSet(`ops:pulse:${dayKey}`, '1', 2 * 24 * 3600);
|
||||
|
||||
const yesterdayEt = opsWatch.dateET(new Date(d.getTime() - 24 * 3600 * 1000));
|
||||
let ledgerRows = null;
|
||||
try { ledgerRows = await countLedgerRows(yesterdayEt); } catch { /* n/a */ }
|
||||
let settles24h = null;
|
||||
try {
|
||||
const logs = [];
|
||||
for (const sp of ['mlb', 'nba', 'wnba', 'soccer']) {
|
||||
const raw = await cacheGet(`outcomes:${sp}:log`);
|
||||
logs.push(Array.isArray(raw) ? raw : (raw && raw.log) || []);
|
||||
}
|
||||
settles24h = opsWatch.countRecentSettles(logs, d.getTime());
|
||||
} catch { /* n/a */ }
|
||||
let quota = null;
|
||||
try { quota = await getQuotaStatus('odds-api'); } catch { /* n/a */ }
|
||||
let health = null;
|
||||
try { health = await getSystemHealth(); } catch { /* n/a */ }
|
||||
|
||||
await notify(
|
||||
opsWatch.buildPulseMessage({ dateEt: opsWatch.dateET(d), ledgerRows, settles24h, quota, health }),
|
||||
{ title: 'VYNDR daily pulse', tags: ['newspaper'] },
|
||||
);
|
||||
// Box health pages separately at high priority — the pulse is a read,
|
||||
// the threshold breach is an action item.
|
||||
const issues = healthIssues(health);
|
||||
if (issues.length > 0) {
|
||||
await notify(`Box health: ${issues.join('; ')}. The record does not fit on a full disk.`, {
|
||||
title: 'VYNDR box', priority: 'high', tags: ['warning'],
|
||||
});
|
||||
}
|
||||
} catch { /* the pulse must never break the scheduler */ }
|
||||
};
|
||||
|
||||
const tick = async () => {
|
||||
await checkOverdue();
|
||||
await pulseTick();
|
||||
const d = now();
|
||||
if (d.getUTCMinutes() !== 0) return;
|
||||
const h = d.getUTCHours();
|
||||
@@ -92,20 +149,47 @@ function startSnapshotScheduler(opts = {}) {
|
||||
const slot = `${d.getUTCFullYear()}-${d.getUTCMonth()}-${d.getUTCDate()}-${h}`;
|
||||
if (slot === lastFiredSlot) return; // fire once per slot
|
||||
lastFiredSlot = slot;
|
||||
// Session 8 — a settle pass that THROWS is a paged event, not a log line.
|
||||
// Settlement failing quietly is how the record dies.
|
||||
try {
|
||||
const settled = await settleAll();
|
||||
const totalSettled = settled.reduce((n, r) => n + (r.settled || 0), 0);
|
||||
console.log(`[outcomes] cron fired ${h}:00 UTC — ${totalSettled} props settled vs real results`);
|
||||
} catch (e) {
|
||||
console.warn('[outcomes] settle run failed:', e.message);
|
||||
await notify(`Outcome settlement THREW at ${h}:00 UTC — ${e.message}. Yesterday's grades are unsettled until the next slot.`, {
|
||||
title: 'VYNDR settlement', priority: 'high', tags: ['rotating_light'],
|
||||
});
|
||||
}
|
||||
let ledgerResults = null;
|
||||
try {
|
||||
const ledger = await settleLedgers();
|
||||
const n = ledger.reduce((t, r) => t + (r.settled || 0), 0);
|
||||
ledgerResults = await settleLedgers();
|
||||
const n = ledgerResults.reduce((t, r) => t + (r.settled || 0), 0);
|
||||
console.log(`[ledger] settle pass — ${n} entries settled (outcome + CLV)`);
|
||||
} catch (e) {
|
||||
console.warn('[ledger] settle run failed:', e.message);
|
||||
await notify(`Ledger settlement THREW at ${h}:00 UTC — ${e.message}. The public record did not advance.`, {
|
||||
title: 'VYNDR settlement', priority: 'high', tags: ['rotating_light'],
|
||||
});
|
||||
}
|
||||
// Session 8 — zero-settle alarm, MORNING slot only (the book-closing pass).
|
||||
// Signal = the ledger settle's own Postgres-backed return values (see
|
||||
// opsWatch.zeroSettleAlarm for why not snapshot:{sport}:previous). Deduped
|
||||
// once per ET date; a genuinely empty yesterday (0 rows fetched) never fires.
|
||||
try {
|
||||
if (h === opsWatch.morningHourUtc(HOURS_UTC) && ledgerResults) {
|
||||
const z = opsWatch.zeroSettleAlarm(ledgerResults);
|
||||
if (z.alarm) {
|
||||
const dk = `ops:settle_zero:${opsWatch.dateET(d)}`;
|
||||
if (!(await cacheGet(dk))) {
|
||||
await cacheSet(dk, '1', 2 * 24 * 3600);
|
||||
await notify(`Morning settle pass closed with 0 settles — ${z.pending} ledger rows from before today are still pending. Yesterday had graded rows; the record did not advance.`, {
|
||||
title: 'VYNDR settlement', priority: 'high', tags: ['rotating_light'],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch { /* alarm evaluation must never break the tick */ }
|
||||
try {
|
||||
const results = await runAll();
|
||||
const ok = results.filter((r) => r.status === 'ok');
|
||||
@@ -116,9 +200,22 @@ function startSnapshotScheduler(opts = {}) {
|
||||
const total = results.reduce((n, r) => n + (r.gradeCount || 0), 0);
|
||||
await notify(`Desk pack ready — ${total} props graded across ${ok.length} sports. vyndr.app/desk`, { title: 'VYNDR desk', tags: ['newspaper'] });
|
||||
}
|
||||
// Session 8 — persistent-failure pager: 3+ CONSECUTIVE erroring slots for
|
||||
// a sport pages once (single-slot errors are normal before lines post).
|
||||
for (const r of results) {
|
||||
const t = failureTracker.record(r.sport, r);
|
||||
if (t.shouldPage) {
|
||||
await notify(`${String(r.sport).toUpperCase()} snapshot has produced nothing for ${t.count} consecutive slots (latest: ${r.status}${r.reason ? ` — ${r.reason}` : ''}). One empty slot is a quiet book; ${t.count} is a broken pipe.`, {
|
||||
title: 'VYNDR pipeline', priority: 'high', tags: ['rotating_light'],
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[snapshot] cron run failed:', e.message);
|
||||
}
|
||||
// Session 8 — quota check after each snapshot run: odds-api >= 80% alerts
|
||||
// once per day (Redis-deduped). Never throws (guarded inside checkQuotaDaily).
|
||||
await opsWatch.checkQuotaDaily({ getStatus: getQuotaStatus, cacheGet, cacheSet, notify, now: () => now() });
|
||||
};
|
||||
|
||||
// Session 60 (night2/D) — Phase 2.5 intraday refresh. Every
|
||||
@@ -158,7 +255,9 @@ function startSnapshotScheduler(opts = {}) {
|
||||
// was invisible at boot, which made "is settlement scheduled?" unanswerable
|
||||
// from logs. This line makes it verifiable forever.
|
||||
console.log(`[settlement] armed — outcomes + ledger settle pass runs FIRST at each snapshot slot (${HOURS_UTC.join(',')} UTC), idempotent re-runs`);
|
||||
return { interval, tick, refreshTick };
|
||||
// Session 8 — same verifiability rule: every watchdog states itself at boot.
|
||||
console.log(`[opsWatch] armed — settle alarms (throw + morning zero-settle), failure pager (${failureTracker.threshold} consecutive), quota daily check, pulse ${PULSE_HOUR_UTC}:00 UTC`);
|
||||
return { interval, tick, refreshTick, pulseTick };
|
||||
}
|
||||
|
||||
module.exports = { startSnapshotScheduler, HOURS_UTC, mostRecentExpectedSlot, isSnapshotOverdue };
|
||||
|
||||
Reference in New Issue
Block a user