'use strict'; /** * opsWatch — pure ops-alarm logic for the scheduler (Session 8, A1 board). * * The record must never die silently. This module holds the TESTABLE pieces * the scheduler wires to ntfy (via opsNotify): * * 1. createFailureTracker — per-sport CONSECUTIVE snapshot-failure counter. * Single-slot errors are normal before books post lines; 3 in a row is a * broken pipeline. Pages exactly once per losing streak (at the * threshold crossing), re-arms on any good outcome. * 2. zeroSettleAlarm — "settle pass finished but the record did not * advance." SIGNAL CHOICE (documented): the ledger settle pass's own * return values. settleLedger fetches `game_date < today AND outcome IS * NULL` from POSTGRES, so the signal survives the Redis TTL expiry that * silently killed morning settles in Session 60 — a * snapshot:{sport}:previous read would vanish in exactly that failure * mode, and it costs zero extra reads. Scoped to SETTLEABLE sports (mlb) * because NBA/WNBA/soccer rows legitimately stay pending until they have * a settled-result feed. A genuinely empty yesterday fetches 0 rows -> * settled 0 / pending 0 -> no alarm, ever. * 3. checkQuotaDaily — odds-api >= 80% -> one alert per day (Redis dedupe * by date key). Complements quotaTracker's once-per-MONTH warn. * 4. buildPulseMessage / countRecentSettles — the 9 AM ET daily pulse, one * notification, VOICE v1.1: deadpan, numbers, no exclamation points. * * Everything here is pure or fully injectable — no requires of redis/ntfy. */ /** Sports with a real settled-result feed (mlb game logs). Keep in sync with * outcomeService/ledgerService's MLB-only settlement until Phase 4.5. */ const SETTLEABLE_SPORTS = ['mlb']; const PAGE_THRESHOLD = 3; /** A snapshot result that means "the pipeline produced nothing": a hard error * or an empty odds feed. 'no grades' (props arrived, grader refused) and 'ok' * are NOT failures for this counter. */ function isBadSnapshotResult(result) { if (!result) return false; if (result.status === 'error') return true; return result.status === 'skipped' && String(result.reason || '') === 'no props'; } /** * Per-sport consecutive-failure counter (in-process; a restart re-arms, which * is acceptable — the missed-cron watchdog covers a crashed scheduler). * record() returns { sport, count, shouldPage }: shouldPage is true ONLY at * the exact threshold crossing, so a sport that keeps failing pages once per * losing streak, not once per slot. */ function createFailureTracker(threshold = PAGE_THRESHOLD) { const counts = new Map(); return { record(sport, result) { const sp = String(sport || '').toLowerCase(); const bad = isBadSnapshotResult(result); const count = bad ? (counts.get(sp) || 0) + 1 : 0; counts.set(sp, count); return { sport: sp, count, shouldPage: bad && count === threshold }; }, count(sport) { return counts.get(String(sport || '').toLowerCase()) || 0; }, threshold, }; } /** The daily "close the book" slot: the first configured UTC hour >= 06 * (hours 0-5 UTC are late-night ET slots of the PREVIOUS ET day). Falls back * to the smallest configured hour when nothing is >= 06. */ function morningHourUtc(hoursUtc) { const hours = (hoursUtc || []).filter((h) => Number.isInteger(h)); if (hours.length === 0) return 14; const daytime = hours.filter((h) => h >= 6); return (daytime.length > 0 ? daytime : hours).sort((a, b) => a - b)[0]; } /** * Evaluate the zero-settle signal from settleAllLedgers results. * alarm === true only when settleable-sport rows EXISTED for settlement * (fetched from Postgres: settled + pending > 0) and none settled. */ function zeroSettleAlarm(ledgerResults, settleable = SETTLEABLE_SPORTS) { const rows = (Array.isArray(ledgerResults) ? ledgerResults : []) .filter((r) => r && settleable.includes(String(r.sport || '').toLowerCase())); const settled = rows.reduce((n, r) => n + (r.settled || 0), 0); const pending = rows.reduce((n, r) => n + (r.pending || 0), 0); return { alarm: settled === 0 && pending > 0, settled, pending }; } /** ET calendar date (YYYY-MM-DD) of a Date; UTC fallback if Intl is absent. */ function dateET(d = new Date()) { try { return new Intl.DateTimeFormat('en-CA', { timeZone: 'America/New_York', year: 'numeric', month: '2-digit', day: '2-digit', }).format(d); } catch { return d.toISOString().slice(0, 10); } } /** * Quota alert: odds-api usage >= thresholdPct -> alert once per UTC day * (Redis dedupe key ops:quota_day:{provider}:{date}). All deps injectable: * { getStatus, cacheGet, cacheSet, notify, now, providerId, thresholdPct }. * Never throws — an ops check must not break the snapshot run. */ async function checkQuotaDaily(deps = {}) { const providerId = deps.providerId || 'odds-api'; const thresholdPct = deps.thresholdPct != null ? deps.thresholdPct : 0.8; try { const status = await deps.getStatus(providerId); const pct = status && Number.isFinite(status.pct) ? status.pct : null; if (pct == null || pct < thresholdPct) return { alerted: false, pct }; const day = (deps.now ? deps.now() : new Date()).toISOString().slice(0, 10); const key = `ops:quota_day:${providerId}:${day}`; if (await deps.cacheGet(key)) return { alerted: false, deduped: true, pct }; await deps.cacheSet(key, '1', 2 * 24 * 3600); await deps.notify( `${providerId} at ${Math.round(pct * 100)}% of ${status.quotaType || 'period'} quota ` + `(${status.used}/${status.limit}). Blocks at 95. PropLine stays primary; ` + `the backup path is thinning.`, { title: 'VYNDR quota', priority: 'high', tags: ['warning', 'chart_decreasing'] }, ); return { alerted: true, pct }; } catch (err) { return { alerted: false, error: err.message }; } } /** Count settled outcomes across logs whose settledAt falls inside the last * windowHours (default 24). Pure — feed it the outcomes:{sport}:log arrays. */ function countRecentSettles(logs, nowMs, windowHours = 24) { const cutoff = nowMs - windowHours * 3600 * 1000; let n = 0; for (const log of logs || []) { for (const o of Array.isArray(log) ? log : []) { const t = o && o.settledAt ? new Date(o.settledAt).getTime() : NaN; if (Number.isFinite(t) && t >= cutoff) n += 1; } } return n; } /** * Assemble the ONE daily-pulse notification body. VOICE v1.1: deadpan, * numbers, no exclamation points. Missing data renders as "n/a" — a pulse * never fabricates a zero. * pieces: { dateEt, ledgerRows (number|null), settles24h (number|null), * quota ({pct, used, limit}|null), health ({disk_pct, mem_pct}|null) } */ function buildPulseMessage(pieces = {}) { const na = (v) => (v == null ? 'n/a' : String(v)); const quota = pieces.quota && Number.isFinite(pieces.quota.pct) ? `${Math.round(pieces.quota.pct * 100)}% (${pieces.quota.used}/${pieces.quota.limit})` : 'n/a'; const h = pieces.health || {}; const disk = Number.isFinite(h.disk_pct) ? `${h.disk_pct}%` : 'n/a'; const mem = Number.isFinite(h.mem_pct) ? `${h.mem_pct}%` : 'n/a'; return [ `VYNDR pulse — ${pieces.dateEt || dateET()}`, `ledger rows yesterday: ${na(pieces.ledgerRows)}`, `settles last 24h: ${na(pieces.settles24h)}`, `odds-api quota: ${quota}`, `disk ${disk} · mem ${mem}`, 'desk pack: see /desk', ].join('\n'); } module.exports = { createFailureTracker, isBadSnapshotResult, zeroSettleAlarm, morningHourUtc, checkQuotaDaily, countRecentSettles, buildPulseMessage, dateET, SETTLEABLE_SPORTS, PAGE_THRESHOLD, };