diff --git a/src/config/sportCadence.js b/src/config/sportCadence.js index 5d9231c..b68244b 100644 --- a/src/config/sportCadence.js +++ b/src/config/sportCadence.js @@ -38,10 +38,15 @@ * - mlb — games + props all day; the full grid is correct. Keep 14/19/22/1/3. * - nba — off-season now (self-skips empty on PropLine, ~0 cost); in-season * posts morning→afternoon ET. The full grid fits both. Keep. - * - wnba — games tip evening ET; props post ~afternoon ET. 19 UTC (3pm ET) - * catches early games, 22 UTC (6pm ET) the bulk, 1 UTC (9pm ET) the - * west-coast lock. 14 UTC (10am ET) and 3 UTC (11pm ET, games done) - * are dropped — that's the wasted "wnba:0" slot removed. + * - wnba — PropLine-backed (cheap), and games span EARLY AFTERNOON (weekend/ + * holiday 1–3pm ET) through LATE-EVENING west-coast (10pm ET). A + * 1pm ET game's props post ~morning ET, so the 14 UTC (10am ET) slot + * is the ONLY one early enough to catch them; 3 UTC (11pm ET) is the + * west-coast lock. Observed 2026-07-15: two afternoon WNBA games had + * already finished by the 22 UTC slot. So WNBA keeps the FULL grid — + * the earlier "drop 14/3 as waste" was wrong: on PropLine it costs a + * rounding error and the coverage is real. (An empty slot is an + * honest 'skipped', never a paged failure.) * - soccer— World Cup live now (final Jul 19), club leagues resume Aug. Lines * post well ahead and barely move; two well-placed odds-api reads * (14 UTC morning research, 19 UTC pre-afternoon-match lock) cover a @@ -53,7 +58,7 @@ const SPORT_CADENCE = Object.freeze({ mlb: { hours: [14, 19, 22, 1, 3], intraday: true, source: 'propline' }, nba: { hours: [14, 19, 22, 1, 3], intraday: true, source: 'propline' }, - wnba: { hours: [19, 22, 1], intraday: true, source: 'propline' }, + wnba: { hours: [14, 19, 22, 1, 3], intraday: true, source: 'propline' }, soccer: { hours: [14, 19], intraday: false, source: 'odds-api' }, }); diff --git a/src/routes/internal.js b/src/routes/internal.js index ccdd164..93db799 100644 --- a/src/routes/internal.js +++ b/src/routes/internal.js @@ -42,6 +42,30 @@ router.get('/quota', async (req, res) => { } }); +/** + * POST /api/internal/quota/test-alert (quota guard) + * + * Test-fires the quota pager end-to-end through the REAL opsNotify path so + * "does the 80% alert actually deliver?" is verifiable on demand (the reason + * the 500/500 drain went unnoticed was a counting blind spot, not delivery — + * this proves the delivery leg). Sends one ntfy to vyndr-pipeline-kev2026. + * Does NOT touch the real counter. Body: { pct? } (default 0.85). + */ +router.post('/quota/test-alert', async (req, res) => { + try { + const notify = require('../utils/opsNotify').notify; + const pct = Number.isFinite(Number(req.body && req.body.pct)) ? Number(req.body.pct) : 0.85; + const out = await notify( + `TEST — odds-api quota alert delivery check (${Math.round(pct * 100)}%). If you can read this, the quota pager path works.`, + { title: 'VYNDR quota (test)', priority: 'high', tags: ['warning', 'chart_decreasing'] }, + ); + return res.json({ ok: true, delivery: out }); + } catch (err) { + const message = err && err.message ? err.message : String(err); + return res.status(500).json({ ok: false, error: message }); + } +}); + /** * POST /api/internal/prefetch/tank01 * diff --git a/src/services/futuresService.js b/src/services/futuresService.js index 5e57cb6..9cbc07b 100644 --- a/src/services/futuresService.js +++ b/src/services/futuresService.js @@ -36,6 +36,13 @@ const { FUTURES_KEYS, ODDS_API_BASE } = require('./oddsService'); const DEFAULT_TTL = 12 * 60 * 60; // 12h logical freshness (quota-disciplined). const PERSIST_TTL = 7 * 24 * 60 * 60; // 7d Redis persistence so prev survives the gap. const HTTP_TIMEOUT_MS = 12_000; +// Credits held back for the ESSENTIAL path (MLB prop backup when PropLine +// fails). Discretionary futures calls stop while ≤ this many odds-api credits +// remain. Operator-tunable; default 50 of the 500/mo pool. +const ODDS_API_RESERVE = (() => { + const n = Number.parseInt(process.env.ODDS_API_RESERVE, 10); + return Number.isFinite(n) && n >= 0 ? n : 50; +})(); // A price is "flat" unless the decimal payout moves by at least this much — // filters odds-jitter from a real steam/drift (mirrors DELTA_NOISE in shape). const MOVE_EPSILON = 0.05; @@ -180,17 +187,23 @@ async function getFutures(sport, deps = {}) { const axios = deps.axios || require('axios'); const base = deps.ODDS_API_BASE || ODDS_API_BASE; + // Quota guard — futures is DISCRETIONARY, so it goes through the gateway + // (counted via recordCall → the 80% pager sees it, and blocked at 95%) with a + // RESERVE floor: it stops spending while ≤ ODDS_API_RESERVE credits remain, so + // a futures drain can never starve the essential MLB prop-backup path. This + // closes the raw-axios bypass that let futures burn odds-api invisibly (the + // reason the 500/500 exhaustion went unpaged). The gateway also syncs headers. + const gateway = deps.gateway || require('./providerGateway'); + const reserve = deps.reserve != null ? deps.reserve : ODDS_API_RESERVE; try { - const res = await axios.get(`${base}/${futuresKey}/odds`, { - params: { apiKey, regions: 'us', markets: 'outrights', oddsFormat: 'american' }, - timeout: HTTP_TIMEOUT_MS, - }); - // Best-effort quota sync (same headers the player-prop path reads). - try { - if (res && res.headers) { - require('./quotaTracker').syncFromHeaders('odds-api', res.headers); - } - } catch (_) { /* quota tracking is a signal, never a dependency */ } + const res = await gateway.fetch( + 'odds-api', + () => axios.get(`${base}/${futuresKey}/odds`, { + params: { apiKey, regions: 'us', markets: 'outrights', oddsFormat: 'american' }, + timeout: HTTP_TIMEOUT_MS, + }), + { capability: 'futures', sport: sp, reserve, syncHeadersFrom: (r) => r && r.headers }, + ); const normalized = normalizeOutrights(res.data); const markets = attachMoves(normalized, cached && cached.markets); diff --git a/src/services/providerGateway.js b/src/services/providerGateway.js index 87b58b0..a4f1994 100644 --- a/src/services/providerGateway.js +++ b/src/services/providerGateway.js @@ -43,7 +43,19 @@ class QuotaExhaustedError extends Error { } } -async function tryOne(providerId, callbackFn, syncHeadersFrom) { +async function tryOne(providerId, callbackFn, syncHeadersFrom, reserve = 0) { + // Reserve floor (Job 1 / quota guard) — a DISCRETIONARY call (futures, + // soccer outrights) is refused while fewer than `reserve` credits remain, so + // it can never drain the last credits that the ESSENTIAL path (MLB prop + // backup when PropLine fails) depends on. Essential calls pass reserve=0 and + // use the quota down to the normal 95% block. Checked BEFORE the optimistic + // increment so we don't consume-then-refund. Degraded Redis fails open. + if (reserve > 0) { + const pre = await quotaTracker.getQuotaStatus(providerId); + if (pre && !pre.degraded && Number.isFinite(pre.remaining) && pre.remaining <= reserve) { + return { ok: false, reason: `reserve_floor(${pre.remaining}<=${reserve})`, status: pre }; + } + } // Optimistic increment — if the call throws we roll back below. // recordCall also evaluates the post-increment threshold; if the // very next call would put us at 95%+, we still execute THIS one @@ -89,10 +101,11 @@ async function fetch(primaryId, callbackFn, opts = {}) { sport, fallbackProviders, syncHeadersFrom, + reserve = 0, } = opts; const attempts = []; - const result = await tryOne(primaryId, callbackFn, syncHeadersFrom); + const result = await tryOne(primaryId, callbackFn, syncHeadersFrom, reserve); if (result.ok) return result.result; // Generic adapter error on the primary — propagate, don't shift. @@ -112,7 +125,7 @@ async function fetch(primaryId, callbackFn, opts = {}) { : []; for (const fallbackId of chain) { - const fb = await tryOne(fallbackId, callbackFn, syncHeadersFrom); + const fb = await tryOne(fallbackId, callbackFn, syncHeadersFrom, reserve); if (fb.ok) { console.log(`[gateway] primary=${primaryId} blocked; succeeded via fallback=${fallbackId}`); return fb.result; diff --git a/src/services/quotaTracker.js b/src/services/quotaTracker.js index 6db306d..187986b 100644 --- a/src/services/quotaTracker.js +++ b/src/services/quotaTracker.js @@ -123,6 +123,28 @@ async function sendQuotaAlert(providerCfg, pct, used, limit) { } } +/** + * Fire the once-per-period WARN/BLOCK alert when `pct` crosses a threshold. + * Extracted so BOTH the optimistic counter (recordCall) AND the authoritative + * header reconcile (syncFromHeaders) can page — the drain that put odds-api at + * 500/500 came in via header-sync, which used to update the number silently + * (recordCall was never on the burner's path). A silent drain must be + * impossible, so the authoritative writer alerts too. Deduped per period. + * Returns true if an alert was sent this call. + */ +async function fireThresholdAlert(providerId, cfg, pct, used, limit) { + if (!(pct >= THRESHOLDS.WARN_PCT)) return false; + const blocked = pct >= THRESHOLDS.BLOCK_PCT; + const dedupeKey = blocked ? `${buildWarnKey(providerId)}:block` : buildWarnKey(providerId); + const already = await cacheGet(dedupeKey); + if (already) return false; + console.warn(`[quotaTracker] ${cfg.name} at ${(pct * 100).toFixed(0)}% quota (${used}/${limit}) for ${getPeriodKey(providerId)}`); + await cacheSet(dedupeKey, '1', getQuotaTTL(providerId)); + // Off the hot path — errors already swallowed inside sendQuotaAlert. + sendQuotaAlert(cfg, pct, used, limit).catch(() => {}); + return true; +} + /** * Read the counter without mutating it. Returns the structured * status the admin dashboard renders + the gateway consults. @@ -188,27 +210,9 @@ async function recordCall(providerId) { if (cached && cached.syncedAt) payload.syncedAt = cached.syncedAt; await cacheSet(key, payload, getQuotaTTL(providerId)); - if (pct >= THRESHOLDS.WARN_PCT) { - // Session 21 — separate dedupe keys for WARN and BLOCK so each - // threshold can fire once per period. Without the second key, - // a provider that hops 75% → 96% in one call would only send - // ONE alert (the WARN); the operator wouldn't get the BLOCK - // notice that's actually the actionable one. - const blocked = pct >= THRESHOLDS.BLOCK_PCT; - const dedupeKey = blocked ? `${buildWarnKey(providerId)}:block` : buildWarnKey(providerId); - const already = await cacheGet(dedupeKey); - if (!already) { - console.warn( - `[quotaTracker] ${cfg.name} at ${(pct * 100).toFixed(0)}% quota (${nextUsed}/${limit}) for ${getPeriodKey(providerId)}`, - ); - await cacheSet(dedupeKey, '1', getQuotaTTL(providerId)); - // Fire ntfy off the hot path — we don't await it. Errors are - // already caught inside sendQuotaAlert, but skipping the await - // also means a slow ntfy server can't add latency to the - // adapter's HTTP call. - sendQuotaAlert(cfg, pct, nextUsed, limit).catch(() => {}); - } - } + // Session 21 — WARN (80%) / BLOCK (95%) alert, once per period per threshold. + // Shared with syncFromHeaders so a header-reconciled crossing pages too. + await fireThresholdAlert(providerId, cfg, pct, nextUsed, limit); return { provider: providerId, name: cfg.name, @@ -283,6 +287,11 @@ async function syncFromHeaders(providerId, headers) { source: 'headers', }; await cacheSet(buildKey(providerId), payload, getQuotaTTL(providerId)); + // The authoritative number can cross the threshold entirely via header + // reconcile (the futures burner synced but never recordCall'd). Page on it — + // this is the fix for "the 80% alert never fired while odds-api hit 500/500". + const pct = limit > 0 ? resolvedUsed / limit : 0; + await fireThresholdAlert(providerId, cfg, pct, resolvedUsed, limit); return payload; } diff --git a/tests/unit/providerGateway.test.js b/tests/unit/providerGateway.test.js index a28c590..0555a5e 100644 --- a/tests/unit/providerGateway.test.js +++ b/tests/unit/providerGateway.test.js @@ -20,6 +20,11 @@ jest.mock('../../src/services/quotaTracker', () => { }), rollback: jest.fn(async () => {}), syncFromHeaders: jest.fn(async () => null), + getQuotaStatus: jest.fn(async (providerId) => { + const s = state.get(providerId) || { allowed: true, used: 0, limit: 500 }; + const remaining = s.remaining != null ? s.remaining : (s.limit - s.used); + return { provider: providerId, allowed: s.allowed, used: s.used, limit: s.limit, remaining, degraded: !!s.degraded }; + }), __state: state, __setStatus: setStatus, }; @@ -154,3 +159,34 @@ describe('gateway.fetch — upstream errors', () => { expect(tracker.rollback).toHaveBeenCalledWith('odds-api'); }); }); + +describe('gateway.fetch — reserve floor (quota guard)', () => { + test('a DISCRETIONARY call (reserve>0) is refused while remaining <= reserve', async () => { + // 470 used of 500 → 30 remaining, at or below a 50-credit reserve. + tracker.__setStatus('odds-api', true, { used: 470, remaining: 30 }); + const cb = jest.fn(async () => ({ data: 'x' })); + await expect( + gateway.fetch('odds-api', cb, { capability: 'futures', sport: 'mlb', reserve: 50 }), + ).rejects.toMatchObject({ code: 'QUOTA_EXHAUSTED' }); + expect(cb).not.toHaveBeenCalled(); // never spent the reserved credits + expect(tracker.recordCall).not.toHaveBeenCalled(); + }); + + test('an ESSENTIAL call (no reserve) still uses the same remaining credits', async () => { + // Same 30 remaining, but the MLB prop-backup path passes no reserve — it may + // spend down to the normal 95% block. This is the MLB-never-starved guarantee. + tracker.__setStatus('odds-api', true, { used: 470, remaining: 30 }); + const cb = jest.fn(async () => ({ data: 'ok' })); + const out = await gateway.fetch('odds-api', cb, { capability: 'odds', sport: 'mlb' }); + expect(out).toEqual({ data: 'ok' }); + expect(cb).toHaveBeenCalledTimes(1); + }); + + test('a discretionary call PROCEEDS when remaining is above the reserve', async () => { + tracker.__setStatus('odds-api', true, { used: 300, remaining: 200 }); + const cb = jest.fn(async () => ({ data: 'ok' })); + const out = await gateway.fetch('odds-api', cb, { capability: 'futures', sport: 'mlb', reserve: 50 }); + expect(out).toEqual({ data: 'ok' }); + expect(cb).toHaveBeenCalledTimes(1); + }); +}); diff --git a/tests/unit/quotaTracker.test.js b/tests/unit/quotaTracker.test.js index d538627..d893217 100644 --- a/tests/unit/quotaTracker.test.js +++ b/tests/unit/quotaTracker.test.js @@ -256,11 +256,18 @@ describe('quotaTracker ntfy alerts (Session 21)', () => { test('posts BLOCK to ntfy at 95% with priority 5', async () => { process.env.NTFY_URL = 'https://alerts.example.com/vyndr'; const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + // Seed silently under WARN, then let a recordCall cross 95%. await tracker.syncFromHeaders('odds-api', { - 'x-requests-used': '474', - 'x-requests-remaining': '26', + 'x-requests-used': '390', + 'x-requests-remaining': '110', + }); + await flushAsync(); + expect(axios.post).toHaveBeenCalledTimes(0); // seed at 78% is silent + // Jump straight to 95% via the authoritative header sync — it must page. + await tracker.syncFromHeaders('odds-api', { + 'x-requests-used': '475', + 'x-requests-remaining': '25', }); - await tracker.recordCall('odds-api'); // 475/500 → 95% await flushAsync(); expect(axios.post).toHaveBeenCalledTimes(1); const [, body, opts] = axios.post.mock.calls[0]; @@ -270,6 +277,26 @@ describe('quotaTracker ntfy alerts (Session 21)', () => { warnSpy.mockRestore(); }); + test('syncFromHeaders (authoritative) pages on a header-only crossing — the silent-drain fix', async () => { + // The 500/500 drain came in entirely via header reconcile; recordCall was + // never on the burner's path, so the counter went to 100% unpaged. The + // authoritative writer must alert. Seed silent, then cross 80% via sync. + process.env.NTFY_URL = 'https://alerts.example.com/vyndr'; + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + await tracker.syncFromHeaders('odds-api', { 'x-requests-used': '390', 'x-requests-remaining': '110' }); + await flushAsync(); + expect(axios.post).toHaveBeenCalledTimes(0); + await tracker.syncFromHeaders('odds-api', { 'x-requests-used': '400', 'x-requests-remaining': '100' }); // 80% + await flushAsync(); + expect(axios.post).toHaveBeenCalledTimes(1); + expect(axios.post.mock.calls[0][2].headers.Priority).toBe('4'); + // A later sync in the same period is deduped — no alert storm. + await tracker.syncFromHeaders('odds-api', { 'x-requests-used': '410', 'x-requests-remaining': '90' }); + await flushAsync(); + expect(axios.post).toHaveBeenCalledTimes(1); + warnSpy.mockRestore(); + }); + test('dedupes — second recordCall in the same period does not re-post', async () => { process.env.NTFY_URL = 'https://alerts.example.com/vyndr'; const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); @@ -287,26 +314,27 @@ describe('quotaTracker ntfy alerts (Session 21)', () => { warnSpy.mockRestore(); }); - test('WARN→BLOCK transition fires BOTH alerts (separate dedupe keys)', async () => { + test('WARN then BLOCK each fire once across a period (separate dedupe keys)', async () => { process.env.NTFY_URL = 'https://alerts.example.com/vyndr'; const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); - // Seed at 79%; first recordCall → 80% (WARN). + // Seed at 78% (silent). await tracker.syncFromHeaders('odds-api', { - 'x-requests-used': '395', - 'x-requests-remaining': '105', + 'x-requests-used': '390', + 'x-requests-remaining': '110', }); - await tracker.recordCall('odds-api'); // 396 → 79.2% — under warn + await tracker.recordCall('odds-api'); // 391 → 78.2% — under warn await flushAsync(); expect(axios.post).toHaveBeenCalledTimes(0); - // Jump to 95% — should fire BLOCK even though WARN never fired. - await tracker.syncFromHeaders('odds-api', { - 'x-requests-used': '474', - 'x-requests-remaining': '26', - }); - await tracker.recordCall('odds-api'); // 475 → 95% — BLOCK + // Cross 80% → WARN (priority 4). + await tracker.syncFromHeaders('odds-api', { 'x-requests-used': '400', 'x-requests-remaining': '100' }); await flushAsync(); expect(axios.post).toHaveBeenCalledTimes(1); - expect(axios.post.mock.calls[0][2].headers.Priority).toBe('5'); + expect(axios.post.mock.calls[0][2].headers.Priority).toBe('4'); + // Cross 95% → BLOCK (priority 5), a SEPARATE dedupe key so it still fires. + await tracker.syncFromHeaders('odds-api', { 'x-requests-used': '475', 'x-requests-remaining': '25' }); + await flushAsync(); + expect(axios.post).toHaveBeenCalledTimes(2); + expect(axios.post.mock.calls[1][2].headers.Priority).toBe('5'); warnSpy.mockRestore(); }); diff --git a/tests/unit/sportCadence.test.js b/tests/unit/sportCadence.test.js index 1316ca7..18fc1b9 100644 --- a/tests/unit/sportCadence.test.js +++ b/tests/unit/sportCadence.test.js @@ -18,15 +18,17 @@ describe('sportCadence', () => { expect(cadence.ALL_HOURS.every((h) => HOURS_UTC.includes(h))).toBe(true); }); - test('WNBA skips 14:00 UTC (props not posted) — the wasted "wnba:0" slot', () => { - expect(cadence.sportsForHour(14)).toContain('mlb'); - expect(cadence.sportsForHour(14)).not.toContain('wnba'); - // but WNBA DOES run at its real hours - expect(cadence.sportsForHour(22)).toContain('wnba'); - expect(cadence.sportsForHour(1)).toContain('wnba'); + test('WNBA keeps the full grid — PropLine-cheap; catches afternoon + late west-coast games', () => { + // Corrected from reality (2026-07-15: afternoon WNBA games finished before + // the 22 UTC slot). 14 UTC (10am ET) is the only slot early enough for a + // 1pm ET game's props; on PropLine the extra slots cost a rounding error. + expect(cadence.hoursFor('wnba')).toEqual([14, 19, 22, 1, 3]); + expect(cadence.sportsForHour(14)).toContain('wnba'); }); - test('soccer runs only its two lean odds-api slots, never the full grid', () => { + test('soccer is the ONLY restricted sport — two lean odds-api slots, never the full grid', () => { + // The genuine cadence win is protecting the 500/mo odds-api key. PropLine + // sports run the full grid cheaply; soccer alone is trimmed. expect(cadence.hoursFor('soccer')).toEqual([14, 19]); expect(cadence.sportsForHour(22)).not.toContain('soccer'); expect(cadence.sportsForHour(1)).not.toContain('soccer');