Quota guard: close the silent odds-api drain + reserve floor for MLB

Diagnosis (why 500/500 went unpaged): the only regular odds-api burner was
futuresService, which called axios DIRECTLY — bypassing the gateway, so it
never hit recordCall (the ONE place the WARN/BLOCK pager fires) and never
respected the 95% block. It only syncFromHeaders, which updated the counter's
number SILENTLY. oddsService (which does go through the gateway) only touches
odds-api when PropLine fails, so recordCall for odds-api effectively never ran.
Result: the counter could reach 100% with neither pager firing.

Fixes (a silent drain is now impossible, not just guarded):
- futuresService routes through gateway.fetch('odds-api', …) → counted, blocked
  at 95%, and reserve-gated. Closes the raw-axios bypass.
- Reserve floor in the gateway: a DISCRETIONARY call (futures/soccer) passes
  reserve=ODDS_API_RESERVE (default 50) and is refused while remaining <= reserve.
  The ESSENTIAL MLB prop-backup passes no reserve and may spend to the 95% block.
  → a futures/soccer drain can NEVER starve MLB's backup path.
- quotaTracker.syncFromHeaders (the AUTHORITATIVE number) now fires the same
  once-per-period WARN/BLOCK alert on a crossing — extracted fireThresholdAlert
  shared with recordCall. The header-only drain now pages.
- POST /api/internal/quota/test-alert (internal-key) test-fires the pager
  end-to-end so ntfy delivery is verifiable on demand.

Also (reality-corrected cadence): WNBA restored to the full grid. 2026-07-15
had two AFTERNOON WNBA games finished before the 22 UTC slot — 14 UTC (10am ET)
is the only slot early enough for a 1pm ET game's props, and on PropLine the
extra slots cost a rounding error. Soccer stays the only trimmed sport (the
real odds-api discipline). Assumption corrected by observed data.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kev
2026-07-15 18:07:14 -04:00
parent 4cd933d83e
commit 2d413cfe1e
8 changed files with 191 additions and 61 deletions
+23 -10
View File
@@ -36,6 +36,13 @@ const { FUTURES_KEYS, ODDS_API_BASE } = require('./oddsService');
const DEFAULT_TTL = 12 * 60 * 60; // 12h logical freshness (quota-disciplined).
const PERSIST_TTL = 7 * 24 * 60 * 60; // 7d Redis persistence so prev survives the gap.
const HTTP_TIMEOUT_MS = 12_000;
// Credits held back for the ESSENTIAL path (MLB prop backup when PropLine
// fails). Discretionary futures calls stop while ≤ this many odds-api credits
// remain. Operator-tunable; default 50 of the 500/mo pool.
const ODDS_API_RESERVE = (() => {
const n = Number.parseInt(process.env.ODDS_API_RESERVE, 10);
return Number.isFinite(n) && n >= 0 ? n : 50;
})();
// A price is "flat" unless the decimal payout moves by at least this much —
// filters odds-jitter from a real steam/drift (mirrors DELTA_NOISE in shape).
const MOVE_EPSILON = 0.05;
@@ -180,17 +187,23 @@ async function getFutures(sport, deps = {}) {
const axios = deps.axios || require('axios');
const base = deps.ODDS_API_BASE || ODDS_API_BASE;
// Quota guard — futures is DISCRETIONARY, so it goes through the gateway
// (counted via recordCall → the 80% pager sees it, and blocked at 95%) with a
// RESERVE floor: it stops spending while ≤ ODDS_API_RESERVE credits remain, so
// a futures drain can never starve the essential MLB prop-backup path. This
// closes the raw-axios bypass that let futures burn odds-api invisibly (the
// reason the 500/500 exhaustion went unpaged). The gateway also syncs headers.
const gateway = deps.gateway || require('./providerGateway');
const reserve = deps.reserve != null ? deps.reserve : ODDS_API_RESERVE;
try {
const res = await axios.get(`${base}/${futuresKey}/odds`, {
params: { apiKey, regions: 'us', markets: 'outrights', oddsFormat: 'american' },
timeout: HTTP_TIMEOUT_MS,
});
// Best-effort quota sync (same headers the player-prop path reads).
try {
if (res && res.headers) {
require('./quotaTracker').syncFromHeaders('odds-api', res.headers);
}
} catch (_) { /* quota tracking is a signal, never a dependency */ }
const res = await gateway.fetch(
'odds-api',
() => axios.get(`${base}/${futuresKey}/odds`, {
params: { apiKey, regions: 'us', markets: 'outrights', oddsFormat: 'american' },
timeout: HTTP_TIMEOUT_MS,
}),
{ capability: 'futures', sport: sp, reserve, syncHeadersFrom: (r) => r && r.headers },
);
const normalized = normalizeOutrights(res.data);
const markets = attachMoves(normalized, cached && cached.markets);
+16 -3
View File
@@ -43,7 +43,19 @@ class QuotaExhaustedError extends Error {
}
}
async function tryOne(providerId, callbackFn, syncHeadersFrom) {
async function tryOne(providerId, callbackFn, syncHeadersFrom, reserve = 0) {
// Reserve floor (Job 1 / quota guard) — a DISCRETIONARY call (futures,
// soccer outrights) is refused while fewer than `reserve` credits remain, so
// it can never drain the last credits that the ESSENTIAL path (MLB prop
// backup when PropLine fails) depends on. Essential calls pass reserve=0 and
// use the quota down to the normal 95% block. Checked BEFORE the optimistic
// increment so we don't consume-then-refund. Degraded Redis fails open.
if (reserve > 0) {
const pre = await quotaTracker.getQuotaStatus(providerId);
if (pre && !pre.degraded && Number.isFinite(pre.remaining) && pre.remaining <= reserve) {
return { ok: false, reason: `reserve_floor(${pre.remaining}<=${reserve})`, status: pre };
}
}
// Optimistic increment — if the call throws we roll back below.
// recordCall also evaluates the post-increment threshold; if the
// very next call would put us at 95%+, we still execute THIS one
@@ -89,10 +101,11 @@ async function fetch(primaryId, callbackFn, opts = {}) {
sport,
fallbackProviders,
syncHeadersFrom,
reserve = 0,
} = opts;
const attempts = [];
const result = await tryOne(primaryId, callbackFn, syncHeadersFrom);
const result = await tryOne(primaryId, callbackFn, syncHeadersFrom, reserve);
if (result.ok) return result.result;
// Generic adapter error on the primary — propagate, don't shift.
@@ -112,7 +125,7 @@ async function fetch(primaryId, callbackFn, opts = {}) {
: [];
for (const fallbackId of chain) {
const fb = await tryOne(fallbackId, callbackFn, syncHeadersFrom);
const fb = await tryOne(fallbackId, callbackFn, syncHeadersFrom, reserve);
if (fb.ok) {
console.log(`[gateway] primary=${primaryId} blocked; succeeded via fallback=${fallbackId}`);
return fb.result;
+30 -21
View File
@@ -123,6 +123,28 @@ async function sendQuotaAlert(providerCfg, pct, used, limit) {
}
}
/**
* Fire the once-per-period WARN/BLOCK alert when `pct` crosses a threshold.
* Extracted so BOTH the optimistic counter (recordCall) AND the authoritative
* header reconcile (syncFromHeaders) can page — the drain that put odds-api at
* 500/500 came in via header-sync, which used to update the number silently
* (recordCall was never on the burner's path). A silent drain must be
* impossible, so the authoritative writer alerts too. Deduped per period.
* Returns true if an alert was sent this call.
*/
async function fireThresholdAlert(providerId, cfg, pct, used, limit) {
if (!(pct >= THRESHOLDS.WARN_PCT)) return false;
const blocked = pct >= THRESHOLDS.BLOCK_PCT;
const dedupeKey = blocked ? `${buildWarnKey(providerId)}:block` : buildWarnKey(providerId);
const already = await cacheGet(dedupeKey);
if (already) return false;
console.warn(`[quotaTracker] ${cfg.name} at ${(pct * 100).toFixed(0)}% quota (${used}/${limit}) for ${getPeriodKey(providerId)}`);
await cacheSet(dedupeKey, '1', getQuotaTTL(providerId));
// Off the hot path — errors already swallowed inside sendQuotaAlert.
sendQuotaAlert(cfg, pct, used, limit).catch(() => {});
return true;
}
/**
* Read the counter without mutating it. Returns the structured
* status the admin dashboard renders + the gateway consults.
@@ -188,27 +210,9 @@ async function recordCall(providerId) {
if (cached && cached.syncedAt) payload.syncedAt = cached.syncedAt;
await cacheSet(key, payload, getQuotaTTL(providerId));
if (pct >= THRESHOLDS.WARN_PCT) {
// Session 21 — separate dedupe keys for WARN and BLOCK so each
// threshold can fire once per period. Without the second key,
// a provider that hops 75% → 96% in one call would only send
// ONE alert (the WARN); the operator wouldn't get the BLOCK
// notice that's actually the actionable one.
const blocked = pct >= THRESHOLDS.BLOCK_PCT;
const dedupeKey = blocked ? `${buildWarnKey(providerId)}:block` : buildWarnKey(providerId);
const already = await cacheGet(dedupeKey);
if (!already) {
console.warn(
`[quotaTracker] ${cfg.name} at ${(pct * 100).toFixed(0)}% quota (${nextUsed}/${limit}) for ${getPeriodKey(providerId)}`,
);
await cacheSet(dedupeKey, '1', getQuotaTTL(providerId));
// Fire ntfy off the hot path — we don't await it. Errors are
// already caught inside sendQuotaAlert, but skipping the await
// also means a slow ntfy server can't add latency to the
// adapter's HTTP call.
sendQuotaAlert(cfg, pct, nextUsed, limit).catch(() => {});
}
}
// Session 21 — WARN (80%) / BLOCK (95%) alert, once per period per threshold.
// Shared with syncFromHeaders so a header-reconciled crossing pages too.
await fireThresholdAlert(providerId, cfg, pct, nextUsed, limit);
return {
provider: providerId, name: cfg.name,
@@ -283,6 +287,11 @@ async function syncFromHeaders(providerId, headers) {
source: 'headers',
};
await cacheSet(buildKey(providerId), payload, getQuotaTTL(providerId));
// The authoritative number can cross the threshold entirely via header
// reconcile (the futures burner synced but never recordCall'd). Page on it —
// this is the fix for "the 80% alert never fired while odds-api hit 500/500".
const pct = limit > 0 ? resolvedUsed / limit : 0;
await fireThresholdAlert(providerId, cfg, pct, resolvedUsed, limit);
return payload;
}