Quota guard: close the silent odds-api drain + reserve floor for MLB
Diagnosis (why 500/500 went unpaged): the only regular odds-api burner was
futuresService, which called axios DIRECTLY — bypassing the gateway, so it
never hit recordCall (the ONE place the WARN/BLOCK pager fires) and never
respected the 95% block. It only syncFromHeaders, which updated the counter's
number SILENTLY. oddsService (which does go through the gateway) only touches
odds-api when PropLine fails, so recordCall for odds-api effectively never ran.
Result: the counter could reach 100% with neither pager firing.
Fixes (a silent drain is now impossible, not just guarded):
- futuresService routes through gateway.fetch('odds-api', …) → counted, blocked
at 95%, and reserve-gated. Closes the raw-axios bypass.
- Reserve floor in the gateway: a DISCRETIONARY call (futures/soccer) passes
reserve=ODDS_API_RESERVE (default 50) and is refused while remaining <= reserve.
The ESSENTIAL MLB prop-backup passes no reserve and may spend to the 95% block.
→ a futures/soccer drain can NEVER starve MLB's backup path.
- quotaTracker.syncFromHeaders (the AUTHORITATIVE number) now fires the same
once-per-period WARN/BLOCK alert on a crossing — extracted fireThresholdAlert
shared with recordCall. The header-only drain now pages.
- POST /api/internal/quota/test-alert (internal-key) test-fires the pager
end-to-end so ntfy delivery is verifiable on demand.
Also (reality-corrected cadence): WNBA restored to the full grid. 2026-07-15
had two AFTERNOON WNBA games finished before the 22 UTC slot — 14 UTC (10am ET)
is the only slot early enough for a 1pm ET game's props, and on PropLine the
extra slots cost a rounding error. Soccer stays the only trimmed sport (the
real odds-api discipline). Assumption corrected by observed data.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -36,6 +36,13 @@ const { FUTURES_KEYS, ODDS_API_BASE } = require('./oddsService');
|
||||
const DEFAULT_TTL = 12 * 60 * 60; // 12h logical freshness (quota-disciplined).
|
||||
const PERSIST_TTL = 7 * 24 * 60 * 60; // 7d Redis persistence so prev survives the gap.
|
||||
const HTTP_TIMEOUT_MS = 12_000;
|
||||
// Credits held back for the ESSENTIAL path (MLB prop backup when PropLine
|
||||
// fails). Discretionary futures calls stop while ≤ this many odds-api credits
|
||||
// remain. Operator-tunable; default 50 of the 500/mo pool.
|
||||
const ODDS_API_RESERVE = (() => {
|
||||
const n = Number.parseInt(process.env.ODDS_API_RESERVE, 10);
|
||||
return Number.isFinite(n) && n >= 0 ? n : 50;
|
||||
})();
|
||||
// A price is "flat" unless the decimal payout moves by at least this much —
|
||||
// filters odds-jitter from a real steam/drift (mirrors DELTA_NOISE in shape).
|
||||
const MOVE_EPSILON = 0.05;
|
||||
@@ -180,17 +187,23 @@ async function getFutures(sport, deps = {}) {
|
||||
|
||||
const axios = deps.axios || require('axios');
|
||||
const base = deps.ODDS_API_BASE || ODDS_API_BASE;
|
||||
// Quota guard — futures is DISCRETIONARY, so it goes through the gateway
|
||||
// (counted via recordCall → the 80% pager sees it, and blocked at 95%) with a
|
||||
// RESERVE floor: it stops spending while ≤ ODDS_API_RESERVE credits remain, so
|
||||
// a futures drain can never starve the essential MLB prop-backup path. This
|
||||
// closes the raw-axios bypass that let futures burn odds-api invisibly (the
|
||||
// reason the 500/500 exhaustion went unpaged). The gateway also syncs headers.
|
||||
const gateway = deps.gateway || require('./providerGateway');
|
||||
const reserve = deps.reserve != null ? deps.reserve : ODDS_API_RESERVE;
|
||||
try {
|
||||
const res = await axios.get(`${base}/${futuresKey}/odds`, {
|
||||
params: { apiKey, regions: 'us', markets: 'outrights', oddsFormat: 'american' },
|
||||
timeout: HTTP_TIMEOUT_MS,
|
||||
});
|
||||
// Best-effort quota sync (same headers the player-prop path reads).
|
||||
try {
|
||||
if (res && res.headers) {
|
||||
require('./quotaTracker').syncFromHeaders('odds-api', res.headers);
|
||||
}
|
||||
} catch (_) { /* quota tracking is a signal, never a dependency */ }
|
||||
const res = await gateway.fetch(
|
||||
'odds-api',
|
||||
() => axios.get(`${base}/${futuresKey}/odds`, {
|
||||
params: { apiKey, regions: 'us', markets: 'outrights', oddsFormat: 'american' },
|
||||
timeout: HTTP_TIMEOUT_MS,
|
||||
}),
|
||||
{ capability: 'futures', sport: sp, reserve, syncHeadersFrom: (r) => r && r.headers },
|
||||
);
|
||||
|
||||
const normalized = normalizeOutrights(res.data);
|
||||
const markets = attachMoves(normalized, cached && cached.markets);
|
||||
|
||||
@@ -43,7 +43,19 @@ class QuotaExhaustedError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
async function tryOne(providerId, callbackFn, syncHeadersFrom) {
|
||||
async function tryOne(providerId, callbackFn, syncHeadersFrom, reserve = 0) {
|
||||
// Reserve floor (Job 1 / quota guard) — a DISCRETIONARY call (futures,
|
||||
// soccer outrights) is refused while fewer than `reserve` credits remain, so
|
||||
// it can never drain the last credits that the ESSENTIAL path (MLB prop
|
||||
// backup when PropLine fails) depends on. Essential calls pass reserve=0 and
|
||||
// use the quota down to the normal 95% block. Checked BEFORE the optimistic
|
||||
// increment so we don't consume-then-refund. Degraded Redis fails open.
|
||||
if (reserve > 0) {
|
||||
const pre = await quotaTracker.getQuotaStatus(providerId);
|
||||
if (pre && !pre.degraded && Number.isFinite(pre.remaining) && pre.remaining <= reserve) {
|
||||
return { ok: false, reason: `reserve_floor(${pre.remaining}<=${reserve})`, status: pre };
|
||||
}
|
||||
}
|
||||
// Optimistic increment — if the call throws we roll back below.
|
||||
// recordCall also evaluates the post-increment threshold; if the
|
||||
// very next call would put us at 95%+, we still execute THIS one
|
||||
@@ -89,10 +101,11 @@ async function fetch(primaryId, callbackFn, opts = {}) {
|
||||
sport,
|
||||
fallbackProviders,
|
||||
syncHeadersFrom,
|
||||
reserve = 0,
|
||||
} = opts;
|
||||
|
||||
const attempts = [];
|
||||
const result = await tryOne(primaryId, callbackFn, syncHeadersFrom);
|
||||
const result = await tryOne(primaryId, callbackFn, syncHeadersFrom, reserve);
|
||||
if (result.ok) return result.result;
|
||||
|
||||
// Generic adapter error on the primary — propagate, don't shift.
|
||||
@@ -112,7 +125,7 @@ async function fetch(primaryId, callbackFn, opts = {}) {
|
||||
: [];
|
||||
|
||||
for (const fallbackId of chain) {
|
||||
const fb = await tryOne(fallbackId, callbackFn, syncHeadersFrom);
|
||||
const fb = await tryOne(fallbackId, callbackFn, syncHeadersFrom, reserve);
|
||||
if (fb.ok) {
|
||||
console.log(`[gateway] primary=${primaryId} blocked; succeeded via fallback=${fallbackId}`);
|
||||
return fb.result;
|
||||
|
||||
@@ -123,6 +123,28 @@ async function sendQuotaAlert(providerCfg, pct, used, limit) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fire the once-per-period WARN/BLOCK alert when `pct` crosses a threshold.
|
||||
* Extracted so BOTH the optimistic counter (recordCall) AND the authoritative
|
||||
* header reconcile (syncFromHeaders) can page — the drain that put odds-api at
|
||||
* 500/500 came in via header-sync, which used to update the number silently
|
||||
* (recordCall was never on the burner's path). A silent drain must be
|
||||
* impossible, so the authoritative writer alerts too. Deduped per period.
|
||||
* Returns true if an alert was sent this call.
|
||||
*/
|
||||
async function fireThresholdAlert(providerId, cfg, pct, used, limit) {
|
||||
if (!(pct >= THRESHOLDS.WARN_PCT)) return false;
|
||||
const blocked = pct >= THRESHOLDS.BLOCK_PCT;
|
||||
const dedupeKey = blocked ? `${buildWarnKey(providerId)}:block` : buildWarnKey(providerId);
|
||||
const already = await cacheGet(dedupeKey);
|
||||
if (already) return false;
|
||||
console.warn(`[quotaTracker] ${cfg.name} at ${(pct * 100).toFixed(0)}% quota (${used}/${limit}) for ${getPeriodKey(providerId)}`);
|
||||
await cacheSet(dedupeKey, '1', getQuotaTTL(providerId));
|
||||
// Off the hot path — errors already swallowed inside sendQuotaAlert.
|
||||
sendQuotaAlert(cfg, pct, used, limit).catch(() => {});
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the counter without mutating it. Returns the structured
|
||||
* status the admin dashboard renders + the gateway consults.
|
||||
@@ -188,27 +210,9 @@ async function recordCall(providerId) {
|
||||
if (cached && cached.syncedAt) payload.syncedAt = cached.syncedAt;
|
||||
await cacheSet(key, payload, getQuotaTTL(providerId));
|
||||
|
||||
if (pct >= THRESHOLDS.WARN_PCT) {
|
||||
// Session 21 — separate dedupe keys for WARN and BLOCK so each
|
||||
// threshold can fire once per period. Without the second key,
|
||||
// a provider that hops 75% → 96% in one call would only send
|
||||
// ONE alert (the WARN); the operator wouldn't get the BLOCK
|
||||
// notice that's actually the actionable one.
|
||||
const blocked = pct >= THRESHOLDS.BLOCK_PCT;
|
||||
const dedupeKey = blocked ? `${buildWarnKey(providerId)}:block` : buildWarnKey(providerId);
|
||||
const already = await cacheGet(dedupeKey);
|
||||
if (!already) {
|
||||
console.warn(
|
||||
`[quotaTracker] ${cfg.name} at ${(pct * 100).toFixed(0)}% quota (${nextUsed}/${limit}) for ${getPeriodKey(providerId)}`,
|
||||
);
|
||||
await cacheSet(dedupeKey, '1', getQuotaTTL(providerId));
|
||||
// Fire ntfy off the hot path — we don't await it. Errors are
|
||||
// already caught inside sendQuotaAlert, but skipping the await
|
||||
// also means a slow ntfy server can't add latency to the
|
||||
// adapter's HTTP call.
|
||||
sendQuotaAlert(cfg, pct, nextUsed, limit).catch(() => {});
|
||||
}
|
||||
}
|
||||
// Session 21 — WARN (80%) / BLOCK (95%) alert, once per period per threshold.
|
||||
// Shared with syncFromHeaders so a header-reconciled crossing pages too.
|
||||
await fireThresholdAlert(providerId, cfg, pct, nextUsed, limit);
|
||||
|
||||
return {
|
||||
provider: providerId, name: cfg.name,
|
||||
@@ -283,6 +287,11 @@ async function syncFromHeaders(providerId, headers) {
|
||||
source: 'headers',
|
||||
};
|
||||
await cacheSet(buildKey(providerId), payload, getQuotaTTL(providerId));
|
||||
// The authoritative number can cross the threshold entirely via header
|
||||
// reconcile (the futures burner synced but never recordCall'd). Page on it —
|
||||
// this is the fix for "the 80% alert never fired while odds-api hit 500/500".
|
||||
const pct = limit > 0 ? resolvedUsed / limit : 0;
|
||||
await fireThresholdAlert(providerId, cfg, pct, resolvedUsed, limit);
|
||||
return payload;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user