Files
vyndr/src/services/providerGateway.js
T
builtbykev 2d413cfe1e Quota guard: close the silent odds-api drain + reserve floor for MLB
Diagnosis (why 500/500 went unpaged): the only regular odds-api burner was
futuresService, which called axios DIRECTLY — bypassing the gateway, so it
never hit recordCall (the ONE place the WARN/BLOCK pager fires) and never
respected the 95% block. It only syncFromHeaders, which updated the counter's
number SILENTLY. oddsService (which does go through the gateway) only touches
odds-api when PropLine fails, so recordCall for odds-api effectively never ran.
Result: the counter could reach 100% with neither pager firing.

Fixes (a silent drain is now impossible, not just guarded):
- futuresService routes through gateway.fetch('odds-api', …) → counted, blocked
  at 95%, and reserve-gated. Closes the raw-axios bypass.
- Reserve floor in the gateway: a DISCRETIONARY call (futures/soccer) passes
  reserve=ODDS_API_RESERVE (default 50) and is refused while remaining <= reserve.
  The ESSENTIAL MLB prop-backup passes no reserve and may spend to the 95% block.
  → a futures/soccer drain can NEVER starve MLB's backup path.
- quotaTracker.syncFromHeaders (the AUTHORITATIVE number) now fires the same
  once-per-period WARN/BLOCK alert on a crossing — extracted fireThresholdAlert
  shared with recordCall. The header-only drain now pages.
- POST /api/internal/quota/test-alert (internal-key) test-fires the pager
  end-to-end so ntfy delivery is verifiable on demand.

Also (reality-corrected cadence): WNBA restored to the full grid. 2026-07-15
had two AFTERNOON WNBA games finished before the 22 UTC slot — 14 UTC (10am ET)
is the only slot early enough for a 1pm ET game's props, and on PropLine the
extra slots cost a rounding error. Soccer stays the only trimmed sport (the
real odds-api discipline). Assumption corrected by observed data.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-15 18:07:14 -04:00

147 lines
5.5 KiB
JavaScript

'use strict';
/**
* Provider gateway (Session 20).
*
* The single entry point every external-data call passes through.
* Adapters call:
*
* const result = await gateway.fetch('odds-api', cbWithProvider, {
* capability: 'odds',
* sport: 'nba',
* fallbackProviders: ['oddspapi'], // optional override
* syncHeadersFrom: (r) => r.headers, // optional
* });
*
* Flow:
* 1. Check primary provider's quota via quotaTracker
* 2. If allowed → invoke callback, sync headers on success
* 3. If blocked → walk the fallback chain (explicit or
* capability-derived from the registry)
* 4. If every provider is exhausted → throw QuotaExhaustedError
* with a structured `attempts` log so the operator can see
* what was tried
* 5. Adapter-thrown errors propagate after rollback
*
* Callback receives the providerId actually being used so it can
* pick the right base URL / API key for fallbacks. For
* single-provider calls, callers can ignore the argument.
*/
const quotaTracker = require('./quotaTracker');
const { getFallbackChain } = require('../config/providers');
class QuotaExhaustedError extends Error {
constructor(primary, sport, attempts) {
super(`All providers exhausted for ${primary}/${sport || '*'}. Tried: ${attempts.map((a) => `${a.provider}=${a.reason}`).join('; ')}`);
this.name = 'QuotaExhaustedError';
this.code = 'QUOTA_EXHAUSTED';
this.statusCode = 503;
this.primary = primary;
this.sport = sport;
this.attempts = attempts;
}
}
async function tryOne(providerId, callbackFn, syncHeadersFrom, reserve = 0) {
// Reserve floor (Job 1 / quota guard) — a DISCRETIONARY call (futures,
// soccer outrights) is refused while fewer than `reserve` credits remain, so
// it can never drain the last credits that the ESSENTIAL path (MLB prop
// backup when PropLine fails) depends on. Essential calls pass reserve=0 and
// use the quota down to the normal 95% block. Checked BEFORE the optimistic
// increment so we don't consume-then-refund. Degraded Redis fails open.
if (reserve > 0) {
const pre = await quotaTracker.getQuotaStatus(providerId);
if (pre && !pre.degraded && Number.isFinite(pre.remaining) && pre.remaining <= reserve) {
return { ok: false, reason: `reserve_floor(${pre.remaining}<=${reserve})`, status: pre };
}
}
// Optimistic increment — if the call throws we roll back below.
// recordCall also evaluates the post-increment threshold; if the
// very next call would put us at 95%+, we still execute THIS one
// (it returned allowed:true before incrementing) and the NEXT
// call will see the block.
const status = await quotaTracker.recordCall(providerId);
if (!status.allowed) {
await quotaTracker.rollback(providerId);
return { ok: false, reason: status.reason || 'blocked', status };
}
try {
const result = await callbackFn(providerId);
// Best-effort header sync — caller signals where the headers
// live on the response object. Failure is non-fatal; the
// optimistic counter remains.
if (typeof syncHeadersFrom === 'function') {
try {
const headers = syncHeadersFrom(result);
if (headers) await quotaTracker.syncFromHeaders(providerId, headers);
} catch (e) {
console.warn(`[gateway] header sync failed for ${providerId}: ${e.message}`);
}
}
return { ok: true, result, provider: providerId };
} catch (err) {
await quotaTracker.rollback(providerId);
return { ok: false, reason: err && err.message ? err.message : 'error', err };
}
}
/**
* Invoke `callbackFn` against the primary provider, falling over
* to alternatives in the fallback chain if quota is exhausted.
*
* IMPORTANT: this only retries fallbacks on QUOTA failures, not on
* generic upstream errors. A network blip on the primary doesn't
* silently shift the entire platform to the fallback (that masks
* outages); it surfaces as the adapter's normal error path.
*/
async function fetch(primaryId, callbackFn, opts = {}) {
const {
capability,
sport,
fallbackProviders,
syncHeadersFrom,
reserve = 0,
} = opts;
const attempts = [];
const result = await tryOne(primaryId, callbackFn, syncHeadersFrom, reserve);
if (result.ok) return result.result;
// Generic adapter error on the primary — propagate, don't shift.
if (result.err) {
attempts.push({ provider: primaryId, reason: result.reason });
throw result.err;
}
attempts.push({ provider: primaryId, reason: result.reason });
// Build the fallback chain. Caller can override; otherwise derive
// from the capability/sport pair in the registry.
const chain = Array.isArray(fallbackProviders) && fallbackProviders.length
? fallbackProviders
: capability
? getFallbackChain(capability, sport, primaryId)
: [];
for (const fallbackId of chain) {
const fb = await tryOne(fallbackId, callbackFn, syncHeadersFrom, reserve);
if (fb.ok) {
console.log(`[gateway] primary=${primaryId} blocked; succeeded via fallback=${fallbackId}`);
return fb.result;
}
attempts.push({ provider: fallbackId, reason: fb.reason });
// Generic error on a fallback → record and continue to the next.
// We don't propagate fallback errors because the user only sees
// one final response, and the original primary was already
// unavailable when we entered this loop.
}
throw new QuotaExhaustedError(primaryId, sport, attempts);
}
module.exports = {
fetch,
QuotaExhaustedError,
};