Session 56: Full audit — PropLine + boxscore + pipeline + sport coverage (2289 tests)
Research (verified against live MLB Stats / ESPN / The Odds APIs): - specs/propline-audit.md — every stat_type mapped against our 4-layer pipeline; real MLB boxscore fields; sport coverage status; pipeline gap analysis. - specs/vyndr-roadmap.md — priority-ordered Sessions 57–64 + coverage targets. - scripts/propline-audit.js + specs/audit-data/ (raw capture). Headline bug: oddsNormalizer mapped batter_rbis → 'rbis' while the whole grade/feature/outcome chain keys on 'rbi' — every PropLine RBI prop silently failed to grade AND settle. Fixed (+ regression test). Phase 4 — wired missing MLB stats end-to-end: - PropLine MLB markets 6 → 12 (+runs, walks, doubles, earned_runs, hits_allowed, outs — same request, no extra quota). - doubles/outs/triples added to featureCache + outcomeService MLB_LOG_FIELD and all three grade whitelists (analyze/scan/validation.py). Phase 6 — pipeline resilience: - opsNotify.js: ntfy alerts (never throws, test-disabled). Snapshot success/ stale/failure alerts; retry-once on hard odds error (not on empty slate). - Missed-cron watchdog (mostRecentExpectedSlot/isSnapshotOverdue); status probe now returns `overdue`. Coverage truth: MLB is the only end-to-end-live sport; outcome settlement is MLB-only (WNBA/NBA/soccer never settle) — documented as the #1 roadmap gap. Backend 2276 → 2289 tests (+13). Web build exit 0. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -61,6 +61,9 @@ const VALID_STAT_TYPES = new Set([
|
||||
'strikeouts', 'hits', 'home_runs', 'rbi', 'total_bases',
|
||||
'walks', 'runs', 'earned_runs', 'innings_pitched',
|
||||
'hits_allowed', 'stolen_bases',
|
||||
// Session 56 audit — settleable against the real boxscore; PropLine now
|
||||
// requests batter_doubles + pitcher_outs. Keep in sync with scan.js + validation.py.
|
||||
'doubles', 'outs',
|
||||
]);
|
||||
|
||||
const VALID_DIRECTIONS = new Set(['over', 'under']);
|
||||
|
||||
@@ -112,7 +112,7 @@ router.post('/snapshot/all', async (req, res) => {
|
||||
*/
|
||||
router.get('/snapshot/status', async (req, res) => {
|
||||
const { cacheGet } = require('../utils/redis');
|
||||
const { HOURS_UTC } = require('../snapshotScheduler');
|
||||
const { HOURS_UTC, isSnapshotOverdue } = require('../snapshotScheduler');
|
||||
const SPORTS = ['mlb', 'nba', 'wnba'];
|
||||
try {
|
||||
const redis_keys = {};
|
||||
@@ -135,10 +135,13 @@ router.get('/snapshot/status', async (req, res) => {
|
||||
}
|
||||
const ticker = await cacheGet('ticker:items');
|
||||
redis_keys['ticker:items'] = !!ticker;
|
||||
// Session 56 — surface the missed-cron signal in the health probe.
|
||||
const mlbTs = last_snapshot.mlb && last_snapshot.mlb.updated_at;
|
||||
return res.json({
|
||||
cron_armed: process.env.SNAPSHOT_CRON === '1',
|
||||
cron_hours_utc: HOURS_UTC,
|
||||
last_snapshot,
|
||||
overdue: isSnapshotOverdue(mlbTs),
|
||||
redis_keys,
|
||||
ticker_count: Array.isArray(ticker) ? ticker.length : 0,
|
||||
});
|
||||
|
||||
@@ -17,6 +17,8 @@ const VALID_STAT_TYPES = new Set([
|
||||
'strikeouts', 'hits', 'home_runs', 'rbi', 'total_bases',
|
||||
'walks', 'runs', 'earned_runs', 'innings_pitched',
|
||||
'hits_allowed', 'stolen_bases',
|
||||
// Session 56 audit — keep in sync with analyze.js + validation.py.
|
||||
'doubles', 'outs',
|
||||
]);
|
||||
const VALID_DIRECTIONS = new Set(['over', 'under']);
|
||||
|
||||
|
||||
@@ -46,7 +46,11 @@ const SPORT_KEYS = {
|
||||
const MARKETS = {
|
||||
nba: ['player_points', 'player_rebounds', 'player_assists', 'player_threes', 'player_blocks', 'player_steals'],
|
||||
wnba: ['player_points', 'player_rebounds', 'player_assists', 'player_threes'],
|
||||
mlb: ['batter_hits', 'batter_home_runs', 'batter_total_bases', 'batter_rbis', 'batter_stolen_bases', 'pitcher_strikeouts'],
|
||||
// Session 56 audit — expanded from 6 to 12 markets. runs/walks/earned_runs/
|
||||
// hits_allowed were already fully supported downstream (whitelist + features +
|
||||
// outcomes) but never requested; doubles/outs are wired this session. All ride
|
||||
// the SAME request (no extra quota) → materially more graded props per slate.
|
||||
mlb: ['batter_hits', 'batter_home_runs', 'batter_total_bases', 'batter_rbis', 'batter_stolen_bases', 'batter_runs', 'batter_walks', 'batter_doubles', 'pitcher_strikeouts', 'pitcher_earned_runs', 'pitcher_hits_allowed', 'pitcher_outs'],
|
||||
nfl: ['player_pass_yds', 'player_rush_yds', 'player_reception_yds', 'player_receptions', 'player_anytime_td', 'player_pass_tds'],
|
||||
nhl: ['player_goals', 'player_shots_on_goal', 'goalie_saves'],
|
||||
ncaab: ['player_points', 'player_rebounds', 'player_assists'],
|
||||
|
||||
@@ -82,6 +82,8 @@ const MLB_LOG_FIELD = {
|
||||
runs: 'runs', stolen_bases: 'stolenBases', walks: 'baseOnBalls',
|
||||
strikeouts: 'strikeOuts', earned_runs: 'earnedRuns', hits_allowed: 'hits',
|
||||
innings_pitched: 'inningsPitched',
|
||||
// Session 56 audit — real boxscore/game-log fields (Braves@Pirates verified).
|
||||
doubles: 'doubles', triples: 'triples', outs: 'outs',
|
||||
};
|
||||
|
||||
function mlbStatValue(statObj, statType) {
|
||||
|
||||
@@ -41,6 +41,8 @@ const MLB_LOG_FIELD = {
|
||||
runs: 'runs', stolen_bases: 'stolenBases', walks: 'baseOnBalls',
|
||||
strikeouts: 'strikeOuts', earned_runs: 'earnedRuns', hits_allowed: 'hits',
|
||||
innings_pitched: 'inningsPitched',
|
||||
// Session 56 audit — confirmed present in the real boxscore/game log.
|
||||
doubles: 'doubles', triples: 'triples', outs: 'outs',
|
||||
};
|
||||
|
||||
function statValue(statObj, statType) {
|
||||
|
||||
@@ -18,7 +18,9 @@ VALID_STAT_TYPES = {
|
||||
'steals', 'blocks', 'turnovers'],
|
||||
'mlb': ['strikeouts', 'hits', 'home_runs', 'rbi', 'total_bases',
|
||||
'walks', 'runs', 'earned_runs', 'innings_pitched',
|
||||
'hits_allowed', 'stolen_bases']
|
||||
'hits_allowed', 'stolen_bases',
|
||||
# Session 56 audit — keep in sync with analyze.js + scan.js.
|
||||
'doubles', 'outs']
|
||||
}
|
||||
|
||||
VALID_SPORTS = ['nba', 'mlb']
|
||||
|
||||
@@ -184,18 +184,36 @@ async function runSnapshot(sport, opts = {}) {
|
||||
cacheSet: opts.cacheSet || require('../utils/redis').cacheSet,
|
||||
now: opts.now || (() => new Date().toISOString()),
|
||||
nowMs: opts.nowMs || (() => Date.now()),
|
||||
// Session 56 — ops alerting (ntfy) + retry-once on a hard odds failure.
|
||||
notify: opts.notify || require('../utils/opsNotify').notify,
|
||||
sleep: opts.sleep || ((ms) => new Promise((r) => setTimeout(r, ms))),
|
||||
retryDelayMs: opts.retryDelayMs != null ? opts.retryDelayMs : 60_000,
|
||||
};
|
||||
const start = deps.nowMs();
|
||||
const ts = deps.now();
|
||||
|
||||
// Session 56 — retry ONCE on a hard failure (thrown error / null response = a
|
||||
// transient PropLine/network blip). A successful-but-empty slate is NOT a
|
||||
// failure (off-hours), so it is not retried — that would waste quota + latency.
|
||||
let odds;
|
||||
try {
|
||||
odds = await deps.getOdds(sp);
|
||||
if (odds == null) throw new Error('null odds response');
|
||||
} catch (e) {
|
||||
return { sport: sp, status: 'error', reason: e.message, gradeCount: 0 };
|
||||
try {
|
||||
await deps.sleep(deps.retryDelayMs);
|
||||
odds = await deps.getOdds(sp);
|
||||
if (odds == null) throw new Error('null odds response (retry)');
|
||||
} catch (e2) {
|
||||
await deps.notify(`❌ ${sp.toUpperCase()} snapshot FAILED: ${e2.message}`, { title: 'VYNDR pipeline', priority: 'high', tags: ['x'] });
|
||||
return { sport: sp, status: 'error', reason: e2.message, gradeCount: 0 };
|
||||
}
|
||||
}
|
||||
const props = (odds && Array.isArray(odds.props)) ? odds.props : [];
|
||||
if (props.length === 0) return { sport: sp, status: 'skipped', reason: 'no props', gradeCount: 0 };
|
||||
if (props.length === 0) {
|
||||
await deps.notify(`⚠️ ${sp.toUpperCase()} snapshot: 0 props (odds unavailable)`, { title: 'VYNDR pipeline', priority: 'low', tags: ['warning'] });
|
||||
return { sport: sp, status: 'skipped', reason: 'no props', gradeCount: 0 };
|
||||
}
|
||||
|
||||
// Grade the slate via the existing service; capture the envelope instead of
|
||||
// letting it write (we re-write an ENRICHED version below).
|
||||
@@ -273,6 +291,18 @@ async function runSnapshot(sport, opts = {}) {
|
||||
const events = generateTickerEvents(sp, enriched, deltas, ts);
|
||||
await pushTickerItems(events, deps);
|
||||
|
||||
// Session 56 — success alert, enriched with the rolling accuracy (if settled).
|
||||
let accPart = '';
|
||||
try {
|
||||
const acc = await deps.cacheGet(`accuracy:${sp}`);
|
||||
const pct = acc && acc.overall && acc.overall.pct;
|
||||
if (pct != null) accPart = `, ${pct}% accuracy (30d)`;
|
||||
} catch { /* accuracy is best-effort in the alert */ }
|
||||
await deps.notify(
|
||||
`✅ ${sp.toUpperCase()} snapshot: ${enriched.length} props graded, ${deltas.length} deltas${accPart}`,
|
||||
{ title: 'VYNDR pipeline', tags: ['white_check_mark'] },
|
||||
);
|
||||
|
||||
return {
|
||||
sport: sp,
|
||||
status: 'ok',
|
||||
|
||||
@@ -17,6 +17,30 @@ const HOURS_UTC = (process.env.SNAPSHOT_HOURS_UTC || '14,19,22,1,3')
|
||||
.map((n) => parseInt(n, 10))
|
||||
.filter((n) => Number.isInteger(n) && n >= 0 && n <= 23);
|
||||
|
||||
// Session 56 — missed-cron detection (pure, testable).
|
||||
// The latest scheduled hour:00 (UTC) at or before `date`. null if none in 48h.
|
||||
function mostRecentExpectedSlot(date, hours = HOURS_UTC) {
|
||||
const d = new Date(date.getTime());
|
||||
d.setUTCMinutes(0, 0, 0);
|
||||
for (let i = 0; i < 48; i += 1) {
|
||||
if (hours.includes(d.getUTCHours())) return new Date(d.getTime());
|
||||
d.setUTCHours(d.getUTCHours() - 1);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// True when we're >graceMin past the most recent expected slot AND the last
|
||||
// recorded snapshot predates that slot (i.e. the run was missed). Never fires on
|
||||
// cold boot (no lastSnapshotIso) — we don't cry wolf before the first snapshot.
|
||||
function isSnapshotOverdue(lastSnapshotIso, now = new Date(), hours = HOURS_UTC, graceMin = 30) {
|
||||
const slot = mostRecentExpectedSlot(now, hours);
|
||||
if (!slot) return false;
|
||||
if (now.getTime() - slot.getTime() < graceMin * 60_000) return false;
|
||||
if (!lastSnapshotIso) return false;
|
||||
const last = new Date(lastSnapshotIso).getTime();
|
||||
return Number.isFinite(last) && last < slot.getTime();
|
||||
}
|
||||
|
||||
function startSnapshotScheduler(opts = {}) {
|
||||
if (process.env.SNAPSHOT_CRON !== '1') {
|
||||
// Session 52 — log the disarmed state so container logs make it unambiguous
|
||||
@@ -31,10 +55,33 @@ function startSnapshotScheduler(opts = {}) {
|
||||
// real (now-completed) results BEFORE grading the fresh slate, so the accuracy
|
||||
// record reflects yesterday's games each cycle.
|
||||
const settleAll = opts.settleAllOutcomes || require('./services/outcomeService').settleAllOutcomes;
|
||||
const notify = opts.notify || require('./utils/opsNotify').notify;
|
||||
const cacheGet = opts.cacheGet || require('./utils/redis').cacheGet;
|
||||
const now = opts.now || (() => new Date());
|
||||
let lastFiredSlot = null;
|
||||
let lastOverdueSlot = null;
|
||||
|
||||
// Session 56 — missed-cron watchdog. Runs every minute (independent of the
|
||||
// fire schedule): if a scheduled slot came and went without a snapshot, alert
|
||||
// ONCE per missed slot.
|
||||
const checkOverdue = async () => {
|
||||
try {
|
||||
const d = now();
|
||||
const slot = mostRecentExpectedSlot(d);
|
||||
if (!slot) return;
|
||||
const slotKey = slot.toISOString();
|
||||
if (slotKey === lastOverdueSlot) return; // already alerted for this slot
|
||||
const latest = await cacheGet('snapshot:mlb:latest');
|
||||
const lastTs = latest && latest.updated_at;
|
||||
if (isSnapshotOverdue(lastTs, d)) {
|
||||
lastOverdueSlot = slotKey;
|
||||
await notify(`⚠️ Snapshot OVERDUE — expected ${slot.getUTCHours()}:00 UTC, last was ${lastTs || 'never'}`, { title: 'VYNDR pipeline', priority: 'high', tags: ['warning'] });
|
||||
}
|
||||
} catch { /* watchdog must never throw */ }
|
||||
};
|
||||
|
||||
const tick = async () => {
|
||||
await checkOverdue();
|
||||
const d = now();
|
||||
if (d.getUTCMinutes() !== 0) return;
|
||||
const h = d.getUTCHours();
|
||||
@@ -64,4 +111,4 @@ function startSnapshotScheduler(opts = {}) {
|
||||
return { interval, tick };
|
||||
}
|
||||
|
||||
module.exports = { startSnapshotScheduler, HOURS_UTC };
|
||||
module.exports = { startSnapshotScheduler, HOURS_UTC, mostRecentExpectedSlot, isSnapshotOverdue };
|
||||
|
||||
@@ -24,7 +24,12 @@ const MARKET_MAP = {
|
||||
batter_hits: 'hits',
|
||||
batter_home_runs: 'home_runs',
|
||||
batter_total_bases: 'total_bases',
|
||||
batter_rbis: 'rbis',
|
||||
// Session 56 audit — must be 'rbi' (singular): the grade whitelists
|
||||
// (analyze/scan/validation.py), featureCache MLB_LOG_FIELD, and outcomeService
|
||||
// all key on 'rbi'. Normalizing to 'rbis' silently dropped every PropLine RBI
|
||||
// prop from grading AND settlement. The streaks/hotlist path uses its own
|
||||
// 'rbis' key built from raw MLB stats — independent of this normalizer.
|
||||
batter_rbis: 'rbi',
|
||||
batter_runs: 'runs',
|
||||
batter_stolen_bases: 'stolen_bases',
|
||||
batter_singles: 'singles',
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* opsNotify (Session 56) — pipeline alerting via ntfy.sh.
|
||||
*
|
||||
* Push a one-line operational alert (snapshot success / failure / stale / missed
|
||||
* cron) to an ntfy topic so a silent pipeline never goes unnoticed. Fire-and-
|
||||
* forget: NEVER throws, NEVER blocks the pipeline on a notify failure.
|
||||
*
|
||||
* Config:
|
||||
* NTFY_URL (default https://ntfy.sh)
|
||||
* NTFY_TOPIC (default vyndr-pipeline-kev2026)
|
||||
* PIPELINE_ALERTS=0 → disable entirely
|
||||
* Disabled automatically under NODE_ENV==='test' unless a fetchImpl is injected
|
||||
* (so the unit tests can assert the call without hitting the network).
|
||||
*/
|
||||
|
||||
const NTFY_URL = () => process.env.NTFY_URL || 'https://ntfy.sh';
|
||||
const NTFY_TOPIC = () => process.env.NTFY_TOPIC || 'vyndr-pipeline-kev2026';
|
||||
|
||||
function enabled(opts = {}) {
|
||||
if (opts.fetchImpl) return true; // tests inject → always "enabled"
|
||||
if (process.env.PIPELINE_ALERTS === '0') return false;
|
||||
if (process.env.NODE_ENV === 'test') return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send an ops alert. `opts`: { title, priority ('min'|'low'|'default'|'high'|
|
||||
* 'urgent'), tags (string[]), fetchImpl }. Resolves { sent: boolean } — never rejects.
|
||||
*/
|
||||
async function notify(message, opts = {}) {
|
||||
if (!enabled(opts)) return { sent: false, reason: 'disabled' };
|
||||
const doFetch = opts.fetchImpl || fetch;
|
||||
const headers = {};
|
||||
if (opts.title) headers.Title = opts.title;
|
||||
if (opts.priority) headers.Priority = opts.priority;
|
||||
if (Array.isArray(opts.tags) && opts.tags.length) headers.Tags = opts.tags.join(',');
|
||||
try {
|
||||
await doFetch(`${NTFY_URL()}/${NTFY_TOPIC()}`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: String(message == null ? '' : message),
|
||||
signal: typeof AbortSignal !== 'undefined' && AbortSignal.timeout ? AbortSignal.timeout(6000) : undefined,
|
||||
});
|
||||
return { sent: true };
|
||||
} catch (err) {
|
||||
// Alerting must never break the pipeline.
|
||||
if (process.env.NODE_ENV !== 'test') console.warn('[opsNotify] failed:', err.message);
|
||||
return { sent: false, reason: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { notify, __internals: { enabled, NTFY_URL, NTFY_TOPIC } };
|
||||
Reference in New Issue
Block a user