Stop persisting missed_window refusals (halt the bleed at source)
missed_window means 'this capture pass ran after first pitch'. It is a fact about our cron cadence, not the market: once a game starts the same prop emits a fresh refusal every ~20 minutes for the rest of the night, per book, per side. Measured over 7 days of production that is 332,608 rows/day - 83.1% of all closing_captures writes, ~66 MB/day - and since B1 filtered both readers, nothing reads them. The filter lives in persist(), not buildCaptureRows(), and that is the whole trick: the caller computes the capture-rate alarm from the full in-memory array, so filtering at build time would have blinded the ops alarm to the exact condition it exists to catch. captureRateAlarm is pure; a test asserts the caller still passes the full array, and that a 10-priced/90-late pass still fires at 0.10. Narrow by design: one_sided_price still persists (liquidity signal), priced captures unchanged, and the rare fault refusals still persist because each names a pipeline fault worth seeing. A missed_window row carrying a price is kept. Growth drops 400,469 -> 67,862 rows/day. The 4.2M historical rows are now static, so the cleanup is a calm decision rather than a race. Nothing deleted. Grade untouched. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -37,6 +37,37 @@ const WINDOW_MINUTES_BEFORE = Number(process.env.CLOSING_WINDOW_MIN || 45);
|
||||
const SHARP_BOOKS = new Set(['pinnacle']);
|
||||
const CAPTURE_RATE_FLOOR = Number(process.env.CLOSING_CAPTURE_FLOOR || 0.6);
|
||||
|
||||
/**
|
||||
* FIX B2 — REFUSALS THAT NOTHING READS ARE NOT PERSISTED.
|
||||
*
|
||||
* `missed_window` means "this capture pass ran after first pitch". It is a fact
|
||||
* about OUR CRON CADENCE, not about the market: the same prop generates a fresh
|
||||
* refusal on every ~20-minute cycle for the rest of the night, x every book,
|
||||
* x both sides. Measured: 4,236,398 rows — 82.7% of `closing_captures` and ~75%
|
||||
* of the entire database — accruing ~465k rows and ~92 MB per day.
|
||||
*
|
||||
* Both readers now filter them out (`attachClosingProb` always did;
|
||||
* `computeDirectionalForRow` since B1), so they are read by nothing. Writing a
|
||||
* row no query will ever select is pure cost.
|
||||
*
|
||||
* NARROW BY DESIGN. `one_sided_price` still persists — a book quoting one side
|
||||
* only is real market-structure information and a standing feature candidate.
|
||||
* Priced captures persist unchanged. `unbound_game_time` /
|
||||
* `doubleheader_ambiguous` / `fetch_failed` also still persist: they are
|
||||
* genuinely rare (0 rows on record) and each names a pipeline fault worth
|
||||
* seeing, which `missed_window` does not.
|
||||
*
|
||||
* Set CLOSING_PERSIST_MISSED_WINDOW=1 to restore the old behaviour.
|
||||
*/
|
||||
const UNREAD_REFUSALS = new Set(['missed_window']);
|
||||
const persistMissedWindow = () => process.env.CLOSING_PERSIST_MISSED_WINDOW === '1';
|
||||
|
||||
/** Is this row one nothing will ever read back? */
|
||||
function isUnreadRefusal(row) {
|
||||
return !!row && UNREAD_REFUSALS.has(row.missed_reason)
|
||||
&& row.over_odds == null && row.under_odds == null;
|
||||
}
|
||||
|
||||
function etDate(iso) {
|
||||
if (!iso) return null;
|
||||
const t = new Date(iso);
|
||||
@@ -222,15 +253,29 @@ function captureRateAlarm({ eligible = 0, captured = 0 } = {}, opts = {}) {
|
||||
* historical fact, and rewriting it would destroy the provenance that C4
|
||||
* lacked in the first place. */
|
||||
async function persist(rows, deps = {}) {
|
||||
const out = { attempted: rows ? rows.length : 0, written: 0, skipped: false, error: null };
|
||||
if (!out.attempted) return out;
|
||||
const all = Array.isArray(rows) ? rows : [];
|
||||
// FIX B2 — drop the rows nothing reads BEFORE the insert. Filtering here (not
|
||||
// in buildCaptureRows) is deliberate: the caller computes the capture-rate
|
||||
// alarm from the FULL in-memory array, so the ops signal must keep seeing
|
||||
// every refusal even though we no longer store them.
|
||||
const keep = (deps.persistMissedWindow ?? persistMissedWindow())
|
||||
? all : all.filter((r) => !isUnreadRefusal(r));
|
||||
const out = {
|
||||
attempted: all.length,
|
||||
to_write: keep.length,
|
||||
skipped_unread: all.length - keep.length,
|
||||
written: 0,
|
||||
skipped: false,
|
||||
error: null,
|
||||
};
|
||||
if (!keep.length) return out;
|
||||
try {
|
||||
const getClient = deps.getClient || require('../utils/supabase').getSupabaseServiceClient;
|
||||
const sb = getClient();
|
||||
if (!sb) { out.skipped = true; return out; }
|
||||
const CHUNK = 250;
|
||||
for (let i = 0; i < rows.length; i += CHUNK) {
|
||||
const chunk = rows.slice(i, i + CHUNK);
|
||||
for (let i = 0; i < keep.length; i += CHUNK) {
|
||||
const chunk = keep.slice(i, i + CHUNK);
|
||||
const { error } = await sb.from('closing_captures').insert(chunk);
|
||||
if (error) { out.error = error.message; break; }
|
||||
out.written += chunk.length;
|
||||
@@ -243,5 +288,6 @@ async function persist(rows, deps = {}) {
|
||||
|
||||
module.exports = {
|
||||
buildCaptureRows, captureWithRetry, captureRateAlarm, persist,
|
||||
isUnreadRefusal, UNREAD_REFUSALS,
|
||||
WINDOW_MINUTES_BEFORE, SHARP_BOOKS,
|
||||
};
|
||||
|
||||
@@ -232,7 +232,11 @@ async function runIntradayRefresh(sport, opts = {}) {
|
||||
const res = await closing.persist(rows);
|
||||
const captured = rows.filter((r) => !r.missed_reason).length;
|
||||
const al = closing.captureRateAlarm({ eligible: rows.length, captured });
|
||||
console.log(`[intraday] closing capture ${sp}: ${captured}/${rows.length} priced, ${res.written} stored${al.alarm ? ' — RATE ALARM' : ''}`);
|
||||
// B2 — `stored` is now less than `eligible` on purpose: post-first-pitch
|
||||
// refusals are no longer persisted. The alarm still sees every one of them.
|
||||
console.log(`[intraday] closing capture ${sp}: ${captured}/${rows.length} priced, `
|
||||
+ `${res.written} stored, ${res.skipped_unread || 0} unread-refusals dropped`
|
||||
+ `${al.alarm ? ' — RATE ALARM' : ''}`);
|
||||
if (al.alarm && deps.notify) {
|
||||
await deps.notify(`Closing capture degraded for ${sp.toUpperCase()} — ${al.reason}`,
|
||||
{ title: 'VYNDR closing capture', priority: 'high', tags: ['rotating_light'] });
|
||||
|
||||
Reference in New Issue
Block a user