Stop persisting missed_window refusals (halt the bleed at source)

missed_window means 'this capture pass ran after first pitch'. It is a fact
about our cron cadence, not the market: once a game starts the same prop emits
a fresh refusal every ~20 minutes for the rest of the night, per book, per
side. Measured over 7 days of production that is 332,608 rows/day - 83.1% of
all closing_captures writes, ~66 MB/day - and since B1 filtered both readers,
nothing reads them.

The filter lives in persist(), not buildCaptureRows(), and that is the whole
trick: the caller computes the capture-rate alarm from the full in-memory
array, so filtering at build time would have blinded the ops alarm to the exact
condition it exists to catch. captureRateAlarm is pure; a test asserts the
caller still passes the full array, and that a 10-priced/90-late pass still
fires at 0.10.

Narrow by design: one_sided_price still persists (liquidity signal), priced
captures unchanged, and the rare fault refusals still persist because each
names a pipeline fault worth seeing. A missed_window row carrying a price is
kept.

Growth drops 400,469 -> 67,862 rows/day. The 4.2M historical rows are now
static, so the cleanup is a calm decision rather than a race.

Nothing deleted. Grade untouched.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kev
2026-08-12 00:34:04 -04:00
parent 2271f46ab2
commit fb0010222d
5 changed files with 250 additions and 5 deletions
+50 -4
View File
@@ -37,6 +37,37 @@ const WINDOW_MINUTES_BEFORE = Number(process.env.CLOSING_WINDOW_MIN || 45);
const SHARP_BOOKS = new Set(['pinnacle']);
const CAPTURE_RATE_FLOOR = Number(process.env.CLOSING_CAPTURE_FLOOR || 0.6);
/**
* FIX B2 — REFUSALS THAT NOTHING READS ARE NOT PERSISTED.
*
* `missed_window` means "this capture pass ran after first pitch". It is a fact
* about OUR CRON CADENCE, not about the market: the same prop generates a fresh
* refusal on every ~20-minute cycle for the rest of the night, x every book,
* x both sides. Measured: 4,236,398 rows — 82.7% of `closing_captures` and ~75%
* of the entire database — accruing ~465k rows and ~92 MB per day.
*
* Both readers now filter them out (`attachClosingProb` always did;
* `computeDirectionalForRow` since B1), so they are read by nothing. Writing a
* row no query will ever select is pure cost.
*
* NARROW BY DESIGN. `one_sided_price` still persists — a book quoting one side
* only is real market-structure information and a standing feature candidate.
* Priced captures persist unchanged. `unbound_game_time` /
* `doubleheader_ambiguous` / `fetch_failed` also still persist: they are
* genuinely rare (0 rows on record) and each names a pipeline fault worth
* seeing, which `missed_window` does not.
*
* Set CLOSING_PERSIST_MISSED_WINDOW=1 to restore the old behaviour.
*/
const UNREAD_REFUSALS = new Set(['missed_window']);
const persistMissedWindow = () => process.env.CLOSING_PERSIST_MISSED_WINDOW === '1';
/** Is this row one nothing will ever read back? */
function isUnreadRefusal(row) {
return !!row && UNREAD_REFUSALS.has(row.missed_reason)
&& row.over_odds == null && row.under_odds == null;
}
function etDate(iso) {
if (!iso) return null;
const t = new Date(iso);
@@ -222,15 +253,29 @@ function captureRateAlarm({ eligible = 0, captured = 0 } = {}, opts = {}) {
* historical fact, and rewriting it would destroy the provenance that C4
* lacked in the first place. */
async function persist(rows, deps = {}) {
const out = { attempted: rows ? rows.length : 0, written: 0, skipped: false, error: null };
if (!out.attempted) return out;
const all = Array.isArray(rows) ? rows : [];
// FIX B2 — drop the rows nothing reads BEFORE the insert. Filtering here (not
// in buildCaptureRows) is deliberate: the caller computes the capture-rate
// alarm from the FULL in-memory array, so the ops signal must keep seeing
// every refusal even though we no longer store them.
const keep = (deps.persistMissedWindow ?? persistMissedWindow())
? all : all.filter((r) => !isUnreadRefusal(r));
const out = {
attempted: all.length,
to_write: keep.length,
skipped_unread: all.length - keep.length,
written: 0,
skipped: false,
error: null,
};
if (!keep.length) return out;
try {
const getClient = deps.getClient || require('../utils/supabase').getSupabaseServiceClient;
const sb = getClient();
if (!sb) { out.skipped = true; return out; }
const CHUNK = 250;
for (let i = 0; i < rows.length; i += CHUNK) {
const chunk = rows.slice(i, i + CHUNK);
for (let i = 0; i < keep.length; i += CHUNK) {
const chunk = keep.slice(i, i + CHUNK);
const { error } = await sb.from('closing_captures').insert(chunk);
if (error) { out.error = error.message; break; }
out.written += chunk.length;
@@ -243,5 +288,6 @@ async function persist(rows, deps = {}) {
module.exports = {
buildCaptureRows, captureWithRetry, captureRateAlarm, persist,
isUnreadRefusal, UNREAD_REFUSALS,
WINDOW_MINUTES_BEFORE, SHARP_BOOKS,
};
+5 -1
View File
@@ -232,7 +232,11 @@ async function runIntradayRefresh(sport, opts = {}) {
const res = await closing.persist(rows);
const captured = rows.filter((r) => !r.missed_reason).length;
const al = closing.captureRateAlarm({ eligible: rows.length, captured });
console.log(`[intraday] closing capture ${sp}: ${captured}/${rows.length} priced, ${res.written} stored${al.alarm ? ' — RATE ALARM' : ''}`);
// B2 — `stored` is now less than `eligible` on purpose: post-first-pitch
// refusals are no longer persisted. The alarm still sees every one of them.
console.log(`[intraday] closing capture ${sp}: ${captured}/${rows.length} priced, `
+ `${res.written} stored, ${res.skipped_unread || 0} unread-refusals dropped`
+ `${al.alarm ? ' — RATE ALARM' : ''}`);
if (al.alarm && deps.notify) {
await deps.notify(`Closing capture degraded for ${sp.toUpperCase()} — ${al.reason}`,
{ title: 'VYNDR closing capture', priority: 'high', tags: ['rotating_light'] });