From fb0010222d0d9b1793dee8d2a0076e60deabf02c Mon Sep 17 00:00:00 2001 From: Kev Date: Wed, 12 Aug 2026 00:34:04 -0400 Subject: [PATCH] Stop persisting missed_window refusals (halt the bleed at source) missed_window means 'this capture pass ran after first pitch'. It is a fact about our cron cadence, not the market: once a game starts the same prop emits a fresh refusal every ~20 minutes for the rest of the night, per book, per side. Measured over 7 days of production that is 332,608 rows/day - 83.1% of all closing_captures writes, ~66 MB/day - and since B1 filtered both readers, nothing reads them. The filter lives in persist(), not buildCaptureRows(), and that is the whole trick: the caller computes the capture-rate alarm from the full in-memory array, so filtering at build time would have blinded the ops alarm to the exact condition it exists to catch. captureRateAlarm is pure; a test asserts the caller still passes the full array, and that a 10-priced/90-late pass still fires at 0.10. Narrow by design: one_sided_price still persists (liquidity signal), priced captures unchanged, and the rare fault refusals still persist because each names a pipeline fault worth seeing. A missed_window row carrying a price is kept. Growth drops 400,469 -> 67,862 rows/day. The 4.2M historical rows are now static, so the cleanup is a calm decision rather than a race. Nothing deleted. Grade untouched. Co-Authored-By: Claude Opus 5 (1M context) --- BUILD-STATE.md | 16 +++ CLAUDE.md | 25 ++++ src/services/closingCapture.js | 54 ++++++++- src/services/intradayRefreshService.js | 6 +- tests/unit/closingCaptureB2.test.js | 154 +++++++++++++++++++++++++ 5 files changed, 250 insertions(+), 5 deletions(-) create mode 100644 tests/unit/closingCaptureB2.test.js diff --git a/BUILD-STATE.md b/BUILD-STATE.md index db41752..96f259d 100755 --- a/BUILD-STATE.md +++ b/BUILD-STATE.md @@ -3,6 +3,22 @@ ## Last Updated 2026-08-12 +## Fix B2 (2026-08-12) — halt the closing_captures bleed at source ✅ +4,793 tests / 373 suites, web build exit 0. **No rows deleted. Grade untouched. +Readers untouched. No R2 involved.** +- `closingCapture.persist()` no longer writes `missed_window` rows (both odds + null). Priced + `one_sided_price` + the rare fault refusals persist unchanged. + Revert with `CLOSING_PERSIST_MISSED_WINDOW=1`. +- **Write reduction, measured on 7 days of production: 400,469 → 67,862 + rows/day (−83.1%); ~80 MB/day → ~13.5 MB/day at 199.6 bytes/row.** +- **Ops alarm intact** — `captureRateAlarm` is pure and the caller still passes + the full in-memory array, so a late pass still pages even though the refusals + are no longer stored. 13 tests, incl. a 10-priced/90-late pass asserting the + alarm still fires at 0.10. +- **The 4,236,398 historical rows are now STATIC** (975 MB of a 1,314 MB DB). + The cleanup is now a calm scheduled decision, not a race — and a delete will + no longer be undone by regrowth. + ## Fix B1 (2026-08-12) — dCLV close-leg filter ✅ 4,780 tests / 372 suites, web build exit 0. **Grade untouched (0 grade-related diff lines); no rows deleted or archived.** diff --git a/CLAUDE.md b/CLAUDE.md index 2d3e4e5..528c17b 100755 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2116,6 +2116,31 @@ phased plan in the Session-57 conversation / BUILD-STATE Next section). - **This makes `missed_window` genuinely unread**, which is the precondition the reclaim order stopped on. The reclaim is still downstream and still needs R2. +## B2 — stop writing the refusals nothing reads (non-obvious) +- **`missed_window` is a fact about OUR CRON CADENCE, not the market.** Once a + game starts, `buildCaptureRows` emits a refusal for that prop on EVERY ~20-min + cycle for the rest of the night, x every book, x BOTH sides. Measured + **332,608 rows/day — 83.1% of all closing_captures writes — ~66 MB/day**, read + by nothing since B1 filtered both readers. +- **THE FILTER LIVES IN `persist()`, NOT `buildCaptureRows()`, AND THAT IS THE + WHOLE TRICK.** The caller computes `captureRateAlarm({eligible: rows.length, + captured})` from the FULL in-memory array. Filtering at build time would have + silently blinded the ops alarm to exactly the condition it exists to catch — + a capture pass running late. `captureRateAlarm` is pure (two numbers, no DB), + so it never touched the stored rows; a test asserts the caller still passes the + full array. +- **NARROW BY DESIGN.** `one_sided_price` still persists (real liquidity signal, + standing feature candidate). Priced captures unchanged. `unbound_game_time` / + `doubleheader_ambiguous` / `fetch_failed` still persist — 0 rows on record, and + each names a pipeline fault worth seeing, which `missed_window` does not. + A `missed_window` row that somehow carries a price is KEPT (`isUnreadRefusal` + requires both odds null) — do not over-cut. +- `CLOSING_PERSIST_MISSED_WINDOW=1` restores the old behaviour. +- **The bleed was the fire; the 4.2M backlog is the scar.** Growth drops + **400,469 → 67,862 rows/day (−83.1%)**, ~80 MB/day → ~13.5 MB/day. The + historical 4,236,398 rows are now STATIC — the cleanup can be scheduled + calmly instead of raced, and it no longer regrows after a delete. + ## Active Skills - vyndr-voice (all user-facing output) - prop-analysis (grading methodology) diff --git a/src/services/closingCapture.js b/src/services/closingCapture.js index 26967b3..96157ab 100644 --- a/src/services/closingCapture.js +++ b/src/services/closingCapture.js @@ -37,6 +37,37 @@ const WINDOW_MINUTES_BEFORE = Number(process.env.CLOSING_WINDOW_MIN || 45); const SHARP_BOOKS = new Set(['pinnacle']); const CAPTURE_RATE_FLOOR = Number(process.env.CLOSING_CAPTURE_FLOOR || 0.6); +/** + * FIX B2 — REFUSALS THAT NOTHING READS ARE NOT PERSISTED. + * + * `missed_window` means "this capture pass ran after first pitch". It is a fact + * about OUR CRON CADENCE, not about the market: the same prop generates a fresh + * refusal on every ~20-minute cycle for the rest of the night, x every book, + * x both sides. Measured: 4,236,398 rows — 82.7% of `closing_captures` and ~75% + * of the entire database — accruing ~465k rows and ~92 MB per day. + * + * Both readers now filter them out (`attachClosingProb` always did; + * `computeDirectionalForRow` since B1), so they are read by nothing. Writing a + * row no query will ever select is pure cost. + * + * NARROW BY DESIGN. `one_sided_price` still persists — a book quoting one side + * only is real market-structure information and a standing feature candidate. + * Priced captures persist unchanged. `unbound_game_time` / + * `doubleheader_ambiguous` / `fetch_failed` also still persist: they are + * genuinely rare (0 rows on record) and each names a pipeline fault worth + * seeing, which `missed_window` does not. + * + * Set CLOSING_PERSIST_MISSED_WINDOW=1 to restore the old behaviour. + */ +const UNREAD_REFUSALS = new Set(['missed_window']); +const persistMissedWindow = () => process.env.CLOSING_PERSIST_MISSED_WINDOW === '1'; + +/** Is this row one nothing will ever read back? */ +function isUnreadRefusal(row) { + return !!row && UNREAD_REFUSALS.has(row.missed_reason) + && row.over_odds == null && row.under_odds == null; +} + function etDate(iso) { if (!iso) return null; const t = new Date(iso); @@ -222,15 +253,29 @@ function captureRateAlarm({ eligible = 0, captured = 0 } = {}, opts = {}) { * historical fact, and rewriting it would destroy the provenance that C4 * lacked in the first place. */ async function persist(rows, deps = {}) { - const out = { attempted: rows ? rows.length : 0, written: 0, skipped: false, error: null }; - if (!out.attempted) return out; + const all = Array.isArray(rows) ? rows : []; + // FIX B2 — drop the rows nothing reads BEFORE the insert. Filtering here (not + // in buildCaptureRows) is deliberate: the caller computes the capture-rate + // alarm from the FULL in-memory array, so the ops signal must keep seeing + // every refusal even though we no longer store them. + const keep = (deps.persistMissedWindow ?? persistMissedWindow()) + ? all : all.filter((r) => !isUnreadRefusal(r)); + const out = { + attempted: all.length, + to_write: keep.length, + skipped_unread: all.length - keep.length, + written: 0, + skipped: false, + error: null, + }; + if (!keep.length) return out; try { const getClient = deps.getClient || require('../utils/supabase').getSupabaseServiceClient; const sb = getClient(); if (!sb) { out.skipped = true; return out; } const CHUNK = 250; - for (let i = 0; i < rows.length; i += CHUNK) { - const chunk = rows.slice(i, i + CHUNK); + for (let i = 0; i < keep.length; i += CHUNK) { + const chunk = keep.slice(i, i + CHUNK); const { error } = await sb.from('closing_captures').insert(chunk); if (error) { out.error = error.message; break; } out.written += chunk.length; @@ -243,5 +288,6 @@ async function persist(rows, deps = {}) { module.exports = { buildCaptureRows, captureWithRetry, captureRateAlarm, persist, + isUnreadRefusal, UNREAD_REFUSALS, WINDOW_MINUTES_BEFORE, SHARP_BOOKS, }; diff --git a/src/services/intradayRefreshService.js b/src/services/intradayRefreshService.js index 0239e4a..b6b32c0 100644 --- a/src/services/intradayRefreshService.js +++ b/src/services/intradayRefreshService.js @@ -232,7 +232,11 @@ async function runIntradayRefresh(sport, opts = {}) { const res = await closing.persist(rows); const captured = rows.filter((r) => !r.missed_reason).length; const al = closing.captureRateAlarm({ eligible: rows.length, captured }); - console.log(`[intraday] closing capture ${sp}: ${captured}/${rows.length} priced, ${res.written} stored${al.alarm ? ' — RATE ALARM' : ''}`); + // B2 — `stored` is now less than `eligible` on purpose: post-first-pitch + // refusals are no longer persisted. The alarm still sees every one of them. + console.log(`[intraday] closing capture ${sp}: ${captured}/${rows.length} priced, ` + + `${res.written} stored, ${res.skipped_unread || 0} unread-refusals dropped` + + `${al.alarm ? ' — RATE ALARM' : ''}`); if (al.alarm && deps.notify) { await deps.notify(`Closing capture degraded for ${sp.toUpperCase()} — ${al.reason}`, { title: 'VYNDR closing capture', priority: 'high', tags: ['rotating_light'] }); diff --git a/tests/unit/closingCaptureB2.test.js b/tests/unit/closingCaptureB2.test.js new file mode 100644 index 0000000..efb6c78 --- /dev/null +++ b/tests/unit/closingCaptureB2.test.js @@ -0,0 +1,154 @@ +'use strict'; + +/** + * B2 — stop persisting refusals nothing reads. + * + * `missed_window` is a fact about our cron cadence, not the market: the same + * prop generates a fresh refusal every ~20 minutes for the rest of the night, + * per book, per side. 4.2M rows, ~465k/day, read by nothing since B1. + * + * The cut must be NARROW — priced captures and `one_sided_price` are read, and + * the capture-rate alarm must keep seeing every refusal even though we no longer + * store them. + */ + +const cap = require('../../src/services/closingCapture'); + +const capture = (calls) => ({ + getClient: () => ({ + from() { + return { insert: async (chunk) => { calls.push(...chunk); return { error: null }; } }; + }, + }), +}); + +const priced = (p = {}) => ({ + sport: 'mlb', player_key: 'a', stat: 'hits', side: 'over', game_date: '2026-08-12', + book: 'draftkings', line: 0.5, over_odds: -120, under_odds: 100, missed_reason: null, ...p, +}); +const refusal = (reason, p = {}) => ({ + sport: 'mlb', player_key: 'a', stat: 'hits', side: 'over', game_date: '2026-08-12', + book: 'draftkings', line: null, over_odds: null, under_odds: null, missed_reason: reason, ...p, +}); + +describe('what persists and what does not', () => { + it('a PRICED capture persists', async () => { + const calls = []; + const out = await cap.persist([priced()], capture(calls)); + expect(out.written).toBe(1); + expect(calls).toHaveLength(1); + expect(out.skipped_unread).toBe(0); + }); + + it('a ONE_SIDED_PRICE refusal persists — it is a real liquidity signal', async () => { + const calls = []; + const out = await cap.persist([refusal('one_sided_price')], capture(calls)); + expect(out.written).toBe(1); + expect(calls[0].missed_reason).toBe('one_sided_price'); + }); + + it('a MISSED_WINDOW refusal is NOT persisted', async () => { + const calls = []; + const out = await cap.persist([refusal('missed_window')], capture(calls)); + expect(out.written).toBe(0); + expect(out.skipped_unread).toBe(1); + expect(calls).toHaveLength(0); + }); + + it('a mixed pass writes exactly the readable rows', async () => { + const calls = []; + const rows = [ + priced(), priced({ player_key: 'b' }), + refusal('one_sided_price', { player_key: 'c' }), + refusal('missed_window', { player_key: 'd' }), + refusal('missed_window', { player_key: 'e' }), + ]; + const out = await cap.persist(rows, capture(calls)); + expect(out.attempted).toBe(5); + expect(out.written).toBe(3); + expect(out.skipped_unread).toBe(2); + expect(calls.map((r) => r.player_key).sort()).toEqual(['a', 'b', 'c']); + }); + + it('OTHER refusals still persist — each names a pipeline fault worth seeing', async () => { + for (const reason of ['unbound_game_time', 'doubleheader_ambiguous', 'fetch_failed']) { + const calls = []; + const out = await cap.persist([refusal(reason)], capture(calls)); + expect(out.written).toBe(1); + } + }); + + it('a missed_window row that somehow CARRIES a price is kept — do not over-cut', async () => { + const calls = []; + const out = await cap.persist([refusal('missed_window', { over_odds: -110, under_odds: -110 })], capture(calls)); + expect(out.written).toBe(1); + }); + + it('the flag restores the old behaviour', async () => { + const calls = []; + const out = await cap.persist([refusal('missed_window')], { ...capture(calls), persistMissedWindow: true }); + expect(out.written).toBe(1); + }); + + it('a pass that is ALL unread refusals never opens a client', async () => { + let opened = false; + const out = await cap.persist([refusal('missed_window'), refusal('missed_window')], { + getClient: () => { opened = true; return null; }, + }); + expect(opened).toBe(false); + expect(out.skipped_unread).toBe(2); + expect(out.written).toBe(0); + }); +}); + +describe('THE OPS SIGNAL SURVIVES — the alarm never read the stored rows', () => { + it('captureRateAlarm is pure: it takes counts, not a database', () => { + const late = cap.captureRateAlarm({ eligible: 100, captured: 10 }); + expect(late.alarm).toBe(true); + expect(late.reason).toMatch(/unrecoverable/); + const fine = cap.captureRateAlarm({ eligible: 100, captured: 90 }); + expect(fine.alarm).toBe(false); + }); + + it('still fires on a late pass whose refusals are now dropped', async () => { + // 100 props, 10 priced, 90 post-first-pitch: nothing but the 10 persists, + // and the alarm must still see 10/100. + const rows = [ + ...Array.from({ length: 10 }, (_, i) => priced({ player_key: `p${i}` })), + ...Array.from({ length: 90 }, (_, i) => refusal('missed_window', { player_key: `m${i}` })), + ]; + const calls = []; + const out = await cap.persist(rows, capture(calls)); + expect(out.written).toBe(10); + expect(out.skipped_unread).toBe(90); + + // The caller computes the alarm from the FULL array, not from what persisted. + const captured = rows.filter((r) => !r.missed_reason).length; + const al = cap.captureRateAlarm({ eligible: rows.length, captured }); + expect(al.alarm).toBe(true); + expect(al.rate).toBeCloseTo(0.10, 5); + }); + + it('the caller still passes the FULL array to the alarm', () => { + const src = require('fs').readFileSync( + require('path').join(__dirname, '..', '..', 'src/services/intradayRefreshService.js'), 'utf8'); + expect(src).toMatch(/captureRateAlarm\(\{ eligible: rows\.length, captured \}\)/); + expect(src).toMatch(/const captured = rows\.filter\(\(r\) => !r\.missed_reason\)\.length/); + }); +}); + +describe('buildCaptureRows is UNCHANGED — the alarm depends on it', () => { + it('still emits missed_window rows in memory', () => { + const started = { player: 'A', stat_type: 'hits', game_time: '2026-08-12T00:00:00Z', book: 'dk' }; + const rows = cap.buildCaptureRows('mlb', [started], { now: new Date('2026-08-12T02:00:00Z') }); + expect(rows).toHaveLength(2); // both sides + expect(rows.every((r) => r.missed_reason === 'missed_window')).toBe(true); + }); + + it('isUnreadRefusal identifies exactly those rows', () => { + expect(cap.isUnreadRefusal(refusal('missed_window'))).toBe(true); + expect(cap.isUnreadRefusal(refusal('one_sided_price'))).toBe(false); + expect(cap.isUnreadRefusal(priced())).toBe(false); + expect(cap.isUnreadRefusal(null)).toBe(false); + }); +});