From 655228166119ba469023e8730fc9759407b84aa2 Mon Sep 17 00:00:00 2001 From: Kev Date: Tue, 28 Jul 2026 18:59:12 -0400 Subject: [PATCH] CLV instrument repair: fix attachClosingProb read + recoverable market_unavailable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The closing_prob funnel collapsed 100k priced captures -> 59 usable. Root cause (VERIFIED against prod, join key is PERFECT with 0 mismatches): - attachClosingProb read closing_captures with .limit(50000) and NO ORDER BY on a 730k-row table that is 86% refusal rows -> saw ~7% for MLB, missed most priced closes and declared 200+ rows closeless that HAD a capture. - market_unavailable_reason was write-once/terminal, so a row wrongly declared (truncated read / premature declaration before the capture was visible) could never recover even once its genuine capture existed. 298 rows (204 MLB + 94 WNBA) were stuck this way. Fix (CLV computation only — no grade/locked_odds/outcome touched): - Read ONLY priced captures (missed_reason IS NULL, both odds NOT NULL), scoped to the candidate rows' game_dates -> small AND complete, no arbitrary truncation. - Drop the market_unavailable exclusion from candidates; make it a re-checkable absence: a genuine close now UPGRADES the row (writes closing_prob, clears the verdict). closing_prob stays write-once (first true close wins). No capture + past game -> still declared absent (honest). No churn on already-absent rows. - New internal trigger POST /api/internal/ledger/attach-closing[/:sport] for backfill + verification (scheduler already runs attach per tick). Recovers ~312 usable closes (59 -> ~371), MLB included. Capture itself was healthy all along (94.9% MLB / 95.8% WNBA per-prop coverage). Full suite 3835 green (17/17 instrument tests incl. 2 new recovery cases), web build exit 0. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01VsztNChZ7vEvSR61AuMhD1 --- src/routes/internal.js | 27 +++++++++++++ src/services/ledgerService.js | 51 +++++++++++++++++++------ tests/unit/projectionInstrument.test.js | 25 ++++++++++++ 3 files changed, 91 insertions(+), 12 deletions(-) diff --git a/src/routes/internal.js b/src/routes/internal.js index 85e2671..9bdd49c 100644 --- a/src/routes/internal.js +++ b/src/routes/internal.js @@ -405,6 +405,33 @@ router.post('/ledger/settle', async (req, res) => { } }); +/** + * POST /api/internal/ledger/attach-closing[/:sport] — CLV instrument repair. + * Attach the de-vigged closing probability from `closing_captures` onto locked + * model-record rows. Recovers rows a truncated read / premature verdict wrongly + * declared closeless (market_unavailable is a re-checkable absence, not terminal). + * Idempotent: `closing_prob` is write-once; a row with a genuine close never + * changes, and a row with no capture stays honestly absent. + */ +async function attachClosingHandler(req, res) { + const ledger = require('../services/ledgerService'); + const ALL = ['mlb', 'wnba', 'nba', 'soccer']; + const only = String(req.params.sport || '').toLowerCase(); + const sports = only ? [only] : ALL; + try { + const results = {}; + for (const sp of sports) results[sp] = await ledger.attachClosingProb(sp, { limit: Number(req.query.limit) || undefined }); + return res.json({ ok: true, results }); + } catch (err) { + const message = err && err.message ? err.message : String(err); + console.error('[internal/ledger/attach-closing] failed:', message); + return res.status(500).json({ ok: false, error: message }); + } +} +// Express 5 (path-to-regexp v8) has no `/:sport?` optional param — register both. +router.post('/ledger/attach-closing', attachClosingHandler); +router.post('/ledger/attach-closing/:sport', attachClosingHandler); + /** * POST /api/internal/newsletter/send (Session S7, a1) — assemble today's * VYNDR REPORT from the pipeline (snapshot signals + streak lens + the diff --git a/src/services/ledgerService.js b/src/services/ledgerService.js index 1e1680d..a2f84e6 100644 --- a/src/services/ledgerService.js +++ b/src/services/ledgerService.js @@ -342,11 +342,17 @@ async function attachClosingProb(sport, opts = {}) { const sb = opts.sb || defaultClient(); const sp = String(sport || '').toLowerCase(); + // Candidates: any locked model-record row still lacking a real close. We do + // NOT exclude rows already stamped market_unavailable_reason — that verdict is + // a RE-CHECKABLE absence, not terminal. A capture we couldn't see before (the + // read was truncated) or that landed after a premature declaration must be + // able to upgrade the row. `closing_prob` itself stays write-once (the filter + // below), so the FIRST true close still wins and is never rewritten. const { data: rows, error } = await sb.from('ledger_entries') - .select('id, player_key, stat, side, game_date') + .select('id, player_key, stat, side, game_date, market_unavailable_reason') .is('user_id', null).eq('sport', sp) - .is('closing_prob', null).is('market_unavailable_reason', null) - .limit(opts.limit || 2000); + .is('closing_prob', null) + .limit(opts.limit || 5000); if (error) return { updated: 0, error: error.message }; if (!rows || !rows.length) return { updated: 0, absent: 0 }; @@ -355,10 +361,20 @@ async function attachClosingProb(sport, opts = {}) { // the grade-time fair price uses, which is what makes lock and close // comparable at all. Asking this table for a `fair_prob` column is a bug: it // has none, and every row then looks closeless. + // + // READ FIX (CLV instrument repair): the table is ~86% refusal rows, and the + // old `.limit(50000)` with no ORDER BY read an arbitrary slice — for MLB + // (730k rows) it saw ~7% and declared 200+ rows closeless that HAD a priced + // capture. Read only PRICED captures, scoped to the candidate rows' game + // dates, so the set is small AND complete. const { devigTwoWay } = require('../utils/devig'); - const { data: caps } = await sb.from('closing_captures') + const dates = [...new Set(rows.map((r) => r.game_date).filter(Boolean))]; + let capsQuery = sb.from('closing_captures') .select('player_key, stat, side, game_date, over_odds, under_odds, captured_at, missed_reason') - .eq('sport', sp).limit(50000); + .eq('sport', sp).is('missed_reason', null) + .not('over_odds', 'is', null).not('under_odds', 'is', null); + if (dates.length) capsQuery = capsQuery.in('game_date', dates); + const { data: caps } = await capsQuery.limit(opts.capLimit || 200000); // Latest usable capture per identity = the TRUE close. const best = new Map(); @@ -378,17 +394,28 @@ async function attachClosingProb(sport, opts = {}) { // rows market-unavailable would be premature absence — as dishonest in the // other direction as imputing one. Only a past game can be declared closeless. const cutoff = opts.beforeDate || todayET(); + let recovered = 0; for (const r of rows) { const hit = best.get(`${r.player_key}|${r.stat}|${r.side}|${r.game_date}`); - if (!hit && String(r.game_date) >= String(cutoff)) continue; // still capturable - const patch = hit - ? { closing_prob: hit.fair_prob, closing_captured_at: hit.captured_at } - : { market_unavailable_reason: 'no_usable_close' }; - const { error: e } = await sb.from('ledger_entries').update(patch).eq('id', r.id); + if (hit) { + // A real close — write it and CLEAR any prior (premature/truncation-bug) + // market-unavailable verdict. This is the recovery path: a row wrongly + // declared closeless is upgraded the moment its genuine capture is seen. + const patch = { closing_prob: hit.fair_prob, closing_captured_at: hit.captured_at, market_unavailable_reason: null }; + const { error: e } = await sb.from('ledger_entries').update(patch).eq('id', r.id); + if (e) continue; + updated += 1; + if (r.market_unavailable_reason) recovered += 1; + continue; + } + // No usable close found for this row. + if (String(r.game_date) >= String(cutoff)) continue; // future/today — a close can still arrive + if (r.market_unavailable_reason) continue; // already honestly declared absent — leave it + const { error: e } = await sb.from('ledger_entries').update({ market_unavailable_reason: 'no_usable_close' }).eq('id', r.id); if (e) continue; - if (hit) updated += 1; else absent += 1; + absent += 1; } - return { updated, absent, candidates: rows.length }; + return { updated, absent, recovered, candidates: rows.length }; } /** diff --git a/tests/unit/projectionInstrument.test.js b/tests/unit/projectionInstrument.test.js index 11114af..c54f9b8 100644 --- a/tests/unit/projectionInstrument.test.js +++ b/tests/unit/projectionInstrument.test.js @@ -78,6 +78,7 @@ describe('attachClosingProb — the market half', () => { from: (t) => ({ select: () => ({ is: function () { return this; }, eq: function () { return this; }, + not: function () { return this; }, in: function () { return this; }, limit: async () => ({ data: t === 'ledger_entries' ? rows : caps, error: null }), }), update: (patch) => ({ eq: async (_c, id) => { updates.push({ id, patch }); return { error: null }; } }), @@ -150,6 +151,30 @@ describe('attachClosingProb — the market half', () => { const fn = src.slice(src.indexOf('async function attachClosingProb')); expect(fn).toMatch(/\.is\('closing_prob', null\)/); }); + + // CLV instrument repair — the write-once verdict on market_unavailable_reason + // was permanent, so a row wrongly declared closeless (truncated read / + // premature declaration) could never recover even though its capture existed. + it('RECOVERS a row previously declared market_unavailable when a real close now exists', async () => { + const sb = makeSb({ + rows: [{ ...ROW, market_unavailable_reason: 'no_usable_close' }], + caps: [{ player_key: 'josh bell', stat: 'hits', side: 'over', game_date: '2026-07-20', over_odds: -210, under_odds: 170, captured_at: '2026-07-20T22:50:00Z' }], + }); + const out = await ledger.attachClosingProb('mlb', { sb, beforeDate: '2026-07-21' }); + expect(out.updated).toBe(1); + expect(out.recovered).toBe(1); + expect(sb.updates[0].patch.closing_prob).toBe(0.647); + // the buggy verdict is CLEARED, not left stale beside a real close. + expect(sb.updates[0].patch.market_unavailable_reason).toBeNull(); + }); + + it('leaves an already-declared row untouched when there is STILL no capture (no churn, no re-write)', async () => { + const sb = makeSb({ rows: [{ ...ROW, market_unavailable_reason: 'no_usable_close' }], caps: [] }); + const out = await ledger.attachClosingProb('mlb', { sb, beforeDate: '2026-07-21' }); + expect(out.updated).toBe(0); + expect(out.absent).toBe(0); + expect(sb.updates).toHaveLength(0); // no write at all + }); }); describe('what stays measurable when the market is absent', () => {