diff --git a/src/services/ledgerService.js b/src/services/ledgerService.js index 52beee0..1257aed 100644 --- a/src/services/ledgerService.js +++ b/src/services/ledgerService.js @@ -37,6 +37,16 @@ const { LEDGER_TAKEABLE_FLOOR, isLedgerTakeable: takeableFor } = require('../con const CONFLICT = 'user_id,player_key,stat,line,side,game_id'; const UPSERT_CHUNK = 200; const SETTLE_FETCH_LIMIT = 500; +/** + * Max ids in a single `.in('id', …)` filter. + * + * PostgREST puts filters in the URL, so an id list is bounded by URL length, not + * by row count: 500 UUIDs is ~18.5 KB and the fetch layer rejects it outright + * with `TypeError: fetch failed`. 100 keeps it near 3.7 KB, comfortably inside + * every proxy default. This is not a tuning knob — it is the guard for the + * defect that silently killed settlement for two days (see settleLedger). + */ +const ID_FILTER_CHUNK = 100; // Session 64 — bounded retry: a row that cannot be resolved after this many // date-targeted attempts becomes 'unrecoverable' rather than pending forever. const SETTLE_ATTEMPT_CAP = Number(process.env.SETTLE_ATTEMPT_CAP || 4); @@ -520,10 +530,23 @@ async function captureClosing(sport, oddsProps, opts = {}) { groups.get(gk).ids.push(row.id); } for (const g of groups.values()) { - const { error: upErr } = await sb.from('ledger_entries') - .update({ closing_line: g.line, closing_odds: g.odds }) - .in('id', g.ids); - if (!upErr) updated += g.ids.length; + // CHUNKED, for the same reason settleLedger no longer refetches by id: an + // `.in('id', …)` filter travels in the URL, and a few hundred UUIDs exceed + // what the fetch layer will send (500 ids ≈ 18.5 KB). That is the defect + // that silently killed settlement on 2026-08-01; this is the same shape, + // one prop-price group deep, and it fails the same way once a single + // line|odds pair collects enough rows on a big slate. + for (let i = 0; i < g.ids.length; i += ID_FILTER_CHUNK) { + const idChunk = g.ids.slice(i, i + ID_FILTER_CHUNK); + const { error: upErr } = await sb.from('ledger_entries') + .update({ closing_line: g.line, closing_odds: g.odds }) + .in('id', idChunk); + if (upErr) { + console.warn(`[ledger] closing capture chunk failed (${idChunk.length} rows): ${upErr.message}`); + continue; + } + updated += idChunk.length; + } } return { updated }; } @@ -614,20 +637,39 @@ async function settleLedger(sport, opts = {}) { const getSchedule = opts.getSchedule || getScheduleFn; const cutoff = opts.beforeDate || todayET(); // settle strictly-before today - const { data: open, error } = await sb.from('ledger_entries') - .select('id, player_key, player_name, stat, line, side, closing_line') + // ONE query, selecting everything the settle loop needs. + // + // THIS USED TO BE TWO QUERIES, and the second one silently killed the entire + // self-learning loop (found 2026-08-03). It refetched by primary key — + // `.in('id', open.map(r => r.id))` — which PostgREST sends as a GET query + // string: 500 UUIDs is an 18,499-character URL, and the fetch layer rejects it + // with `TypeError: fetch failed`. The result was destructured as + // `const { data: rows } = ...` with NO error binding, so `rows` came back null, + // the loop body never executed, and settleLedger returned + // `{settled:0, voided:0, unrecoverable:0, pending:0}` — BYTE-IDENTICAL to a + // clean "nothing to settle". + // + // It stayed invisible because it is volume-triggered: daily volume ran 20–260 + // rows and settled perfectly for weeks. 2026-08-01 was the first day over the + // 500-row fetch limit, and settlement died that night — 1,444 rows with + // `settle_attempts = 0`, never even attempted. Worse, the zero-settle ops alarm + // reads THESE return values, so `pending: 0` told the watchdog there was + // nothing pending and nobody was paged. + // + // The refetch existed only to add game_date/settle_attempts/dclv_computed_at. + // Selecting them up front removes the URL entirely — there is no id list to + // send at any volume. + const { data: rows, error } = await sb.from('ledger_entries') + .select('id, player_key, player_name, stat, line, side, closing_line, game_date, settle_attempts, dclv_computed_at') .eq('sport', sp) .is('outcome', null) .lt('game_date', cutoff) .order('game_date', { ascending: true }) .limit(SETTLE_FETCH_LIMIT); + // An errored fetch is NOT an empty backlog. Returning zeros here is what made + // a dead loop look like a healthy one for two days — surface it instead. if (error) return { settled: 0, pending: 0, error: error.message }; - if (!open || open.length === 0) return { settled: 0, pending: 0 }; - - // We need each row's game_date for the log match — refetch with it included. - const { data: rows } = await sb.from('ledger_entries') - .select('id, player_name, stat, line, side, closing_line, game_date, settle_attempts, dclv_computed_at, player_key') - .in('id', open.map((r) => r.id)); + if (!rows || rows.length === 0) return { settled: 0, pending: 0 }; // One game-log fetch per unique player. const players = [...new Set((rows || []).map((r) => r.player_name))]; diff --git a/tests/unit/clvServerGate.test.js b/tests/unit/clvServerGate.test.js index d7ffd46..becbc9e 100644 --- a/tests/unit/clvServerGate.test.js +++ b/tests/unit/clvServerGate.test.js @@ -78,6 +78,13 @@ describe('IMMUTABILITY — a shown badge never silently flips', () => { expect(svc).toMatch(/row\.dclv_computed_at\s*\n?\s*\?\s*null/); }); test('the settle read fetches dclv_computed_at so the guard can see it', () => { - expect(svc).toMatch(/dclv_computed_at, player_key/); + // Asserted on the SETTLE SELECT itself rather than on an adjacent column + // pair: the settle read is now a single query (the id-refetch that used to + // follow it overflowed the URL and silently returned null), so column order + // changed while the requirement did not. What matters is that the column the + // immutability guard reads is actually fetched. + const settleSelect = svc.match(/\.select\('id, player_key[^']*'\)/g) || []; + expect(settleSelect.length).toBeGreaterThan(0); + expect(settleSelect.some((sel) => sel.includes('dclv_computed_at'))).toBe(true); }); }); diff --git a/tests/unit/ledgerService.test.js b/tests/unit/ledgerService.test.js index 69448c3..3262feb 100644 --- a/tests/unit/ledgerService.test.js +++ b/tests/unit/ledgerService.test.js @@ -157,9 +157,9 @@ describe('computeClv — signed by side (Phase 1 amendment)', () => { describe('settleLedger — outcome + CLV vs the real result', () => { test('settles hit/miss/push from the game log and stamps CLV', async () => { const sb = fakeSb(); - // 1st select: open ids; 2nd: full rows. + // ONE select — the id-refetch that used to follow it is gone (it built an + // 18.5 KB URL and silently returned null; see settleLedger's header). sb._state.selectResults = [ - [{ id: 'r1' }, { id: 'r2' }], [ { id: 'r1', player_name: 'Aaron Judge', stat: 'home_runs', line: 0.5, side: 'over', closing_line: 0.5, game_date: '2026-07-09' }, { id: 'r2', player_name: 'Aaron Judge', stat: 'hits', line: 1.5, side: 'over', closing_line: 2.5, game_date: '2026-07-09' }, @@ -182,7 +182,6 @@ describe('settleLedger — outcome + CLV vs the real result', () => { test('no row + game FINAL → VOID (genuine DNP; dates verified correct)', async () => { const sb = fakeSb(); sb._state.selectResults = [ - [{ id: 'r1' }], [{ id: 'r1', player_name: 'Aaron Judge', stat: 'home_runs', line: 0.5, side: 'over', closing_line: null, game_date: '2026-07-09' }], ]; const getPlayerStats = async () => ({ found: true, last10: [{ date: '2026-07-08', stat: { homeRuns: 2 } }] }); @@ -198,7 +197,6 @@ describe('settleLedger — outcome + CLV vs the real result', () => { test('no row + game NOT FINAL → stays pending, never voided', async () => { const sb = fakeSb(); sb._state.selectResults = [ - [{ id: 'r1' }], [{ id: 'r1', player_name: 'Aaron Judge', stat: 'home_runs', line: 0.5, side: 'over', closing_line: null, game_date: '2026-07-09' }], ]; const getPlayerStats = async () => ({ found: true, last10: [{ date: '2026-07-08', stat: { homeRuns: 2 } }] }); @@ -216,7 +214,6 @@ describe('settleLedger — outcome + CLV vs the real result', () => { test('undetermined state becomes UNRECOVERABLE at the retry cap', async () => { const sb = fakeSb(); sb._state.selectResults = [ - [{ id: 'r1' }], [{ id: 'r1', player_name: 'Aaron Judge', stat: 'home_runs', line: 0.5, side: 'over', closing_line: null, game_date: '2026-07-09', settle_attempts: 3 }], ]; const getPlayerStats = async () => ({ found: true, last10: [] }); @@ -435,3 +432,113 @@ describe('Session 64 Order 2 — quarantine vs analysis_flags (two scopes)', () expect(agg).toMatch(/quarantine_reason/); }); }); + +/** + * THE SILENT-SETTLEMENT REGRESSION (2026-08-03). + * + * settleLedger used to fetch open ids, then REFETCH the full rows by + * `.in('id', ids)`. PostgREST puts filters in the URL, so 500 UUIDs became an + * 18,499-character request that the fetch layer rejected outright. The result + * was destructured as `const { data: rows } = ...` with no error binding, so + * `rows` was null, the loop never ran, and the function returned + * `{settled:0, voided:0, unrecoverable:0, pending:0}` — byte-identical to a + * healthy "nothing to settle". + * + * It was invisible for two days because it is VOLUME-triggered: daily volume ran + * 20–260 rows and settled perfectly for weeks; 2026-08-01 was the first day past + * the 500-row fetch limit and 1,444 rows were left with `settle_attempts = 0`, + * never attempted. The zero-settle ops alarm reads these same return values, so + * `pending: 0` told the watchdog the backlog was empty and nobody was paged. + * + * These tests assert the two properties that would have caught it: settlement + * issues NO id-list filter at any volume, and a failed fetch is never reported + * as an empty backlog. + */ +describe('settleLedger — the silent-settlement regression stays fixed', () => { + function bigSb(n) { + const calls = { selects: [], updates: [], inFilters: [] }; + const rows = Array.from({ length: n }, (_, i) => ({ + id: `00000000-0000-4000-8000-${String(i).padStart(12, '0')}`, + player_name: 'Aaron Judge', stat: 'home_runs', line: 0.5, side: 'over', + closing_line: 0.5, game_date: '2026-07-09', + })); + let served = false; + function builder() { + const b = { + _update: null, + update(v) { b._update = v; return b; }, + select() { return b; }, eq() { return b; }, is() { return b; }, + not() { return b; }, lt() { return b; }, gte() { return b; }, gt() { return b; }, + in(col, ids) { + calls.inFilters.push({ col, count: ids.length }); + if (b._update) { calls.updates.push({ values: b._update, ids }); return Promise.resolve({ error: null }); } + return terminal(); + }, + order() { return b; }, + limit() { return terminal(); }, + then(res, rej) { return terminal().then(res, rej); }, + }; + function terminal() { + if (b._update) { calls.updates.push({ values: b._update }); return Promise.resolve({ error: null }); } + if (served) return Promise.resolve({ data: [], error: null }); + served = true; + return Promise.resolve({ data: rows, error: null }); + } + return b; + } + return { from: () => builder(), _calls: calls }; + } + + test('settles a 500-row backlog WITHOUT ever sending an id-list filter', async () => { + const sb = bigSb(500); + const getPlayerStats = async () => ({ + found: true, last10: [{ date: '2026-07-09', stat: { homeRuns: 1 } }], + }); + const res = await ledger.settleLedger('mlb', { + sb, getPlayerStats, now: () => NOW, beforeDate: '2026-07-10', + }); + expect(res.settled).toBe(500); + // The whole defect in one assertion: no `.in('id', …)` at read time means no + // URL to overflow, at any volume. + expect(sb._calls.inFilters.filter((f) => f.col === 'id')).toHaveLength(0); + }); + + test('a FAILED fetch is surfaced, never reported as an empty backlog', async () => { + // The exact production failure: the query rejects, so `data` is null. + const sb = { from: () => ({ + select() { return this; }, eq() { return this; }, is() { return this; }, + lt() { return this; }, order() { return this; }, + limit() { return Promise.resolve({ data: null, error: { message: 'TypeError: fetch failed' } }); }, + }) }; + const res = await ledger.settleLedger('mlb', { + sb, getPlayerStats: async () => ({ found: false }), now: () => NOW, beforeDate: '2026-07-10', + }); + // A dead loop must not be indistinguishable from a healthy one. + expect(res.error).toBe('TypeError: fetch failed'); + }); +}); + +describe('captureClosing — id filters are chunked (same defect family)', () => { + test('a 250-row same-price group updates in bounded chunks, all rows counted', async () => { + const calls = { updates: [] }; + const rows = Array.from({ length: 250 }, (_, i) => ({ + id: `id-${i}`, player_key: 'aaron judge', stat: 'home_runs', side: 'over', + })); + let served = false; + const sb = { from: () => { + const b = { + _update: null, + update(v) { b._update = v; return b; }, + select() { return b; }, eq() { return b; }, is() { return b; }, + in(col, ids) { calls.updates.push({ values: b._update, ids }); return Promise.resolve({ error: null }); }, + limit() { if (served) return Promise.resolve({ data: [], error: null }); served = true; return Promise.resolve({ data: rows, error: null }); }, + }; + return b; + } }; + const res = await ledger.captureClosing('mlb', [PROP], { sb, gameDate: '2026-07-10' }); + expect(res.updated).toBe(250); + // Every chunk stays small enough that the id list cannot overflow the URL. + expect(calls.updates.length).toBeGreaterThan(1); + for (const u of calls.updates) expect(u.ids.length).toBeLessThanOrEqual(100); + }); +}); diff --git a/tests/unit/nbaSettlement.test.js b/tests/unit/nbaSettlement.test.js index cbd8966..b2f0012 100644 --- a/tests/unit/nbaSettlement.test.js +++ b/tests/unit/nbaSettlement.test.js @@ -175,8 +175,9 @@ describe('ledgerService.settleLedger — WNBA settles vs the ESPN game log', () test('settles a WNBA row (points) via the ESPN gamelog ET-date match', async () => { const sb = fakeSb(); + // ONE select — settleLedger no longer refetches rows by id (that filter + // built an 18.5 KB URL and silently returned null; see ledgerService). sb._state.selectResults = [ - [{ id: 'r1' }], [{ id: 'r1', player_name: "A'ja Wilson", stat: 'points', line: 19.5, side: 'over', closing_line: 19.5, game_date: '2026-07-12' }], ]; // ESPN ISO date normalizes to ET 2026-07-12 → matches game_date.