'use strict'; /** * safePaginate — spec: specs/read-integrity-harness.md §7. * Fake query builders throughout; nothing here touches a network. */ const { paginate } = require('../../src/utils/safePaginate'); /** * A fake PostgREST table. `rows` may be mutated between pages to simulate * concurrent writes. Honours order+range the way Postgres does WITH an ORDER BY. */ function fakeTable(rows, opts = {}) { const state = { rows, orderCalls: 0, rangeCalls: 0 }; state.make = () => { let orderKey = null; let asc = true; const q = { order(k, o) { orderKey = k; asc = !o || o.ascending !== false; state.orderCalls += 1; return q; }, async range(from, to) { state.rangeCalls += 1; if (opts.errorOnRange && opts.errorOnRange(from)) return { data: null, error: { message: 'connection reset' } }; if (opts.beforeRange) opts.beforeRange(from, state); // Unordered simulation: return physical order (the production defect). let out = [...state.rows]; if (orderKey) { out.sort((a, b) => (a[orderKey] < b[orderKey] ? -1 : a[orderKey] > b[orderKey] ? 1 : 0)); if (!asc) out.reverse(); } return { data: out.slice(from, to + 1), error: null }; }, }; return q; }; return state; } const row = (id) => ({ id, v: `r${id}` }); const ids = (a, b) => Array.from({ length: b - a + 1 }, (_, i) => row(a + i)); describe('safePaginate — (a) returns the full set, in order', () => { it('walks every page and returns each row exactly once, ascending by key', async () => { const t = fakeTable(ids(1, 2490)); const out = await paginate(t.make, { pageSize: 1000 }); expect(out).toHaveLength(2490); expect(new Set(out.map((r) => r.id)).size).toBe(2490); expect(out.map((r) => r.id)).toEqual(ids(1, 2490).map((r) => r.id)); // in order expect(t.rangeCalls).toBe(3); }); it('always applies the order — every page is ordered, not just the first', async () => { const t = fakeTable(ids(1, 2490)); await paginate(t.make, { pageSize: 1000 }); expect(t.orderCalls).toBe(t.rangeCalls); }); it('orders by the configured key, and can descend', async () => { const t = fakeTable(ids(1, 5)); const out = await paginate(t.make, { pageSize: 10, ascending: false }); expect(out.map((r) => r.id)).toEqual([5, 4, 3, 2, 1]); }); it('returns [] for an empty table without throwing', async () => { const out = await paginate(fakeTable([]).make, { pageSize: 10 }); expect(out).toEqual([]); }); }); describe('safePaginate — (b) a mid-walk error THROWS, never a silent end', () => { it('throws when page 2 fails instead of returning page 1 as the whole set', async () => { // THE :117 DEFECT: `if (error || !data) break` would return 1,000 rows here // and the caller would fit a map on them believing it had 2,490. const t = fakeTable(ids(1, 2490), { errorOnRange: (from) => from === 1000 }); await expect(paginate(t.make, { pageSize: 1000, label: 'cal' })) .rejects.toThrow(/read failed at range 1000-1999 — connection reset/); }); it('tags the error so a caller can tell a read failure from thin history', async () => { const t = fakeTable(ids(1, 10), { errorOnRange: (from) => from === 0 }); await expect(paginate(t.make, { pageSize: 5 })).rejects.toMatchObject({ code: 'READ_FAILED' }); }); it('names itself in the message', async () => { const t = fakeTable(ids(1, 10), { errorOnRange: () => true }); await expect(paginate(t.make, { pageSize: 5, label: 'fromLedger(hits)' })) .rejects.toThrow(/^fromLedger\(hits\):/); }); it('refuses a runaway walk rather than reporting a truncated read as complete', async () => { // Fresh unique ids every page, so the runaway guard is what fires — not the // duplicate guard. let n = 0; const t = { make: () => ({ order() { return this; }, async range() { const page = ids(n * 10 + 1, n * 10 + 10); n += 1; return { data: page, error: null }; }, }), }; await expect(paginate(t.make, { pageSize: 10, maxPages: 3 })).rejects.toThrow(/maxPages/); }); }); describe('safePaginate — (c) set-identity vs an ordered full-table control', () => { const control = (rows) => [...rows].sort((a, b) => a.id - b.id).map((r) => r.id); it('is set-identical to the control on a static table', async () => { const rows = ids(1, 4321); const out = await paginate(fakeTable(rows).make, { pageSize: 1000 }); expect(out.map((r) => r.id)).toEqual(control(rows)); }); it('is set-identical when the physical order is shuffled under it', async () => { // The planner returning rows in an arbitrary physical order is exactly the // production condition. A stable ORDER BY makes it irrelevant. const rows = ids(1, 3000); const t = fakeTable([...rows].reverse()); const out = await paginate(t.make, { pageSize: 1000 }); expect(out.map((r) => r.id)).toEqual(control(rows)); }); it('THROWS on a duplicate key rather than returning a corrupted set', async () => { // A non-unique ordering key: ties may come back in any order, so pages // overlap. An .order() clause on such a column is not a fix. const dup = [{ id: 1 }, { id: 2 }, { id: 2 }, { id: 3 }]; await expect(paginate(fakeTable(dup).make, { pageSize: 2 })) .rejects.toThrow(/duplicate key \(id\)=2/); }); }); describe('safePaginate — concurrent append (the regression this fix exists for)', () => { it('returns every pre-existing row exactly once while rows are APPENDED mid-walk', async () => { // The snapshot cron appends to ledger_entries at 14/19/22/1/3 UTC. With a // stable ascending key, appended rows land after the cursor: nothing already // returned can be returned again, and nothing pending can be skipped. const original = ids(1, 2500); let next = 10000; const t = fakeTable([...original], { beforeRange: (from, state) => { if (from > 0) state.rows.push(row(next++)); // a write lands mid-walk }, }); const out = await paginate(t.make, { pageSize: 1000 }); const seen = out.map((r) => r.id); expect(new Set(seen).size).toBe(seen.length); // no duplicates for (const r of original) expect(seen).toContain(r.id); // nothing dropped }); it('the OLD unordered walk fails that same simulation — the test has teeth', async () => { // Same table, paged over PHYSICAL order with no ORDER BY. An UPDATE in // Postgres writes a new tuple version at the end of the heap, so a row that // was already returned moves AFTER the cursor (returned twice) and every row // behind it shifts back by one (one falls across the page boundary unseen). // This is the production shape, and it is what the fix removes. const original = ids(1, 2500); const rows = [...original]; const unorderedWalk = async () => { const out = []; for (let from = 0; ; from += 1000) { if (from > 0) rows.push(rows.splice(0, 1)[0]); // a row is re-written mid-walk const page = rows.slice(from, from + 1000); if (page.length === 0) break; out.push(...page); if (page.length < 1000) break; } return out; }; const seen = (await unorderedWalk()).map((r) => r.id); expect(new Set(seen).size).toBeLessThan(seen.length); // duplicates appear const missing = original.filter((r) => !seen.includes(r.id)); expect(missing.length).toBeGreaterThan(0); // and rows are lost }); }); // ── FIX A2b — COMPOSITE KEYS ──────────────────────────────────────────────── const { normalizeKey } = require('../../src/utils/safePaginate'); const { uniqueKeyFor } = require('../../src/utils/tableKeys'); /** A fake table keyed by a composite tuple, ordered the way Postgres would. */ function fakeComposite(rows, opts = {}) { const state = { rows, orderCols: [], rangeCalls: 0 }; state.make = () => { const cols = []; const q = { order(c) { cols.push(c); return q; }, async range(from, to) { state.rangeCalls += 1; state.orderCols = [...cols]; if (opts.errorOnRange && opts.errorOnRange(from)) return { data: null, error: { message: 'reset' } }; const out = [...state.rows].sort((a, b) => { for (const c of cols) { if (a[c] < b[c]) return -1; if (a[c] > b[c]) return 1; } return 0; }); return { data: out.slice(from, to + 1), error: null }; }, }; return q; }; return state; } const KEY = ['as_of_date', 'sport', 'season', 'player_key']; const composite = (n) => Array.from({ length: n }, (_, i) => ({ as_of_date: `2026-08-${String((i % 9) + 1).padStart(2, '0')}`, sport: 'mlb', season: 2026, player_key: `p${i}`, v: i, })); describe('safePaginate — composite keys (A2b)', () => { it('normalizeKey treats a string and a one-column list identically', () => { expect(normalizeKey('id')).toEqual(['id']); expect(normalizeKey(['id'])).toEqual(['id']); expect(normalizeKey(undefined)).toEqual(['id']); expect(normalizeKey(KEY)).toEqual(KEY); }); it('rejects an empty key rather than silently defaulting to id', () => { expect(() => normalizeKey([])).toThrow(/must be a column name/); expect(() => normalizeKey([null])).toThrow(/must be a column name/); }); it('returns the full set exactly once, ordered by the whole tuple', async () => { const t = fakeComposite(composite(2500)); const rows = await paginate(t.make, { key: KEY, pageSize: 1000 }); expect(rows).toHaveLength(2500); expect(new Set(rows.map((r) => r.player_key)).size).toBe(2500); // sorted by as_of_date first for (let i = 1; i < rows.length; i += 1) { expect(rows[i].as_of_date >= rows[i - 1].as_of_date).toBe(true); } }); it('orders EVERY column of the key, in the declared order', async () => { const t = fakeComposite(composite(10)); await paginate(t.make, { key: KEY, pageSize: 100 }); expect(t.orderCols).toEqual(KEY); }); it('a mid-walk error still THROWS with a composite key', async () => { const t = fakeComposite(composite(2500), { errorOnRange: (f) => f === 1000 }); await expect(paginate(t.make, { key: KEY, pageSize: 1000, label: 'ctx' })) .rejects.toThrow(/ctx: read failed at range 1000-1999/); }); it('THROWS on a duplicate TUPLE, naming every column', async () => { const dup = [ { as_of_date: '2026-08-01', sport: 'mlb', season: 2026, player_key: 'a' }, { as_of_date: '2026-08-01', sport: 'mlb', season: 2026, player_key: 'a' }, ]; await expect(paginate(fakeComposite(dup).make, { key: KEY, pageSize: 10 })) .rejects.toThrow(/duplicate key \(as_of_date,sport,season,player_key\)/); }); it('does NOT throw when only a PREFIX of the tuple repeats', async () => { // Same day, same sport, different player — a legitimate row, not a duplicate. const rows = [ { as_of_date: '2026-08-01', sport: 'mlb', season: 2026, player_key: 'a' }, { as_of_date: '2026-08-01', sport: 'mlb', season: 2026, player_key: 'b' }, ]; await expect(paginate(fakeComposite(rows).make, { key: KEY, pageSize: 10 })) .resolves.toHaveLength(2); }); it('the single-key path is unchanged — backward compatible', async () => { const t = fakeTable(ids(1, 2490)); const out = await paginate(t.make, { pageSize: 1000 }); // no key => 'id' expect(out).toHaveLength(2490); expect(new Set(out.map((r) => r.id)).size).toBe(2490); }); }); describe('tableKeys — the constraints come from the schema, not a guess', () => { it('gives the real composite key for each context table', () => { expect(uniqueKeyFor('batter_spray')).toEqual(['as_of_date', 'sport', 'season', 'source_id']); expect(uniqueKeyFor('team_defense')).toEqual(['as_of_date', 'sport', 'season', 'team']); expect(uniqueKeyFor('statcast_aggregates')).toEqual(['sport', 'season', 'source_id', 'role']); expect(uniqueKeyFor('platoon_splits')).toEqual(['as_of_date', 'sport', 'season', 'player_key']); expect(uniqueKeyFor('park_dimensions')).toEqual(['as_of_date', 'sport', 'venue_id']); expect(uniqueKeyFor('hitter_opportunity')).toEqual(['as_of_date', 'sport', 'season', 'player_key']); expect(uniqueKeyFor('lineup_context')).toEqual(['as_of_date', 'sport', 'game_pk', 'player_key']); }); it('keeps the single-column keys single', () => { expect(uniqueKeyFor('ledger_entries')).toEqual(['id']); expect(uniqueKeyFor('model_snapshots')).toEqual(['id']); expect(uniqueKeyFor('game_context')).toEqual(['game_id']); }); it('THROWS for an unknown table rather than defaulting to id', () => { expect(() => uniqueKeyFor('some_new_table')).toThrow(/no unique key recorded/); }); });