vyndr/tests/unit/safePaginate.test.js

'use strict';

/**
 * safePaginate — spec: specs/read-integrity-harness.md §7.
 * Fake query builders throughout; nothing here touches a network.
 */

const { paginate } = require('../../src/utils/safePaginate');

/**
 * A fake PostgREST table. `rows` may be mutated between pages to simulate
 * concurrent writes. Honours order+range the way Postgres does WITH an ORDER BY.
 */
function fakeTable(rows, opts = {}) {
  const state = { rows, orderCalls: 0, rangeCalls: 0 };
  state.make = () => {
    let orderKey = null; let asc = true;
    const q = {
      order(k, o) { orderKey = k; asc = !o || o.ascending !== false; state.orderCalls += 1; return q; },
      async range(from, to) {
        state.rangeCalls += 1;
        if (opts.errorOnRange && opts.errorOnRange(from)) return { data: null, error: { message: 'connection reset' } };
        if (opts.beforeRange) opts.beforeRange(from, state);
        // Unordered simulation: return physical order (the production defect).
        let out = [...state.rows];
        if (orderKey) {
          out.sort((a, b) => (a[orderKey] < b[orderKey] ? -1 : a[orderKey] > b[orderKey] ? 1 : 0));
          if (!asc) out.reverse();
        }
        return { data: out.slice(from, to + 1), error: null };
      },
    };
    return q;
  };
  return state;
}

const row = (id) => ({ id, v: `r${id}` });
const ids = (a, b) => Array.from({ length: b - a + 1 }, (_, i) => row(a + i));

describe('safePaginate — (a) returns the full set, in order', () => {
  it('walks every page and returns each row exactly once, ascending by key', async () => {
    const t = fakeTable(ids(1, 2490));
    const out = await paginate(t.make, { pageSize: 1000 });
    expect(out).toHaveLength(2490);
    expect(new Set(out.map((r) => r.id)).size).toBe(2490);
    expect(out.map((r) => r.id)).toEqual(ids(1, 2490).map((r) => r.id));   // in order
    expect(t.rangeCalls).toBe(3);
  });

  it('always applies the order — every page is ordered, not just the first', async () => {
    const t = fakeTable(ids(1, 2490));
    await paginate(t.make, { pageSize: 1000 });
    expect(t.orderCalls).toBe(t.rangeCalls);
  });

  it('orders by the configured key, and can descend', async () => {
    const t = fakeTable(ids(1, 5));
    const out = await paginate(t.make, { pageSize: 10, ascending: false });
    expect(out.map((r) => r.id)).toEqual([5, 4, 3, 2, 1]);
  });

  it('returns [] for an empty table without throwing', async () => {
    const out = await paginate(fakeTable([]).make, { pageSize: 10 });
    expect(out).toEqual([]);
  });
});

describe('safePaginate — (b) a mid-walk error THROWS, never a silent end', () => {
  it('throws when page 2 fails instead of returning page 1 as the whole set', async () => {
    // THE :117 DEFECT: `if (error || !data) break` would return 1,000 rows here
    // and the caller would fit a map on them believing it had 2,490.
    const t = fakeTable(ids(1, 2490), { errorOnRange: (from) => from === 1000 });
    await expect(paginate(t.make, { pageSize: 1000, label: 'cal' }))
      .rejects.toThrow(/read failed at range 1000-1999 — connection reset/);
  });

  it('tags the error so a caller can tell a read failure from thin history', async () => {
    const t = fakeTable(ids(1, 10), { errorOnRange: (from) => from === 0 });
    await expect(paginate(t.make, { pageSize: 5 })).rejects.toMatchObject({ code: 'READ_FAILED' });
  });

  it('names itself in the message', async () => {
    const t = fakeTable(ids(1, 10), { errorOnRange: () => true });
    await expect(paginate(t.make, { pageSize: 5, label: 'fromLedger(hits)' }))
      .rejects.toThrow(/^fromLedger\(hits\):/);
  });

  it('refuses a runaway walk rather than reporting a truncated read as complete', async () => {
    // Fresh unique ids every page, so the runaway guard is what fires — not the
    // duplicate guard.
    let n = 0;
    const t = {
      make: () => ({
        order() { return this; },
        async range() { const page = ids(n * 10 + 1, n * 10 + 10); n += 1; return { data: page, error: null }; },
      }),
    };
    await expect(paginate(t.make, { pageSize: 10, maxPages: 3 })).rejects.toThrow(/maxPages/);
  });
});

describe('safePaginate — (c) set-identity vs an ordered full-table control', () => {
  const control = (rows) => [...rows].sort((a, b) => a.id - b.id).map((r) => r.id);

  it('is set-identical to the control on a static table', async () => {
    const rows = ids(1, 4321);
    const out = await paginate(fakeTable(rows).make, { pageSize: 1000 });
    expect(out.map((r) => r.id)).toEqual(control(rows));
  });

  it('is set-identical when the physical order is shuffled under it', async () => {
    // The planner returning rows in an arbitrary physical order is exactly the
    // production condition. A stable ORDER BY makes it irrelevant.
    const rows = ids(1, 3000);
    const t = fakeTable([...rows].reverse());
    const out = await paginate(t.make, { pageSize: 1000 });
    expect(out.map((r) => r.id)).toEqual(control(rows));
  });

  it('THROWS on a duplicate key rather than returning a corrupted set', async () => {
    // A non-unique ordering key: ties may come back in any order, so pages
    // overlap. An .order() clause on such a column is not a fix.
    const dup = [{ id: 1 }, { id: 2 }, { id: 2 }, { id: 3 }];
    await expect(paginate(fakeTable(dup).make, { pageSize: 2 }))
      .rejects.toThrow(/duplicate key \(id\)=2/);
  });
});

describe('safePaginate — concurrent append (the regression this fix exists for)', () => {
  it('returns every pre-existing row exactly once while rows are APPENDED mid-walk', async () => {
    // The snapshot cron appends to ledger_entries at 14/19/22/1/3 UTC. With a
    // stable ascending key, appended rows land after the cursor: nothing already
    // returned can be returned again, and nothing pending can be skipped.
    const original = ids(1, 2500);
    let next = 10000;
    const t = fakeTable([...original], {
      beforeRange: (from, state) => {
        if (from > 0) state.rows.push(row(next++));       // a write lands mid-walk
      },
    });
    const out = await paginate(t.make, { pageSize: 1000 });
    const seen = out.map((r) => r.id);

    expect(new Set(seen).size).toBe(seen.length);          // no duplicates
    for (const r of original) expect(seen).toContain(r.id); // nothing dropped
  });

  it('the OLD unordered walk fails that same simulation — the test has teeth', async () => {
    // Same table, paged over PHYSICAL order with no ORDER BY. An UPDATE in
    // Postgres writes a new tuple version at the end of the heap, so a row that
    // was already returned moves AFTER the cursor (returned twice) and every row
    // behind it shifts back by one (one falls across the page boundary unseen).
    // This is the production shape, and it is what the fix removes.
    const original = ids(1, 2500);
    const rows = [...original];
    const unorderedWalk = async () => {
      const out = [];
      for (let from = 0; ; from += 1000) {
        if (from > 0) rows.push(rows.splice(0, 1)[0]);   // a row is re-written mid-walk
        const page = rows.slice(from, from + 1000);
        if (page.length === 0) break;
        out.push(...page);
        if (page.length < 1000) break;
      }
      return out;
    };
    const seen = (await unorderedWalk()).map((r) => r.id);
    expect(new Set(seen).size).toBeLessThan(seen.length);  // duplicates appear
    const missing = original.filter((r) => !seen.includes(r.id));
    expect(missing.length).toBeGreaterThan(0);             // and rows are lost
  });
});

// ── FIX A2b — COMPOSITE KEYS ────────────────────────────────────────────────
const { normalizeKey } = require('../../src/utils/safePaginate');
const { uniqueKeyFor } = require('../../src/utils/tableKeys');

/** A fake table keyed by a composite tuple, ordered the way Postgres would. */
function fakeComposite(rows, opts = {}) {
  const state = { rows, orderCols: [], rangeCalls: 0 };
  state.make = () => {
    const cols = [];
    const q = {
      order(c) { cols.push(c); return q; },
      async range(from, to) {
        state.rangeCalls += 1;
        state.orderCols = [...cols];
        if (opts.errorOnRange && opts.errorOnRange(from)) return { data: null, error: { message: 'reset' } };
        const out = [...state.rows].sort((a, b) => {
          for (const c of cols) {
            if (a[c] < b[c]) return -1;
            if (a[c] > b[c]) return 1;
          }
          return 0;
        });
        return { data: out.slice(from, to + 1), error: null };
      },
    };
    return q;
  };
  return state;
}

const KEY = ['as_of_date', 'sport', 'season', 'player_key'];
const composite = (n) => Array.from({ length: n }, (_, i) => ({
  as_of_date: `2026-08-${String((i % 9) + 1).padStart(2, '0')}`,
  sport: 'mlb', season: 2026, player_key: `p${i}`, v: i,
}));

describe('safePaginate — composite keys (A2b)', () => {
  it('normalizeKey treats a string and a one-column list identically', () => {
    expect(normalizeKey('id')).toEqual(['id']);
    expect(normalizeKey(['id'])).toEqual(['id']);
    expect(normalizeKey(undefined)).toEqual(['id']);
    expect(normalizeKey(KEY)).toEqual(KEY);
  });

  it('rejects an empty key rather than silently defaulting to id', () => {
    expect(() => normalizeKey([])).toThrow(/must be a column name/);
    expect(() => normalizeKey([null])).toThrow(/must be a column name/);
  });

  it('returns the full set exactly once, ordered by the whole tuple', async () => {
    const t = fakeComposite(composite(2500));
    const rows = await paginate(t.make, { key: KEY, pageSize: 1000 });
    expect(rows).toHaveLength(2500);
    expect(new Set(rows.map((r) => r.player_key)).size).toBe(2500);
    // sorted by as_of_date first
    for (let i = 1; i < rows.length; i += 1) {
      expect(rows[i].as_of_date >= rows[i - 1].as_of_date).toBe(true);
    }
  });

  it('orders EVERY column of the key, in the declared order', async () => {
    const t = fakeComposite(composite(10));
    await paginate(t.make, { key: KEY, pageSize: 100 });
    expect(t.orderCols).toEqual(KEY);
  });

  it('a mid-walk error still THROWS with a composite key', async () => {
    const t = fakeComposite(composite(2500), { errorOnRange: (f) => f === 1000 });
    await expect(paginate(t.make, { key: KEY, pageSize: 1000, label: 'ctx' }))
      .rejects.toThrow(/ctx: read failed at range 1000-1999/);
  });

  it('THROWS on a duplicate TUPLE, naming every column', async () => {
    const dup = [
      { as_of_date: '2026-08-01', sport: 'mlb', season: 2026, player_key: 'a' },
      { as_of_date: '2026-08-01', sport: 'mlb', season: 2026, player_key: 'a' },
    ];
    await expect(paginate(fakeComposite(dup).make, { key: KEY, pageSize: 10 }))
      .rejects.toThrow(/duplicate key \(as_of_date,sport,season,player_key\)/);
  });

  it('does NOT throw when only a PREFIX of the tuple repeats', async () => {
    // Same day, same sport, different player — a legitimate row, not a duplicate.
    const rows = [
      { as_of_date: '2026-08-01', sport: 'mlb', season: 2026, player_key: 'a' },
      { as_of_date: '2026-08-01', sport: 'mlb', season: 2026, player_key: 'b' },
    ];
    await expect(paginate(fakeComposite(rows).make, { key: KEY, pageSize: 10 }))
      .resolves.toHaveLength(2);
  });

  it('the single-key path is unchanged — backward compatible', async () => {
    const t = fakeTable(ids(1, 2490));
    const out = await paginate(t.make, { pageSize: 1000 });          // no key => 'id'
    expect(out).toHaveLength(2490);
    expect(new Set(out.map((r) => r.id)).size).toBe(2490);
  });
});

describe('tableKeys — the constraints come from the schema, not a guess', () => {
  it('gives the real composite key for each context table', () => {
    expect(uniqueKeyFor('batter_spray')).toEqual(['as_of_date', 'sport', 'season', 'source_id']);
    expect(uniqueKeyFor('team_defense')).toEqual(['as_of_date', 'sport', 'season', 'team']);
    expect(uniqueKeyFor('statcast_aggregates')).toEqual(['sport', 'season', 'source_id', 'role']);
    expect(uniqueKeyFor('platoon_splits')).toEqual(['as_of_date', 'sport', 'season', 'player_key']);
    expect(uniqueKeyFor('park_dimensions')).toEqual(['as_of_date', 'sport', 'venue_id']);
    expect(uniqueKeyFor('hitter_opportunity')).toEqual(['as_of_date', 'sport', 'season', 'player_key']);
    expect(uniqueKeyFor('lineup_context')).toEqual(['as_of_date', 'sport', 'game_pk', 'player_key']);
  });

  it('keeps the single-column keys single', () => {
    expect(uniqueKeyFor('ledger_entries')).toEqual(['id']);
    expect(uniqueKeyFor('model_snapshots')).toEqual(['id']);
    expect(uniqueKeyFor('game_context')).toEqual(['game_id']);
  });

  it('THROWS for an unknown table rather than defaulting to id', () => {
    expect(() => uniqueKeyFor('some_new_table')).toThrow(/no unique key recorded/);
  });
});