vyndr/tests/unit/lineageFamilyLookup.test.js

// LINEAGE FAMILY LOOKUP — validity, scale and failure atomicity.
//
// The first canary began from an EMPTY graph and passed. The second failed
// because history existed: the family lookup sent 100 natural keys as a
// PostgREST IN-list, the planner had no statistics for `read_natural_key`
// (its last autoanalyze predated the column ever being populated), estimated
// 172,409 rows and chose a sequential scan of 344,818 -- 8.5s, then 57014.
//
// The repair is not a smaller chunk. It is a lookup whose cost is bounded by
// ONE SLATE, plus a predicate that says what a completed lineage action IS.

const ret = require('../../src/services/retentionService');
const R = require('../../src/services/read/readLineage');
const { isValidLineageAction, VALID_LINEAGE_ACTION_FIELDS, familyScopesFrom } = ret;

const CAND = {
  sport: 'mlb', game_date: '2026-08-28', player_key: 'aaron judge', stat: 'hits',
  side: 'over', line: 0.5, canonical_event_id: 'mlb:gamepk:1',
  captured_at: '2026-08-28T23:00:00.000Z', published: true, p_win: 0.9, grade: 'B+',
};
const KEY = R.readNaturalKey(CAND);

const validRow = (over = {}) => ({
  id: 1, read_id: '11111111-1111-4111-8111-111111111111', read_natural_key: KEY,
  game_id: 'g', claim_digest: 'OLDDIGEST', revision_ordinal: 0, lineage_action: 'ORIGIN',
  supersedes_id: null, captured_at: '2026-08-28T19:00:00.000Z', lineage_state: 'LIVE',
  lineage_version: 'lin@1', claim_schema_version: 'claim@1',
  digest_algorithm_version: 'sha256-json-sorted@1',
  // Claim fields sit at TOP LEVEL, exactly as the lookup projection returns
  // them: `attachLineage` rebuilds the nested `claim` from these columns, so a
  // fixture that only nests them is testing a shape the database never emits.
  canonical_event_id: 'mlb:gamepk:1', line: 0.5, side: 'over', p_win: 0.5, grade: 'C',
  ...over,
});
// The EXACT shape the 2026-08-28 failure wrote 1,119 times: a natural key and
// publication stamps, every lineage action field NULL.
const partialRow = (over = {}) => ({
  id: 2, read_id: null, read_natural_key: KEY, game_id: 'g', claim_digest: null,
  revision_ordinal: null, lineage_action: null, supersedes_id: null,
  captured_at: '2026-08-28T23:00:00.000Z', lineage_state: null, lineage_version: null,
  claim_schema_version: null, digest_algorithm_version: null,
  publication_id: 'pub-1', published_at: '2026-08-28T19:03:29.579Z',
  canonical_event_id: 'mlb:gamepk:1', line: 0.5, side: 'over',
  ...over,
});

describe('STEP 2 — semantic result and persisted action are different layers', () => {
  test('the repository maps NEW/CHANGED/UNCHANGED onto ORIGIN/REVISION/RECAPTURE', () => {
    const noHistory = R.resolveLineage({ candidate: CAND, existing: [], mintReadId: () => 'M' });
    expect(noHistory.change_type).toBe(R.CHANGE_TYPE.INITIAL_PUBLICATION);
    expect(noHistory.action).toBe(R.LINEAGE_ACTION.ORIGIN);

    const head = validRow();
    const changed = R.resolveLineage({ candidate: CAND, existing: [head], mintReadId: () => 'M' });
    expect(changed.action).toBe(R.LINEAGE_ACTION.REVISION);
    expect(changed.change_type).not.toBe(R.CHANGE_TYPE.NO_MATERIAL_PUBLISHED_CHANGE);

    const same = R.resolveLineage({
      candidate: CAND,
      existing: [validRow({ claim_digest: R.claimDigest(CAND) })],
      mintReadId: () => 'M',
    });
    expect(same.action).toBe(R.LINEAGE_ACTION.RECAPTURE);
    expect(same.change_type).toBe(R.CHANGE_TYPE.NO_MATERIAL_PUBLISHED_CHANGE);
  });

  test('NO_MATERIAL_CHANGE and RECAPTURE describe ONE claim, not two', () => {
    // The semantic result and the persisted action are two views of the same
    // row. Counting them as separate populations would double-report a slate.
    const r = R.resolveLineage({
      candidate: CAND,
      existing: [validRow({ claim_digest: R.claimDigest(CAND) })],
      mintReadId: () => 'M',
    });
    expect(r.action).toBe('RECAPTURE');
    expect(r.change_type).toBe('NO_MATERIAL_PUBLISHED_CHANGE');
    // One resolution object. One claim.
    expect(Object.keys(r).filter((k) => k === 'action' || k === 'change_type')).toHaveLength(2);
  });
});

describe('STEP 5 — VALID_LINEAGE_ACTION_PREDICATE', () => {
  test('it states what a completed action IS, field by field', () => {
    expect(VALID_LINEAGE_ACTION_FIELDS).toEqual(expect.arrayContaining([
      'read_id', 'read_natural_key', 'lineage_action', 'claim_digest',
      'revision_ordinal', 'lineage_state', 'lineage_version',
      'claim_schema_version', 'digest_algorithm_version',
    ]));
    expect(isValidLineageAction(validRow())).toBe(true);
    // Removing ANY required field disqualifies the row — the predicate is not
    // shaped around one known batch.
    for (const f of VALID_LINEAGE_ACTION_FIELDS) {
      expect(isValidLineageAction({ ...validRow(), [f]: null })).toBe(false);
    }
  });

  test('a read_natural_key alone is NOT lineage history', () => {
    expect(isValidLineageAction({ read_natural_key: KEY })).toBe(false);
    expect(isValidLineageAction(partialRow())).toBe(false);
    // Publication provenance does not make it one either.
    expect(isValidLineageAction({ read_natural_key: KEY, publication_id: 'p', published_at: 'x' })).toBe(false);
  });

  test('ordinal 0 is a real ordinal, not an absence', () => {
    expect(isValidLineageAction(validRow({ revision_ordinal: 0 }))).toBe(true);
  });
});

describe('STEP 7/8 — partial rows may not shadow a valid head', () => {
  const withClaim = (r) => ({ ...r, claim: { canonical_event_id: r.canonical_event_id, line: r.line, side: r.side, p_win: r.p_win, grade: r.grade } });
  const resolve = (existing) => R.resolveLineage({
    candidate: CAND, existing: existing.filter(isValidLineageAction).map(withClaim), mintReadId: () => 'MINT',
  });

  test('A valid ORIGIN + newer partial row, partial FIRST: head, parent and read_id are correct', () => {
    const r = resolve([partialRow(), validRow()]);
    expect(r.action).toBe('REVISION');
    expect(r.read_id).toBe('11111111-1111-4111-8111-111111111111');
    expect(r.supersedes_id).toBe(1);
    expect(r.revision_ordinal).toBe(1);
    expect(r.lineage_state).toBe('LIVE');
  });

  test('B partial rows ONLY behaves as NO VALID LINEAGE HISTORY', () => {
    const r = resolve([partialRow()]);
    expect(r.action).toBe('ORIGIN');
    expect(r.revision_ordinal).toBe(0);
    expect(r.supersedes_id).toBeNull();
    // LIVE, not LEGACY_UNVERIFIED: no lineage action ever existed here, so
    // claiming unverified history would be a false statement about the past.
    expect(r.lineage_state).toBe('LIVE');
  });

  test('C valid ORIGIN only, and D no history, are unchanged', () => {
    expect(resolve([validRow()]).action).toBe('REVISION');
    expect(resolve([]).action).toBe('ORIGIN');
    expect(resolve([]).lineage_state).toBe('LIVE');
  });

  test('THE PRE-REPAIR DEFECTS, reproduced on an unfiltered family', () => {
    // Fed the raw rows the old lookup returned, the resolver produced a
    // REVISION with a NULL read_id (an orphaned chain node) and mislabelled a
    // brand-new Read as LEGACY_UNVERIFIED. Both are reachable by construction.
    const withClaim = (r) => ({ ...r, claim: { canonical_event_id: r.canonical_event_id, line: r.line, side: r.side, p_win: r.p_win, grade: r.grade } });
    const orphan = R.resolveLineage({ candidate: CAND, existing: [withClaim(partialRow()), withClaim(validRow())], mintReadId: () => 'MINT' });
    expect(orphan.action).toBe('REVISION');
    expect(orphan.read_id).toBeNull();
    const mislabelled = R.resolveLineage({ candidate: CAND, existing: [withClaim(partialRow())], mintReadId: () => 'MINT' });
    expect(mislabelled.lineage_state).toBe('LEGACY_UNVERIFIED');
  });
});

describe('STEP 12/13 — the lookup scope is lossless and bounded by one slate', () => {
  test('sport and game_date are COMPONENTS OF THE KEY, so the bound cannot lose a family member', () => {
    // Derived from the key builder itself, not from a parallel assumption.
    const k = R.readNaturalKey(CAND);
    const [sport, gameDate] = k.split('|');
    expect(sport).toBe(CAND.sport);
    expect(gameDate).toBe(CAND.game_date);
    expect(familyScopesFrom([k])).toEqual([{ sport: 'mlb', game_date: '2026-08-28' }]);
  });

  test('one scope per distinct (sport, date) however many keys are requested', () => {
    const keys = [];
    for (let i = 0; i < 5000; i += 1) {
      keys.push(R.readNaturalKey({ ...CAND, player_key: `p${i}` }));
    }
    // 5,000 keys -> ONE index-backed range, not 50 chunked IN-lists.
    expect(familyScopesFrom(keys)).toHaveLength(1);
  });

  test('a batch spanning two dates produces exactly two scopes', () => {
    const a = R.readNaturalKey(CAND);
    const b = R.readNaturalKey({ ...CAND, game_date: '2026-08-29' });
    expect(familyScopesFrom([a, b])).toHaveLength(2);
  });

  test('a malformed key contributes no scope rather than a wildcard one', () => {
    expect(familyScopesFrom(['', 'mlb', null, undefined])).toEqual([]);
  });
});

describe('STEP 15 — failure atomicity', () => {
  test('a lookup that throws leaves NO lineage-specific state behind', async () => {
    const rows = [{ ...CAND, snapshot_id: 's' }];
    const out = await ret.attachLineage(rows, {
      fetchExisting: async () => { throw new Error('57014 canceling statement due to statement timeout'); },
    });
    expect(out.error).toMatch(/57014/);
    // The row still persists -- retention is not lost -- but it carries nothing
    // that could later be mistaken for history.
    expect(rows[0].read_natural_key).toBeNull();
    expect(rows[0].read_id).toBeNull();
    expect(rows[0].lineage_action).toBeNull();
    expect(isValidLineageAction(rows[0])).toBe(false);
  });

  test('publication provenance is NOT erased by a lineage failure', async () => {
    const rows = [{ ...CAND, snapshot_id: 's', publication_id: 'pub-1', published_at: 'T' }];
    await ret.attachLineage(rows, { fetchExisting: async () => { throw new Error('boom'); } });
    // The slate really was published. Deleting that true fact to tidy up a
    // false one would be the wrong repair.
    expect(rows[0].publication_id).toBe('pub-1');
    expect(rows[0].published_at).toBe('T');
  });

  test('an unpublished capture is cleared too — it was never an action', async () => {
    const rows = [{ ...CAND, published: false, snapshot_id: 's' }];
    const out = await ret.attachLineage(rows, { fetchExisting: async () => [] });
    expect(out.not_published).toBe(1);
    expect(rows[0].read_natural_key).toBeNull();
  });

  test('a successful resolution keeps every field', async () => {
    const rows = [{ ...CAND, snapshot_id: 's' }];
    await ret.attachLineage(rows, { fetchExisting: async () => [], mintReadId: () => 'MINT' });
    expect(rows[0].read_natural_key).toBe(KEY);
    expect(rows[0].lineage_action).toBe('ORIGIN');
    expect(isValidLineageAction(rows[0])).toBe(true);
  });
});

describe('STEP 21 — deterministic in-batch, digest and fork semantics', () => {
  const run = async (rows, existing) => {
    const out = await ret.attachLineage(rows, {
      fetchExisting: async () => existing, mintReadId: () => 'MINT-UUID',
    });
    return out;
  };

  test('two claims of one family in a single batch: ORIGIN then RECAPTURE/REVISION', async () => {
    const a = { ...CAND, snapshot_id: 's' };
    const b = { ...CAND, snapshot_id: 's' };            // identical claim
    await run([a, b], []);
    expect(a.lineage_action).toBe('ORIGIN');
    expect(b.lineage_action).toBe('RECAPTURE');
    expect(b.read_id).toBe(a.read_id);                   // no forked mint
    const c = { ...CAND, snapshot_id: 's' };
    const d = { ...CAND, snapshot_id: 's', p_win: 0.2, grade: 'F' }; // changed claim
    await run([c, d], []);
    expect(c.lineage_action).toBe('ORIGIN');
    expect(d.lineage_action).toBe('REVISION');
    expect(d.revision_ordinal).toBe(1);
  });

  test('an unchanged digest against an existing valid head is a RECAPTURE at the same ordinal', async () => {
    const row = { ...CAND, snapshot_id: 's' };
    await run([row], [validRow({ claim_digest: R.claimDigest(CAND) })]);
    expect(row.lineage_action).toBe('RECAPTURE');
    expect(row.revision_ordinal).toBe(0);
    expect(row.supersedes_id).toBeNull();
    expect(row.recaptures_id).toBe(1);
  });

  test('supersedes uniqueness is a DATABASE guard, and the writer names it', () => {
    const src = require('fs').readFileSync(
      require('path').join(__dirname, '../../src/services/retentionService.js'), 'utf8');
    expect(src).toContain('model_snapshots_supersedes_unique');
  });
});

describe('each layer of the defence is pinned INDEPENDENTLY', () => {
  // Removing either layer alone left the suite green: the other caught it. That
  // is defence in depth working and test coverage failing, so each layer now
  // has its own proof.

  test('LAYER 2 — attachLineage excludes an invalid row a fetcher hands it', async () => {
    const row = { ...CAND, snapshot_id: 's' };
    const out = await ret.attachLineage([row], {
      fetchExisting: async () => [partialRow()], mintReadId: () => 'MINT',
    });
    expect(out.invalid_rows_excluded).toBe(1);
    expect(row.lineage_action).toBe('ORIGIN');
    expect(row.lineage_state).toBe('LIVE');
  });

  test('LAYER 1 — the default lookup asks the database for valid actions only', async () => {
    const applied = { eq: [], not: [], select: null, table: null };
    const rows = [validRow(), partialRow()];
    const builder = {
      select(c) { applied.select = c; return this; },
      eq(col, val) { applied.eq.push([col, val]); return this; },
      not(col, op, val) { applied.not.push([col, op, val]); return this; },
      order() { return this; },
      // The fake honours the filter it was given rather than merely accepting it.
      range() {
        const keep = applied.not.some(([c, o]) => c === 'lineage_action' && o === 'is')
          ? rows.filter((r) => r.lineage_action !== null) : rows;
        return Promise.resolve({ data: keep, error: null });
      },
    };
    const supabase = { from(t) { applied.table = t; return builder; } };
    const row = { ...CAND, snapshot_id: 's' };
    const out = await ret.attachLineage([row], { getClient: () => supabase, mintReadId: () => 'MINT' });

    expect(applied.table).toBe('model_snapshots');
    expect(applied.not).toContainEqual(['lineage_action', 'is', null]);
    expect(applied.eq).toContainEqual(['sport', 'mlb']);
    expect(applied.eq).toContainEqual(['game_date', '2026-08-28']);
    // The projection must carry every field the predicate needs, or the guard
    // would judge a row on columns it never asked for.
    for (const f of VALID_LINEAGE_ACTION_FIELDS) expect(applied.select).toContain(f);
    expect(out.lookup).toMatchObject({ scopes: 1 });
    expect(out.error).toBeNull();
  });

  test('LAYER 1 — the in-loop check rejects an invalid row even if the query returned one', async () => {
    const rows = [partialRow()];
    const builder = {
      select() { return this; }, eq() { return this; }, not() { return this; }, order() { return this; },
      range() { return Promise.resolve({ data: rows, error: null }); },
    };
    const row = { ...CAND, snapshot_id: 's' };
    const out = await ret.attachLineage([row], { getClient: () => ({ from: () => builder }), mintReadId: () => 'MINT' });
    expect(out.lookup.invalid_excluded).toBe(1);
    expect(out.lookup.valid_actions).toBe(0);
    expect(row.lineage_state).toBe('LIVE');
  });
});

describe('the claim digest is frozen', () => {
  test('a known claim hashes to a known digest', () => {
    // Pins the digest itself. Dropping or adding a CLAIM field silently changes
    // every future comparison, and nothing else in the suite would notice.
    const claim = {
      line: 0.5, side: 'over', book: 'draftkings', over_odds: -115, under_odds: 100,
      p_win: 0.9, grade: 'B+', confidence: 90, confidence_basis: 'p_win',
      ev_pct: 1.5, projection: 1.2, edge_pct: 3, takeable: true, value: false,
      fair_odds: -122, fair_prob: 0.55, refused: false, refusal_reason: null,
    };
    expect(R.claimDigest(claim)).toBe(R.claimDigest({ ...claim }));
    expect(R.CLAIM_MARKET_FIELDS).toEqual(['line', 'side', 'book', 'locked_odds', 'over_odds', 'under_odds']);
    // Every market term must move the digest: a field that cannot change it is
    // a field the chronology is blind to.
    for (const f of R.CLAIM_MARKET_FIELDS) {
      const bumped = { ...claim, [f]: f === 'side' ? 'under' : 999 };
      expect(R.claimDigest(bumped)).not.toBe(R.claimDigest(claim));
    }
    expect(R.CLAIM_SCHEMA_VERSION).toBe('claim@1');
    expect(R.DIGEST_ALGORITHM_VERSION).toBe('sha256-json-sorted@1');
  });
});