vyndr/tests/unit/retentionMaterialization.test.js

'use strict';

/**
 * TRANSPORT TRUTH vs MATERIALIZATION TRUTH.
 *
 * Transport truth says every intended write operation succeeded. Materialization
 * truth says every identity that should exist actually exists. They are not the
 * same fact, because the write is
 *
 *   upsert(rows, { onConflict: 'snapshot_id,player_key,stat,line,side',
 *                  ignoreDuplicates: true })
 *
 * against the production index
 *
 *   model_snapshots_cycle_prop_uniq UNIQUE (snapshot_id, player_key, stat, line, side)
 *
 * so two outbound rows sharing that tuple collapse to ONE stored row, silently,
 * while `written` counts both and the terminal status reads COMPLETE.
 *
 * Measured against production 2026-08-27: no key column is ever NULL
 * (0 of 328,262 rows), so NULLS DISTINCT never applies; `line` is an
 * unconstrained numeric, which is why identity normalises it.
 */

const retention = require('../../src/services/retentionService');
const { TERMINAL, MATERIALIZATION } = retention;

const SNAP = '11111111-1111-1111-1111-111111111111';
const OTHER_SNAP = '22222222-2222-2222-2222-222222222222';

const row = (o = {}) => ({
  snapshot_id: SNAP, player_key: 'nolan arenado', stat: 'hits', line: 0.5, side: 'over', ...o,
});

/** The identity a real database row would present when read back. */
const actualSetFrom = (rows) => new Set(rows.map(retention.rowIdentity));

describe('THE CONFLICT IDENTITY IS THE REAL DATABASE IDENTITY', () => {
  test('it is exactly model_snapshots_cycle_event_prop_uniq', () => {
    // WAS (snapshot_id, player_key, stat, line, side) — no event component,
    // which is what merged doubleheaders. Migration 048 adds the event.
    expect(retention.CONFLICT_IDENTITY).toEqual(
      ['snapshot_id', 'game_id', 'canonical_event_id', 'player_key', 'stat', 'line', 'side']);
  });

  test('the production upsert names that same identity', () => {
    const src = require('fs').readFileSync(require.resolve('../../src/services/retentionService'), 'utf8');
    const body = src.slice(src.indexOf('async function persist('));
    expect(body).toMatch(/upsertSnapshotChunk\(supabase, chunk, \{ ignoreDuplicates: true \}\)/);
    const bridge = src.slice(src.indexOf('async function upsertSnapshotChunk'));
    expect(bridge).toMatch(/onConflict: RETENTION_CONFLICT/);
  });

  test('numeric line normalises — 0.5 and "0.50" are ONE identity', () => {
    expect(retention.rowIdentity(row({ line: 0.5 })))
      .toBe(retention.rowIdentity(row({ line: '0.50' })));
  });

  test('a null key value is marked, never collapsed into an empty string', () => {
    expect(retention.rowIdentity(row({ side: null })))
      .not.toBe(retention.rowIdentity(row({ side: '' })));
  });
});

describe('PRIOR-CYCLE COLLISION IS STRUCTURALLY IMPOSSIBLE', () => {
  test('snapshot_id is part of the identity', () => {
    expect(retention.CONFLICT_IDENTITY).toContain('snapshot_id');
  });

  test('captured_at is NOT part of the identity', () => {
    expect(retention.CONFLICT_IDENTITY).not.toContain('captured_at');
  });

  test('the same prop in a different cycle is a DIFFERENT identity', () => {
    expect(retention.rowIdentity(row()))
      .not.toBe(retention.rowIdentity(row({ snapshot_id: OTHER_SNAP })));
  });

  test('snapshot_id is freshly generated per cycle', () => {
    expect(retention.newSnapshotId()).not.toBe(retention.newSnapshotId());
    const src = require('fs').readFileSync(require.resolve('../../src/services/snapshotService'), 'utf8');
    expect(src).toMatch(/retention\.newSnapshotId\(\)/);
  });

  test('so a new-cycle row can never be suppressed by an old-cycle row', () => {
    // Both cycles' rows coexist: append-only chronology across cycles is safe.
    const cycleA = [row()];
    const cycleB = [row({ snapshot_id: OTHER_SNAP })];
    const ids = actualSetFrom([...cycleA, ...cycleB]);
    expect(ids.size).toBe(2);
  });
});

describe('EXPECTED MATERIALIZATION', () => {
  test('NO DUPLICATES — every expected identity materializes', () => {
    const rows = [row({ side: 'over' }), row({ side: 'under' }), row({ stat: 'total_bases' })];
    const exp = retention.expectedMaterialization(rows);
    expect(exp.outbound_rows).toBe(3);
    expect(exp.expected_identities).toBe(3);
    expect(exp.collision_count).toBe(0);
    const rec = retention.reconcileMaterialization({
      expected: exp, actualIdentities: actualSetFrom(rows), transportStatus: TERMINAL.COMPLETE,
    });
    expect(rec.status).toBe(MATERIALIZATION.COMPLETE);
    expect([rec.missing_identity_count, rec.extra_identity_count]).toEqual([0, 0]);
  });

  test('expected is derived from the payload, NOT from attempted', () => {
    // `attempted` counts rows SENT. Using it as the denominator would make a
    // collided cohort look short by exactly the rows the database discarded,
    // and a deduped one look broken.
    const rows = [row(), row()];               // identical -> one identity
    const exp = retention.expectedMaterialization(rows);
    expect(exp.outbound_rows).toBe(2);
    expect(exp.expected_identities).toBe(1);
  });

  test('it refuses anything but an expectedMaterialization result', () => {
    expect(() => retention.reconcileMaterialization({ expected: null })).toThrow(TypeError);
    expect(() => retention.reconcileMaterialization({ expected: { count: 3 } })).toThrow(TypeError);
  });
});

describe('THE DOUBLEHEADER — repaired, and still defended', () => {
  // canonical_event_id is now IN the conflict identity, so the same hitter's
  // same line in two real games is TWO propositions. Before migration 048 these
  // four rows produced two identities and two rows were silently discarded.
  const rows = [
    row({ canonical_event_id: 'mlb:gamepk:824514', side: 'over' }),
    row({ canonical_event_id: 'mlb:gamepk:824514', side: 'under' }),
    row({ canonical_event_id: 'mlb:gamepk:824478', side: 'over' }),
    row({ canonical_event_id: 'mlb:gamepk:824478', side: 'under' }),
  ];

  test('two distinct games are now two distinct propositions', () => {
    const exp = retention.expectedMaterialization(rows);
    expect(exp.outbound_rows).toBe(4);
    expect(exp.expected_identities).toBe(4);   // was 2
    expect(exp.collision_count).toBe(0);       // was 2
  });

  test('the cohort qualifies once both games materialize', () => {
    const exp = retention.expectedMaterialization(rows);
    const rec = retention.reconcileMaterialization({
      expected: exp, actualIdentities: exp.identities, transportStatus: TERMINAL.COMPLETE,
    });
    expect(rec.status).toBe(MATERIALIZATION.COMPLETE);
    expect([rec.missing_identity_count, rec.extra_identity_count]).toEqual([0, 0]);
  });

  test('while the legacy index still bridges, the loss is REPORTED, not hidden', () => {
    // The bridge keeps the pipeline running before migration 049, but the
    // second game is still discarded by the legacy index. The reconciliation
    // sees 4 expected and 2 actual and refuses the cohort.
    const exp = retention.expectedMaterialization(rows);
    const legacyActual = new Set([...exp.identities].slice(0, 2));
    const rec = retention.reconcileMaterialization({
      expected: exp, actualIdentities: legacyActual, transportStatus: TERMINAL.COMPLETE,
    });
    expect(rec.missing_identity_count).toBe(2);
    expect(rec.status).toBe(MATERIALIZATION.MISSING);
    expect(rec.status).not.toBe(MATERIALIZATION.COMPLETE);
  });

  test('a GENUINE duplicate still collides and still disqualifies', () => {
    const dup = retention.expectedMaterialization([row(), row()]);
    expect(dup.collision_count).toBe(1);
    expect(retention.reconcileMaterialization({
      expected: dup, actualIdentities: dup.identities, transportStatus: TERMINAL.COMPLETE,
    }).status).toBe(MATERIALIZATION.COLLISION);
  });

  test('transport reporting COMPLETE is still not evidence on its own', () => {
    const st = retention.classifyPersist({ attempted: 4, written: 4, skipped: false, error: null });
    expect(st).toBe(TERMINAL.COMPLETE);
    expect(retention.isRetentionFailure(st)).toBe(false);
  });
});

describe('MATERIALIZATION FAILURES', () => {
  const rows = [row({ side: 'over' }), row({ side: 'under' }), row({ stat: 'rbi' })];

  test('COMPLETE TRANSPORT + MISSING EXPECTED IDENTITY -> fails', () => {
    const exp = retention.expectedMaterialization(rows);
    const actual = actualSetFrom(rows.slice(0, 2));
    const rec = retention.reconcileMaterialization({
      expected: exp, actualIdentities: actual, transportStatus: TERMINAL.COMPLETE,
    });
    expect(rec.missing_identity_count).toBe(1);
    expect(rec.status).toBe(MATERIALIZATION.MISSING);
  });

  test('COMPLETE TRANSPORT + EXTRA UNEXPECTED IDENTITY -> fails', () => {
    const exp = retention.expectedMaterialization(rows);
    const actual = actualSetFrom([...rows, row({ stat: 'runs' })]);
    const rec = retention.reconcileMaterialization({
      expected: exp, actualIdentities: actual, transportStatus: TERMINAL.COMPLETE,
    });
    expect(rec.extra_identity_count).toBe(1);
    expect(rec.status).toBe(MATERIALIZATION.EXTRA);
  });

  test('EQUAL COUNTS, DIFFERENT SETS -> fails (count equality is not set equality)', () => {
    const exp = retention.expectedMaterialization(rows);
    const swapped = actualSetFrom([rows[0], rows[1], row({ stat: 'runs' })]);
    const rec = retention.reconcileMaterialization({
      expected: exp, actualIdentities: swapped, transportStatus: TERMINAL.COMPLETE,
    });
    expect(rec.expected_materialized_count).toBe(rec.actual_materialized_count);  // counts agree
    expect(rec.missing_identity_count).toBe(1);
    expect(rec.extra_identity_count).toBe(1);
    expect(rec.status).not.toBe(MATERIALIZATION.COMPLETE);
  });

  test('PARTIAL CHUNK FAILURE -> cohort cannot qualify however many rows persisted', () => {
    const exp = retention.expectedMaterialization(rows);
    const rec = retention.reconcileMaterialization({
      expected: exp,
      actualIdentities: actualSetFrom(rows),      // rows ARE all present
      transportStatus: TERMINAL.FAILED_PARTIAL,
    });
    expect(rec.status).toBe(MATERIALIZATION.TRANSPORT_FAILED);
    expect(rec.status).not.toBe(MATERIALIZATION.COMPLETE);
  });

  test('row presence alone never qualifies a cohort', () => {
    for (const st of [TERMINAL.FAILED_PARTIAL, TERMINAL.FAILED_ZERO_WRITE, TERMINAL.FAILED_UNRESOLVED_ERROR]) {
      const rec = retention.reconcileMaterialization({
        expected: retention.expectedMaterialization(rows),
        actualIdentities: actualSetFrom(rows),
        transportStatus: st,
      });
      expect(rec.status).toBe(MATERIALIZATION.TRANSPORT_FAILED);
    }
  });
});

describe('PER-SPORT TERMINAL EVIDENCE', () => {
  beforeEach(() => retention.resetTerminal());

  test('last_retention is keyed per sport, not a single global slot', async () => {
    const ok = { from: () => ({ upsert: async () => ({ error: null }) }) };
    const rowsA = [row({ side: 'over' })];
    const rowsB = [row({ snapshot_id: OTHER_SNAP, side: 'under' })];
    retention.recordTerminal({
      sport: 'mlb', snapshotId: SNAP, rows: rowsA,
      result: await retention.persist(rowsA, { getClient: () => ok }),
    });
    retention.recordTerminal({
      sport: 'wnba', snapshotId: OTHER_SNAP, rows: rowsB,
      result: await retention.persist(rowsB, { getClient: () => ok }),
    });
    const last = retention.lastRetention();
    // A later sport must not overwrite the target sport's evidence.
    expect(last.mlb.snapshot_id).toBe(SNAP);
    expect(last.wnba.snapshot_id).toBe(OTHER_SNAP);
    expect(Object.keys(last).sort()).toEqual(['mlb', 'wnba']);
  });

  test('the terminal record carries the expected-materialization facts', () => {
    // Two BYTE-IDENTICAL rows — a genuine duplicate, which still collides.
    const rows = [row(), row()];
    const e = retention.recordTerminal({
      sport: 'mlb', snapshotId: SNAP, rows,
      result: { attempted: 2, written: 2, skipped: false, error: null },
    });
    expect(e.outbound_rows).toBe(2);
    expect(e.expected_materialized_count).toBe(1);
    expect(e.outbound_collision_count).toBe(1);
    expect(e.expected_identity_digest).toMatch(/^[0-9a-f]{32}$/);
  });

  test('it exposes counts and a digest, never the identities themselves', () => {
    const e = retention.recordTerminal({
      sport: 'mlb', snapshotId: SNAP, rows: [row()],
      result: { attempted: 1, written: 1, skipped: false, error: null },
    });
    expect(JSON.stringify(e)).not.toMatch(/nolan arenado/);
    expect(e).not.toHaveProperty('identities');
  });

  test('the call site hands the outbound payload to the recorder', () => {
    const src = require('fs').readFileSync(require.resolve('../../src/services/snapshotService'), 'utf8');
    const i = src.indexOf('retention.recordTerminal');
    const block = src.slice(i, i + 500);
    expect(block).toMatch(/\brows,/);
    expect(block).toMatch(/result: r,/);
  });
});

describe('AN UNEXPECTED COLLISION IS ANNOUNCED, NOT INFERRED', () => {
  const src = require('fs').readFileSync(require.resolve('../../src/services/snapshotService'), 'utf8');
  const block = src.slice(src.indexOf('UNEXPECTED COLLISION IS SILENT DATA LOSS'),
    src.indexOf('UNEXPECTED COLLISION IS SILENT DATA LOSS') + 1800);

  test('it alerts even though the transport status is COMPLETE', () => {
    expect(block).toMatch(/terminal\.outbound_collision_count > 0/);
    expect(block).toMatch(/priority: 'high'/);
  });

  test('the alert names the counts, the cycle and the build', () => {
    for (const f of ['${terminal.outbound_collision_count}', '${terminal.outbound_rows}',
      'transport=${terminal.status}', 'expected_identities=${terminal.expected_materialized_count}',
      'snapshot_id=${terminal.snapshot_id}', 'code_sha=${terminal.code_sha']) {
      expect(block).toContain(f);
    }
    expect(block).toMatch(/NOT evidence-complete/);
  });
});