'use strict'; /** * SEMANTIC RETENTION PROPOSITION IDENTITY, and the database enforcement of it. * * The semantic question comes FIRST: when are two outbound rows the same * retention proposition within one cycle? Answer — same cycle, same EVENT, same * participant, same stat, same line, same side. Provider/book is deliberately * absent: collapsing books is `dedupeProps`'s job and the price anchor is * chosen later. * * The database index is ENFORCEMENT of that answer, never the definition of it. * `model_snapshots_cycle_prop_uniq` carried no event component, so a real MLB * doubleheader collapsed to one identity — verified on production: 2026-08-17 * St. Louis @ Cincinnati holds 1,738 retained rows under a single game_id, and * replaying 1,000 real propositions across both real gamePks materialized 1,000 * of 2,000 rows under the old index and 2,000 of 2,000 under the new one. */ const fs = require('fs'); const path = require('path'); const retention = require('../../src/services/retentionService'); const ROOT = path.resolve(__dirname, '..', '..'); const SNAP = '11111111-1111-1111-1111-111111111111'; const G1 = 'mlb:gamepk:824514'; const G2 = 'mlb:gamepk:824478'; const row = (o = {}) => ({ snapshot_id: SNAP, game_id: 'mlb:2026-08-17:St.LouisCardinals@CincinnatiReds', canonical_event_id: G1, player_key: 'nolan arenado', stat: 'hits', line: 0.5, side: 'over', sport: 'mlb', ...o, }); describe('SEMANTIC IDENTITY — the contract, stated before the schema', () => { test('the conflict target names cycle, event, participant, stat, line, side', () => { expect(retention.RETENTION_CONFLICT.split(',')).toEqual( ['snapshot_id', 'game_id', 'canonical_event_id', 'player_key', 'stat', 'line', 'side']); }); test('BOOK is deliberately NOT part of identity', () => { expect(retention.RETENTION_CONFLICT).not.toMatch(/book/); // …and that is upstream's documented job, not an oversight. const gs = fs.readFileSync(path.join(ROOT, 'src/services/gradeSlateService.js'), 'utf8'); expect(gs).toMatch(/Provider is collapsed on\s+\* purpose/); }); test('SAME proposition -> same identity', () => { expect(retention.rowIdentity(row())).toBe(retention.rowIdentity(row())); }); test('DIFFERENT EVENT -> DISTINCT (the doubleheader)', () => { expect(retention.rowIdentity(row({ canonical_event_id: G1 }))) .not.toBe(retention.rowIdentity(row({ canonical_event_id: G2 }))); }); for (const [what, patch] of [ ['LINE', { line: 1.5 }], ['SIDE', { side: 'under' }], ['STAT', { stat: 'total_bases' }], ['PARTICIPANT', { player_key: 'paul goldschmidt' }], ['CYCLE', { snapshot_id: '22222222-2222-2222-2222-222222222222' }], ['GAME (no canonical)', { canonical_event_id: null, game_id: 'nba:2026-08-27:LAL@NYK' }], ]) { test(`DIFFERENT ${what} -> DISTINCT`, () => { expect(retention.rowIdentity(row())).not.toBe(retention.rowIdentity(row(patch))); }); } }); describe('SEMANTIC IDENTITY <=> DATABASE CONFLICT IDENTITY', () => { test('the JS identity uses EXACTLY the columns the DB conflict target names', () => { expect(retention.CONFLICT_IDENTITY).toEqual(retention.RETENTION_CONFLICT.split(',')); }); test('the migration creates an index on exactly those columns', () => { const sql = fs.readFileSync(path.join(ROOT, 'supabase/migrations/048_retention_event_aware_identity_expand.sql'), 'utf8'); const stmt = sql.slice(sql.indexOf('create unique index')); const cols = stmt.slice(stmt.indexOf('(snapshot_id'), stmt.indexOf('nulls not distinct')) .replace(/[()\s]/g, ''); expect(cols.replace(/,$/, '')).toBe(retention.RETENTION_CONFLICT); }); test('NULLS NOT DISTINCT is present, and it is load-bearing', () => { // canonical_event_id is NULL for every non-MLB row. Under PostgreSQL's // DEFAULT semantics two NULLs are DISTINCT, so the same NBA proposition // would insert again on every retry — measured: 2 rows. NULLS NOT DISTINCT // makes it 1. const sql = fs.readFileSync(path.join(ROOT, 'supabase/migrations/048_retention_event_aware_identity_expand.sql'), 'utf8'); expect(sql).toMatch(/nulls not distinct/i); }); }); describe('CANONICAL EVENT AVAILABILITY BY ROW CLASS', () => { const gs = fs.readFileSync(path.join(ROOT, 'src/services/gradeSlateService.js'), 'utf8'); test('MLB: only RESOLVED rows WITH a canonical id are admitted', () => { expect(gs).toMatch(/status === 'RESOLVED' && p\.canonical_event_id/); }); test('UNRESOLVED / AMBIGUOUS / CONTRADICTED never reach retention at all', () => { // They are rejected BEFORE grading, and onGraded is what feeds retention. expect(gs).toMatch(/const gate = admitForGrading\(props, sport\)/); expect(gs).toMatch(/dedupeProps\(gate\.admitted, limit\)/); expect(gs).not.toMatch(/dedupeProps\(gate\.rejected/); }); test('a sport with no resolver is admitted unchanged — canonical stays NULL', () => { expect(gs).toMatch(/!status \|\| status === 'UNSUPPORTED'/); }); test('game_id is the always-present fallback, so identity is never NULL-only', () => { // NOT NULL in the schema; measured 0 nulls / 0 empties across 328,262 rows. const contract = JSON.parse(fs.readFileSync(path.join(ROOT, 'supabase/schema/model_snapshots.columns.json'), 'utf8')); expect(contract.columns).toContain('game_id'); expect(retention.RETENTION_CONFLICT).toMatch(/game_id/); }); }); describe('MIXED-FLEET BRIDGE', () => { const err = (code, message) => ({ code, message }); test('a missing new index (pre-EXPAND) is bridged', () => { expect(retention.isLegacyConflictBlock( err('42P10', 'there is no unique or exclusion constraint matching the ON CONFLICT specification'))).toBe(true); }); test('the LEGACY index rejecting a doubleheader row is bridged', () => { expect(retention.isLegacyConflictBlock( err('23505', 'duplicate key value violates unique constraint "model_snapshots_cycle_prop_uniq"'))).toBe(true); }); test('a SUPERSEDES conflict is NOT bridged — the fork guard must still fire', () => { // Also a 23505. Swallowing it would destroy the forked-history guard, which // is why the bridge requires the legacy index to be NAMED. expect(retention.isLegacyConflictBlock( err('23505', 'duplicate key value violates unique constraint "model_snapshots_supersedes_unique"'))).toBe(false); }); test('unrelated errors are never bridged', () => { expect(retention.isLegacyConflictBlock(err('23502', 'null value in column "captured_at"'))).toBe(false); expect(retention.isLegacyConflictBlock(err('PGRST204', "Could not find the 'x' column"))).toBe(false); expect(retention.isLegacyConflictBlock(null)).toBe(false); }); test('it targets the event-aware identity FIRST', async () => { const seen = []; const sb = { from: () => ({ upsert: async (rows, o) => { seen.push(o.onConflict); return { error: null }; } }) }; const res = await retention.upsertSnapshotChunk(sb, [row()], { ignoreDuplicates: true }); expect(seen).toEqual([retention.RETENTION_CONFLICT]); expect(res.fellBack).toBe(false); }); test('it retries the SAME chunk on the legacy target when bridged', async () => { const seen = []; const sb = { from: () => ({ upsert: async (rows, o) => { seen.push({ target: o.onConflict, n: rows.length }); return seen.length === 1 ? { error: err('23505', 'duplicate key value violates unique constraint "model_snapshots_cycle_prop_uniq"') } : { error: null }; }, }), }; const chunk = [row({ canonical_event_id: G1 }), row({ canonical_event_id: G2 })]; const res = await retention.upsertSnapshotChunk(sb, chunk, { ignoreDuplicates: true }); expect(seen.map((s) => s.target)).toEqual([retention.RETENTION_CONFLICT, retention.LEGACY_CONFLICT]); expect(seen[1].n).toBe(chunk.length); // the SAME chunk, whole expect(res.error).toBeNull(); expect(res.fellBack).toBe(true); }); test('a non-bridgeable error is returned, never retried', async () => { const seen = []; const sb = { from: () => ({ upsert: async (r, o) => { seen.push(o.onConflict); return { error: err('23502', 'null value') }; } }) }; const res = await retention.upsertSnapshotChunk(sb, [row()], {}); expect(seen).toHaveLength(1); expect(res.error.code).toBe('23502'); expect(res.fellBack).toBe(false); }); test('persist reports when the bridge fired — it is a bridge, not a resting place', async () => { let first = true; const sb = { from: () => ({ upsert: async () => (first ? (first = false, { error: err('23505', 'duplicate key value violates unique constraint "model_snapshots_cycle_prop_uniq"') }) : { error: null }), }), }; const out = await retention.persist([row()], { getClient: () => sb }); expect(out.legacy_conflict_fallback).toBe(true); expect(out.written).toBe(1); }); test('EVERY model_snapshots write goes through the bridge', () => { const src = fs.readFileSync(path.join(ROOT, 'src/services/retentionService.js'), 'utf8') .replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:])\/\/.*$/gm, '$1'); // No hardcoded legacy target may survive outside the constant itself. const hard = src.match(/onConflict: 'snapshot_id,player_key,stat,line,side'/g) || []; expect(hard).toHaveLength(0); const upserts = src.match(/\.upsert\(/g) || []; const bridged = src.match(/upsertSnapshotChunk\(/g) || []; // One .upsert lives inside the bridge helper (two calls: primary + fallback). expect(upserts.length).toBe(2); expect(bridged.length).toBeGreaterThanOrEqual(4); // definition + 3 call sites }); }); describe('MIGRATION SAFETY', () => { const expand = fs.readFileSync(path.join(ROOT, 'supabase/migrations/048_retention_event_aware_identity_expand.sql'), 'utf8'); const contract = fs.readFileSync(path.join(ROOT, 'supabase/migrations/049_retention_event_aware_identity_contract.sql'), 'utf8'); test('EXPAND is additive and idempotent, and retires nothing', () => { expect(expand).toMatch(/create unique index if not exists/i); expect(expand).not.toMatch(/drop\s+index/i); expect(expand).not.toMatch(/alter table/i); }); test('EXPAND performs no backfill', () => { expect(expand).not.toMatch(/\bupdate\b/i); expect(expand).not.toMatch(/\binsert\b/i); }); test('CONTRACT only drops the legacy index, and documents its rollback', () => { expect(contract).toMatch(/drop index if exists public\.model_snapshots_cycle_prop_uniq/i); expect(contract).not.toMatch(/\bupdate\b/i); expect(contract).toMatch(/ROLLBACK/); expect(contract).toMatch(/APPLY THIS ONLY AFTER FLEET CONVERGENCE IS PROVEN/); }); test('the two phases are separate files — they must not land together', () => { expect(expand).not.toMatch(/drop index if exists public\.model_snapshots_cycle_prop_uniq/i); }); }); describe('MATERIALIZATION DEFENCE SURVIVES THE REPAIR', () => { test('the collision detector is still wired and still disqualifies a cohort', () => { const rows = [row({ canonical_event_id: G1 }), row({ canonical_event_id: G2 })]; // Under the NEW identity these are two propositions, so no collision. const exp = retention.expectedMaterialization(rows); expect(exp.expected_identities).toBe(2); expect(exp.collision_count).toBe(0); // And a genuine duplicate still collides. const dup = retention.expectedMaterialization([row(), row()]); expect(dup.collision_count).toBe(1); expect(retention.reconcileMaterialization({ expected: dup, actualIdentities: dup.identities, transportStatus: retention.TERMINAL.COMPLETE, }).status).toBe(retention.MATERIALIZATION.COLLISION); }); test('transport vs materialization, and per-sport evidence, are intact', () => { expect(typeof retention.classifyPersist).toBe('function'); expect(typeof retention.reconcileMaterialization).toBe('function'); expect(typeof retention.lastRetention).toBe('function'); expect(retention.TERMINAL.FAILED_PARTIAL).toBeTruthy(); }); });