diff --git a/src/services/retentionService.js b/src/services/retentionService.js index 31e6adf..05c8d07 100644 --- a/src/services/retentionService.js +++ b/src/services/retentionService.js @@ -450,9 +450,7 @@ async function recoverFromFork(chunk, deps = {}) { // eslint-disable-next-line no-await-in-loop await attachLineage(chunk, deps); // eslint-disable-next-line no-await-in-loop - const { error } = await supabase - .from('model_snapshots') - .upsert(chunk, { onConflict: 'snapshot_id,player_key,stat,line,side' }); + const { error } = await upsertSnapshotChunk(supabase, chunk); if (!error) { out.recovered = true; out.written = chunk.length; return out; } if (!isSupersedesConflict(error)) { out.error = error.message; return out; } out.error = error.message; @@ -521,9 +519,7 @@ async function commitPublication(spec, deps = {}) { for (let i = 0; i < served.length; i += CHUNK) { const chunk = served.slice(i, i + CHUNK); // eslint-disable-next-line no-await-in-loop - const { error } = await supabase - .from('model_snapshots') - .upsert(chunk, { onConflict: 'snapshot_id,player_key,stat,line,side' }); + const { error } = await upsertSnapshotChunk(supabase, chunk); if (!error) { out.published += chunk.length; continue; } // ── CONCURRENCY RECOVERY ──────────────────────────────────────────── @@ -606,12 +602,9 @@ async function persist(rows, deps = {}) { const CHUNK = 250; for (let i = 0; i < rows.length; i += CHUNK) { const chunk = rows.slice(i, i + CHUNK); - const { error } = await supabase - .from('model_snapshots') - .upsert(chunk, { - onConflict: 'snapshot_id,player_key,stat,line,side', - ignoreDuplicates: true, - }); + const res = await upsertSnapshotChunk(supabase, chunk, { ignoreDuplicates: true }); + if (res.fellBack) out.legacy_conflict_fallback = true; + const { error } = res; if (error) { out.error = error.message; break; } out.written += chunk.length; } @@ -727,14 +720,98 @@ function newSnapshotId() { return crypto.randomUUID(); } +/* ------------------------------------------------------------------ * + * RETENTION CONFLICT TARGET — event-aware, mixed-fleet safe + * + * SEMANTIC IDENTITY. Two outbound rows are the SAME retention proposition + * within one cycle when they share: the cycle, the EVENT, the participant, the + * stat, the line and the side. Provider/book is deliberately NOT part of it — + * collapsing books is `dedupeProps`'s actual job, and the price anchor is + * chosen later. + * + * The EVENT component uses the strongest truthful label available and never + * fabricates one: `canonical_event_id` where a sport has a resolver (MLB, where + * `admitForGrading` makes it non-null for every row that can reach retention), + * and `game_id` otherwise (NOT NULL in the schema, and the only event label + * sports without a resolver possess). Both columns are in the identity, so the + * weaker label still discriminates where the stronger one is absent. + * + * NULLS NOT DISTINCT is load-bearing. `canonical_event_id` is NULL for every + * non-MLB row, and under PostgreSQL's DEFAULT null semantics two NULLs are + * DISTINCT — measured: the same NBA proposition inserted twice produced TWO + * rows, i.e. every retry would duplicate for ever. With NULLS NOT DISTINCT the + * same test produces one row and retry idempotency holds. + * + * MIXED-FLEET SAFETY. A rollout serves old and new containers at once + * (measured: 11 of 12 probes new, 1 old). The two writers need different + * indexes, and no schema state satisfies both: + * + * old index present -> old writer works; new writer ERRORS 23505 on a + * doubleheader (the legacy index rejects a row the + * event-aware identity considers distinct) + * old index dropped -> new writer works; old writer ERRORS 42P10 + * + * A bare `ON CONFLICT DO NOTHING` would have bridged this, but PostgREST does + * NOT emit one: `ignoreDuplicates` without `onConflict` was measured raising a + * real duplicate-key error, so that bridge does not exist through this client. + * + * So the writer bridges it instead. It targets the event-aware identity and, on + * exactly the two errors that mean "the schema is not in the state I expect", + * retries the SAME chunk on the legacy target. A failed chunk rolls back + * atomically (measured: 0 rows), so the retry cannot double-write. The result + * is a writer that is correct in every schema state: + * + * legacy only -> 42P10 -> legacy target -> legacy semantics, no outage + * both present -> 23505 -> legacy target -> legacy semantics, no outage + * new only -> primary succeeds -> doubleheaders kept + * + * The fallback is a BRIDGE, not a resting place: while it fires, doubleheader + * rows are still lost, and `outbound_collision_count` still reports it. + * ------------------------------------------------------------------ */ + +const LEGACY_CONFLICT_INDEX = 'model_snapshots_cycle_prop_uniq'; +const LEGACY_CONFLICT = 'snapshot_id,player_key,stat,line,side'; +const RETENTION_CONFLICT = 'snapshot_id,game_id,canonical_event_id,player_key,stat,line,side'; + +/** + * Is this error "the schema is not in the state the event-aware target + * expects"? Deliberately narrow: a supersedes conflict is also a 23505, and + * swallowing THAT would destroy the forked-history guard, so the legacy index + * must be named. + */ +function isLegacyConflictBlock(error) { + if (!error) return false; + const code = String(error.code || ''); + const msg = String(error.message || ''); + if (code === '42P10' || /no unique or exclusion constraint matching/i.test(msg)) return true; + return (code === '23505' || /duplicate key value/i.test(msg)) && msg.includes(LEGACY_CONFLICT_INDEX); +} + +/** + * One chunk write. Returns the error (never throws) plus which target actually + * carried it, so a caller can report that the bridge is still in use. + */ +async function upsertSnapshotChunk(supabase, chunk, opts = {}) { + const first = await supabase.from('model_snapshots') + .upsert(chunk, { onConflict: RETENTION_CONFLICT, ...opts }); + if (!first.error) return { error: null, target: RETENTION_CONFLICT, fellBack: false }; + if (!isLegacyConflictBlock(first.error)) { + return { error: first.error, target: RETENTION_CONFLICT, fellBack: false }; + } + const second = await supabase.from('model_snapshots') + .upsert(chunk, { onConflict: LEGACY_CONFLICT, ...opts }); + return { error: second.error || null, target: LEGACY_CONFLICT, fellBack: true }; +} + /* ------------------------------------------------------------------ * * MATERIALIZATION IDENTITY * * TRANSPORT COMPLETE and MATERIALIZATION COMPLETE are different facts. * - * The write is `upsert(..., { onConflict: 'snapshot_id,player_key,stat,line,side', - * ignoreDuplicates: true })`, and the production index behind it is + * The write is `upsert(..., { onConflict: RETENTION_CONFLICT, ignoreDuplicates: + * true })`. Until migration 049 retires it, the LEGACY index * model_snapshots_cycle_prop_uniq UNIQUE (snapshot_id, player_key, stat, line, side) + * is still enforced and the writer falls back to it (see the bridge above) * so two outbound rows sharing that tuple collapse to ONE stored row and the * loser is discarded WITHOUT AN ERROR. `written` counts rows in committed * chunks, so transport reports both as written and the status reads COMPLETE. @@ -753,8 +830,13 @@ function newSnapshotId() { * The conflict identity is NOT changed here. This only makes the loss visible. * ------------------------------------------------------------------ */ -/** The exact columns of model_snapshots_cycle_prop_uniq, in index order. */ -const CONFLICT_IDENTITY = Object.freeze(['snapshot_id', 'player_key', 'stat', 'line', 'side']); +/** + * The exact columns the database conflict target names, in index order — + * DERIVED from RETENTION_CONFLICT rather than restated, so the identity the + * materialization check expects can never drift from the identity the database + * enforces. Restating it is how the two silently disagreed before. + */ +const CONFLICT_IDENTITY = Object.freeze(RETENTION_CONFLICT.split(',')); const IDENTITY_SEP = '\u001f'; const IDENTITY_NULL = '\u0000NULL'; @@ -956,6 +1038,11 @@ function resetTerminal() { lastTerminal.clear(); } module.exports = { MODEL_VERSION, TERMINAL, + RETENTION_CONFLICT, + LEGACY_CONFLICT, + LEGACY_CONFLICT_INDEX, + isLegacyConflictBlock, + upsertSnapshotChunk, MATERIALIZATION, CONFLICT_IDENTITY, rowIdentity, diff --git a/supabase/migrations/048_retention_event_aware_identity_expand.sql b/supabase/migrations/048_retention_event_aware_identity_expand.sql new file mode 100644 index 0000000..0123759 --- /dev/null +++ b/supabase/migrations/048_retention_event_aware_identity_expand.sql @@ -0,0 +1,47 @@ +-- 048 — EXPAND: event-aware retention proposition identity. +-- +-- WHY. The retention conflict identity is +-- model_snapshots_cycle_prop_uniq (snapshot_id, player_key, stat, line, side) +-- which carries no event component. An MLB doubleheader — the same hitter, the +-- same stat, the same line, in two genuinely different games — is therefore ONE +-- identity, and `ignoreDuplicates` discards the second row with no error while +-- transport reports COMPLETE. Verified on production data: 2026-08-17 +-- St. Louis @ Cincinnati holds 1,738 retained rows under a single game_id. +-- +-- WHAT. An additive unique index that adds the EVENT and nothing else. +-- +-- game_id AND canonical_event_id are both included, on purpose: +-- * canonical_event_id splits a doubleheader (game_id is byte-identical for +-- both halves), and MLB's admission gate makes it non-null for every row +-- that can reach retention; +-- * game_id is NOT NULL in the schema and is the only event label sports +-- without a canonical resolver possess, so the weaker label still +-- discriminates where the stronger one is absent. Nothing is fabricated. +-- +-- NULLS NOT DISTINCT is required, not stylistic. canonical_event_id is NULL for +-- every non-MLB row, and under PostgreSQL's default NULL semantics two NULLs +-- are DISTINCT — measured on a disposable PG17: the same NBA proposition +-- inserted twice produced TWO rows, i.e. every retry would duplicate for ever. +-- With NULLS NOT DISTINCT the same test yields one row. +-- +-- SAFE TO CREATE. Measured over all 434 real cohorts / 328,262 retained rows, +-- the new identity produces exactly 328,262 identities — identical to the old +-- count, 0 cohorts changed. Adding columns can only split, and legacy rows all +-- carry canonical_event_id IS NULL with one game_id per proposition, so no +-- existing row can violate it. +-- +-- THIS MIGRATION DOES NOT RETIRE THE OLD INDEX. Both coexist deliberately so a +-- rolling deploy keeps working; the old index is dropped in 049 only after the +-- fleet is proven converged onto a writer that no longer needs it. +-- +-- NO BACKFILL. No historical value is written or rewritten. Legacy rows remain +-- LEGACY EVENT-AGNOSTIC RETENTION, and their NULL canonical_event_id continues +-- to say exactly that. + +create unique index if not exists model_snapshots_cycle_event_prop_uniq + on public.model_snapshots + (snapshot_id, game_id, canonical_event_id, player_key, stat, line, side) + nulls not distinct; + +comment on index public.model_snapshots_cycle_event_prop_uniq is + 'Event-aware retention proposition identity. Supersedes model_snapshots_cycle_prop_uniq, which merged doubleheaders. NULLS NOT DISTINCT so a NULL canonical_event_id (every non-MLB row) still deduplicates on retry.'; diff --git a/supabase/migrations/049_retention_event_aware_identity_contract.sql b/supabase/migrations/049_retention_event_aware_identity_contract.sql new file mode 100644 index 0000000..09614b3 --- /dev/null +++ b/supabase/migrations/049_retention_event_aware_identity_contract.sql @@ -0,0 +1,26 @@ +-- 049 — CONTRACT: retire the event-agnostic retention conflict identity. +-- +-- APPLY THIS ONLY AFTER FLEET CONVERGENCE IS PROVEN. +-- +-- While model_snapshots_cycle_prop_uniq exists it keeps rejecting rows the +-- event-aware identity considers distinct: measured through the real +-- supabase-js -> PostgREST -> Postgres path, an event-aware write of a real +-- doubleheader chunk fails 23505 naming this index and the whole chunk rolls +-- back. The retention writer bridges that by retrying on the legacy target, so +-- there is no outage — but while the bridge fires, doubleheader rows are still +-- lost. Dropping this index is what actually ends the loss. +-- +-- CONVERSELY, a writer that still names these five columns fails 42P10 ("there +-- is no unique or exclusion constraint matching the ON CONFLICT specification") +-- the moment this index is gone. That is why convergence must be PROVEN by +-- sampling the runtime probe, not assumed from a fast rollout: a single probe +-- response during a rolling deploy was measured returning the OLD build. +-- +-- ROLLBACK. Re-create it with: +-- create unique index model_snapshots_cycle_prop_uniq +-- on public.model_snapshots (snapshot_id, player_key, stat, line, side); +-- It is safe to re-create only while no doubleheader cohort has been retained +-- under the event-aware identity; after that, two legitimately distinct rows +-- exist and the five-column uniqueness no longer holds. + +drop index if exists public.model_snapshots_cycle_prop_uniq; diff --git a/tests/unit/retentionIdentity.test.js b/tests/unit/retentionIdentity.test.js new file mode 100644 index 0000000..c97b543 --- /dev/null +++ b/tests/unit/retentionIdentity.test.js @@ -0,0 +1,264 @@ +'use strict'; + +/** + * SEMANTIC RETENTION PROPOSITION IDENTITY, and the database enforcement of it. + * + * The semantic question comes FIRST: when are two outbound rows the same + * retention proposition within one cycle? Answer — same cycle, same EVENT, same + * participant, same stat, same line, same side. Provider/book is deliberately + * absent: collapsing books is `dedupeProps`'s job and the price anchor is + * chosen later. + * + * The database index is ENFORCEMENT of that answer, never the definition of it. + * `model_snapshots_cycle_prop_uniq` carried no event component, so a real MLB + * doubleheader collapsed to one identity — verified on production: 2026-08-17 + * St. Louis @ Cincinnati holds 1,738 retained rows under a single game_id, and + * replaying 1,000 real propositions across both real gamePks materialized 1,000 + * of 2,000 rows under the old index and 2,000 of 2,000 under the new one. + */ + +const fs = require('fs'); +const path = require('path'); +const retention = require('../../src/services/retentionService'); + +const ROOT = path.resolve(__dirname, '..', '..'); +const SNAP = '11111111-1111-1111-1111-111111111111'; +const G1 = 'mlb:gamepk:824514'; +const G2 = 'mlb:gamepk:824478'; + +const row = (o = {}) => ({ + snapshot_id: SNAP, game_id: 'mlb:2026-08-17:St.LouisCardinals@CincinnatiReds', + canonical_event_id: G1, player_key: 'nolan arenado', stat: 'hits', line: 0.5, + side: 'over', sport: 'mlb', ...o, +}); + +describe('SEMANTIC IDENTITY — the contract, stated before the schema', () => { + test('the conflict target names cycle, event, participant, stat, line, side', () => { + expect(retention.RETENTION_CONFLICT.split(',')).toEqual( + ['snapshot_id', 'game_id', 'canonical_event_id', 'player_key', 'stat', 'line', 'side']); + }); + + test('BOOK is deliberately NOT part of identity', () => { + expect(retention.RETENTION_CONFLICT).not.toMatch(/book/); + // …and that is upstream's documented job, not an oversight. + const gs = fs.readFileSync(path.join(ROOT, 'src/services/gradeSlateService.js'), 'utf8'); + expect(gs).toMatch(/Provider is collapsed on\s+\* purpose/); + }); + + test('SAME proposition -> same identity', () => { + expect(retention.rowIdentity(row())).toBe(retention.rowIdentity(row())); + }); + + test('DIFFERENT EVENT -> DISTINCT (the doubleheader)', () => { + expect(retention.rowIdentity(row({ canonical_event_id: G1 }))) + .not.toBe(retention.rowIdentity(row({ canonical_event_id: G2 }))); + }); + + for (const [what, patch] of [ + ['LINE', { line: 1.5 }], + ['SIDE', { side: 'under' }], + ['STAT', { stat: 'total_bases' }], + ['PARTICIPANT', { player_key: 'paul goldschmidt' }], + ['CYCLE', { snapshot_id: '22222222-2222-2222-2222-222222222222' }], + ['GAME (no canonical)', { canonical_event_id: null, game_id: 'nba:2026-08-27:LAL@NYK' }], + ]) { + test(`DIFFERENT ${what} -> DISTINCT`, () => { + expect(retention.rowIdentity(row())).not.toBe(retention.rowIdentity(row(patch))); + }); + } +}); + +describe('SEMANTIC IDENTITY <=> DATABASE CONFLICT IDENTITY', () => { + test('the JS identity uses EXACTLY the columns the DB conflict target names', () => { + expect(retention.CONFLICT_IDENTITY).toEqual(retention.RETENTION_CONFLICT.split(',')); + }); + + test('the migration creates an index on exactly those columns', () => { + const sql = fs.readFileSync(path.join(ROOT, + 'supabase/migrations/048_retention_event_aware_identity_expand.sql'), 'utf8'); + const stmt = sql.slice(sql.indexOf('create unique index')); + const cols = stmt.slice(stmt.indexOf('(snapshot_id'), stmt.indexOf('nulls not distinct')) + .replace(/[()\s]/g, ''); + expect(cols.replace(/,$/, '')).toBe(retention.RETENTION_CONFLICT); + }); + + test('NULLS NOT DISTINCT is present, and it is load-bearing', () => { + // canonical_event_id is NULL for every non-MLB row. Under PostgreSQL's + // DEFAULT semantics two NULLs are DISTINCT, so the same NBA proposition + // would insert again on every retry — measured: 2 rows. NULLS NOT DISTINCT + // makes it 1. + const sql = fs.readFileSync(path.join(ROOT, + 'supabase/migrations/048_retention_event_aware_identity_expand.sql'), 'utf8'); + expect(sql).toMatch(/nulls not distinct/i); + }); +}); + +describe('CANONICAL EVENT AVAILABILITY BY ROW CLASS', () => { + const gs = fs.readFileSync(path.join(ROOT, 'src/services/gradeSlateService.js'), 'utf8'); + + test('MLB: only RESOLVED rows WITH a canonical id are admitted', () => { + expect(gs).toMatch(/status === 'RESOLVED' && p\.canonical_event_id/); + }); + + test('UNRESOLVED / AMBIGUOUS / CONTRADICTED never reach retention at all', () => { + // They are rejected BEFORE grading, and onGraded is what feeds retention. + expect(gs).toMatch(/const gate = admitForGrading\(props, sport\)/); + expect(gs).toMatch(/dedupeProps\(gate\.admitted, limit\)/); + expect(gs).not.toMatch(/dedupeProps\(gate\.rejected/); + }); + + test('a sport with no resolver is admitted unchanged — canonical stays NULL', () => { + expect(gs).toMatch(/!status \|\| status === 'UNSUPPORTED'/); + }); + + test('game_id is the always-present fallback, so identity is never NULL-only', () => { + // NOT NULL in the schema; measured 0 nulls / 0 empties across 328,262 rows. + const contract = JSON.parse(fs.readFileSync(path.join(ROOT, + 'supabase/schema/model_snapshots.columns.json'), 'utf8')); + expect(contract.columns).toContain('game_id'); + expect(retention.RETENTION_CONFLICT).toMatch(/game_id/); + }); +}); + +describe('MIXED-FLEET BRIDGE', () => { + const err = (code, message) => ({ code, message }); + + test('a missing new index (pre-EXPAND) is bridged', () => { + expect(retention.isLegacyConflictBlock( + err('42P10', 'there is no unique or exclusion constraint matching the ON CONFLICT specification'))).toBe(true); + }); + + test('the LEGACY index rejecting a doubleheader row is bridged', () => { + expect(retention.isLegacyConflictBlock( + err('23505', 'duplicate key value violates unique constraint "model_snapshots_cycle_prop_uniq"'))).toBe(true); + }); + + test('a SUPERSEDES conflict is NOT bridged — the fork guard must still fire', () => { + // Also a 23505. Swallowing it would destroy the forked-history guard, which + // is why the bridge requires the legacy index to be NAMED. + expect(retention.isLegacyConflictBlock( + err('23505', 'duplicate key value violates unique constraint "model_snapshots_supersedes_unique"'))).toBe(false); + }); + + test('unrelated errors are never bridged', () => { + expect(retention.isLegacyConflictBlock(err('23502', 'null value in column "captured_at"'))).toBe(false); + expect(retention.isLegacyConflictBlock(err('PGRST204', "Could not find the 'x' column"))).toBe(false); + expect(retention.isLegacyConflictBlock(null)).toBe(false); + }); + + test('it targets the event-aware identity FIRST', async () => { + const seen = []; + const sb = { from: () => ({ upsert: async (rows, o) => { seen.push(o.onConflict); return { error: null }; } }) }; + const res = await retention.upsertSnapshotChunk(sb, [row()], { ignoreDuplicates: true }); + expect(seen).toEqual([retention.RETENTION_CONFLICT]); + expect(res.fellBack).toBe(false); + }); + + test('it retries the SAME chunk on the legacy target when bridged', async () => { + const seen = []; + const sb = { + from: () => ({ + upsert: async (rows, o) => { + seen.push({ target: o.onConflict, n: rows.length }); + return seen.length === 1 + ? { error: err('23505', 'duplicate key value violates unique constraint "model_snapshots_cycle_prop_uniq"') } + : { error: null }; + }, + }), + }; + const chunk = [row({ canonical_event_id: G1 }), row({ canonical_event_id: G2 })]; + const res = await retention.upsertSnapshotChunk(sb, chunk, { ignoreDuplicates: true }); + expect(seen.map((s) => s.target)).toEqual([retention.RETENTION_CONFLICT, retention.LEGACY_CONFLICT]); + expect(seen[1].n).toBe(chunk.length); // the SAME chunk, whole + expect(res.error).toBeNull(); + expect(res.fellBack).toBe(true); + }); + + test('a non-bridgeable error is returned, never retried', async () => { + const seen = []; + const sb = { from: () => ({ upsert: async (r, o) => { seen.push(o.onConflict); return { error: err('23502', 'null value') }; } }) }; + const res = await retention.upsertSnapshotChunk(sb, [row()], {}); + expect(seen).toHaveLength(1); + expect(res.error.code).toBe('23502'); + expect(res.fellBack).toBe(false); + }); + + test('persist reports when the bridge fired — it is a bridge, not a resting place', async () => { + let first = true; + const sb = { + from: () => ({ + upsert: async () => (first + ? (first = false, { error: err('23505', 'duplicate key value violates unique constraint "model_snapshots_cycle_prop_uniq"') }) + : { error: null }), + }), + }; + const out = await retention.persist([row()], { getClient: () => sb }); + expect(out.legacy_conflict_fallback).toBe(true); + expect(out.written).toBe(1); + }); + + test('EVERY model_snapshots write goes through the bridge', () => { + const src = fs.readFileSync(path.join(ROOT, 'src/services/retentionService.js'), 'utf8') + .replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:])\/\/.*$/gm, '$1'); + // No hardcoded legacy target may survive outside the constant itself. + const hard = src.match(/onConflict: 'snapshot_id,player_key,stat,line,side'/g) || []; + expect(hard).toHaveLength(0); + const upserts = src.match(/\.upsert\(/g) || []; + const bridged = src.match(/upsertSnapshotChunk\(/g) || []; + // One .upsert lives inside the bridge helper (two calls: primary + fallback). + expect(upserts.length).toBe(2); + expect(bridged.length).toBeGreaterThanOrEqual(4); // definition + 3 call sites + }); +}); + +describe('MIGRATION SAFETY', () => { + const expand = fs.readFileSync(path.join(ROOT, + 'supabase/migrations/048_retention_event_aware_identity_expand.sql'), 'utf8'); + const contract = fs.readFileSync(path.join(ROOT, + 'supabase/migrations/049_retention_event_aware_identity_contract.sql'), 'utf8'); + + test('EXPAND is additive and idempotent, and retires nothing', () => { + expect(expand).toMatch(/create unique index if not exists/i); + expect(expand).not.toMatch(/drop\s+index/i); + expect(expand).not.toMatch(/alter table/i); + }); + + test('EXPAND performs no backfill', () => { + expect(expand).not.toMatch(/\bupdate\b/i); + expect(expand).not.toMatch(/\binsert\b/i); + }); + + test('CONTRACT only drops the legacy index, and documents its rollback', () => { + expect(contract).toMatch(/drop index if exists public\.model_snapshots_cycle_prop_uniq/i); + expect(contract).not.toMatch(/\bupdate\b/i); + expect(contract).toMatch(/ROLLBACK/); + expect(contract).toMatch(/APPLY THIS ONLY AFTER FLEET CONVERGENCE IS PROVEN/); + }); + + test('the two phases are separate files — they must not land together', () => { + expect(expand).not.toMatch(/drop index if exists public\.model_snapshots_cycle_prop_uniq/i); + }); +}); + +describe('MATERIALIZATION DEFENCE SURVIVES THE REPAIR', () => { + test('the collision detector is still wired and still disqualifies a cohort', () => { + const rows = [row({ canonical_event_id: G1 }), row({ canonical_event_id: G2 })]; + // Under the NEW identity these are two propositions, so no collision. + const exp = retention.expectedMaterialization(rows); + expect(exp.expected_identities).toBe(2); + expect(exp.collision_count).toBe(0); + // And a genuine duplicate still collides. + const dup = retention.expectedMaterialization([row(), row()]); + expect(dup.collision_count).toBe(1); + expect(retention.reconcileMaterialization({ + expected: dup, actualIdentities: dup.identities, transportStatus: retention.TERMINAL.COMPLETE, + }).status).toBe(retention.MATERIALIZATION.COLLISION); + }); + + test('transport vs materialization, and per-sport evidence, are intact', () => { + expect(typeof retention.classifyPersist).toBe('function'); + expect(typeof retention.reconcileMaterialization).toBe('function'); + expect(typeof retention.lastRetention).toBe('function'); + expect(retention.TERMINAL.FAILED_PARTIAL).toBeTruthy(); + }); +}); diff --git a/tests/unit/retentionMaterialization.test.js b/tests/unit/retentionMaterialization.test.js index 1a43dcf..1ff8b90 100644 --- a/tests/unit/retentionMaterialization.test.js +++ b/tests/unit/retentionMaterialization.test.js @@ -36,16 +36,19 @@ const row = (o = {}) => ({ const actualSetFrom = (rows) => new Set(rows.map(retention.rowIdentity)); describe('THE CONFLICT IDENTITY IS THE REAL DATABASE IDENTITY', () => { - test('it is exactly model_snapshots_cycle_prop_uniq', () => { + test('it is exactly model_snapshots_cycle_event_prop_uniq', () => { + // WAS (snapshot_id, player_key, stat, line, side) — no event component, + // which is what merged doubleheaders. Migration 048 adds the event. expect(retention.CONFLICT_IDENTITY).toEqual( - ['snapshot_id', 'player_key', 'stat', 'line', 'side']); + ['snapshot_id', 'game_id', 'canonical_event_id', 'player_key', 'stat', 'line', 'side']); }); test('the production upsert names that same identity', () => { const src = require('fs').readFileSync(require.resolve('../../src/services/retentionService'), 'utf8'); const body = src.slice(src.indexOf('async function persist(')); - expect(body).toMatch(/onConflict: 'snapshot_id,player_key,stat,line,side'/); - expect(body).toMatch(/ignoreDuplicates: true/); + expect(body).toMatch(/upsertSnapshotChunk\(supabase, chunk, \{ ignoreDuplicates: true \}\)/); + const bridge = src.slice(src.indexOf('async function upsertSnapshotChunk')); + expect(bridge).toMatch(/onConflict: RETENTION_CONFLICT/); }); test('numeric line normalises — 0.5 and "0.50" are ONE identity', () => { @@ -118,10 +121,10 @@ describe('EXPECTED MATERIALIZATION', () => { }); }); -describe('UNEXPECTED COLLISION — the doubleheader', () => { - // canonical_event_id is NOT in the conflict identity, so the same hitter's - // same line in two REAL games is one identity. Before event-aware dedupe the - // second game was dropped before grading; now both grade, and both collide. +describe('THE DOUBLEHEADER — repaired, and still defended', () => { + // canonical_event_id is now IN the conflict identity, so the same hitter's + // same line in two real games is TWO propositions. Before migration 048 these + // four rows produced two identities and two rows were silently discarded. const rows = [ row({ canonical_event_id: 'mlb:gamepk:824514', side: 'over' }), row({ canonical_event_id: 'mlb:gamepk:824514', side: 'under' }), @@ -129,27 +132,45 @@ describe('UNEXPECTED COLLISION — the doubleheader', () => { row({ canonical_event_id: 'mlb:gamepk:824478', side: 'under' }), ]; - test('two semantically distinct records collapse under the database identity', () => { + test('two distinct games are now two distinct propositions', () => { const exp = retention.expectedMaterialization(rows); expect(exp.outbound_rows).toBe(4); - expect(exp.expected_identities).toBe(2); - expect(exp.collision_count).toBe(2); + expect(exp.expected_identities).toBe(4); // was 2 + expect(exp.collision_count).toBe(0); // was 2 }); - test('the cohort does NOT silently qualify, even though set equality holds', () => { + test('the cohort qualifies once both games materialize', () => { const exp = retention.expectedMaterialization(rows); - // The discarded rows were never in the expected set, so a naive set - // comparison passes while two real records were lost. const rec = retention.reconcileMaterialization({ expected: exp, actualIdentities: exp.identities, transportStatus: TERMINAL.COMPLETE, }); - expect(rec.missing_identity_count).toBe(0); - expect(rec.extra_identity_count).toBe(0); - expect(rec.status).toBe(MATERIALIZATION.COLLISION); + expect(rec.status).toBe(MATERIALIZATION.COMPLETE); + expect([rec.missing_identity_count, rec.extra_identity_count]).toEqual([0, 0]); + }); + + test('while the legacy index still bridges, the loss is REPORTED, not hidden', () => { + // The bridge keeps the pipeline running before migration 049, but the + // second game is still discarded by the legacy index. The reconciliation + // sees 4 expected and 2 actual and refuses the cohort. + const exp = retention.expectedMaterialization(rows); + const legacyActual = new Set([...exp.identities].slice(0, 2)); + const rec = retention.reconcileMaterialization({ + expected: exp, actualIdentities: legacyActual, transportStatus: TERMINAL.COMPLETE, + }); + expect(rec.missing_identity_count).toBe(2); + expect(rec.status).toBe(MATERIALIZATION.MISSING); expect(rec.status).not.toBe(MATERIALIZATION.COMPLETE); }); - test('transport still reports COMPLETE — which is why transport alone is not evidence', () => { + test('a GENUINE duplicate still collides and still disqualifies', () => { + const dup = retention.expectedMaterialization([row(), row()]); + expect(dup.collision_count).toBe(1); + expect(retention.reconcileMaterialization({ + expected: dup, actualIdentities: dup.identities, transportStatus: TERMINAL.COMPLETE, + }).status).toBe(MATERIALIZATION.COLLISION); + }); + + test('transport reporting COMPLETE is still not evidence on its own', () => { const st = retention.classifyPersist({ attempted: 4, written: 4, skipped: false, error: null }); expect(st).toBe(TERMINAL.COMPLETE); expect(retention.isRetentionFailure(st)).toBe(false); @@ -237,9 +258,8 @@ describe('PER-SPORT TERMINAL EVIDENCE', () => { }); test('the terminal record carries the expected-materialization facts', () => { - const rows = [ - row({ canonical_event_id: 'g1' }), row({ canonical_event_id: 'g2' }), // collide - ]; + // Two BYTE-IDENTICAL rows — a genuine duplicate, which still collides. + const rows = [row(), row()]; const e = retention.recordTerminal({ sport: 'mlb', snapshotId: SNAP, rows, result: { attempted: 2, written: 2, skipped: false, error: null },