builtbykev/vyndr
1
0
Fork 0
Code Issues 1 Pull Requests Actions 5 Packages Projects Releases Wiki Activity

MLB canonical event identity, impossible-binding refusal, event-aware dedupe, publication commit

Browse Source 
Release-isolated slice built from 41ba38e. Ships ONLY the event-integrity +
publication + lineage-canary closure; the 90-path development tree stays
undeployed.

- canonical MLB event identity from statsapi gamePk (mlb:gamepk:<pk>), with
  event_identity_source/method/version recorded. The id is canonical; the
  binding is derived and says so.
- IMPOSSIBLE-BINDING REFUSAL. Verified in prod 2026-08-26: Joe Mack (Marlins)
  was bound to Dodgers@Braves and Yandy Diaz (Rays) to Rangers@WhiteSox, both
  from one book in the 01:00/03:01 UTC cycles after their own games began. Root
  cause is source market data, not the binder. A prop whose player's team is not
  an event participant now refuses; unknown team preserves uncertainty.
- event-aware dedupe: books still collapse, events no longer do. An unresolved
  MLB event fractures rather than falling back to the collision-prone
  date+teams key.
- publication commit moved AFTER the authoritative Redis slate write, with
  exact parity-gap identity when the product publishes and the record does not.
- lineage dual-write behind LINEAGE_CANARY_SPORTS, DISABLED for this deploy.

Excluded deliberately: WNBA feed/chain, market ontology, PerformanceDistribution,
calibration certification, truth diagnostics, applyRevision Phase-1, and the
analyzeViaEngine1 confidence-rounding change (a served field).

Suite 380/5,040/0 from this worktree; web tsc exit 0; champion output identical
to production.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CQJeAG8vcDoL5zkiaJyVb8
This commit is contained in:
Kev
2026-08-27 16:27:36 -04:00
parent 41ba38ed4e
commit 352016790a
16 changed files with 4217 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/dedupe-parity.js
+142
View File
@@ -0,0 +1,142 @@
'use strict';
/**
* DEDUPE PARITY — old key vs new key on REAL prop shapes.
*
* The event-aware dedupe changes candidate identity, so a single champion
* probability check is not sufficient evidence that ordinary slates are
* unaffected. This replays BOTH keys over real prop shapes reconstructed from
* production rows and reports every difference.
*
* Writes nothing. Deterministic.
*
* PARITY_DATE=2026-08-26 node scripts/dedupe-parity.js
*/
const { createClient } = require('@supabase/supabase-js');
const { paginate } = require('../src/utils/safePaginate');
const E = require('../src/services/event/eventIdentity');
const SPORT = process.env.PARITY_SPORT || 'mlb';
const DATE = process.env.PARITY_DATE || '2026-08-26';
function client() {
const url = process.env.SUPABASE_URL;
const key = process.env.SUPABASE_SERVICE_KEY || process.env.SUPABASE_SERVICE_ROLE_KEY;
if (!url || !key) throw new Error('SUPABASE_URL + service key required');
return createClient(url, key, { auth: { persistSession: false } });
}
/** The OLD key, verbatim: player::stat_type::line. */
const oldKey = (p) => `${p.player}::${p.stat_type}::${p.line}`;
/** The NEW key: event leads. */
function newKey(p) {
const ev = p.canonical_event_id
|| (p.event_identity_method && p.event_identity_method !== 'UNSUPPORTED_SPORT'
? `unresolved-event:${p.event_identity_method}:${p.__row}`
: `legacy:${p.game_date}:${String(p.away_team || '').replace(/\s+/g, '')}@${String(p.home_team || '').replace(/\s+/g, '')}`);
return `${ev}::${p.player}::${p.stat_type}::${p.line}`;
}
function teamsFromGameId(gameId) {
const tail = String(gameId || '').split(':').pop();
const [away, home] = String(tail).split('@');
return { away_team: away || null, home_team: home || null };
}
async function main() {
const sb = client();
const rows = await paginate(
() => sb.from('model_snapshots')
.select('id, player_name, player_key, stat, line, side, game_id, game_date, book')
.eq('sport', SPORT).eq('game_date', DATE),
{ key: 'id', label: 'model_snapshots:dedupe-parity' },
);
// Schedule + rosters straight from statsapi (free, no cache layer) so the
// replay does not depend on Redis being reachable.
const sched = await (await fetch(`https://statsapi.mlb.com/api/v1/schedule?sportId=1&date=${DATE}`)).json();
const games = (sched.dates || []).flatMap((d) => d.games || []).map((g) => ({
gamePk: g.gamePk, gameNumber: g.gameNumber, gameDate: g.gameDate,
home: { team: g.teams.home.team.name, teamId: g.teams.home.team.id },
away: { team: g.teams.away.team.name, teamId: g.teams.away.team.id },
}));
// PARTICIPANT EVIDENCE — the same index production builds.
const built = await E.buildPlayerTeamIndex(games, {
getTeamRoster: async (id) => {
const r = await (await fetch(`https://statsapi.mlb.com/api/v1/teams/${id}/roster?rosterType=active`)).json();
return (r.roster || []).map((x) => ({ id: x.person && x.person.id, name: x.person && x.person.fullName }));
},
});
const playerTeams = built.index;
// Reconstruct prop shapes. One row per (player, stat, line, book) — the side
// is not part of the dedupe key, so both sides collapse to the same prop.
const seen = new Set();
const props = [];
for (const r of rows) {
const k = `${r.player_key}|${r.stat}|${r.line}|${r.book}`;
if (seen.has(k)) continue;
seen.add(k);
props.push({
__row: props.length,
player: r.player_name || r.player_key,
stat_type: r.stat,
line: r.line,
book: r.book,
game_date: r.game_date,
...teamsFromGameId(r.game_id),
});
}
// The schedule has one game per pair on a non-doubleheader date, so teams
// alone resolve; no game_time is needed and none is reconstructed.
const ident = E.attachEventIdentity(SPORT, props, games, playerTeams);
const oldSet = new Set(props.map(oldKey));
const newSet = new Set(props.map(newKey));
// Which OLD keys collapsed rows that the NEW key separates?
const byOld = new Map();
for (const p of props) {
const k = oldKey(p);
if (!byOld.has(k)) byOld.set(k, new Set());
byOld.get(k).add(newKey(p));
}
const split = [...byOld.entries()].filter(([, s]) => s.size > 1);
console.log(`\n=== DEDUPE PARITY — ${SPORT} ${DATE} ===`);
console.log(`snapshot rows read : ${rows.length}`);
console.log(`reconstructed prop shapes : ${props.length}`);
console.log(`participant evidence : ${built.stats.players} players / ${built.stats.teams} rosters`);
console.log(`canonical event resolution : ${ident.canonical}/${ident.total}`
+ ` (unresolved ${ident.unresolved}, unsupported ${ident.unsupported})`);
console.log(`IMPOSSIBLE-BINDING refusals : ${ident.impossible}`);
if (Object.keys(ident.reasons).length) console.log(` unresolved reasons : ${JSON.stringify(ident.reasons)}`);
console.log(`distinct events : ${new Set(props.map((p) => p.canonical_event_id).filter(Boolean)).size}`);
console.log('');
console.log(`OLD key -> unique candidates : ${oldSet.size}`);
console.log(`NEW key -> unique candidates : ${newSet.size}`);
console.log(`difference : ${newSet.size - oldSet.size}`);
console.log(`OLD keys the NEW key SPLITS : ${split.length}`);
for (const [k, s] of split.slice(0, 5)) {
console.log(` ${k}`);
for (const nk of s) console.log(` -> ${nk}`);
}
console.log('');
if (split.length === 0) {
console.log('VERDICT: NO CANDIDATE-UNIVERSE DIFFERENCE on this slate.');
console.log(' Every old key mapped to exactly one new key, so the event');
console.log(' component changed nothing here. Expected on a non-doubleheader date.');
} else {
console.log('VERDICT: CANDIDATE-UNIVERSE DATA-INTEGRITY CORRECTION.');
console.log(` ${split.length} old key(s) collapsed rows from more than one real event.`);
}
console.log('');
}
if (require.main === module) {
main().then(() => process.exit(0)).catch((e) => { console.error('FAILED:', e.message); process.exit(1); });
}
module.exports = { oldKey, newKey };
src/services/evaluator/gradeFreeze.js
+509
View File
@@ -0,0 +1,509 @@
'use strict';
/**
* gradeFreeze — is the served forecast genuinely untouched by the shadows?
*
* ── THE INVARIANT ────────────────────────────────────────────────────────
* Shadow layers (A5 `factor_inputs`, A6 `would_fire`, chain-v1 `chain_shadow`)
* exist so a challenger can be measured against the served number without
* becoming it. Every one of them is documented as "reaches nothing" — and every
* one of those statements is currently the generator's own account of its own
* code. A5 is the reason that is not good enough: three factors were built,
* correct, wired, described as live for two sessions, and fired on 0 of 596
* rows. Nobody was lying; nobody had measured.
*
* So this module re-derives the invariant from what is actually persisted and
* from what the real code actually emits, by four independent arms:
*
* 1. DIFFERENTIAL perturb the shadow's VALUES, re-run the real serving
* code, and require the served output to be byte-
* identical. Perturbation, not absence: switching a
* shadow OFF only proves its presence is harmless;
* changing what it says proves its values cannot travel.
* 2. STRUCTURAL the served letter and confidence must be exactly the
* functions of the served p_win that they are declared
* to be. A leak that moved the letter without moving the
* number is caught here and nowhere else.
* 3. CO-DETERMINATION rows with an IDENTICAL champion feature vector but a
* DIFFERENT shadow value must carry an identical p_win.
* This is the arm that cannot be satisfied by agreement-
* with-itself: it needs the shadow to actually vary.
* 4. CONTAINMENT every stored shadow block declares itself unservable,
* and no served column carries a shadow-only key.
*
* ── WHAT IS DELIBERATELY *NOT* AN ARM ────────────────────────────────────
* `chain_shadow.counter_p` equals the served `p_win` on every row. That looks
* like a perfect freeze check and it is worthless: `mergeChainShadow` COPIES
* `p_win` into `counter_p`, so the equality is an identity, not evidence. It is
* named here so nobody adds it later mistaking a tautology for a test.
*
* PURE. Takes rows, returns verdicts. No I/O, no database, no clock.
*/
const { VERDICT, arm } = require('./verdict');
/**
* ── FROZEN ANCHOR: the served-grade bands, restated independently ────────
* A deliberate second copy of `servedGrade.BANDS`. Re-deriving the letter from
* the module that produced it would be asking the generator whether it agrees
* with itself. The meta-guard asserts this copy still matches the live table —
* so a band change fails a test and has to be re-affirmed by a human, instead
* of silently redefining what "correct" means for every historical row.
*/
const BANDS = Object.freeze([
Object.freeze({ letter: 'B+', min: 0.780 }),
Object.freeze({ letter: 'B', min: 0.700 }),
Object.freeze({ letter: 'C+', min: 0.640 }),
Object.freeze({ letter: 'C', min: 0.560 }),
Object.freeze({ letter: 'C-', min: 0.480 }),
Object.freeze({ letter: 'D', min: 0.350 }),
Object.freeze({ letter: 'F', min: 0.000 }),
]);
/** Letters the served scale declares structurally unissuable. */
const UNISSUABLE = Object.freeze(['A+', 'A', 'A-']);
/**
* ── FROZEN ANCHOR: tolerances ────────────────────────────────────────────
* Every number a gate is allowed to forgive, in one place, so loosening one is
* a visible edit to a frozen constant rather than a `+ 0.001` buried in a
* comparison. The meta-guard asserts these EXACT values.
*
* MAX_* are counts of violations tolerated, and they are ZERO. A freeze that
* tolerates a leak is not a freeze. The only non-zero number here is the
* minimum evidence a PASS requires.
*/
const TOLERANCES = Object.freeze({
/** Served letter vs the band table: no mismatch is acceptable. */
MAX_LETTER_MISMATCH: 0,
/** Served confidence vs the exact-decimal re-derivation: no mismatch is
* acceptable. Pre-repair rows on the four known half-way values are counted
* separately (see CONFIDENCE_REPAIR), never forgiven as a tolerance here. */
MAX_CONFIDENCE_MISMATCH: 0,
/** Structurally unissuable letters actually emitted. */
MAX_UNISSUABLE_EMITTED: 0,
/** Co-determination groups where the shadow moved and p_win moved with it. */
MAX_CODETERMINATION_VIOLATIONS: 0,
/** Served fields that differ when the shadow's values are perturbed. */
MAX_DIFFERENTIAL_DIVERGENCES: 0,
/** Stored shadow blocks failing to declare themselves unservable. */
MAX_UNDECLARED_SHADOW_BLOCKS: 0,
/**
* A structural arm needs rows to be evidence of anything. Below this the arm
* reports INSUFFICIENT — not PASS. "We checked 3 rows and found nothing" is
* not the same claim as "this holds".
*/
MIN_STRUCTURAL_ROWS: 100,
/**
* The co-determination arm needs the shadow to have ACTUALLY VARIED while the
* champion inputs held still. One such group is a real observation; the arm
* reports how thin it is rather than rounding thinness up to confidence.
*/
MIN_CODETERMINATION_GROUPS: 1,
/**
* The differential arm needs the perturbation to have actually changed the
* shadow. If perturbing produced an identical shadow, the arm proved nothing
* about propagation and must say so.
*/
MIN_PERTURBED_ROWS: 1,
});
/**
* The fields a user, the ledger, or the public record actually consumes.
* A leak matters exactly insofar as it reaches one of these.
*/
const SERVED_FIELDS = Object.freeze([
'p_win', 'grade', 'engine_grade', 'confidence', 'confidence_basis',
'ev_pct', 'projection', 'edge_pct', 'takeable', 'value',
'kelly', 'model_odds', 'fair_odds', 'fair_prob', 'alt_lines',
'refused', 'insufficient_data', 'refusal_reason',
]);
/**
* Keys that exist only to carry shadow evidence. None may appear as, or feed,
* a served field. Listed explicitly so adding a new shadow column is a
* conscious act with a test attached.
*/
const SHADOW_KEYS = Object.freeze([
'factor_inputs', 'chain_shadow', 'would_fire', 'shadow_inputs',
'p_win_isotonic_shadow', 'p_win_calibrated', 'p_win_lowparam',
'chain_p', 'chain_p_over', 'chain_projected_value',
]);
const num = (v) => {
if (v === null || v === undefined || v === '') return null;
const n = Number(v);
return Number.isFinite(n) ? n : null;
};
/** The letter the served number implies, re-derived from the frozen band table. */
function expectedLetter(pWin) {
const p = num(pWin);
if (p === null) return null;
const band = BANDS.find((b) => p >= b.min);
return band ? band.letter : BANDS[BANDS.length - 1].letter;
}
/**
* The confidence the served number implies.
*
* ── THE VERIFIER MIRRORS THE PRODUCER, AND MOVED WITH IT ─────────────────
* This originally reproduced the producer's IEEE-754 `Math.round(p_win * 100)`
* on purpose: re-deriving in exact decimal reported 465 real production rows as
* violations of an invariant they satisfied, because the checker was using
* BETTER arithmetic than the thing it checked. Mirroring is the rule.
*
* On 2026-08-17 the PRODUCER was corrected to exact-decimal round-half-up
* (analyzeViaEngine1.js — 0.565 now yields 57, as a reader intends), so this
* moved in the same commit. If only one of the two had changed, the gate would
* have reported the corrected rows as a leak — which is precisely the failure
* mode this pairing exists to avoid.
*
* The implementation is deliberately a SECOND, INDEPENDENT copy rather than an
* import from the engine: asking the producer to confirm the producer proves
* nothing. A unit test cross-checks the two agree on every boundary case.
*/
function expectedConfidence(pWin) {
const p = num(pWin);
if (p === null) return null;
// p_win is quantised to three decimals by the producer, so its exact value is
// an integer number of thousandths and the rounding becomes integer maths.
const thousandths = Math.round(p * 1000);
return Math.floor((thousandths + 5) / 10);
}
/**
* ── FROZEN ANCHOR: the rounding-repair allowance ─────────────────────────
* Rows graded BEFORE the exact-decimal repair carry the float value and are not
* wrong — they were correct under the arithmetic that produced them. Rows
* graded after carry the repaired value. Both are legitimate, so the check
* accepts either, and ONLY on the four half-way values where the two can
* differ at all. Every other deviation still fails.
*
* This is not a blanket +/-1 tolerance: outside these four p_win values a
* one-off confidence is a real violation and blocks. The allowance is retired
* once no pre-repair rows remain in the scored window, which is a deliberate
* later edit, not a decay.
*/
const CONFIDENCE_REPAIR = Object.freeze({
repaired_at: '2026-08-17',
producer: 'src/services/intelligence/analyzeViaEngine1.js',
reason: 'Math.round(p_win*100) rounded the IEEE-754 artifact (0.565*100 = 56.49999999999999) '
+ 'rather than the value; replaced with exact-decimal round-half-up over integer thousandths',
measured: Object.freeze({
rows_affected: 488,
letters_changed: 0,
p_win_values: Object.freeze([0.145, 0.285, 0.565, 0.575]),
}),
});
/** Is this the known pre-repair value for one of the four half-way cases? */
function isPreRepairConfidence(pWin, stored) {
const p = num(pWin);
const c = num(stored);
if (p === null || c === null) return false;
if (!CONFIDENCE_REPAIR.measured.p_win_values.includes(p)) return false;
// The float result is exactly one below the exact-decimal result on these.
return c === expectedConfidence(p) - 1;
}
/**
* ARM 2 — STRUCTURAL IDENTITY.
*
* The served letter and confidence are declared to be pure functions of the
* served p_win. If a shadow ever reaches the grade surface without passing
* through p_win, this is where it shows up.
*
* @param {Array} rows { p_win, grade, confidence, confidence_basis, refused }
*/
function structuralIdentity(rows = [], opts = {}) {
const scoped = rows.filter((r) => num(r.p_win) !== null && !r.refused && r.grade);
const letterMismatch = [];
const confMismatch = [];
const unissuable = [];
// Rows that fall inside a declared, dated deploy boundary. Counted and
// reported, never silently dropped — see `opts.boundary` below.
let boundaryRows = 0;
const boundaryMismatch = [];
// Rows still carrying the pre-repair float confidence — legitimate, counted.
let preRepairConfidence = 0;
const onBoundary = (r) => {
if (!opts.boundary) return false;
const stamp = String(r[opts.boundary.field] || '');
return stamp.slice(0, 10) === opts.boundary.date;
};
for (const r of scoped) {
const inBoundary = onBoundary(r);
if (inBoundary) boundaryRows += 1;
const exp = expectedLetter(r.p_win);
if (exp !== r.grade) {
const rec = { id: r.id, p_win: num(r.p_win), grade: r.grade, expected: exp };
if (inBoundary) boundaryMismatch.push(rec); else letterMismatch.push(rec);
}
if (UNISSUABLE.includes(r.grade)) {
const rec = { id: r.id, grade: r.grade, p_win: num(r.p_win) };
if (inBoundary) boundaryMismatch.push(rec); else unissuable.push(rec);
}
if (r.confidence_basis === 'p_win') {
const expC = expectedConfidence(r.p_win);
if (num(r.confidence) !== expC) {
// A pre-repair row carrying the float value on one of the four known
// half-way p_win values is counted separately, not as a violation.
if (isPreRepairConfidence(r.p_win, r.confidence)) preRepairConfidence += 1;
else confMismatch.push({ id: r.id, p_win: num(r.p_win), confidence: num(r.confidence), expected: expC });
}
}
}
const numbers = {
rows_scoped: scoped.length,
letter_mismatches: letterMismatch.length,
confidence_mismatches: confMismatch.length,
unissuable_emitted: unissuable.length,
boundary_date: opts.boundary ? opts.boundary.date : null,
boundary_rows: boundaryRows,
boundary_anomalies: boundaryMismatch.length,
pre_repair_confidence_rows: preRepairConfidence,
};
// A BOUNDARY THAT GREW IS NOT A BOUNDARY. The declared exception carries the
// exact count measured when it was declared; anything above it is a new
// violation wearing an old excuse, and it blocks.
if (opts.boundary && boundaryMismatch.length > opts.boundary.max_anomalies) {
return arm('grade_freeze.structural', VERDICT.FAIL,
`the declared ${opts.boundary.date} deploy boundary holds ${boundaryMismatch.length} anomalies, `
+ `above the ${opts.boundary.max_anomalies} recorded when it was declared — this is new, not historical`,
numbers, { samples: { boundary: boundaryMismatch.slice(0, 5) } });
}
if (scoped.length < TOLERANCES.MIN_STRUCTURAL_ROWS) {
return arm('grade_freeze.structural', VERDICT.INSUFFICIENT,
`only ${scoped.length} graded rows in scope (need ${TOLERANCES.MIN_STRUCTURAL_ROWS}) — too few to call this held`,
numbers, { samples: { letterMismatch: letterMismatch.slice(0, 5) } });
}
const violations = letterMismatch.length + confMismatch.length + unissuable.length;
if (violations > 0) {
return arm('grade_freeze.structural', VERDICT.FAIL,
`${letterMismatch.length} letters, ${confMismatch.length} confidences and ${unissuable.length} unissuable grades do not follow from the served p_win`,
numbers, {
samples: {
letterMismatch: letterMismatch.slice(0, 5),
confMismatch: confMismatch.slice(0, 5),
unissuable: unissuable.slice(0, 5),
},
});
}
return arm('grade_freeze.structural', VERDICT.PASS,
`${scoped.length - boundaryRows} graded rows outside the declared boundary: every letter and confidence `
+ `follows exactly from the served p_win`
+ (boundaryRows ? ` (${boundaryRows} rows on the declared ${opts.boundary.date} boundary carry ${boundaryMismatch.length} known anomalies)` : ''),
numbers, { ...(opts.extra || {}) });
}
/**
* ARM 3 — CO-DETERMINATION.
*
* Group rows by the CHAMPION's inputs (the identity plus a hash of the feature
* vector). Within a group the champion cannot produce two answers. So if the
* shadow value varies inside a group and `p_win` also varies, the served number
* moved with something other than the champion — which is a leak.
*
* This arm is the one that cannot be passed by construction. Its power comes
* entirely from groups where the shadow genuinely varied; with none, it reports
* INSUFFICIENT and says how many groups it had.
*
* @param {Array} rows { fingerprint, shadow_value, p_win }
*/
function coDetermination(rows = []) {
const groups = new Map();
for (const r of rows) {
const fp = r.fingerprint;
const sv = num(r.shadow_value);
const pw = num(r.p_win);
if (!fp || sv === null || pw === null) continue;
if (!groups.has(fp)) groups.set(fp, { shadow: new Set(), pwin: new Set(), n: 0 });
const g = groups.get(fp);
g.shadow.add(sv);
g.pwin.add(pw);
g.n += 1;
}
const discriminating = [];
const violations = [];
for (const [fp, g] of groups) {
if (g.shadow.size < 2) continue;
const rec = {
fingerprint: fp,
rows: g.n,
distinct_shadow: g.shadow.size,
distinct_p_win: g.pwin.size,
shadow_spread: Math.max(...g.shadow) - Math.min(...g.shadow),
};
discriminating.push(rec);
if (g.pwin.size > 1) violations.push(rec);
}
const numbers = {
rows: rows.length,
groups: groups.size,
discriminating_groups: discriminating.length,
violations: violations.length,
max_shadow_spread: discriminating.length
? Math.max(...discriminating.map((d) => d.shadow_spread)) : null,
};
if (violations.length > TOLERANCES.MAX_CODETERMINATION_VIOLATIONS) {
return arm('grade_freeze.codetermination', VERDICT.FAIL,
`${violations.length} group(s) where the champion inputs were identical, the shadow moved, and the served p_win moved with it`,
numbers, { samples: violations.slice(0, 5) });
}
if (discriminating.length < TOLERANCES.MIN_CODETERMINATION_GROUPS) {
return arm('grade_freeze.codetermination', VERDICT.INSUFFICIENT,
`${groups.size} groups examined but the shadow never varied inside one — nothing here can distinguish a frozen grade from a leaking one`,
numbers);
}
return arm('grade_freeze.codetermination', VERDICT.PASS,
`${discriminating.length} group(s) where the shadow moved (spread up to ${numbers.max_shadow_spread}) on identical champion inputs: served p_win held constant in every one`,
numbers, { samples: discriminating.slice(0, 5) });
}
/**
* ARM 4 — CONTAINMENT.
*
* A stored shadow block must declare itself unservable IN THE PAYLOAD, and no
* served field may carry a shadow-only key. The declaration matters because the
* data outlives the comment: a later query joining these columns has only what
* the row says about itself.
*
* @param {Array} blocks { id, block, served } block = the stored shadow jsonb
*/
/**
* ── WHAT BLOCKS, AND WHAT IS ONLY REPORTED ───────────────────────────────
* The first live run of this arm found 2,043 A6 `would_fire` blocks carrying a
* multiplier with no `servable:false` — because the two shadow layers were
* built in different sessions and only `chainShadow` adopted the self-declaring
* convention. That is a real gap and it is reported with its exact count.
*
* It does not block, and the distinction is substantive rather than
* convenient: a stored PROBABILITY is the thing that can be mistaken for a
* forecast when someone queries the column a year from now, which is the harm
* the declaration exists to prevent. A multiplier is not a forecast — it cannot
* be read as one — and its own inputs are stored beside it, so it stays
* re-derivable either way. Blocking on it would fail every build from now until
* a writer changes, which is how a real gate gets switched off.
*/
const PROBABILITY_KEYS = Object.freeze(['chain_p', 'chain_p_over', 'p_calibrated']);
const MULTIPLIER_KEYS = Object.freeze(['multiplier']);
function containment(blocks = []) {
const undeclared = [];
const undeclaredMultiplier = [];
const contaminated = [];
for (const b of blocks) {
const block = b.block || {};
const carriesProbability = PROBABILITY_KEYS.some((k) => num(block[k]) !== null);
const carriesMultiplier = MULTIPLIER_KEYS.some((k) => num(block[k]) !== null);
if (carriesProbability && block.servable !== false) {
undeclared.push({ id: b.id, servable: block.servable ?? null, keys: Object.keys(block).slice(0, 8) });
} else if (carriesMultiplier && block.servable !== false) {
undeclaredMultiplier.push({ id: b.id, keys: Object.keys(block).slice(0, 8) });
}
const served = b.served || {};
for (const k of SHADOW_KEYS) {
if (Object.prototype.hasOwnProperty.call(served, k)) {
contaminated.push({ id: b.id, key: k });
}
}
}
const numbers = {
blocks: blocks.length,
undeclared_shadow_blocks: undeclared.length,
served_fields_carrying_shadow_keys: contaminated.length,
// REPORTED, NOT BLOCKING — the standing gap named above.
undeclared_multiplier_blocks: undeclaredMultiplier.length,
};
if (!blocks.length) {
return arm('grade_freeze.containment', VERDICT.INSUFFICIENT,
'no shadow blocks present to inspect — the invariant is not violated, but it is also not demonstrated', numbers);
}
if (undeclared.length > TOLERANCES.MAX_UNDECLARED_SHADOW_BLOCKS || contaminated.length > 0) {
return arm('grade_freeze.containment', VERDICT.FAIL,
`${undeclared.length} shadow block(s) carry a probability without declaring servable:false; `
+ `${contaminated.length} served field(s) carry a shadow-only key`,
numbers, { samples: { undeclared: undeclared.slice(0, 5), contaminated: contaminated.slice(0, 5) } });
}
return arm('grade_freeze.containment', VERDICT.PASS,
`${blocks.length} shadow block(s): none carries an undeclared probability and no served field carries a shadow key`
+ (undeclaredMultiplier.length
? `. STANDING FINDING: ${undeclaredMultiplier.length} A6 would_fire block(s) carry a multiplier without the servable:false declaration chainShadow adopted — reported, not blocking`
: ''),
numbers, undeclaredMultiplier.length ? { samples: { undeclared_multiplier: undeclaredMultiplier.slice(0, 3) } } : {});
}
/**
* ARM 1 — DIFFERENTIAL.
*
* Compare the served output of two runs of the REAL serving code that differ
* ONLY in what the shadow said. Any difference is a leak, by definition.
*
* @param {Array} pairs { id, base, perturbed, shadow_changed }
* base/perturbed are the two served result objects.
*/
function differential(pairs = [], fields = SERVED_FIELDS) {
const divergences = [];
let perturbedCount = 0;
for (const p of pairs) {
if (p.shadow_changed) perturbedCount += 1;
for (const f of fields) {
const a = p.base ? p.base[f] : undefined;
const b = p.perturbed ? p.perturbed[f] : undefined;
const sa = JSON.stringify(a === undefined ? null : a);
const sb = JSON.stringify(b === undefined ? null : b);
if (sa !== sb) divergences.push({ id: p.id, field: f, base: a, perturbed: b });
}
}
const numbers = {
pairs: pairs.length,
fields_compared: fields.length,
shadow_actually_perturbed: perturbedCount,
divergences: divergences.length,
};
if (divergences.length > TOLERANCES.MAX_DIFFERENTIAL_DIVERGENCES) {
return arm('grade_freeze.differential', VERDICT.FAIL,
`${divergences.length} served field(s) changed when only the shadow's values changed — the shadow reaches the served path`,
numbers, { samples: divergences.slice(0, 8) });
}
if (perturbedCount < TOLERANCES.MIN_PERTURBED_ROWS) {
// The perturbation did not take. Reporting PASS here would be the exact
// failure this layer exists to catch, one level up: a check that ran, found
// nothing, and had nothing to find.
return arm('grade_freeze.differential', VERDICT.INSUFFICIENT,
`the perturbation changed the shadow on ${perturbedCount} of ${pairs.length} rows — with the shadow unchanged, identical output proves nothing`,
numbers);
}
return arm('grade_freeze.differential', VERDICT.PASS,
`${pairs.length} rows re-run with the shadow's values perturbed on ${perturbedCount} of them: all ${fields.length} served fields byte-identical`,
numbers);
}
module.exports = {
BANDS, UNISSUABLE, TOLERANCES, SERVED_FIELDS, SHADOW_KEYS,
expectedLetter, expectedConfidence, isPreRepairConfidence, CONFIDENCE_REPAIR,
structuralIdentity, coDetermination, containment, differential,
};
src/services/evaluator/verdict.js
+101
View File
@@ -0,0 +1,101 @@
'use strict';
/**
* verdict — the evaluator's shared vocabulary.
*
* ── WHY THIS IS ITS OWN MODULE ────────────────────────────────────────────
* Every gate in this layer has to answer the same question the same way, and
* the single most dangerous thing a gate can do is turn "I could not check
* this" into "this is fine". That is the shape of the 2026-08-01 settlement
* outage (`{settled:0,pending:0}` was byte-identical to a healthy nothing-to-do)
* and of the `fielding_oaa: 0 rows` wiring bug (a failed feed degrading to an
* honest-looking absence). So INSUFFICIENT is a first-class verdict here, it is
* NEVER folded into PASS, and `combine()` refuses to report an overall PASS
* when any arm could not be measured.
*
* PURE. No I/O, no imports. The gate scripts and the unit tests share it.
*/
const VERDICT = Object.freeze({
/** Measured, and the invariant held. */
PASS: 'PASS',
/** Measured, and the invariant was violated. Blocking. */
FAIL: 'FAIL',
/** Not measurable right now (no DB, no rows, no discriminating sample).
* Not a pass. Blocking only where the caller says the arm is required. */
INSUFFICIENT: 'INSUFFICIENT',
/** The check itself broke. An error is an error — never an empty result. */
ERROR: 'ERROR',
});
/** A verdict that may not be treated as success under any circumstance. */
const NOT_PASSING = Object.freeze([VERDICT.FAIL, VERDICT.INSUFFICIENT, VERDICT.ERROR]);
/**
* Build one arm's result.
*
* `numbers` is mandatory in spirit: a verdict with no measured quantities
* behind it is an attestation, which is the thing this whole layer exists to
* replace. Callers pass what they counted.
*/
function arm(id, verdict, reason, numbers = {}, extra = {}) {
if (!Object.values(VERDICT).includes(verdict)) {
throw new Error(`unknown verdict '${verdict}' for arm '${id}' — add it explicitly rather than passing it through`);
}
return { id, verdict, reason, numbers, ...extra };
}
/**
* Combine arms into one blocking answer.
*
* @param {Array} arms
* @param {object} opts
* - requiredArms: ids that MUST reach PASS. An INSUFFICIENT on a required arm
* is a FAIL of the whole gate: a gate you cannot run is not a gate you
* passed. Arms not in this list may report INSUFFICIENT without blocking,
* and the summary says so out loud rather than rounding it up.
*/
function combine(arms, opts = {}) {
const required = new Set(opts.requiredArms || []);
const failed = arms.filter((a) => a.verdict === VERDICT.FAIL);
const errored = arms.filter((a) => a.verdict === VERDICT.ERROR);
const insufficient = arms.filter((a) => a.verdict === VERDICT.INSUFFICIENT);
const requiredNotPassing = arms.filter(
(a) => required.has(a.id) && a.verdict !== VERDICT.PASS,
);
let verdict = VERDICT.PASS;
let reason = `all ${arms.length} arms passed`;
if (errored.length) {
verdict = VERDICT.ERROR;
reason = `${errored.length} arm(s) errored: ${errored.map((a) => a.id).join(', ')}`;
} else if (failed.length) {
verdict = VERDICT.FAIL;
reason = `${failed.length} arm(s) FAILED: ${failed.map((a) => a.id).join(', ')}`;
} else if (requiredNotPassing.length) {
verdict = VERDICT.FAIL;
reason = `required arm(s) did not pass: ${requiredNotPassing.map((a) => `${a.id}=${a.verdict}`).join(', ')}`;
} else if (insufficient.length) {
// Not a pass, not a block — stated as itself so a green line never means
// "we checked" when what happened was "we could not check".
verdict = VERDICT.INSUFFICIENT;
reason = `${arms.length - insufficient.length} passed; ${insufficient.length} could not be measured: ${insufficient.map((a) => a.id).join(', ')}`;
}
return {
verdict,
reason,
blocking: verdict === VERDICT.FAIL || verdict === VERDICT.ERROR,
counts: {
arms: arms.length,
pass: arms.filter((a) => a.verdict === VERDICT.PASS).length,
fail: failed.length,
insufficient: insufficient.length,
error: errored.length,
},
arms,
};
}
module.exports = { VERDICT, NOT_PASSING, arm, combine };
src/services/event/eventIdentity.js
+331
View File
@@ -0,0 +1,331 @@
'use strict';
/**
* CANONICAL EVENT IDENTITY — a sport-neutral contract, an MLB vertical slice.
*
* -- THE DEFECT THIS CLOSES ----------------------------------------------
* `ledgerService.gameIdFor` derives an event label as `sport:date:away@home`.
* That is a LABEL, not an identity: it carries no occurrence component, so the
* two halves of a doubleheader produce a BYTE-IDENTICAL string.
*
* VERIFIED against the authoritative source, MLB 2026 through 2026-08-27:
* 19 doubleheaders / 38 real games collapse into 19 derived labels.
* Real case 2026-08-17, St. Louis Cardinals @ Cincinnati Reds:
* gamePk 824514 (gameNumber 1, 17:40Z) and 824478 (gameNumber 2, 22:40Z).
* VYNDR recorded ONE id, `mlb:2026-08-17:St.LouisCardinals@CincinnatiReds`,
* holding 171 ledger rows and FOUR distinct starting pitchers (Pallante,
* Mathews, Lowder, Emanuel) — i.e. both games merged into one event.
*
* -- WHY gamePk AND NOT THE ESPN EVENT ID --------------------------------
* `gameBinder` binds through `scheduleService`, which is ESPN. ESPN ids are
* also per-game, but statsapi is the AUTHORITATIVE MLB source, is already
* fetched inside the snapshot pipeline (`snapshotService` calls
* `mlbStatsAdapter.getScheduleWithPitchers`), is free and unlimited, and
* carries `gameNumber` / `doubleHeader` alongside `gamePk` so the occurrence is
* self-describing. Using the source of record avoids a second identity space.
*
* -- SPORT-NEUTRAL CONTRACT, ONE IMPLEMENTED SPORT ------------------------
* The FIELD is generic. Only MLB has a verified resolver. Every other sport
* returns UNSUPPORTED rather than a guessed identifier — an invented id would
* be worse than none, because downstream code would trust it.
*/
const IDENTITY_VERSION = 'evid@1';
/** Where an identity came from. */
const IDENTITY_SOURCE = Object.freeze({
MLB_STATSAPI_GAMEPK: 'MLB_STATSAPI_GAMEPK',
LEGACY_DERIVED: 'LEGACY_DERIVED',
UNKNOWN: 'UNKNOWN',
});
/** How strongly it identifies the event. */
const IDENTITY_METHOD = Object.freeze({
CANONICAL: 'CANONICAL',
LEGACY_DERIVED: 'LEGACY_DERIVED',
UNRESOLVED: 'UNRESOLVED',
UNSUPPORTED_SPORT: 'UNSUPPORTED_SPORT',
});
const SUPPORTED = Object.freeze(['mlb']);
/** Namespaced so two sports' numeric ids can never collide. */
function canonicalEventId(sport, nativeId) {
if (!sport || nativeId === null || nativeId === undefined || nativeId === '') return null;
return `${String(sport).toLowerCase()}:gamepk:${nativeId}`;
}
const refusal = (method, reason) => Object.freeze({
canonical_event_id: null,
event_identity_source: IDENTITY_SOURCE.UNKNOWN,
event_identity_method: method,
event_identity_version: IDENTITY_VERSION,
event_occurrence: null,
reason: reason || null,
});
const norm = (v) => String(v || '').toLowerCase().replace(/[^a-z]/g, '');
/**
* TEAM RESOLUTION — canonical, not fuzzy.
*
* Real prop feeds spell one team several ways: `Cincinnati Reds`,
* `CINReds`, `CIN`. A prefix test fails on all the abbreviated forms
* (`newyorkmets`.startsWith(`nymets`) is false), which is why the first draft
* resolved nothing for them.
*
* So resolution is anchored on `environmentContext.NAME_TO_ABBR` — the
* repository's existing canonical 31-team map — and every spelling is reduced
* to an ABBREVIATION before comparison:
* 1. the flattened full name `cincinnatireds` -> CIN
* 2. the canonical nickname as SUFFIX `cinreds` -> CIN
* 3. the abbreviation itself `cin` -> CIN
*
* Nicknames are matched LONGEST-FIRST because `whitesox` and `redsox` both end
* in `sox`; shortest-first would map every Chicago White Sox prop to Boston.
*
* This narrows candidates only. Choosing BETWEEN two candidates is never done
* by name — that is decided by start time, below.
*/
const { NAME_TO_ABBR } = require('../environmentContext');
const { nameKey } = require('../../utils/playerName');
const TEAM_INDEX = (() => {
const byFlatName = new Map();
const byAbbr = new Map();
const nicknames = [];
for (const [full, abbr] of Object.entries(NAME_TO_ABBR || {})) {
byFlatName.set(norm(full), abbr);
byAbbr.set(norm(abbr), abbr);
const words = String(full).trim().split(/\s+/);
// The nickname is everything after the city. Two-word nicknames
// (`red sox`, `white sox`, `blue jays`) need both trailing words.
for (const take of [2, 1]) {
if (words.length > take) nicknames.push({ nick: norm(words.slice(-take).join('')), abbr });
}
}
nicknames.sort((a, b) => b.nick.length - a.nick.length);
return { byFlatName, byAbbr, nicknames };
})();
/** Any spelling -> canonical abbreviation, or null. */
function teamAbbr(name) {
const f = norm(name);
if (!f) return null;
const exact = TEAM_INDEX.byFlatName.get(f);
if (exact) return exact;
const asAbbr = TEAM_INDEX.byAbbr.get(f);
if (asAbbr) return asAbbr;
for (const { nick, abbr } of TEAM_INDEX.nicknames) {
if (f.endsWith(nick)) return abbr;
}
return null;
}
/**
* Does a schedule game match a prop's team pair?
* Both sides must resolve to a canonical abbreviation and both must agree. An
* unresolvable name matches NOTHING rather than falling back to a text compare.
*/
function teamsMatch(game, homeTeam, awayTeam) {
const gh = teamAbbr(game && game.home && game.home.team);
const ga = teamAbbr(game && game.away && game.away.team);
const ph = teamAbbr(homeTeam);
const pa = teamAbbr(awayTeam);
if (!gh || !ga || !ph || !pa) return false;
return gh === ph && ga === pa;
}
/**
* -- IMPOSSIBLE-BINDING INVARIANT -----------------------------------------
* A player proposition must never be bound to an event his team is not in.
*
* VERIFIED IN PRODUCTION, mlb 2026-08-26. Two props were bound to real games
* their players were not playing in:
*
* Joe Mack (statsapi id 691788, MIAMI MARLINS)
* -> mlb:2026-08-26:LosAngelesDodgers@AtlantaBraves (pinnacle, 03:01 UTC)
* Yandy Diaz (statsapi id 650490, TAMPA BAY RAYS)
* -> mlb:2026-08-26:TexasRangers@ChicagoWhiteSox (pinnacle, 01:00/03:01 UTC)
*
* ROOT CAUSE IS SOURCE MARKET DATA, NOT THE BINDER. `gameMatchesTeams` is
* correctly guarded (empty team names return false), and the stored `game_id`
* is built from the PROP'S OWN home/away fields — so the provider delivered
* these players nested inside the wrong event. Both players' real games had
* already started (Marlins 22:40Z, Rays 17:10Z) while the events they were
* mis-nested into were still open (Braves 23:15Z, White Sox 23:40Z), which is
* why it only appears in the 01:00 and 03:01 UTC cycles and only from one book.
*
* The prop's own team fields are therefore NOT independent evidence — they are
* the wrong event's teams. The check needs the PLAYER'S team from a source that
* cannot be wrong in the same way, and refuses when they disagree.
*
* UNKNOWN TEAM IS NOT A VIOLATION. A player we cannot place is left alone:
* refusing on absent evidence would drop real props for a data gap.
*/
function checkParticipantEvidence(prop, game, playerTeams) {
if (!playerTeams || typeof playerTeams.get !== 'function') return null;
const key = nameKey(prop && (prop.player || prop.player_name));
if (!key) return null;
const teams = playerTeams.get(key);
if (!teams || teams.size === 0) return null; // unknown team -> not a violation
const home = teamAbbr(game && game.home && game.home.team);
const away = teamAbbr(game && game.away && game.away.team);
if (!home || !away) return null; // unreadable event -> not a violation
for (const t of teams) if (t === home || t === away) return null;
return { violated: true, player_teams: [...teams], event_teams: [away, home] };
}
/**
* Build nameKey -> Set(team abbreviation) for the teams playing on a slate.
* Injectable; production supplies the statsapi roster reader. A team whose
* roster cannot be read contributes nothing rather than an empty answer that
* would look like 'this player has no team'.
*/
async function buildPlayerTeamIndex(games, deps) {
const d = deps || {};
const getRoster = d.getTeamRoster;
const index = new Map();
const out = { teams: 0, players: 0, failed: 0 };
if (typeof getRoster !== 'function') return { index, stats: out };
const ids = new Map();
for (const g of Array.isArray(games) ? games : []) {
for (const side of ['home', 'away']) {
const t = g && g[side];
if (t && t.teamId != null) ids.set(t.teamId, t.team);
}
}
for (const [id, name] of ids) {
let roster = null;
try { roster = await getRoster(id); } catch { roster = null; }
if (!Array.isArray(roster)) { out.failed += 1; continue; }
out.teams += 1;
const abbr = teamAbbr(name);
if (!abbr) continue;
for (const p of roster) {
const k = nameKey(p && (p.name || p.fullName || p.player_name));
if (!k) continue;
if (!index.has(k)) index.set(k, new Set());
index.get(k).add(abbr);
out.players += 1;
}
}
return { index, stats: out };
}
/**
* Resolve ONE prop to a canonical MLB event.
*
* -- THE DISAMBIGUATOR IS TIME, AND IT IS REQUIRED -----------------------
* When teams match TWO games the pair alone cannot choose, and choosing wrong
* attributes a claim to the game that did not happen. `game_time` separates
* them (17:40Z vs 22:40Z in the verified case). Without a usable time the
* result is UNRESOLVED — a fracture, never a guess. A false fracture is
* recoverable; a false merge destroys the receipt.
*
* @param {object} prop needs home_team, away_team, optionally game_time
* @param {Array} games mlbStatsAdapter.getScheduleWithPitchers() output
*/
function resolveMlbEvent(prop, games, playerTeams) {
const p = prop || {};
const list = Array.isArray(games) ? games.filter((g) => g && g.gamePk != null) : [];
if (list.length === 0) return refusal(IDENTITY_METHOD.UNRESOLVED, 'no_schedule');
const matches = list.filter((g) => teamsMatch(g, p.home_team, p.away_team));
if (matches.length === 0) return refusal(IDENTITY_METHOD.UNRESOLVED, 'no_team_match');
// Refuse an event the player's own team is not in, BEFORE any other
// disambiguation. Time proximity must never override an impossible
// team/event relationship.
const possible = matches.filter((g) => !checkParticipantEvidence(prop, g, playerTeams));
if (possible.length === 0) {
const ev = checkParticipantEvidence(prop, matches[0], playerTeams);
return Object.freeze({
canonical_event_id: null,
event_identity_source: IDENTITY_SOURCE.UNKNOWN,
event_identity_method: IDENTITY_METHOD.UNRESOLVED,
event_identity_version: IDENTITY_VERSION,
event_occurrence: null,
reason: 'impossible_binding',
evidence: ev || null,
});
}
if (possible.length !== matches.length) matches.length = 0, matches.push(...possible);
let chosen = matches[0];
if (matches.length > 1) {
const t = Date.parse(p.game_time || '');
if (!Number.isFinite(t)) {
return refusal(IDENTITY_METHOD.UNRESOLVED, 'ambiguous_no_game_time');
}
let best = null; let bestDelta = Infinity;
for (const g of matches) {
const gt = Date.parse(g.gameDate || '');
if (!Number.isFinite(gt)) continue;
const d = Math.abs(gt - t);
if (d < bestDelta) { bestDelta = d; best = g; }
}
if (!best) return refusal(IDENTITY_METHOD.UNRESOLVED, 'ambiguous_no_schedule_time');
// A start time that matches neither game within an hour is not a match at
// all. Refusing beats attributing the claim to the nearer wrong game.
if (bestDelta > 60 * 60 * 1000) {
return refusal(IDENTITY_METHOD.UNRESOLVED, 'ambiguous_time_too_far');
}
chosen = best;
}
return Object.freeze({
canonical_event_id: canonicalEventId('mlb', chosen.gamePk),
event_identity_source: IDENTITY_SOURCE.MLB_STATSAPI_GAMEPK,
event_identity_method: IDENTITY_METHOD.CANONICAL,
event_identity_version: IDENTITY_VERSION,
// Self-describing occurrence, straight from the source. Never inferred.
event_occurrence: chosen.gameNumber ?? null,
reason: null,
});
}
/**
* Sport-neutral entry point. Only MLB resolves; everything else is
* UNSUPPORTED_SPORT, which is a truthful state and not a failure.
*/
function resolveEvent(sport, prop, games, playerTeams) {
const sp = String(sport || '').toLowerCase();
if (!SUPPORTED.includes(sp)) return refusal(IDENTITY_METHOD.UNSUPPORTED_SPORT, `sport_${sp || 'unknown'}`);
return resolveMlbEvent(prop, games, playerTeams);
}
/**
* Attach identity to a slate's props IN PLACE, returning counts so a silent
* resolution collapse is visible rather than quietly reintroducing collisions.
*/
function attachEventIdentity(sport, props, games, playerTeams) {
const list = Array.isArray(props) ? props : [];
const out = { total: list.length, canonical: 0, unresolved: 0, unsupported: 0, impossible: 0, reasons: {} };
for (const p of list) {
if (!p) continue;
const r = resolveEvent(sport, p, games, playerTeams);
// Declared on EVERY prop, resolved or not.
p.canonical_event_id = r.canonical_event_id;
p.event_identity_source = r.event_identity_source;
p.event_identity_method = r.event_identity_method;
p.event_identity_version = r.event_identity_version;
p.event_occurrence = r.event_occurrence;
if (r.event_identity_method === IDENTITY_METHOD.CANONICAL) out.canonical += 1;
else if (r.event_identity_method === IDENTITY_METHOD.UNSUPPORTED_SPORT) out.unsupported += 1;
else {
out.unresolved += 1;
if (r.reason === 'impossible_binding') out.impossible += 1;
if (r.reason) out.reasons[r.reason] = (out.reasons[r.reason] || 0) + 1;
}
}
return out;
}
module.exports = {
resolveEvent, resolveMlbEvent, attachEventIdentity, canonicalEventId, teamsMatch, teamAbbr,
checkParticipantEvidence, buildPlayerTeamIndex,
IDENTITY_SOURCE, IDENTITY_METHOD, IDENTITY_VERSION, SUPPORTED,
};
src/services/gradeSlateService.js
+89 -1
View File
@@ -77,13 +77,86 @@ const DEFAULT_TTL = 7200; // 2 hours — matches the spec's grades-cache TTL.
// and before the limit — which makes the graded set byte-identical to what it // and before the limit — which makes the graded set byte-identical to what it
// was before the widening. This gate lifts only when the MLB calibration is // was before the widening. This gate lifts only when the MLB calibration is
// re-run on the consensus ruler and v2 is promoted. // re-run on the consensus ruler and v2 is promoted.
/**
* WHAT DUPLICATES IS THIS FUNCTION SUPPOSED TO REMOVE?
*
* ONE: the same proposition offered by SEVERAL BOOKS. Provider is collapsed on
* purpose — grading the same player/stat/line once per book would multiply the
* work with no new information, and the price anchor is chosen later.
*
* WHAT IT WAS ALSO REMOVING, WRONGLY
*
* The key was `player::stat_type::line` with NO event component, so the SAME
* proposition in TWO DIFFERENT GAMES also collapsed — and first-row-wins
* silently discarded the second real game.
*
* VERIFIED on production: 2026-08-17 St. Louis @ Cincinnati was a doubleheader
* (gamePk 824514 / 824478). VYNDR recorded ONE derived id holding FOUR distinct
* starting pitchers, i.e. both games merged. 19 doubleheaders / 38 games are
* affected across the 2026 season to date.
*
* The key therefore adds the EVENT, and nothing else. Provider stays collapsed
* because collapsing books is the function's actual job.
*
* `canonical_event_id` is used when identity resolved. When it did not, the
* fallback is the legacy derived label, which reproduces the previous behaviour
* exactly — it does not pretend to distinguish what it cannot.
*/
let fractureSeq = 0;
/**
* The EVENT component of the dedupe key.
*
* -- FAIL CLOSED WHERE CANONICAL IDENTITY IS EXPECTED --------------------
* A first version fell back to `legacy:{date}:{away}@{home}` whenever identity
* did not resolve. That is precisely the collision-prone key this whole change
* exists to remove: on a doubleheader it is byte-identical for two real games,
* so an unresolved MLB prop could still merge two events.
*
* So for a sport where canonical identity is EXPECTED (MLB), an unresolved prop
* FRACTURES: it gets a key unique to that row, which cannot merge with anything.
* The cost is that books stop collapsing for that prop, so it may be graded more
* than once. That is a coverage/efficiency cost, and it is the right direction:
*
* UNKNOWN EVENT IDENTITY MAY LOSE COVERAGE.
* IT MAY NOT MERGE TWO REAL EVENTS.
*
* MEASURED: over MLB 2026-08-20..27, 101 of 101 team-pairs resolve from teams
* alone; only a doubleheader pair needs the start time to disambiguate. So the
* fracture path is rare in normal operation and bites only where identity is
* genuinely unknown.
*
* -- SPORTS WITHOUT A RESOLVER ARE UNCHANGED ----------------------------
* WNBA, NBA and the rest have no canonical adapter and never did. Fracturing
* them would multiply their grading load for no safety gain, because their
* identity was never canonical to begin with. They keep the legacy label, which
* is the exact behaviour that existed before this change, and their lineage is
* excluded from the canary for the same reason.
*/
function propositionEventKey(p) {
if (p && p.canonical_event_id) return p.canonical_event_id;
const method = p && p.event_identity_method;
const expectsCanonical = method && method !== 'UNSUPPORTED_SPORT';
if (expectsCanonical) {
// Unique per row: this prop can never share an event key with another.
fractureSeq += 1;
return `unresolved-event:${method}:${fractureSeq}`;
}
const away = String((p && p.away_team) || '').replace(/\s+/g, '');
const home = String((p && p.home_team) || '').replace(/\s+/g, '');
const date = (p && p.game_date) || '';
return `legacy:${date}:${away}@${home}`;
}
function dedupeProps(props, limit) { function dedupeProps(props, limit) {
const seen = new Set(); const seen = new Set();
const out = []; const out = [];
for (const p of props || []) { for (const p of props || []) {
if (!p || !p.player || !p.stat_type || p.line == null) continue; if (!p || !p.player || !p.stat_type || p.line == null) continue;
if (!isModelBook(p.book)) continue; if (!isModelBook(p.book)) continue;
const key = `${p.player}::${p.stat_type}::${p.line}`; const key = `${propositionEventKey(p)}::${p.player}::${p.stat_type}::${p.line}`;
if (seen.has(key)) continue; if (seen.has(key)) continue;
seen.add(key); seen.add(key);
out.push(p); out.push(p);
@@ -128,6 +201,13 @@ async function gradeBestSide(grade, prop, sport, opts = {}) {
game_time: prop.game_time ?? null, game_time: prop.game_time ?? null,
home_team: prop.home_team ?? null, home_team: prop.home_team ?? null,
away_team: prop.away_team ?? null, away_team: prop.away_team ?? null,
// CANONICAL EVENT IDENTITY rides with the grade so retention and lineage
// key on the real event rather than re-deriving a label from team names.
canonical_event_id: prop.canonical_event_id ?? null,
event_identity_source: prop.event_identity_source ?? null,
event_identity_method: prop.event_identity_method ?? null,
event_identity_version: prop.event_identity_version ?? null,
event_occurrence: prop.event_occurrence ?? null,
}; };
const sides = await Promise.all([ const sides = await Promise.all([
Promise.resolve() Promise.resolve()
@@ -155,6 +235,14 @@ async function gradeBestSide(grade, prop, sport, opts = {}) {
// Strip the internal retention fields so they never reach a cache or payload. // Strip the internal retention fields so they never reach a cache or payload.
delete winner._features; delete winner._features;
delete winner._grade_11; delete winner._grade_11;
// PUBLICATION SIGNAL. `onGraded` above fired with both sides before any
// filtering; this fires ONLY for the side that becomes the served Read, so
// retention can tell a published claim from a captured model state. Measured:
// 64.9% of captured rows describe a state no user ever saw.
if (typeof opts.onPublished === 'function') {
try { opts.onPublished(base, winner); } catch { /* never breaks the slate */ }
}
// CARRY THE GAME (2026-08-01). The legacy grade shape drops home/away, so by // CARRY THE GAME (2026-08-01). The legacy grade shape drops home/away, so by
// the time the challenger runs, nothing on the grade says WHICH GAME it is — // the time the challenger runs, nothing on the grade says WHICH GAME it is —
// measured: `team` was null on 416/416 stored grades, so the park/weather // measured: `team` was null on 416/416 stored grades, so the park/weather
src/services/ledgerService.js
+11 -2
View File
@@ -244,7 +244,7 @@ function archetypeVectorOf(g) {
return null; return null;
} }
function rowsFromSnapshot(sport, grades, oddsProps, nowIso) { function rowsFromSnapshot(sport, grades, oddsProps, nowIso, lineageIndex) {
const sp = String(sport || '').toLowerCase(); const sp = String(sport || '').toLowerCase();
let skippedUnbound = 0; let skippedUnbound = 0;
// TWO indexes, two roles — see indexProps. `byKey` supplies GAME FACTS from // TWO indexes, two roles — see indexProps. `byKey` supplies GAME FACTS from
@@ -375,6 +375,15 @@ function rowsFromSnapshot(sport, grades, oddsProps, nowIso) {
ruler_version: CURRENT_RULER_VERSION, ruler_version: CURRENT_RULER_VERSION,
game_id: gameIdFor(sp, prop, gameDate), game_id: gameIdFor(sp, prop, gameDate),
game_date: gameDate, game_date: gameDate,
// SHADOW LINEAGE LINKAGE. Which logical Read this receipt belongs to.
// Declared on EVERY row (PostgREST builds the bulk insert from the first
// row's shape), null when lineage did not resolve — unknown, never
// guessed. Read by nothing; settlement is unaffected.
read_id: lineageIndex
? (lineageIndex.get([sp, gameDate, nameKey(player), stat, side,
Number(line).toFixed(6)].join('|')) || null)
: null,
read_revision_id: null,
}); });
} }
if (skippedUnbound > 0) { if (skippedUnbound > 0) {
@@ -392,7 +401,7 @@ async function recordPipelineGrades(sport, grades, oddsProps, opts = {}) {
if (!opts.sb && !isConfigured()) return { skipped: 'supabase not configured', written: 0 }; if (!opts.sb && !isConfigured()) return { skipped: 'supabase not configured', written: 0 };
const sb = opts.sb || defaultClient(); const sb = opts.sb || defaultClient();
const nowIso = (opts.now || (() => new Date().toISOString()))(); const nowIso = (opts.now || (() => new Date().toISOString()))();
const rows = rowsFromSnapshot(sport, grades, oddsProps, nowIso); const rows = rowsFromSnapshot(sport, grades, oddsProps, nowIso, opts.lineageIndex || null);
if (rows.length === 0) return { written: 0 }; if (rows.length === 0) return { written: 0 };
let written = 0; let written = 0;
for (let i = 0; i < rows.length; i += UPSERT_CHUNK) { for (let i = 0; i < rows.length; i += UPSERT_CHUNK) {
src/services/read/readLineage.js
+602
View File
@@ -0,0 +1,602 @@
'use strict';
/**
* READ LINEAGE — the append-only chronology of what VYNDR published.
*
* ── THE FINDING THIS IS BUILT ON ─────────────────────────────────────────
* VYNDR already stores immutable published claims. `model_snapshots` is
* upserted with `ignoreDuplicates: true` on
* (snapshot_id, player_key, stat, line, side), so a row is written once per
* snapshot cycle and NEVER overwritten; `captured_at` is NOT NULL, and
* `model_version` / `code_sha` / `ruler_version` record provenance. The only
* post-insert write is settlement (outcome, actual_value, settled_at), which is
* not part of the claim.
*
* MEASURED (mlb, 2026-08-26): 5,376 props, 4,286 with more than one row, avg
* 2.42 rows per prop, and 183 props whose GRADE CHANGED across them.
*
* So the immutable revisions exist. What was missing is LINEAGE:
* - a stable logical Read identity linking a prop's rows across cycles,
* - an explicit supersession pointer,
* - a way for a receipt to name the exact revision it settles.
*
* This module is that linkage. It does NOT create a second claim store, and it
* does not create a grade-history table — the claim already lives on the row.
*
* ── CAPTURE IS NOT REVISION, AND CONFLATING THEM WOULD FABRICATE HISTORY ─
* Most cycles re-capture an unchanged claim. On the measured date only 183 of
* 5,376 props actually changed. If every capture became a "revision", the
* chronology would report ~13,000 published claims where roughly 183 occurred.
* A row is a REVISION only when its claim DIGEST differs from the standing one;
* otherwise it is a RECAPTURE of the same published state and is labelled so.
*
* ── IDENTITY MUST SURVIVE UNSTABLE game_id SPELLING ──────────────────────
* The same real game appears as `mlb:2026-08-22:DetroitTigers@KansasCityRoyals`
* from one feed and as an abbreviated spelling from another. Logical Read
* identity therefore does NOT include the raw game_id.
*
* MEASURED over 30,746 identity groups: 1,328 carried more than one game_id
* spelling, and only 18 carried genuinely different team pairs (real distinct
* events — e.g. `jac caglianone` on 2026-08-22 appearing under both
* DetroitTigers@KansasCityRoyals and MinnesotaTwins@SanDiegoPadres). The prefix
* rule below reconciles 1,310 of the 1,328 and isolates exactly those 18, which
* MUST stay separate Reads.
*/
const crypto = require('crypto');
const { SERVED_FIELDS } = require('../evaluator/gradeFreeze');
const LINEAGE_VERSION = 'lin@1';
/**
* ── CLAIM SCHEMA VERSIONING ──────────────────────────────────────────────
* A digest is only interpretable against the claim definition that produced
* it. `SERVED_FIELDS` will change; when it does, digests computed afterwards
* mean something different from digests computed before, and nothing in the
* bytes says so.
*
* This bit us already: the first digest listed only `locked_odds` while
* `model_snapshots` stores `over_odds`/`under_odds`, so it was blind to price
* on exactly the rows it digested. That was caught by a replay crashing on a
* missing column — i.e. by luck. A version stamp is what makes the next such
* change visible instead of silent.
*
* Both are stored on every row. A historical digest is NEVER recomputed under a
* newer definition and claimed to be original.
*/
const CLAIM_SCHEMA_VERSION = 'claim@1';
const DIGEST_ALGORITHM_VERSION = 'sha256-json-sorted@1';
/** What a row is, relative to the chronology. */
const LINEAGE_ACTION = Object.freeze({
ORIGIN: 'ORIGIN', // the first published state of a logical Read
REVISION: 'REVISION', // a materially different claim; supersedes the prior
RECAPTURE: 'RECAPTURE', // same claim, observed again — NOT a new revision
FORK_DETECTED: 'FORK_DETECTED', // two writers superseded the same parent
});
/** Honest states for rows whose provenance predates lineage. */
const LINEAGE_STATE = Object.freeze({
LIVE: 'LIVE',
LEGACY_UNVERIFIED: 'LEGACY_UNVERIFIED',
});
/**
* Normalise a derived game_id into a comparable team pair.
* Returns null when the string is not in the derived shape — a null is honest;
* a guessed pair would silently merge two events.
*/
function eventFingerprint(gameId) {
const s = String(gameId || '');
const tail = s.split(':').pop();
if (!tail || !tail.includes('@')) return null;
const [awayRaw, homeRaw] = tail.split('@');
const norm = (v) => String(v || '').toLowerCase().replace(/[^a-z]/g, '');
const away = norm(awayRaw);
const home = norm(homeRaw);
if (!away || !home) return null;
return { away, home };
}
/** One side matches if either spelling is a prefix of the other. */
function sideMatches(a, b) {
if (!a || !b) return false;
return a === b || a.startsWith(b) || b.startsWith(a);
}
/**
* Are two game_id spellings the same real event?
* Unknown fingerprints are NOT assumed equal — an unparseable id cannot be
* proven to match anything, and merging on a guess is the failure this exists
* to prevent.
*/
function sameEvent(gameIdA, gameIdB) {
if (gameIdA === gameIdB) return true;
const a = eventFingerprint(gameIdA);
const b = eventFingerprint(gameIdB);
if (!a || !b) return false;
return sideMatches(a.away, b.away) && sideMatches(a.home, b.home);
}
const norm = (v) => (v === undefined ? null : v);
/**
* The digest of the MATERIAL CLAIM.
*
* Built from `gradeFreeze.SERVED_FIELDS` — the repository's existing canonical
* definition of what a user, the ledger and the public record actually consume.
* Reusing it means the claim the lineage tracks cannot drift from the claim the
* evaluator polices; a new served field joins both at once.
*
* The market terms the claim is expressed against (line, side, book, odds) are
* included because changing any of them changes what the user was told.
*/
/**
* Both odds spellings are listed because the two stores name price differently:
* `ledger_entries` has `locked_odds`, `model_snapshots` has `over_odds` /
* `under_odds`. Listing only one made the digest BLIND TO PRICE on exactly the
* rows it digests — a price move would have read as an unchanged claim. A row
* supplies whichever fields it has; the rest are null on both sides of any
* comparison, so the digest stays stable within a store.
*/
const CLAIM_MARKET_FIELDS = Object.freeze([
'line', 'side', 'book', 'locked_odds', 'over_odds', 'under_odds',
]);
/**
* ── THE CLAIM, CLASSIFIED ────────────────────────────────────────────────
* "Did price count as a revision?" is the wrong question because it has no
* single answer. VYNDR's BELIEF can hold still while the MARKET moves, and the
* composed PUBLISHED state changes either way. A chronology that cannot tell
* those apart reports 4,206 price moves and 301 belief moves as one undifferen-
* tiated number.
*
* So every claim field carries a class, and the change type is derived from
* WHICH classes moved.
*/
const FIELD_CLASS = Object.freeze({
BELIEF: 'BELIEF',
MARKET: 'MARKET',
COMPARISON: 'COMPARISON',
PROVENANCE: 'PROVENANCE',
PRESENTATION: 'PRESENTATION',
});
const CLAIM_FIELD_CLASSES = Object.freeze({
// What VYNDR independently believes will happen.
p_win: FIELD_CLASS.BELIEF,
projection: FIELD_CLASS.BELIEF,
grade: FIELD_CLASS.BELIEF,
engine_grade: FIELD_CLASS.BELIEF,
confidence: FIELD_CLASS.BELIEF,
fair_prob: FIELD_CLASS.BELIEF,
fair_odds: FIELD_CLASS.BELIEF,
model_odds: FIELD_CLASS.BELIEF,
// NOTE: canonical_event_id is deliberately NOT a claim field. Correcting a
// previously-derived event label is an IDENTITY correction, not a change of
// opinion or of price — classifying it as BELIEF would report every identity
// repair as VYNDR changing its mind.
//
// What the market offered.
line: FIELD_CLASS.MARKET,
side: FIELD_CLASS.MARKET,
book: FIELD_CLASS.MARKET,
locked_odds: FIELD_CLASS.MARKET,
over_odds: FIELD_CLASS.MARKET,
under_odds: FIELD_CLASS.MARKET,
// Belief measured against the market — moves when either side moves, which
// is why it is its own class and not evidence of a belief change.
ev_pct: FIELD_CLASS.COMPARISON,
edge_pct: FIELD_CLASS.COMPARISON,
value: FIELD_CLASS.COMPARISON,
kelly: FIELD_CLASS.COMPARISON,
takeable: FIELD_CLASS.COMPARISON,
// How the claim was produced.
confidence_basis: FIELD_CLASS.PROVENANCE,
// What the user was shown about refusal / alternatives.
alt_lines: FIELD_CLASS.PRESENTATION,
refused: FIELD_CLASS.PRESENTATION,
insufficient_data: FIELD_CLASS.PRESENTATION,
refusal_reason: FIELD_CLASS.PRESENTATION,
});
/** Why the published state advanced. Ordinal says WHEN; this says WHY. */
const CHANGE_TYPE = Object.freeze({
INITIAL_PUBLICATION: 'INITIAL_PUBLICATION',
BELIEF_CHANGE: 'BELIEF_CHANGE',
MARKET_REPRICE: 'MARKET_REPRICE',
MARKET_LINE_CHANGE: 'MARKET_LINE_CHANGE',
COMPARISON_CHANGE: 'COMPARISON_CHANGE',
PROVENANCE_CHANGE: 'PROVENANCE_CHANGE',
PRESENTATION_CHANGE: 'PRESENTATION_CHANGE',
MIXED_CHANGE: 'MIXED_CHANGE',
NO_MATERIAL_PUBLISHED_CHANGE: 'NO_MATERIAL_PUBLISHED_CHANGE',
});
/** Which claim fields differ between two published states, and their classes. */
function classifyChange(prev, next) {
const changed = [];
const classes = new Set();
for (const f of [...CLAIM_MARKET_FIELDS, ...SERVED_FIELDS]) {
const a = norm(prev ? prev[f] : null);
const b = norm(next ? next[f] : null);
const same = (typeof a === 'number' && typeof b === 'number')
? a === b : String(a) === String(b);
if (!same) {
changed.push(f);
classes.add(CLAIM_FIELD_CLASSES[f] || FIELD_CLASS.PRESENTATION);
}
}
if (changed.length === 0) {
return Object.freeze({
change_type: CHANGE_TYPE.NO_MATERIAL_PUBLISHED_CHANGE,
changed_fields: Object.freeze([]), changed_classes: Object.freeze([]),
belief_changed: false, market_changed: false,
});
}
const has = (c) => classes.has(c);
const beliefChanged = has(FIELD_CLASS.BELIEF);
const marketChanged = has(FIELD_CLASS.MARKET);
// COMPARISON alone is derived, so it is not promoted to a belief change.
const substantive = [...classes].filter((c) => c !== FIELD_CLASS.COMPARISON);
let type;
if (substantive.length === 0) type = CHANGE_TYPE.COMPARISON_CHANGE;
else if (substantive.length > 1) type = CHANGE_TYPE.MIXED_CHANGE;
else if (beliefChanged) type = CHANGE_TYPE.BELIEF_CHANGE;
else if (marketChanged) {
// A line move and a reprice are different market events and are not merged.
type = changed.includes('line') ? CHANGE_TYPE.MARKET_LINE_CHANGE : CHANGE_TYPE.MARKET_REPRICE;
} else if (has(FIELD_CLASS.PROVENANCE)) type = CHANGE_TYPE.PROVENANCE_CHANGE;
else type = CHANGE_TYPE.PRESENTATION_CHANGE;
return Object.freeze({
change_type: type,
changed_fields: Object.freeze(changed),
changed_classes: Object.freeze([...classes]),
belief_changed: beliefChanged,
market_changed: marketChanged,
});
}
function claimDigest(row) {
const r = row || {};
const material = {};
for (const f of [...CLAIM_MARKET_FIELDS, ...SERVED_FIELDS].sort()) {
const v = norm(r[f]);
// Numbers are stringified at fixed precision so 1.5 and "1.5" cannot
// produce two digests for one claim.
material[f] = typeof v === 'number' ? v.toFixed(6) : v;
}
return crypto.createHash('sha256').update(JSON.stringify(material)).digest('hex').slice(0, 32);
}
/**
* The natural key of a logical Read, EXCLUDING event spelling.
* Event compatibility is decided separately by `sameEvent`, because it is a
* fuzzy match and a hash cannot express one.
*/
/**
* -- EVENT IDENTITY METHOD, RECORDED NOT ASSUMED --------------------------
* A heuristic identity must carry its method, version and a refusal state
* rather than passing as truth.
*
* CANONICAL_EVENT_ID a provider-native event id was available and used.
* DERIVED_TEAM_PAIR identity came from `sport:date:away@home` plus the
* prefix rule. This is the CURRENT state for every row.
*/
const IDENTITY_METHOD = Object.freeze({
CANONICAL_EVENT_ID: 'CANONICAL_EVENT_ID',
DERIVED_TEAM_PAIR: 'DERIVED_TEAM_PAIR',
});
const IDENTITY_VERSION = 'ident@1';
/**
* A provider-native event occurrence, if the row carries one.
*
* -- WHY THIS HOOK EXISTS AND IS CURRENTLY ALWAYS NULL --------------------
* `ledgerService.gameIdFor` emits `sport:date:away@home` with NO game number,
* so the two halves of a doubleheader produce a BYTE-IDENTICAL id. No rule
* over that string can separate them, and `model_snapshots` carries no other
* event column.
*
* `mlbStatsAdapter` DOES expose `gamePk` (the canonical MLB game id), but it
* is not threaded onto the prop or the retention row. When it is, this reads
* it and the natural key separates the two games automatically.
*
* Until then identity is DERIVED_TEAM_PAIR and the doubleheader case is a
* reported limit, never a silent merge.
*/
function eventOccurrence(row) {
const r = row || {};
const v = r.canonical_event_id ?? r.game_pk ?? r.gamePk ?? r.event_id ?? null;
return v === null || v === undefined || v === '' ? null : String(v);
}
function identityMethod(row) {
return eventOccurrence(row)
? IDENTITY_METHOD.CANONICAL_EVENT_ID
: IDENTITY_METHOD.DERIVED_TEAM_PAIR;
}
function readNaturalKey(row) {
const r = row || {};
// A canonical occurrence, when present, is PART OF IDENTITY -- that is what
// makes two doubleheader games two Reads rather than one.
const occ = eventOccurrence(r);
const parts = [
String(r.sport || '').toLowerCase(),
String(r.game_date || ''),
String(r.player_key || ''),
String(r.stat || '').toLowerCase(),
String(r.side || '').toLowerCase(),
r.line === null || r.line === undefined ? '' : Number(r.line).toFixed(6),
];
if (parts.some((p) => p === '')) return null; // incomplete identity ⇒ no key
return occ ? `${parts.join('|')}|#${occ}` : parts.join('|');
}
/**
* Resolve where a candidate row belongs in the chronology.
*
* @param {object} spec
* candidate the row being written (needs sport/game_date/player_key/stat/
* side/line/game_id/captured_at + the claim fields)
* existing rows already in this logical Read's lineage, any order. Each
* needs: id, read_id, game_id, claim_digest, revision_ordinal,
* lineage_action, captured_at.
* @returns {object} never throws
*/
function resolveLineage(spec) {
const s = spec || {};
const cand = s.candidate || {};
const key = readNaturalKey(cand);
if (!key) {
return Object.freeze({
ok: false, refused: 'incomplete_read_identity',
lineage_version: LINEAGE_VERSION,
});
}
if (!cand.captured_at) {
return Object.freeze({ ok: false, refused: 'no_capture_clock', lineage_version: LINEAGE_VERSION });
}
// ── PUBLICATION GATE ────────────────────────────────────────────────────
// A capture is not a publication. `onGraded` fires with BOTH sides, graded
// AND refused, before any filtering; only the higher-confidence graded side
// becomes the served Read. MEASURED on mlb 2026-08-26: 8,443 of 13,012
// captured rows (64.9%) describe a state no user was ever shown.
//
// Publication chronology therefore advances ONLY for rows the collector's
// publication signal marked. An unmarked row is retained as model history
// with NULL lineage — truthful, and not a published claim.
if (cand.published !== true) {
return Object.freeze({
ok: false, refused: 'not_published', lineage_version: LINEAGE_VERSION,
});
}
const digest = claimDigest(cand);
const stamp = {
claim_schema_version: CLAIM_SCHEMA_VERSION,
digest_algorithm_version: DIGEST_ALGORITHM_VERSION,
};
// Only rows whose EVENT is compatible belong to this Read. Two genuinely
// different team pairs on one date are two Reads, not one.
const candOcc = eventOccurrence(cand);
const family = (Array.isArray(s.existing) ? s.existing : []).filter((r) => {
if (!r) return false;
const rOcc = eventOccurrence(r.claim || r);
// When BOTH sides carry a canonical occurrence it is the authority, and
// the team-name heuristic is not consulted at all.
if (candOcc && rOcc) return candOcc === rOcc;
return sameEvent(r.game_id, cand.game_id);
});
if (family.length === 0) {
return Object.freeze({
ok: true,
lineage_version: LINEAGE_VERSION,
action: LINEAGE_ACTION.ORIGIN,
read_id: s.mintReadId ? s.mintReadId() : null,
read_natural_key: key,
claim_digest: digest,
revision_ordinal: 0,
supersedes_id: null,
lineage_state: LINEAGE_STATE.LIVE,
change_type: CHANGE_TYPE.INITIAL_PUBLICATION,
identity_method: identityMethod(cand),
identity_version: IDENTITY_VERSION,
...stamp,
});
}
const readId = family[0].read_id || null;
// The standing published state = the highest-ordinal REVISION/ORIGIN row.
// Recaptures do not advance the chain, so they are not candidates for it.
const chain = family
.filter((r) => r.lineage_action === LINEAGE_ACTION.ORIGIN || r.lineage_action === LINEAGE_ACTION.REVISION)
.sort((a, b) => (a.revision_ordinal - b.revision_ordinal)
|| String(a.captured_at).localeCompare(String(b.captured_at)));
if (chain.length === 0) {
// Rows exist but none carries a chain position — legacy data. Do not invent
// an ordering for it; attach the candidate as an origin and say the history
// before it is unverified.
return Object.freeze({
ok: true,
lineage_version: LINEAGE_VERSION,
action: LINEAGE_ACTION.ORIGIN,
read_id: readId || (s.mintReadId ? s.mintReadId() : null),
read_natural_key: key,
claim_digest: digest,
revision_ordinal: 0,
supersedes_id: null,
lineage_state: LINEAGE_STATE.LEGACY_UNVERIFIED,
change_type: CHANGE_TYPE.INITIAL_PUBLICATION,
...stamp,
});
}
const head = chain[chain.length - 1];
// IDEMPOTENCY + NO-OP, one rule. An identical claim is the SAME published
// state however many times it is observed or retried.
if (head.claim_digest === digest) {
return Object.freeze({
ok: true,
lineage_version: LINEAGE_VERSION,
action: LINEAGE_ACTION.RECAPTURE,
read_id: readId,
read_natural_key: key,
claim_digest: digest,
// The recapture belongs to the standing revision; it does not advance it.
revision_ordinal: head.revision_ordinal,
supersedes_id: null,
recaptures_id: head.id,
lineage_state: LINEAGE_STATE.LIVE,
change_type: CHANGE_TYPE.NO_MATERIAL_PUBLISHED_CHANGE,
...stamp,
});
}
// ── WHY THERE IS NO IN-MEMORY FORK CHECK HERE ──────────────────────────
// A first draft refused when `family` already contained a row superseding the
// head. That check is unreachable: the head is BY DEFINITION the highest
// ordinal, so anything superseding it would already be the head. A resolver
// that can see its rival is not racing it — it appends after it, correctly.
//
// The real race is two resolvers that each read the same `existing` before
// either wrote. Neither can see the other, so no amount of in-memory logic
// detects it. That is enforced in the DATABASE by a UNIQUE index on
// supersedes_id (migration 045): only one row may supersede a given parent,
// so the second writer's insert fails loudly instead of forking history.
// `chronology()` also reports a fork after the fact.
// WHY the published state advanced, not merely THAT it did.
const delta = classifyChange(head.claim || head, cand);
return Object.freeze({
ok: true,
lineage_version: LINEAGE_VERSION,
action: LINEAGE_ACTION.REVISION,
read_id: readId,
read_natural_key: key,
claim_digest: digest,
revision_ordinal: head.revision_ordinal + 1,
supersedes_id: head.id,
lineage_state: LINEAGE_STATE.LIVE,
change_type: delta.change_type,
identity_method: identityMethod(cand),
identity_version: IDENTITY_VERSION,
changed_fields: delta.changed_fields,
belief_changed: delta.belief_changed,
market_changed: delta.market_changed,
...stamp,
});
}
/**
* Walk a Read's rows into its published chronology.
* Recaptures are attached to the revision they re-observed rather than dropped
* — they are evidence of when a claim was still standing.
*/
function chronology(rows) {
const list = (Array.isArray(rows) ? rows : []).filter(Boolean);
if (list.length === 0) return Object.freeze({ ok: false, reason: 'no rows', revisions: Object.freeze([]) });
const ids = new Set(list.map((r) => String(r.read_id)));
if (ids.size > 1) {
return Object.freeze({ ok: false, reason: 'rows span multiple read_ids', revisions: Object.freeze([]) });
}
const chain = list
.filter((r) => r.lineage_action === LINEAGE_ACTION.ORIGIN || r.lineage_action === LINEAGE_ACTION.REVISION)
.sort((a, b) => a.revision_ordinal - b.revision_ordinal);
const origins = chain.filter((r) => r.lineage_action === LINEAGE_ACTION.ORIGIN);
if (origins.length !== 1) {
return Object.freeze({ ok: false, reason: `expected exactly 1 ORIGIN, found ${origins.length}`, revisions: Object.freeze([]) });
}
for (let i = 1; i < chain.length; i += 1) {
if (String(chain[i].supersedes_id) !== String(chain[i - 1].id)) {
return Object.freeze({ ok: false, reason: `broken supersession at ordinal ${chain[i].revision_ordinal}`, revisions: Object.freeze([]) });
}
}
const recaptures = list.filter((r) => r.lineage_action === LINEAGE_ACTION.RECAPTURE);
return Object.freeze({
ok: true, reason: null,
read_id: chain[0] ? chain[0].read_id : null,
revisions: Object.freeze(chain),
original: chain[0] || null,
current: chain[chain.length - 1] || null,
revision_count: Math.max(0, chain.length - 1),
recapture_count: recaptures.length,
});
}
/** The revision standing at an instant — "what were we saying at 7pm?" */
function revisionAt(rows, instant) {
const c = chronology(rows);
if (!c.ok) return null;
let found = null;
for (const r of c.revisions) {
if (String(r.captured_at) <= String(instant)) found = r;
else break;
}
return found;
}
/**
* ── SETTLEMENT / CLAIM OWNERSHIP BOUNDARY ────────────────────────────────
* `model_snapshots` legitimately receives post-hoc evaluation after the event.
* That is enrichment of a historical row, not a change to what was claimed.
*
* The boundary is stated here rather than left implicit, because both are
* columns on the same table and nothing else distinguishes them. A settle pass
* that touched a claim-owned column would silently rewrite a receipt.
*/
const POST_HOC_FIELDS = Object.freeze([
'outcome', 'actual_value', 'settled_at', 'settlement_source',
'settlement_version', 're_settled_at', 'settle_attempts',
]);
/** Every field the published claim owns. Immutable after publication. */
function claimOwnedFields() {
return Object.freeze([
...CLAIM_MARKET_FIELDS,
...SERVED_FIELDS,
'read_id', 'read_natural_key', 'claim_digest', 'revision_ordinal',
'supersedes_id', 'lineage_action', 'change_type',
'claim_schema_version', 'digest_algorithm_version',
'captured_at', 'model_version', 'code_sha', 'published',
]);
}
/**
* Is a proposed post-event update confined to evaluation fields?
* Returns the violating keys rather than a bare boolean — a guard that cannot
* name what it caught is hard to act on.
*/
function settlementUpdateIsSafe(patch) {
const keys = Object.keys(patch || {});
const owned = new Set(claimOwnedFields());
const violations = keys.filter((k) => owned.has(k));
return Object.freeze({
safe: violations.length === 0,
violations: Object.freeze(violations),
// A key that is neither claim-owned nor a known evaluation field is
// reported: unrecognised is not the same as safe.
unrecognised: Object.freeze(keys.filter((k) => !owned.has(k) && !POST_HOC_FIELDS.includes(k))),
});
}
module.exports = {
resolveLineage, chronology, revisionAt, classifyChange,
claimDigest, readNaturalKey, eventFingerprint, sameEvent,
LINEAGE_ACTION, LINEAGE_STATE, LINEAGE_VERSION, CLAIM_MARKET_FIELDS,
CLAIM_SCHEMA_VERSION, DIGEST_ALGORITHM_VERSION,
CLAIM_FIELD_CLASSES, FIELD_CLASS, CHANGE_TYPE,
IDENTITY_METHOD, IDENTITY_VERSION, eventOccurrence, identityMethod,
POST_HOC_FIELDS, claimOwnedFields, settlementUpdateIsSafe,
};
src/services/retentionService.js
+436 -1
View File
@@ -88,6 +88,10 @@ function boolOrNull(v) {
*/ */
function rowsFromSides(base, sides, ctx = {}) { function rowsFromSides(base, sides, ctx = {}) {
const rows = []; const rows = [];
// `published` is declared on every row and defaults FALSE. A capture is not a
// publication until the winner signal says so; defaulting true would assert
// that 65% of these rows were shown to a user.
const list = Array.isArray(sides) ? sides : []; const list = Array.isArray(sides) ? sides : [];
for (const s of list) { for (const s of list) {
if (!s) continue; if (!s) continue;
@@ -143,6 +147,17 @@ function rowsFromSides(base, sides, ctx = {}) {
archetype: s.archetype || null, archetype: s.archetype || null,
refused, refused,
// Set TRUE only by the collector's publication signal (see createCollector).
published: false,
// CANONICAL EVENT IDENTITY. Declared on every row; null when the sport
// has no resolver or the event could not be resolved. `game_id` below
// stays as the LEGACY DERIVED label — useful for diagnostics and
// compatibility, never authoritative for identity.
canonical_event_id: (s.canonical_event_id ?? base.canonical_event_id) ?? null,
event_identity_source: (s.event_identity_source ?? base.event_identity_source) ?? null,
event_identity_method: (s.event_identity_method ?? base.event_identity_method) ?? null,
event_identity_version: (s.event_identity_version ?? base.event_identity_version) ?? null,
event_occurrence: (s.event_occurrence ?? base.event_occurrence) ?? null,
refusal_reason: refused refusal_reason: refused
? (s.suppressed_reason || (s.insufficient_data ? 'insufficient_data' : 'no_grade')) ? (s.suppressed_reason || (s.insufficient_data ? 'insufficient_data' : 'no_grade'))
: null, : null,
@@ -171,16 +186,397 @@ function rowsFromSides(base, sides, ctx = {}) {
/** A collector to hand to gradeAndCacheSlate's `onGraded` hook. */ /** A collector to hand to gradeAndCacheSlate's `onGraded` hook. */
function createCollector(ctx) { function createCollector(ctx) {
const rows = []; const rows = [];
// Index by (player|stat|line|side) so the publication signal can find the row
// it already collected without re-deriving the winner rule.
const index = new Map();
const keyOf = (r) => [r.player_key, r.stat, r.line, r.side].join('|');
return { return {
rows, rows,
onGraded(base, sides) { onGraded(base, sides) {
try { try {
rows.push(...rowsFromSides(base, sides, ctx)); const made = rowsFromSides(base, sides, ctx);
for (const r of made) index.set(keyOf(r), r);
rows.push(...made);
} catch { /* collection never affects grading */ } } catch { /* collection never affects grading */ }
}, },
/**
* PUBLICATION SIGNAL — fired only for the side that actually reaches the
* slate.
*
* ── WHY THIS EXISTS ──────────────────────────────────────────────────
* `onGraded` fires with BOTH sides, graded AND refused, BEFORE any
* filtering. Only the higher-confidence graded side becomes the served
* Read. MEASURED on mlb 2026-08-26: of 13,012 collected rows, 3,859 were
* refusals and only 4,569 matched a prop that reached the ledger — 8,443
* (64.9%) describe a state the user was never shown.
*
* Nothing on the row said which was which, so treating every capture as a
* published claim would have built "publication history" for model activity
* that was never published. This marks the difference at the one place that
* knows it: the moment the winner is chosen.
*/
onPublished(base, winner) {
try {
if (!winner) return;
const made = rowsFromSides(base, [winner], ctx);
for (const r of made) {
const hit = index.get(keyOf(r));
if (hit) { hit.published = true; hit.published_side = true; }
}
} catch { /* signalling never affects grading */ }
},
}; };
} }
/**
* DUAL-WRITE: attach append-only Read lineage to rows about to be persisted.
*
* ── WHY THIS IS BEST-EFFORT AND WHY IT MUST DECLARE EVERY KEY ────────────
* Retention is already best-effort relative to the ledger; lineage sits one
* layer further out, so a lineage failure must never cost a retention row and
* never reach the grade. Every failure path below leaves the row persistable
* with NULL lineage, which reads as "unknown" — the honest value.
*
* Lineage keys are declared on EVERY row even when unresolved. PostgREST builds
* a bulk insert from the FIRST row's shape, so a key present on only some rows
* is dropped for the whole batch — the same defect that silently killed
* retention for three days when `chain_shadow` was missing from the schema.
*
* NOT AUTHORITATIVE. Nothing reads these columns to serve a product surface;
* this records truth in parallel so it can be compared against the live path
* before any authority changes.
*/
const LINEAGE_KEYS = Object.freeze([
'read_id', 'read_natural_key', 'claim_digest', 'revision_ordinal',
'supersedes_id', 'recaptures_id', 'lineage_action', 'lineage_state',
'lineage_version', 'change_type', 'claim_schema_version',
'digest_algorithm_version',
]);
/** Claim fields the resolver needs from EXISTING rows to classify a change. */
const LINEAGE_FETCH_CLAIM = Object.freeze([
'canonical_event_id',
'line', 'side', 'book', 'over_odds', 'under_odds',
'p_win', 'grade', 'confidence', 'projection', 'edge_pct', 'ev_pct',
'takeable', 'value', 'fair_odds', 'fair_prob', 'confidence_basis',
'refused', 'refusal_reason',
]);
function blankLineage() {
const o = {};
for (const k of LINEAGE_KEYS) o[k] = null;
return o;
}
/**
* ── DUAL-WRITE FAILURE CONTRACT ──────────────────────────────────────────
* Lineage lives on the SAME ROW as the retention record and is attached before
* the single upsert that writes it. That is not a convenience; it is what makes
* two of the four failure quadrants structurally impossible:
*
* legacy OK + lineage OK one insert carries both.
* legacy OK + lineage FAILS the row still persists with NULL lineage. The
* claim is retained; only its position in the
* chronology is unknown, and `out.lineage.error`
* plus the not_published/refused counters make
* the gap measurable rather than silent.
* legacy FAILS + lineage OK IMPOSSIBLE. There is no separate lineage
* write, so a failed retention insert cannot
* leave behind a lineage record claiming a Read
* was published. This is the quadrant that would
* manufacture false publication history, and the
* write ordering removes it rather than guarding
* against it.
* legacy FAILS + lineage n/a nothing is written at all.
*
* Ordering matters and is deliberate: resolve first, then one write. A separate
* lineage table written after the fact would reintroduce the impossible
* quadrant as a real one.
*/
async function attachLineage(rows, deps = {}) {
const out = {
attempted: Array.isArray(rows) ? rows.length : 0,
origins: 0, revisions: 0, recaptures: 0, forks: 0, refused: 0, unresolved: 0,
// A capture that was never the served Read is NOT a lineage failure. It is
// counted separately so the parity gap stays legible: lumping it into
// `refused` would make a healthy slate look broken.
not_published: 0,
change_types: {},
error: null,
};
const list = Array.isArray(rows) ? rows : [];
// Declare the keys first, unconditionally. Even a total failure leaves a
// persistable batch shape.
for (const r of list) Object.assign(r, blankLineage());
if (list.length === 0) return out;
try {
const lineage = deps.lineage || require('./read/readLineage');
const mintReadId = deps.mintReadId || (() => require('crypto').randomUUID());
const keyed = [];
for (const r of list) {
const key = lineage.readNaturalKey(r);
if (!key) { out.unresolved += 1; continue; }
r.read_natural_key = key;
keyed.push(r);
}
if (keyed.length === 0) return out;
// ONE bounded read of the existing families for these natural keys.
const keys = [...new Set(keyed.map((r) => r.read_natural_key))];
const fetchExisting = deps.fetchExisting || (async (naturalKeys) => {
const getClient = deps.getClient || require('../utils/supabase').getSupabaseServiceClient;
const supabase = getClient();
if (!supabase) return null;
const found = [];
const CH = 100; // never send an unbounded id list — it becomes a URL
for (let i = 0; i < naturalKeys.length; i += CH) {
const { data, error } = await supabase
.from('model_snapshots')
.select(['id', 'read_id', 'read_natural_key', 'game_id', 'claim_digest',
'revision_ordinal', 'lineage_action', 'supersedes_id', 'captured_at',
...LINEAGE_FETCH_CLAIM].join(', '))
.in('read_natural_key', naturalKeys.slice(i, i + CH));
if (error) throw new Error(error.message);
if (Array.isArray(data)) found.push(...data);
}
return found;
});
const existing = await fetchExisting(keys);
if (existing === null) return out; // no database configured — leave NULL
const byKey = new Map();
for (const e of existing) {
// The prior claim rides alongside so `classifyChange` can say WHY the
// published state advanced, not merely that the digest differs.
const claim = {};
for (const f of LINEAGE_FETCH_CLAIM) claim[f] = e[f];
const entry = { ...e, claim };
if (!byKey.has(e.read_natural_key)) byKey.set(e.read_natural_key, []);
byKey.get(e.read_natural_key).push(entry);
}
for (const r of keyed) {
const res = lineage.resolveLineage({
candidate: r,
existing: byKey.get(r.read_natural_key) || [],
mintReadId,
});
if (!res.ok) {
if (res.refused === 'not_published') out.not_published += 1;
else if (res.action === lineage.LINEAGE_ACTION.FORK_DETECTED) out.forks += 1;
else out.refused += 1;
// A refusal leaves NULL lineage on a row that is still persisted. The
// claim is retained; only its position in the chronology is unknown.
continue;
}
r.read_id = res.read_id;
r.claim_digest = res.claim_digest;
r.revision_ordinal = res.revision_ordinal;
r.supersedes_id = res.supersedes_id ?? null;
r.recaptures_id = res.recaptures_id ?? null;
r.lineage_action = res.action;
r.lineage_state = res.lineage_state;
r.lineage_version = res.lineage_version;
r.change_type = res.change_type || null;
r.claim_schema_version = res.claim_schema_version || null;
r.digest_algorithm_version = res.digest_algorithm_version || null;
if (res.change_type) {
out.change_types[res.change_type] = (out.change_types[res.change_type] || 0) + 1;
}
if (res.action === lineage.LINEAGE_ACTION.ORIGIN) out.origins += 1;
else if (res.action === lineage.LINEAGE_ACTION.REVISION) out.revisions += 1;
else if (res.action === lineage.LINEAGE_ACTION.RECAPTURE) out.recaptures += 1;
// Newly minted read_ids must be visible to later rows in the SAME batch,
// or two rows of one Read would each mint an id and fork it immediately.
const fam = byKey.get(r.read_natural_key) || [];
const claim = {};
for (const f of LINEAGE_FETCH_CLAIM) claim[f] = r[f];
fam.push({
id: null, read_id: r.read_id, read_natural_key: r.read_natural_key,
game_id: r.game_id, claim_digest: r.claim_digest,
revision_ordinal: r.revision_ordinal, lineage_action: r.lineage_action,
supersedes_id: r.supersedes_id, captured_at: r.captured_at, claim,
});
byKey.set(r.read_natural_key, fam);
}
} catch (e) {
out.error = e && e.message ? e.message : String(e);
}
return out;
}
/** The unique index that prevents a forked chain, named so the guard is legible. */
const SUPERSEDES_CONSTRAINT = 'model_snapshots_supersedes_unique';
const FORK_RETRY_LIMIT = 2;
function isSupersedesConflict(error) {
if (!error) return false;
const m = `${error.message || ''} ${error.details || ''}`;
return m.includes(SUPERSEDES_CONSTRAINT) || (error && error.code === '23505' && m.includes('supersedes'));
}
/**
* Recover a batch that lost a supersedes race.
*
* Bounded on purpose: two attempts, then an explicit failure. An unbounded loop
* against a writer that keeps winning would spin forever, and the correct answer
* after a couple of misses is to report the gap rather than keep trying.
*
* Sequence per attempt: re-resolve the batch against the CURRENT head, then
* write. Re-resolution is what makes this safe -- if the rival already published
* an identical claim, `attachLineage` returns RECAPTURE and the row lands
* idempotently; if our claim is still materially different it appends after the
* new head instead of trying to supersede a parent that is no longer the head.
*/
async function recoverFromFork(chunk, deps = {}) {
const out = { recovered: false, written: 0, attempts: 0, error: null };
const supabase = deps.supabase;
for (let attempt = 1; attempt <= FORK_RETRY_LIMIT; attempt += 1) {
out.attempts = attempt;
try {
// Clear stale lineage so re-resolution sees the row as a fresh candidate.
for (const r of chunk) Object.assign(r, blankLineage());
// eslint-disable-next-line no-await-in-loop
await attachLineage(chunk, deps);
// eslint-disable-next-line no-await-in-loop
const { error } = await supabase
.from('model_snapshots')
.upsert(chunk, { onConflict: 'snapshot_id,player_key,stat,line,side' });
if (!error) { out.recovered = true; out.written = chunk.length; return out; }
if (!isSupersedesConflict(error)) { out.error = error.message; return out; }
out.error = error.message;
} catch (e) {
out.error = e && e.message ? e.message : String(e);
return out;
}
}
return out;
}
/**
* COMMIT PUBLICATION - the only place a claim becomes "published".
*
* -- WHY THIS IS SEPARATE FROM THE CAPTURE WRITE -------------------------
* The authoritative served slate is ONE atomic Redis write,
* `cacheSet('snapshot:{sport}:latest', snapshot)`. Retention persists BEFORE
* that write (measured: persistRetention at snapshotService:1137, the slate
* write at :1152), and the winner is selected earlier still. So neither capture
* nor winner-selection can mean "published":
*
* winner selected -> intended for the slate
* retention persisted -> the capture is on the record
* REDIS WRITE SUCCEEDS -> the claim was actually made available <-- HERE
*
* Setting the marker any earlier lets lineage assert a publication that a failed
* Redis write never made. That is the one thing lineage must never do.
*
* -- PUBLICATION IS SLATE-LEVEL, BECAUSE THE WRITE IS --------------------
* The whole envelope commits or none of it does, so every served row shares one
* `publication_id` and one `published_at`. Modelling per-row publication would
* claim a granularity the transport does not have.
*
* -- FAILURE DIRECTION IS DELIBERATE ------------------------------------
* If this update fails after a successful Redis write, rows stay
* `published: false` with NULL lineage. The record then UNDERSTATES what was
* served, which is recoverable and observable. The opposite error - claiming a
* publication that did not happen - is not.
*/
async function commitPublication(spec, deps = {}) {
const s = spec || {};
const out = {
publication_id: s.publicationId || null,
published_at: s.publishedAt || null,
candidates: 0, published: 0, skipped: false, error: null, lineage: null,
};
const served = Array.isArray(s.rows) ? s.rows.filter((r) => r && r.published === true) : [];
out.candidates = served.length;
if (served.length === 0) return out;
try {
const getClient = deps.getClient || require('../utils/supabase').getSupabaseServiceClient;
const supabase = getClient();
if (!supabase) { out.skipped = true; return out; }
for (const r of served) {
r.published_at = out.published_at;
r.publication_id = out.publication_id;
}
// Lineage resolves ONLY over confirmed-published rows.
if (deps.lineage !== false) {
out.lineage = await attachLineage(served, { ...deps, getClient });
}
const CHUNK = 250;
for (let i = 0; i < served.length; i += CHUNK) {
const chunk = served.slice(i, i + CHUNK);
// eslint-disable-next-line no-await-in-loop
const { error } = await supabase
.from('model_snapshots')
.upsert(chunk, { onConflict: 'snapshot_id,player_key,stat,line,side' });
if (!error) { out.published += chunk.length; continue; }
// ── CONCURRENCY RECOVERY ────────────────────────────────────────────
// `model_snapshots_supersedes_unique` rejects a second row superseding the
// same parent. That constraint is the authority against a forked history,
// and hitting it means another writer advanced the chain between our
// resolve and our write. The loser must not silently drop a materially new
// published state, and must not loop.
if (!isSupersedesConflict(error)) { out.error = error.message; break; }
// eslint-disable-next-line no-await-in-loop
const rec = await recoverFromFork(chunk, { ...deps, getClient, supabase });
out.conflicts = (out.conflicts || 0) + 1;
if (rec.recovered) {
out.recovered = (out.recovered || 0) + 1;
out.published += rec.written;
} else {
out.recovery_failed = (out.recovery_failed || 0) + 1;
out.error = rec.error || 'fork recovery exhausted';
break;
}
}
} catch (e) {
out.error = e && e.message ? e.message : String(e);
}
// ── EXACT PARITY-GAP IDENTITY ──────────────────────────────────────────
// "lineage failed" is not an answer. When the product published successfully
// and the record did not, the system must be able to say WHICH EXACT SERVED
// CLAIM is missing, without re-deriving it from player/stat/line/date.
//
// Everything needed to recover it deterministically is captured here from the
// in-memory rows that were actually served.
if (out.error || (out.lineage && out.lineage.error) || out.published < out.candidates) {
out.parity_gap = Object.freeze({
sport: served[0] ? served[0].sport : null,
publication_id: out.publication_id,
published_at: out.published_at,
attempted_at: (deps.now || (() => new Date().toISOString()))(),
reason: out.error || (out.lineage && out.lineage.error) || 'partial_commit',
missing_count: out.candidates - out.published,
// The exact rows, by the identity the retention table itself keys on.
missing: Object.freeze(served.slice(out.published).map((r) => Object.freeze({
snapshot_id: r.snapshot_id || null,
player_key: r.player_key || null,
stat: r.stat || null,
line: r.line === undefined ? null : r.line,
side: r.side || null,
canonical_event_id: r.canonical_event_id || null,
read_id: r.read_id || null,
read_natural_key: r.read_natural_key || null,
claim_digest: r.claim_digest || null,
captured_at: r.captured_at || null,
}))),
});
}
return out;
}
/** /**
* Persist rows. Chunked, upsert-on-conflict-ignore so a retried cycle can never * Persist rows. Chunked, upsert-on-conflict-ignore so a retried cycle can never
* duplicate. No-ops (successfully) without Supabase env, so tests and local dev * duplicate. No-ops (successfully) without Supabase env, so tests and local dev
@@ -193,6 +589,15 @@ async function persist(rows, deps = {}) {
const getClient = deps.getClient || require('../utils/supabase').getSupabaseServiceClient; const getClient = deps.getClient || require('../utils/supabase').getSupabaseServiceClient;
const supabase = getClient(); const supabase = getClient();
if (!supabase) { out.skipped = true; return out; } if (!supabase) { out.skipped = true; return out; }
// NOTE: lineage is NOT resolved here any more.
//
// Capture happens BEFORE the authoritative Redis slate write
// (`snapshot:{sport}:latest`), so resolving lineage at this point would
// record a published claim for a slate that may never be served. Publication
// is committed by `commitPublication()` after that write succeeds.
//
// Rows are persisted with `published: false` and NULL lineage, which is the
// truthful state of a capture that has not yet been served.
const CHUNK = 250; const CHUNK = 250;
for (let i = 0; i < rows.length; i += CHUNK) { for (let i = 0; i < rows.length; i += CHUNK) {
const chunk = rows.slice(i, i + CHUNK); const chunk = rows.slice(i, i + CHUNK);
@@ -253,6 +658,29 @@ function mergeEnrichment(rows, enrichedGrades) {
}); });
} }
/**
* Attach the SHADOW CHAIN read to collected rows — the (chain_p, counter_p,
* outcome) triple's first two thirds.
*
* Like `mergeEnrichment` this fills ONE field and touches nothing else. It runs
* later than grade time for the same structural reason the archetype does: the
* chain needs the statcast rows and park factors the enrichment pass loaded, and
* pulling that forward into the grader would put per-prop I/O on the serving
* path.
*
* THE SIDE ALIGNMENT IS THE LOAD-BEARING PART. The chain computes P(over the
* line); `p_win` on the row is expressed for the graded SIDE. Storing the raw
* over-probability against an under row's `p_win` would invert every comparison
* made from it afterwards, silently — so `alignToSide` is given the row's own
* side and the row's own counter probability, and both halves of the triple end
* up pointing the same way. `outcome` is written by the ordinary settle pass and
* is already side-aligned, which completes it.
*
* A row the chain could not read is left NULL. It is not a zero probability and
* not an average hitter — the chain refusing to read someone is a fact worth
* keeping, and a fabricated third of a triple would poison the adjudication this
* column exists to enable.
*/
/** /**
* Attach the SHADOW CHAIN read to collected rows — the (chain_p, counter_p, * Attach the SHADOW CHAIN read to collected rows — the (chain_p, counter_p,
* outcome) triple's first two thirds. * outcome) triple's first two thirds.
@@ -303,6 +731,13 @@ module.exports = {
mergeEnrichment, mergeEnrichment,
mergeChainShadow, mergeChainShadow,
persist, persist,
attachLineage,
commitPublication,
recoverFromFork,
isSupersedesConflict,
FORK_RETRY_LIMIT,
LINEAGE_KEYS,
LINEAGE_FETCH_CLAIM,
newSnapshotId, newSnapshotId,
__internals: { numOrNull, intOrNull, boolOrNull }, __internals: { numOrNull, intOrNull, boolOrNull },
}; };
src/services/snapshotService.js
+135 -1
View File
@@ -299,6 +299,25 @@ async function loadPitcherArsenals(sport) {
* flatten it. Both now serve RAW. * flatten it. Both now serve RAW.
*/ */
const CALIBRATION_DEPLOYED = Object.freeze([]); const CALIBRATION_DEPLOYED = Object.freeze([]);
/**
* Which sports record publication lineage. MLB only by default — it is the only
* sport with a verified canonical event identity.
*/
// DARK BY DEFAULT. This release deploys the event-integrity and
// publication-commit corrections with the lineage OBSERVER OFF, so the repaired
// pipeline is exposed to production before anything starts writing history.
// Enabling is a deliberate configuration act: LINEAGE_CANARY_SPORTS=mlb.
//
// The flag governs the observer only. It does NOT control canonical event
// identity, the impossible-binding refusal, event-aware dedupe, or
// publication-commit ordering — those are integrity repairs and ship active.
const LINEAGE_CANARY_SPORTS = String(process.env.LINEAGE_CANARY_SPORTS || '')
.split(',').map((x) => x.trim().toLowerCase()).filter(Boolean);
function lineageCanaryEnabled(sport) {
return LINEAGE_CANARY_SPORTS.includes(String(sport || '').toLowerCase());
}
/** /**
* NOTHING IS SERVED CALIBRATED, AND THIS IS DELIBERATE. * NOTHING IS SERVED CALIBRATED, AND THIS IS DELIBERATE.
* *
@@ -412,6 +431,52 @@ async function runSnapshot(sport, opts = {}) {
const binder = deps.gameBinder || require('./gameBinder'); const binder = deps.gameBinder || require('./gameBinder');
const b = await binder.attachGameTimes(sp, props, { gradedAt: ts }); const b = await binder.attachGameTimes(sp, props, { gradedAt: ts });
console.log(`[snapshot] game binding ${sp}: ${b.bound} bound, ${b.alreadyHad} already had times, ${b.unresolved} UNRESOLVED, ${b.ambiguous} ambiguous(doubleheader)`); console.log(`[snapshot] game binding ${sp}: ${b.bound} bound, ${b.alreadyHad} already had times, ${b.unresolved} UNRESOLVED, ${b.ambiguous} ambiguous(doubleheader)`);
// CANONICAL EVENT IDENTITY (MLB). The bound game_time above is what makes
// this resolvable: two halves of a doubleheader share teams and date and
// differ only by start time, so identity is decided by the clock, never by
// the team pair. Attached BEFORE dedupe, because dedupe is where two real
// games were previously collapsed into one.
//
// Sport-neutral field, MLB-only resolver: every other sport records
// UNSUPPORTED_SPORT rather than a guessed id.
if (sp === 'mlb') {
try {
const evid = deps.eventIdentity || require('./event/eventIdentity');
const mlbAdapter = deps.mlbAdapter || require('./adapters/mlbStatsAdapter');
const dates = [...new Set(props.map((p) => p && p.game_date).filter(Boolean))];
const games = [];
for (const d of dates) {
// eslint-disable-next-line no-await-in-loop
const g = await mlbAdapter.getScheduleWithPitchers(d);
if (Array.isArray(g)) games.push(...g);
}
// PARTICIPANT EVIDENCE. The prop's own team fields cannot police this:
// when the provider nests a player under the wrong event, those fields
// ARE the wrong event's teams. The player's real team has to come from
// a source that cannot be wrong the same way — the slate's rosters.
// Best-effort: no index means no refusals, i.e. the previous behaviour.
let playerTeams = null;
try {
const built = await evid.buildPlayerTeamIndex(games, {
getTeamRoster: (id) => mlbAdapter.getTeamRoster(id),
});
playerTeams = built.index;
console.log(`[snapshot] participant evidence ${sp}: ${built.stats.players} players`
+ ` across ${built.stats.teams} rosters${built.stats.failed ? `, ${built.stats.failed} unreadable` : ''}`);
} catch (e3) {
console.warn(`[snapshot] participant evidence unavailable for ${sp}:`, e3.message);
}
const e = evid.attachEventIdentity(sp, props, games, playerTeams);
console.log(`[snapshot] event identity ${sp}: ${e.canonical}/${e.total} canonical, `
+ `${e.unresolved} unresolved, ${e.impossible} impossible-binding refusals`
+ `${Object.keys(e.reasons).length ? ' ' + JSON.stringify(e.reasons) : ''}`);
} catch (e2) {
// Identity is additive: a failure leaves props on legacy identity, the
// exact behaviour that existed before. It must never cost a slate.
console.warn(`[snapshot] event identity failed for ${sp} (slate continues):`, e2.message);
}
}
if (b.unresolved > 0 && b.bound === 0 && b.alreadyHad === 0) { if (b.unresolved > 0 && b.bound === 0 && b.alreadyHad === 0) {
await deps.notify(`Game binding produced NOTHING for ${sp.toUpperCase()} — ${b.unresolved} props could not be tied to a scheduled game. Their rows will be skipped rather than mis-dated.`, { await deps.notify(`Game binding produced NOTHING for ${sp.toUpperCase()} — ${b.unresolved} props could not be tied to a scheduled game. Their rows will be skipped rather than mis-dated.`, {
title: 'VYNDR pipeline', priority: 'high', tags: ['rotating_light'], title: 'VYNDR pipeline', priority: 'high', tags: ['rotating_light'],
@@ -539,6 +604,7 @@ async function runSnapshot(sport, opts = {}) {
now: deps.now, now: deps.now,
cacheSet: async (_k, v) => { envelope = v; }, cacheSet: async (_k, v) => { envelope = v; },
onGraded: collector ? collector.onGraded : undefined, onGraded: collector ? collector.onGraded : undefined,
onPublished: collector ? collector.onPublished : undefined,
}); });
// Retention is COLLECTED here (grade time — features must be exactly what the // Retention is COLLECTED here (grade time — features must be exactly what the
@@ -547,6 +613,13 @@ async function runSnapshot(sport, opts = {}) {
// empty-slate early return: a slate that graded nothing but refused // empty-slate early return: a slate that graded nothing but refused
// everything is exactly the case worth recording. // everything is exactly the case worth recording.
let retentionRows = 0; let retentionRows = 0;
// SHADOW LEDGER LINKAGE. retention persists BEFORE the ledger write, so the
// read_ids it assigned are already in memory — the join costs no query.
// read_revision_id is deliberately NOT resolved here: the upsert does not
// return row ids, and issuing a second query in the hot path to obtain one
// would be a real cost for a column nothing reads yet.
let lineageIndex = null;
let persistedRows = null;
const persistRetention = async (enrichedGrades, chainShadow = null) => { const persistRetention = async (enrichedGrades, chainShadow = null) => {
if (!retention || !collector || !collector.rows.length) return; if (!retention || !collector || !collector.rows.length) return;
try { try {
@@ -563,6 +636,13 @@ async function runSnapshot(sport, opts = {}) {
} }
const r = await retention.persist(rows); const r = await retention.persist(rows);
retentionRows = r.written || 0; retentionRows = r.written || 0;
// Held for the publication commit, which happens only after the
// authoritative slate write succeeds.
persistedRows = rows;
// NOTE: no lineage index is built here. Lineage is assigned by
// commitPublication AFTER the slate write, so at this point no row has a
// read_id yet — building an index here would produce an empty one and
// quietly leave every ledger row unlinked.
console.log(`[snapshot] retention ${sp}: ${r.written}/${r.attempted} rows${r.skipped ? ' (skipped — no supabase env)' : ''}${r.error ? ` ERROR: ${r.error}` : ''}`); console.log(`[snapshot] retention ${sp}: ${r.written}/${r.attempted} rows${r.skipped ? ' (skipped — no supabase env)' : ''}${r.error ? ` ERROR: ${r.error}` : ''}`);
} catch (e) { } catch (e) {
console.warn(`[snapshot] retention write failed for ${sp} (snapshot continues):`, e.message); console.warn(`[snapshot] retention write failed for ${sp} (snapshot continues):`, e.message);
@@ -1013,6 +1093,7 @@ async function runSnapshot(sport, opts = {}) {
} }
} }
// LINEUP + BASERUNNER CONTEXT — the input RBI and runs have always needed. // LINEUP + BASERUNNER CONTEXT — the input RBI and runs have always needed.
// Best-effort and dated: a context failure must never break a snapshot, and a // Best-effort and dated: a context failure must never break a snapshot, and a
// lineup is a PRE-GAME fact that changes by the hour, so what we knew at grade // lineup is a PRE-GAME fact that changes by the hour, so what we knew at grade
@@ -1046,6 +1127,57 @@ async function runSnapshot(sport, opts = {}) {
// what produced the "SIGNAL LIVE vs STALE 8h" contradiction. // what produced the "SIGNAL LIVE vs STALE 8h" contradiction.
const snapshot = { sport: sp, updated_at: ts, refreshed_at: ts, grades: enriched, deltas, gradeCount: enriched.length }; const snapshot = { sport: sp, updated_at: ts, refreshed_at: ts, grades: enriched, deltas, gradeCount: enriched.length };
await deps.cacheSet(`snapshot:${sp}:latest`, snapshot, SNAP_TTL); await deps.cacheSet(`snapshot:${sp}:latest`, snapshot, SNAP_TTL);
// ── PUBLICATION COMMITTED ───────────────────────────────────────────────
// The line above is the authoritative act: it is the single atomic write that
// makes the slate available to `GET /api/snapshot/:sport`. Only now is a
// claim "published", so only now may lineage record one.
//
// If the cacheSet above throws, execution never reaches here and no row is
// marked published — which is the entire point of the ordering. If THIS block
// fails, rows stay unpublished: the record understates what was served, which
// is observable and recoverable.
// ── CANARY SCOPE ────────────────────────────────────────────────────────
// Lineage dual-write runs for MLB only, because MLB is the one sport with a
// VERIFIED canonical event identity (statsapi gamePk). A sport without one
// cannot separate a doubleheader, and recording publication history on an
// identity that can merge two real games is the failure this whole layer
// exists to prevent.
//
// Env-gated both ways, repo-native (same shape as SNAPSHOT_CRON /
// INTRADAY_REFRESH): `LINEAGE_CANARY_SPORTS=` disables it entirely,
// `LINEAGE_CANARY_SPORTS=mlb,wnba` would widen it. Disabling needs no
// migration revert and touches no historical row.
if (retention && retention.commitPublication && persistedRows && lineageCanaryEnabled(sp)) {
try {
const pub = await retention.commitPublication({
rows: persistedRows,
publicationId: retentionCtx.snapshotId,
publishedAt: deps.now(),
});
const lin = pub.lineage || {};
console.log(`[publication] ${sp}: ${pub.published}/${pub.candidates} served rows committed`
+ `${pub.skipped ? ' (skipped — no supabase env)' : ''}${pub.error ? ` ERROR: ${pub.error}` : ''}`
+ ` · lineage origins=${lin.origins || 0} revisions=${lin.revisions || 0}`
+ ` recaptures=${lin.recaptures || 0} not_published=${lin.not_published || 0}`
+ ` forks=${lin.forks || 0}${lin.error ? ` lineage_error=${lin.error}` : ''}`);
if (pub.error || (lin && lin.error)) {
// A parity gap must be loud: the product published, the record did not.
await deps.notify(
`Lineage parity gap on ${sp.toUpperCase()}: slate published but lineage did not record it `
+ `(${pub.error || lin.error}). The product is unaffected; the historical record is incomplete.`,
{ title: 'VYNDR pipeline', priority: 'high', tags: ['rotating_light'] },
);
}
const idx = new Map();
for (const row of persistedRows) {
if (row && row.read_natural_key && row.read_id) idx.set(row.read_natural_key, row.read_id);
}
if (idx.size > 0) lineageIndex = idx;
} catch (e) {
console.warn(`[publication] commit failed for ${sp} (product unaffected):`, e.message);
}
}
// Session 59 — grades:{sport} must outlive the gap between cron runs (up to // Session 59 — grades:{sport} must outlive the gap between cron runs (up to
// 5h) or team rosters / Explore / leaders go dark mid-day. SNAP_TTL (6h), // 5h) or team rosters / Explore / leaders go dark mid-day. SNAP_TTL (6h),
// NOT the legacy 2h gradeSlateService TTL — that gap was why /team showed // NOT the legacy 2h gradeSlateService TTL — that gap was why /team showed
@@ -1063,7 +1195,7 @@ async function runSnapshot(sport, opts = {}) {
// Best-effort: the ledger must never break the snapshot. // Best-effort: the ledger must never break the snapshot.
let ledgerWritten = 0; let ledgerWritten = 0;
try { try {
const rec = await deps.ledger.recordPipelineGrades(sp, withChallenger, props, { now: deps.now }); const rec = await deps.ledger.recordPipelineGrades(sp, withChallenger, props, { now: deps.now, lineageIndex });
ledgerWritten = rec.written || 0; ledgerWritten = rec.written || 0;
await deps.ledger.captureClosing(sp, props); await deps.ledger.captureClosing(sp, props);
} catch (e) { } catch (e) {
@@ -1134,6 +1266,8 @@ async function runAllSnapshots(opts = {}) {
} }
module.exports = { module.exports = {
lineageCanaryEnabled,
LINEAGE_CANARY_SPORTS,
runSnapshot, runSnapshot,
runAllSnapshots, runAllSnapshots,
computeLineDeltas, computeLineDeltas,
supabase/migrations/045_read_lineage.sql
+89
View File
@@ -0,0 +1,89 @@
-- Migration 045: append-only Read lineage (ADDITIVE ONLY).
--
-- Every statement is ADD COLUMN IF NOT EXISTS or CREATE INDEX. No column is
-- dropped, renamed, retyped or backfilled. No historical row is rewritten.
--
-- ── WHY THIS IS COLUMNS AND NOT A NEW CLAIM TABLE ──────────────────────────
-- `model_snapshots` is ALREADY an append-only store of published claims: it is
-- upserted with ignoreDuplicates on (snapshot_id, player_key, stat, line, side),
-- so a row is written once per snapshot cycle and never overwritten, and
-- `captured_at` is NOT NULL. Its only post-insert write is settlement
-- (outcome / actual_value / settled_at), which is not part of the claim.
--
-- MEASURED (mlb, 2026-08-26): 5,376 props, 4,286 with more than one row, and
-- 183 whose GRADE CHANGED across those rows. The immutable claims exist. What
-- was missing is the lineage over them, which is what these columns are.
--
-- Building a separate revision table would have duplicated a claim that is
-- already immutable, and given VYNDR two stores that can disagree.
--
-- ── THE ROW'S OWN id IS THE REVISION IDENTITY ─────────────────────────────
-- `model_snapshots.id` is a bigserial primary key on an append-only row. It is
-- already immutable and already unique, so a parallel text `revision_id` would
-- be a second spelling of the same fact. `supersedes_id` and `recaptures_id`
-- point at it directly.
--
-- ── CAPTURE IS NOT REVISION ───────────────────────────────────────────────
-- Most cycles re-observe an unchanged claim. `claim_digest` (sha256 over the
-- market terms plus gradeFreeze.SERVED_FIELDS) is what separates a genuine new
-- published state from a re-observation, so the chronology reports the ~183
-- revisions that happened rather than the ~13,000 captures that were written.
-- The logical Read: stable across cycles, and deliberately NOT derived from the
-- game_id string. Measured over 30,746 identity groups, 1,328 carried more than
-- one game_id spelling for the same real game while only 18 carried genuinely
-- different team pairs. Identity that included the raw spelling would have
-- fractured 1,310 Reads that are one Read.
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS read_id text;
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS read_natural_key text;
-- The published state.
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS claim_digest text;
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS revision_ordinal integer;
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS supersedes_id bigint;
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS recaptures_id bigint;
-- ORIGIN | REVISION | RECAPTURE. A row says which it is rather than leaving a
-- reader to infer it from ordinals.
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS lineage_action text;
-- LIVE | LEGACY_UNVERIFIED. Rows written before this migration keep NULL, which
-- is the honest value: their position in a chronology is unknown and is not
-- reconstructed.
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS lineage_state text;
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS lineage_version text;
-- Walking one Read's chronology in order.
CREATE INDEX IF NOT EXISTS model_snapshots_lineage_idx
ON model_snapshots (read_id, revision_ordinal) WHERE read_id IS NOT NULL;
-- Resolving a candidate's family on write.
CREATE INDEX IF NOT EXISTS model_snapshots_natural_key_idx
ON model_snapshots (read_natural_key) WHERE read_natural_key IS NOT NULL;
-- FORK PREVENTION, not merely fork detection.
--
-- Only ONE row may supersede a given parent. Two workers that each read the
-- same standing revision before either wrote cannot see one another, so no
-- in-memory check can catch that race — the second insert must fail, and this
-- is what makes it fail. Without it the chain could silently become
-- R1 -> {R2a, R2b} and no reader could say which claim VYNDR actually published.
CREATE UNIQUE INDEX IF NOT EXISTS model_snapshots_supersedes_unique
ON model_snapshots (supersedes_id) WHERE supersedes_id IS NOT NULL;
-- ── LEDGER LINKAGE (SHADOW) ───────────────────────────────────────────────
-- A receipt should be able to name the EXACT revision it settles. These are
-- written in shadow alongside the existing settlement path and are read by
-- nothing; current settlement outcomes are unchanged. NULL on every existing
-- row, and no historical receipt is re-pointed.
ALTER TABLE ledger_entries ADD COLUMN IF NOT EXISTS read_id text;
ALTER TABLE ledger_entries ADD COLUMN IF NOT EXISTS read_revision_id bigint;
CREATE INDEX IF NOT EXISTS ledger_entries_read_id_idx
ON ledger_entries (read_id) WHERE read_id IS NOT NULL;
COMMENT ON COLUMN model_snapshots.read_id IS
'The stable identity of a logical Read across every published revision. Deliberately NOT derived from game_id: measured, 1,328 of 30,746 identity groups carry more than one game_id spelling for the same real game.';
COMMENT ON COLUMN model_snapshots.claim_digest IS
'sha256 over the market terms (line, side, book, locked_odds) plus gradeFreeze.SERVED_FIELDS. Separates a genuine new published claim from a re-observation of the standing one.';
COMMENT ON COLUMN model_snapshots.lineage_action IS
'ORIGIN | REVISION | RECAPTURE. Most cycles re-observe an unchanged claim; only a changed claim_digest advances the chain.';
COMMENT ON COLUMN ledger_entries.read_revision_id IS
'SHADOW: the exact model_snapshots row this receipt settles. Read by nothing; settlement outcomes are unchanged by migration 045.';
supabase/migrations/046_publication_semantics.sql
+52
View File
@@ -0,0 +1,52 @@
-- Migration 046: publication semantics (ADDITIVE ONLY).
--
-- Every statement is ADD COLUMN IF NOT EXISTS or CREATE INDEX. Nothing is
-- dropped, renamed, retyped or backfilled. Historical rows keep NULL.
--
-- ── WHY 045 WAS NOT ENOUGH ────────────────────────────────────────────────
-- Migration 045 gave `model_snapshots` a lineage. Tracing the publication path
-- afterwards showed the lineage would have described the wrong population:
-- `gradeSlateService` fires its retention hook with BOTH sides, graded AND
-- refused, BEFORE any filtering, and only the higher-confidence graded side
-- becomes the served Read.
--
-- MEASURED, mlb 2026-08-26: of 13,012 captured rows, 3,859 were refusals and
-- only 4,569 matched a prop that reached the ledger. **8,443 rows (64.9%)
-- describe a state no user was ever shown**, and nothing on the row said which
-- was which. Building chronology over all of them would have produced
-- "publication history" for model activity that was never published.
--
-- `published` is the smallest missing link. It is set by the collector's
-- publication signal at the one place that knows the answer — the moment the
-- winner is selected — and defaults FALSE, because a capture is not a
-- publication until something says it is.
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS published boolean;
-- ── WHY THE PUBLISHED STATE ADVANCED ──────────────────────────────────────
-- The revision ordinal says WHEN. It cannot say WHY, and the difference is the
-- whole product question: VYNDR's belief can hold still while the market moves.
-- A replay of one real date found 4,507 digest changes of which only 301 moved
-- the grade — 4,206 were price alone. One undifferentiated "revision" count
-- conflates a changed opinion with a changed price.
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS change_type text;
-- ── CLAIM SEMANTICS ARE VERSIONED ─────────────────────────────────────────
-- A digest is only interpretable against the claim definition that produced it.
-- The first digest listed only `locked_odds` while this table stores
-- `over_odds`/`under_odds`, so it was blind to price on exactly the rows it
-- digested — caught by a replay crashing on a missing column, i.e. by luck.
-- These stamps make the next such change visible instead of silent. A
-- historical digest is never recomputed under a newer definition.
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS claim_schema_version text;
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS digest_algorithm_version text;
-- Publication chronology is read published-only; the partial index matches.
CREATE INDEX IF NOT EXISTS model_snapshots_published_lineage_idx
ON model_snapshots (read_id, revision_ordinal) WHERE published IS TRUE;
COMMENT ON COLUMN model_snapshots.published IS
'TRUE only for the side that actually became the served Read. Measured: 64.9% of captured rows describe a state no user was shown, and before this column nothing on the row distinguished them. NULL on rows written before migration 046 - unknown, not reconstructed.';
COMMENT ON COLUMN model_snapshots.change_type IS
'INITIAL_PUBLICATION | BELIEF_CHANGE | MARKET_REPRICE | MARKET_LINE_CHANGE | COMPARISON_CHANGE | PROVENANCE_CHANGE | PRESENTATION_CHANGE | MIXED_CHANGE | NO_MATERIAL_PUBLISHED_CHANGE. Ordinal says when; this says why.';
COMMENT ON COLUMN model_snapshots.claim_schema_version IS
'The claim definition in force when the digest was computed. Never recompute a historical digest under a newer definition and present it as original.';
supabase/migrations/047_canonical_event_publication.sql
+58
View File
@@ -0,0 +1,58 @@
-- Migration 047: canonical event identity + publication commit (ADDITIVE ONLY).
--
-- Every statement is ADD COLUMN IF NOT EXISTS or CREATE INDEX. Nothing is
-- dropped, renamed, retyped or backfilled. Historical rows keep NULL.
--
-- ── WHY: EVENT LABELS ARE NOT EVENT IDENTITY ─────────────────────────────
-- `ledgerService.gameIdFor` derives `sport:date:away@home`. That has no
-- occurrence component, so both halves of a doubleheader produce a
-- BYTE-IDENTICAL string.
--
-- VERIFIED against statsapi, MLB 2026 through 2026-08-27: 19 doubleheaders /
-- 38 real games collapse into 19 derived labels. Real case 2026-08-17,
-- St. Louis @ Cincinnati: gamePk 824514 (game 1, 17:40Z) and 824478 (game 2,
-- 22:40Z). VYNDR recorded ONE id holding 171 ledger rows and FOUR distinct
-- starting pitchers -- both games merged into one event.
--
-- `canonical_event_id` is namespaced (`mlb:gamepk:824514`) so two sports'
-- numeric ids can never collide. The legacy `game_id` is KEPT for diagnostics
-- and compatibility and is explicitly not authoritative.
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS canonical_event_id text;
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS event_identity_source text;
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS event_identity_method text;
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS event_identity_version text;
-- The source's own occurrence number (statsapi `gameNumber`). Never inferred.
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS event_occurrence integer;
-- ── WHY: PUBLISHED MUST MEAN COMMITTED ───────────────────────────────────
-- The authoritative served slate is ONE atomic Redis write,
-- `cacheSet('snapshot:{sport}:latest')`. Retention persists BEFORE it
-- (snapshotService:1137 vs :1152) and the winner is chosen earlier still, so
-- neither capture nor selection means published. `published_at` records the
-- commit instant and `publication_id` the slate it committed in -- the write is
-- slate-atomic, so publication is slate-level and every served row shares one.
--
-- `captured_at` answers when the model state was captured. `published_at`
-- answers when the composed Read became authoritative. They are different
-- questions and are stored separately.
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS published_at timestamptz;
ALTER TABLE model_snapshots ADD COLUMN IF NOT EXISTS publication_id text;
-- Chronology is walked per canonical event.
CREATE INDEX IF NOT EXISTS model_snapshots_canonical_event_idx
ON model_snapshots (canonical_event_id) WHERE canonical_event_id IS NOT NULL;
-- Publication parity is read per slate.
CREATE INDEX IF NOT EXISTS model_snapshots_publication_idx
ON model_snapshots (publication_id) WHERE publication_id IS NOT NULL;
-- Ledger linkage to the canonical event, additive and read by nothing.
ALTER TABLE ledger_entries ADD COLUMN IF NOT EXISTS canonical_event_id text;
CREATE INDEX IF NOT EXISTS ledger_entries_canonical_event_idx
ON ledger_entries (canonical_event_id) WHERE canonical_event_id IS NOT NULL;
COMMENT ON COLUMN model_snapshots.canonical_event_id IS
'Namespaced canonical event identity, e.g. mlb:gamepk:824514. MLB only; other sports record UNSUPPORTED_SPORT rather than a guessed id. The legacy game_id is a derived LABEL and is not authoritative -- it cannot separate a doubleheader.';
COMMENT ON COLUMN model_snapshots.published_at IS
'When the composed Read became authoritative -- i.e. when the atomic slate write to snapshot:{sport}:latest succeeded. NOT the capture time, and never set by winner selection alone.';
COMMENT ON COLUMN model_snapshots.publication_id IS
'The slate whose atomic Redis write committed this row. Publication is slate-level because the write is.';
tests/unit/chainShadowAccrual.test.js
+21 -1
View File
@@ -423,7 +423,15 @@ describe('0 LEAKS — measured against the source, not asserted in prose', () =>
}); });
it('the ONLY writers are the shadow service, retention and the snapshot', () => { it('the ONLY writers are the shadow service, retention and the snapshot', () => {
const writers = walk('src').filter((f) => /chain_shadow|chainShadow/.test(read(f))); // The EVALUATOR names the column and must: `gradeFreeze` exists to verify
// that the shadow reaches nothing served, and a verifier that cannot say
// the name of the thing it verifies is not a verifier. It is excluded from
// the writer list and held to the stricter rule below — it may NAME the
// column, never assign to it.
const EVALUATOR = 'src/services/evaluator/';
const writers = walk('src')
.filter((f) => /chain_shadow|chainShadow/.test(read(f)))
.filter((f) => !f.startsWith(EVALUATOR));
expect(writers.sort()).toEqual([ expect(writers.sort()).toEqual([
'src/services/model/chainShadow.js', 'src/services/model/chainShadow.js',
'src/services/retentionService.js', 'src/services/retentionService.js',
@@ -431,6 +439,18 @@ describe('0 LEAKS — measured against the source, not asserted in prose', () =>
]); ]);
}); });
it('the evaluator NAMES the shadow column but never assigns to it', () => {
// Excluding it above is only safe if this holds. Proven, not assumed.
const dir = 'src/services/evaluator/';
const files = walk('src').filter((f) => f.startsWith(dir));
expect(files.length).toBeGreaterThan(0);
for (const f of files) {
const src = read(f);
expect(src).not.toMatch(/chain_shadow\s*[:=][^=]/);
expect(src).not.toMatch(/\.chain_shadow\s*=/);
}
});
it('the shadow runs with requireCalibrated FALSE and says so where it is stored', () => { it('the shadow runs with requireCalibrated FALSE and says so where it is stored', () => {
expect(read('src/services/model/chainShadow.js')).toMatch(/requireCalibrated:\s*false/); expect(read('src/services/model/chainShadow.js')).toMatch(/requireCalibrated:\s*false/);
expect(cs.SHADOW_VERSION).toBe('chain-shadow@1'); expect(cs.SHADOW_VERSION).toBe('chain-shadow@1');
tests/unit/eventPublication.test.js
+734
View File
@@ -0,0 +1,734 @@
'use strict';
/**
* CANONICAL EVENT IDENTITY + PUBLICATION COMMIT TRUTH.
*
* Two findings drive this suite, both verified against production:
*
* 1. EVENT LABELS ARE NOT EVENT IDENTITY. `gameIdFor` derives
* `sport:date:away@home`, which has no occurrence component, so both halves
* of a doubleheader produce a byte-identical string. MLB 2026 through
* 2026-08-27: 19 doubleheaders / 38 real games collapse into 19 labels. The
* 2026-08-17 St. Louis @ Cincinnati pair (gamePk 824514 / 824478) landed in
* ONE id holding 171 ledger rows and FOUR distinct starting pitchers.
*
* 2. PUBLISHED MUST MEAN COMMITTED. Retention persisted at
* snapshotService:1137 while the authoritative slate write is at :1152, and
* the winner was marked published earlier still.
*/
const path = require('path');
const fs = require('fs');
const E = require('../../src/services/event/eventIdentity');
const L = require('../../src/services/read/readLineage');
const retention = require('../../src/services/retentionService');
const ROOT = path.resolve(__dirname, '..', '..');
// The REAL doubleheader, from statsapi.
const DH_GAMES = [
{ gamePk: 824514, gameNumber: 1, gameDate: '2026-08-17T17:40:00Z', home: { team: 'Cincinnati Reds' }, away: { team: 'St. Louis Cardinals' } },
{ gamePk: 824478, gameNumber: 2, gameDate: '2026-08-17T22:40:00Z', home: { team: 'Cincinnati Reds' }, away: { team: 'St. Louis Cardinals' } },
];
const dhProp = (time, over = {}) => ({
player: 'Andre Pallante', stat_type: 'strikeouts', line: 4.5, book: 'draftkings',
home_team: 'Cincinnati Reds', away_team: 'St. Louis Cardinals',
game_date: '2026-08-17', game_time: time, ...over,
});
describe('EVENT IDENTITY — canonical, sport-neutral contract', () => {
test('MLB resolves to a namespaced canonical id from statsapi gamePk', () => {
const r = E.resolveEvent('mlb', dhProp('2026-08-17T17:40:00Z'), DH_GAMES);
expect(r.canonical_event_id).toBe('mlb:gamepk:824514');
expect(r.event_identity_source).toBe(E.IDENTITY_SOURCE.MLB_STATSAPI_GAMEPK);
expect(r.event_identity_method).toBe(E.IDENTITY_METHOD.CANONICAL);
expect(r.event_occurrence).toBe(1);
});
test('an unsupported sport records UNSUPPORTED_SPORT, never a guessed id', () => {
for (const sp of ['wnba', 'nba', 'soccer', 'nfl']) {
const r = E.resolveEvent(sp, dhProp('2026-08-17T17:40:00Z'), DH_GAMES);
expect(r.event_identity_method).toBe(E.IDENTITY_METHOD.UNSUPPORTED_SPORT);
expect(r.canonical_event_id).toBeNull();
}
});
test('team resolution is canonical, handling every real spelling', () => {
const cases = [
['Cincinnati Reds', 'CIN'], ['CINReds', 'CIN'], ['CIN', 'CIN'],
['St. Louis Cardinals', 'STL'], ['St.LouisCardinals', 'STL'],
['New York Mets', 'NYM'], ['NYMets', 'NYM'], ['CLEGuardians', 'CLE'],
// The trap: both end in "sox". Longest nickname must win.
['Chicago White Sox', 'CWS'], ['ChicagoWhiteSox', 'CWS'],
['Boston Red Sox', 'BOS'], ['BOSRedSox', 'BOS'],
['Toronto Blue Jays', 'TOR'], ['TORBlueJays', 'TOR'],
];
for (const [name, abbr] of cases) expect(E.teamAbbr(name)).toBe(abbr);
expect(E.teamAbbr('Not A Team')).toBeNull();
expect(E.teamAbbr('')).toBeNull();
});
test('an unresolvable team name matches NOTHING rather than falling back to text', () => {
expect(E.teamsMatch(DH_GAMES[0], 'Cincinnati Reds', 'Some Fake Club')).toBe(false);
});
});
describe('DOUBLEHEADER SAFETY — the mandatory proof', () => {
test('two games of one doubleheader receive DIFFERENT canonical ids', () => {
const g1 = E.resolveEvent('mlb', dhProp('2026-08-17T17:40:00Z'), DH_GAMES);
const g2 = E.resolveEvent('mlb', dhProp('2026-08-17T22:40:00Z'), DH_GAMES);
expect(g1.canonical_event_id).toBe('mlb:gamepk:824514');
expect(g2.canonical_event_id).toBe('mlb:gamepk:824478');
expect(g1.canonical_event_id).not.toBe(g2.canonical_event_id);
expect(g1.event_occurrence).toBe(1);
expect(g2.event_occurrence).toBe(2);
});
test('both survive dedupe — same player, stat and line, different games', () => {
const gs = require('../../src/services/gradeSlateService');
const props = [dhProp('2026-08-17T17:40:00Z'), dhProp('2026-08-17T22:40:00Z')];
E.attachEventIdentity('mlb', props, DH_GAMES);
const kept = gs.__internals
? gs.__internals.dedupeProps(props, 100)
: null;
expect(kept).not.toBeNull();
expect(kept).toHaveLength(2);
expect(new Set(kept.map((p) => p.canonical_event_id)).size).toBe(2);
});
test('their LEGACY derived label collides, and that no longer matters', () => {
const gameIdFor = require('../../src/services/ledgerService').__internals.gameIdFor;
const prop = { away_team: 'St. Louis Cardinals', home_team: 'Cincinnati Reds' };
const a = gameIdFor('mlb', prop, '2026-08-17');
const b = gameIdFor('mlb', prop, '2026-08-17');
expect(a).toBe(b); // the label cannot separate them...
// ...but canonical identity does, and lineage keys on that.
const base = {
sport: 'mlb', game_date: '2026-08-17', player_key: 'andre pallante',
stat: 'strikeouts', side: 'over', line: 4.5, game_id: a,
captured_at: '2026-08-17T14:00:00Z', published: true, grade: 'B', p_win: 0.6,
};
const r1 = { ...base, canonical_event_id: 'mlb:gamepk:824514' };
const r2 = { ...base, canonical_event_id: 'mlb:gamepk:824478' };
expect(L.readNaturalKey(r1)).not.toBe(L.readNaturalKey(r2));
});
test('neither game can supersede or overwrite the other', () => {
const base = {
sport: 'mlb', game_date: '2026-08-17', player_key: 'andre pallante',
stat: 'strikeouts', side: 'over', line: 4.5, game_id: 'mlb:2026-08-17:StL@Cin',
captured_at: '2026-08-17T14:00:00Z', published: true, grade: 'B', p_win: 0.6,
};
const g1 = { ...base, canonical_event_id: 'mlb:gamepk:824514' };
const g2 = { ...base, canonical_event_id: 'mlb:gamepk:824478' };
const r1 = L.resolveLineage({ candidate: g1, existing: [], mintReadId: () => 'RA' });
const row1 = {
id: 101, read_id: 'RA', read_natural_key: r1.read_natural_key, game_id: base.game_id,
claim_digest: r1.claim_digest, revision_ordinal: 0, lineage_action: 'ORIGIN',
supersedes_id: null, captured_at: base.captured_at, claim: g1,
};
const r2 = L.resolveLineage({ candidate: g2, existing: [row1], mintReadId: () => 'RB' });
expect(r2.read_id).toBe('RB');
expect(r2.action).toBe(L.LINEAGE_ACTION.ORIGIN);
expect(r2.supersedes_id).toBeNull();
});
test('identity is decided by TIME, and refuses when time cannot decide', () => {
const noTime = E.resolveEvent('mlb', dhProp(null), DH_GAMES);
expect(noTime.event_identity_method).toBe(E.IDENTITY_METHOD.UNRESOLVED);
expect(noTime.reason).toBe('ambiguous_no_game_time');
// A time matching neither game is refused rather than snapped to the nearer.
const farOff = E.resolveEvent('mlb', dhProp('2026-08-17T05:00:00Z'), DH_GAMES);
expect(farOff.event_identity_method).toBe(E.IDENTITY_METHOD.UNRESOLVED);
expect(farOff.reason).toBe('ambiguous_time_too_far');
});
test('a single-game day needs no time to resolve', () => {
const r = E.resolveEvent('mlb', dhProp(null), [DH_GAMES[0]]);
expect(r.canonical_event_id).toBe('mlb:gamepk:824514');
});
});
describe('DEDUPE SEMANTICS — collapse books, never events', () => {
const gs = require('../../src/services/gradeSlateService');
const dedupe = (props, n) => gs.__internals.dedupeProps(props, n);
test('the same proposition from several books still collapses to one', () => {
const props = ['draftkings', 'fanduel', 'betmgm'].map((book) =>
({ ...dhProp('2026-08-17T17:40:00Z'), book, canonical_event_id: 'mlb:gamepk:824514' }));
expect(dedupe(props, 100)).toHaveLength(1);
});
test('the same proposition in two events does NOT collapse', () => {
const props = [
{ ...dhProp('2026-08-17T17:40:00Z'), canonical_event_id: 'mlb:gamepk:824514' },
{ ...dhProp('2026-08-17T22:40:00Z'), canonical_event_id: 'mlb:gamepk:824478' },
];
expect(dedupe(props, 100)).toHaveLength(2);
});
test('without canonical identity behaviour is UNCHANGED from before', () => {
// Same teams, same date, no canonical id: the legacy fallback collapses
// them exactly as the old key did. The fix does not pretend to distinguish
// what it cannot.
const props = [dhProp('2026-08-17T17:40:00Z'), dhProp('2026-08-17T22:40:00Z')];
expect(dedupe(props, 100)).toHaveLength(1);
});
test('different games on the same date never collapse even without canonical id', () => {
const a = { ...dhProp('t'), home_team: 'Cincinnati Reds', away_team: 'St. Louis Cardinals' };
const b = { ...dhProp('t'), home_team: 'Boston Red Sox', away_team: 'Arizona Diamondbacks' };
expect(dedupe([a, b], 100)).toHaveLength(2);
});
});
describe('PUBLICATION COMMIT TRUTH', () => {
const served = () => ({
sport: 'mlb', game_date: '2026-08-17', player_key: 'x', stat: 'hits', side: 'over',
line: 0.5, game_id: 'g', canonical_event_id: 'mlb:gamepk:824514',
captured_at: '2026-08-17T14:00:00Z', grade: 'B', p_win: 0.61, published: true,
});
const okClient = () => ({ from: () => ({ upsert: async () => ({ error: null }) }) });
test('capture does NOT resolve lineage — persist leaves it null', () => {
const src = fs.readFileSync(path.join(ROOT, 'src/services/retentionService.js'), 'utf8');
const persistBody = src.slice(src.indexOf('async function persist('));
expect(persistBody).not.toMatch(/attachLineage\(/);
});
test('the slate write PRECEDES the publication commit', () => {
const src = fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8');
const persistAt = src.indexOf('await persistRetention(enriched');
const slateAt = src.indexOf('cacheSet(`snapshot:${sp}:latest`');
const commitAt = src.indexOf('commitPublication({');
expect(persistAt).toBeGreaterThan(-1);
expect(slateAt).toBeGreaterThan(persistAt);
expect(commitAt).toBeGreaterThan(slateAt);
});
test('WINNER SELECTED + REDIS OK -> the exact snapshot becomes published', async () => {
const rows = [served()];
const out = await retention.commitPublication(
{ rows, publicationId: 'slate-1', publishedAt: '2026-08-17T15:00:00Z' },
{ getClient: okClient, fetchExisting: async () => [], mintReadId: () => 'R1' },
);
expect(out.published).toBe(1);
expect(rows[0].published_at).toBe('2026-08-17T15:00:00Z');
expect(rows[0].publication_id).toBe('slate-1');
expect(rows[0].lineage_action).toBe('ORIGIN');
});
test('WINNER SELECTED + REDIS FAILS -> nothing is published', () => {
// The commit is unreachable when the slate write throws: it sits after the
// awaited cacheSet in the same function, with no catch between them.
const src = fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8');
const slateAt = src.indexOf('cacheSet(`snapshot:${sp}:latest`');
const commitAt = src.indexOf('commitPublication({');
const between = src.slice(slateAt, commitAt);
expect(between).not.toMatch(/catch\s*\(/);
});
test('a row that was never served is not committed even in a served batch', async () => {
const rows = [served(), { ...served(), side: 'under', published: false }];
const out = await retention.commitPublication(
{ rows, publicationId: 'slate-1', publishedAt: 't' },
{ getClient: okClient, fetchExisting: async () => [], mintReadId: () => 'R1' },
);
expect(out.candidates).toBe(1);
expect(rows[1].published_at).toBeUndefined();
expect(rows[1].lineage_action).toBeUndefined();
});
test('REDIS OK + PUBLICATION UPDATE FAILS -> understated, never overstated', async () => {
const rows = [served()];
const failing = () => ({ from: () => ({ upsert: async () => ({ error: { message: 'db down' } }) }) });
const out = await retention.commitPublication(
{ rows, publicationId: 'slate-1', publishedAt: 't' },
{ getClient: failing, fetchExisting: async () => [], mintReadId: () => 'R1' },
);
expect(out.published).toBe(0);
expect(out.error).toBe('db down');
});
test('no database configured skips without throwing', async () => {
const out = await retention.commitPublication(
{ rows: [served()], publicationId: 's', publishedAt: 't' },
{ getClient: () => null },
);
expect(out.skipped).toBe(true);
expect(out.error).toBeNull();
});
test('retrying the commit does not manufacture a second publication', async () => {
const rows = [served()];
const deps = { getClient: okClient, fetchExisting: async () => [], mintReadId: () => 'R1' };
await retention.commitPublication({ rows, publicationId: 's', publishedAt: 't' }, deps);
const first = rows[0].claim_digest;
// Second run sees the first as existing history.
const existing = [{
id: 1, read_id: rows[0].read_id, read_natural_key: rows[0].read_natural_key,
game_id: rows[0].game_id, claim_digest: first, revision_ordinal: 0,
lineage_action: 'ORIGIN', supersedes_id: null, captured_at: rows[0].captured_at,
claim: rows[0],
}];
const rows2 = [served()];
const out2 = await retention.commitPublication(
{ rows: rows2, publicationId: 's', publishedAt: 't' },
{ ...deps, fetchExisting: async () => existing },
);
expect(out2.lineage.recaptures).toBe(1);
expect(out2.lineage.revisions).toBe(0);
});
});
describe('PUBLICATION CLOCK', () => {
test('captured_at and published_at are different fields answering different questions', async () => {
const rows = [{
sport: 'mlb', game_date: '2026-08-17', player_key: 'x', stat: 'hits', side: 'over',
line: 0.5, game_id: 'g', canonical_event_id: 'mlb:gamepk:824514',
captured_at: '2026-08-17T14:00:00Z', grade: 'B', p_win: 0.61, published: true,
}];
await retention.commitPublication(
{ rows, publicationId: 's', publishedAt: '2026-08-17T15:07:00Z' },
{ getClient: () => ({ from: () => ({ upsert: async () => ({ error: null }) }) }), fetchExisting: async () => [], mintReadId: () => 'R1' },
);
expect(rows[0].captured_at).toBe('2026-08-17T14:00:00Z');
expect(rows[0].published_at).toBe('2026-08-17T15:07:00Z');
expect(rows[0].published_at).not.toBe(rows[0].captured_at);
});
});
describe('CONCURRENCY RECOVERY — bounded', () => {
const served = () => ({
sport: 'mlb', game_date: '2026-08-17', player_key: 'x', stat: 'hits', side: 'over',
line: 0.5, game_id: 'g', canonical_event_id: 'mlb:gamepk:824514',
captured_at: 't', grade: 'B', p_win: 0.61, published: true,
});
const conflict = { message: 'duplicate key value violates unique constraint "model_snapshots_supersedes_unique"' };
test('the supersedes conflict is recognised and other errors are not', () => {
expect(retention.isSupersedesConflict(conflict)).toBe(true);
expect(retention.isSupersedesConflict({ message: 'connection refused' })).toBe(false);
expect(retention.isSupersedesConflict(null)).toBe(false);
});
test('a losing writer re-resolves against the new head and succeeds', async () => {
let n = 0;
const sb = { from: () => ({ upsert: async () => { n += 1; return n === 1 ? { error: conflict } : { error: null }; } }) };
const out = await retention.commitPublication(
{ rows: [served()], publicationId: 's', publishedAt: 't' },
{ getClient: () => sb, fetchExisting: async () => [], mintReadId: () => 'R1' },
);
expect(out.conflicts).toBe(1);
expect(out.recovered).toBe(1);
expect(out.published).toBe(1);
});
test('a permanent conflict fails EXPLICITLY and is bounded', async () => {
let n = 0;
const sb = { from: () => ({ upsert: async () => { n += 1; return { error: conflict }; } }) };
const out = await retention.commitPublication(
{ rows: [served()], publicationId: 's', publishedAt: 't' },
{ getClient: () => sb, fetchExisting: async () => [], mintReadId: () => 'R1' },
);
expect(out.recovery_failed).toBe(1);
expect(out.error).toBeTruthy();
// 1 initial + FORK_RETRY_LIMIT attempts. Bounded, never a spin.
expect(n).toBe(1 + retention.FORK_RETRY_LIMIT);
expect(retention.FORK_RETRY_LIMIT).toBeLessThanOrEqual(3);
});
test('a non-conflict error is NOT retried', async () => {
let n = 0;
const sb = { from: () => ({ upsert: async () => { n += 1; return { error: { message: 'db down' } }; } }) };
const out = await retention.commitPublication(
{ rows: [served()], publicationId: 's', publishedAt: 't' },
{ getClient: () => sb, fetchExisting: async () => [], mintReadId: () => 'R1' },
);
expect(n).toBe(1);
expect(out.error).toBe('db down');
expect(out.conflicts).toBeUndefined();
});
});
describe('CHANGE SEMANTICS — identity is not an opinion', () => {
test('correcting a derived event label is NOT a BELIEF change', () => {
const a = { line: 0.5, side: 'over', p_win: 0.61, grade: 'B', canonical_event_id: null };
const b = { ...a, canonical_event_id: 'mlb:gamepk:824514' };
expect(L.CLAIM_FIELD_CLASSES.canonical_event_id).toBeUndefined();
expect(L.classifyChange(a, b).change_type).toBe(L.CHANGE_TYPE.NO_MATERIAL_PUBLISHED_CHANGE);
expect(L.classifyChange(a, b).belief_changed).toBe(false);
});
});
describe('SETTLEMENT SEPARATION still holds after propagation', () => {
test('canonical event, publication and lineage fields are claim-owned', () => {
for (const f of ['published', 'claim_digest', 'revision_ordinal', 'change_type']) {
expect(L.settlementUpdateIsSafe({ outcome: 'hit', [f]: 'x' }).violations).toContain(f);
}
});
test('the real production settle patch still passes', () => {
const src = fs.readFileSync(path.join(ROOT, 'src/services/snapshotSettlementService.js'), 'utf8');
const m = src.match(/\.update\(\{([\s\S]*?)\}\)/);
const keys = [...m[1].matchAll(/^\s*([a-z_0-9]+)\s*:/gm)].map((x) => x[1]);
expect(L.settlementUpdateIsSafe(Object.fromEntries(keys.map((k) => [k, 1])))).toMatchObject({ safe: true });
});
});
describe('PRODUCTION PRESERVATION', () => {
test('no product path imports event identity or lineage', () => {
const walk = (dir, out = []) => {
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
if (e.name === 'node_modules' || e.name.startsWith('.')) continue;
const full = path.join(dir, e.name);
if (e.isDirectory()) walk(full, out);
else if (/\.(js|jsx|ts|tsx)$/.test(e.name)) out.push(full);
}
return out;
};
const allowed = new Set([
path.join(ROOT, 'src/services/read/readLineage.js'),
path.join(ROOT, 'src/services/retentionService.js'),
path.join(ROOT, 'src/services/event/eventIdentity.js'),
path.join(ROOT, 'src/services/snapshotService.js'),
]);
const files = [
...walk(path.join(ROOT, 'src/routes')),
...(fs.existsSync(path.join(ROOT, 'web/src')) ? walk(path.join(ROOT, 'web/src')) : []),
].filter((f) => !allowed.has(f));
const offenders = files.filter((f) => {
const src = fs.readFileSync(f, 'utf8')
.replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:])\/\/.*$/gm, '$1');
return /(require\(|from\s+)['"`][^'"`]*(readLineage|eventIdentity)['"`]/.test(src);
}).map((f) => path.relative(ROOT, f));
expect(offenders).toEqual([]);
});
test('the served envelope still comes from the cache', () => {
const src = fs.readFileSync(path.join(ROOT, 'src/routes/snapshot.js'), 'utf8');
expect(src).toMatch(/cacheGet/);
expect(src).not.toMatch(/model_snapshots/);
expect(src).not.toMatch(/canonical_event_id/);
});
});
describe('CANARY SCOPE — bounded and reversible', () => {
const load = (val) => {
const prev = process.env.LINEAGE_CANARY_SPORTS;
if (val === undefined) delete process.env.LINEAGE_CANARY_SPORTS;
else process.env.LINEAGE_CANARY_SPORTS = val;
jest.resetModules();
// eslint-disable-next-line global-require
const m = require('../../src/services/snapshotService');
if (prev === undefined) delete process.env.LINEAGE_CANARY_SPORTS;
else process.env.LINEAGE_CANARY_SPORTS = prev;
return m;
};
test('DARK BY DEFAULT — an unset flag enables lineage for nothing', () => {
// This release deploys the integrity repairs with the observer off.
const m = load(undefined);
expect(m.LINEAGE_CANARY_SPORTS).toEqual([]);
for (const sp of ['mlb', 'wnba', 'nba', 'soccer']) {
expect(m.lineageCanaryEnabled(sp)).toBe(false);
}
});
test('MLB is enabled only by explicit configuration', () => {
const m = load('mlb');
expect(m.lineageCanaryEnabled('mlb')).toBe(true);
for (const sp of ['wnba', 'nba', 'soccer']) {
expect(m.lineageCanaryEnabled(sp)).toBe(false);
}
});
test('an empty value is a kill switch needing no migration revert', () => {
const m = load('');
expect(m.LINEAGE_CANARY_SPORTS).toEqual([]);
expect(m.lineageCanaryEnabled('mlb')).toBe(false);
});
test('a sport without a verified event resolver cannot be canaried into safety', () => {
// Widening the flag does NOT create identity: eventIdentity still refuses,
// so a widened sport records UNSUPPORTED_SPORT and its lineage stays honest.
const m = load('mlb,wnba');
expect(m.lineageCanaryEnabled('wnba')).toBe(true);
const r = E.resolveEvent('wnba', dhProp('2026-08-17T17:40:00Z'), DH_GAMES);
expect(r.canonical_event_id).toBeNull();
expect(r.event_identity_method).toBe(E.IDENTITY_METHOD.UNSUPPORTED_SPORT);
});
test('the commit is gated on the flag in the live path', () => {
const src = fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8');
expect(src).toMatch(/commitPublication && persistedRows && lineageCanaryEnabled\(sp\)/);
});
});
describe('UNRESOLVED EVENT SAFETY — may lose coverage, may never merge', () => {
const gs = require('../../src/services/gradeSlateService');
const dedupe = (props, n) => gs.__internals.dedupeProps(props, n);
const base = (o) => ({
player: 'Andre Pallante', stat_type: 'strikeouts', line: 4.5, book: 'draftkings',
home_team: 'Cincinnati Reds', away_team: 'St. Louis Cardinals',
game_date: '2026-08-17', ...o,
});
test('an UNRESOLVED MLB prop never merges with another — it fractures', () => {
// Two rows that could be two different games. Merging them would be the
// exact defect this change removes, so they must both survive.
const props = [
base({ event_identity_method: 'UNRESOLVED' }),
base({ event_identity_method: 'UNRESOLVED', book: 'fanduel' }),
];
expect(dedupe(props, 100)).toHaveLength(2);
});
test('the unresolved key is NOT date+teams and NOT the legacy label', () => {
const src = fs.readFileSync(path.join(ROOT, 'src/services/gradeSlateService.js'), 'utf8');
const fn = src.slice(src.indexOf('function propositionEventKey'), src.indexOf('function dedupeProps'));
// The legacy label is reachable ONLY for sports with no resolver.
expect(fn).toMatch(/expectsCanonical/);
expect(fn).toMatch(/unresolved-event:/);
// And the fracture branch is returned BEFORE the legacy label is built.
expect(fn.indexOf('unresolved-event:')).toBeLessThan(fn.indexOf('legacy:'));
});
test('a resolved event still collapses books — the real dedupe job survives', () => {
const props = ['draftkings', 'fanduel', 'betmgm'].map((book) =>
base({ book, canonical_event_id: 'mlb:gamepk:824514' }));
expect(dedupe(props, 100)).toHaveLength(1);
});
test('a sport with NO resolver keeps legacy behaviour, unchanged', () => {
// WNBA never had canonical identity; fracturing it would multiply grading
// load for no safety gain, and its lineage is excluded from the canary.
const props = [
base({ event_identity_method: 'UNSUPPORTED_SPORT' }),
base({ event_identity_method: 'UNSUPPORTED_SPORT', book: 'fanduel' }),
];
expect(dedupe(props, 100)).toHaveLength(1);
});
test('unresolved MLB props from two REAL games cannot collapse', () => {
// Same player/stat/line/teams/date, identity unknown for both.
const g1 = base({ event_identity_method: 'UNRESOLVED', game_time: '2026-08-17T17:40:00Z' });
const g2 = base({ event_identity_method: 'UNRESOLVED', game_time: '2026-08-17T22:40:00Z' });
expect(dedupe([g1, g2], 100)).toHaveLength(2);
});
});
describe('EXACT PARITY-GAP IDENTITY', () => {
const served = () => ({
sport: 'mlb', snapshot_id: 'snap-1', game_date: '2026-08-17', player_key: 'judge',
stat: 'hits', side: 'over', line: 0.5, game_id: 'g',
canonical_event_id: 'mlb:gamepk:824514', captured_at: '2026-08-17T14:00:00Z',
grade: 'B', p_win: 0.61, published: true,
});
test('a failed commit names WHICH served claim is missing', async () => {
const failing = () => ({ from: () => ({ upsert: async () => ({ error: { message: 'db down' } }) }) });
const out = await retention.commitPublication(
{ rows: [served()], publicationId: 'slate-9', publishedAt: '2026-08-17T15:00:00Z' },
{ getClient: failing, fetchExisting: async () => [], mintReadId: () => 'R1', now: () => 't5' },
);
expect(out.parity_gap).toBeTruthy();
expect(out.parity_gap.reason).toBe('db down');
expect(out.parity_gap.missing_count).toBe(1);
const m = out.parity_gap.missing[0];
// Deterministic recovery, not "lineage failed".
for (const k of ['snapshot_id', 'player_key', 'stat', 'line', 'side', 'canonical_event_id', 'captured_at']) {
expect(m[k]).not.toBeNull();
}
expect(out.parity_gap.publication_id).toBe('slate-9');
expect(out.parity_gap.published_at).toBe('2026-08-17T15:00:00Z');
expect(out.parity_gap.attempted_at).toBe('t5');
});
test('a clean commit records NO parity gap', async () => {
const ok = () => ({ from: () => ({ upsert: async () => ({ error: null }) }) });
const out = await retention.commitPublication(
{ rows: [served()], publicationId: 's', publishedAt: 't' },
{ getClient: ok, fetchExisting: async () => [], mintReadId: () => 'R1' },
);
expect(out.parity_gap).toBeUndefined();
expect(out.published).toBe(1);
});
});
describe('PROPAGATION — canonical identity survives every hop', () => {
test('the graded base carries identity from the prop', () => {
// Source assertion on the ONE place the prop→base copy happens. A test that
// only built props with the field already set could not catch it being
// dropped here, which is exactly what the teeth run exposed.
const src = fs.readFileSync(path.join(ROOT, 'src/services/gradeSlateService.js'), 'utf8');
const base = src.slice(src.indexOf('const base = {'), src.indexOf('const sides = await'));
expect(base).toMatch(/canonical_event_id:\s*prop\.canonical_event_id/);
expect(base).toMatch(/event_identity_method:\s*prop\.event_identity_method/);
expect(base).not.toMatch(/canonical_event_id:\s*null\s*,/);
});
test('the retention row carries identity from the graded base', () => {
const c = retention.createCollector({
snapshotId: 's1', capturedAt: '2026-08-17T14:00:00Z',
gameDate: '2026-08-17', gameIdFor: () => 'mlb:2026-08-17:StL@Cin',
});
const base = {
player: 'Andre Pallante', stat_type: 'strikeouts', line: 4.5, sport: 'mlb',
canonical_event_id: 'mlb:gamepk:824514',
event_identity_source: 'MLB_STATSAPI_GAMEPK',
event_identity_method: 'CANONICAL',
event_identity_version: 'evid@1',
event_occurrence: 1,
};
c.onGraded(base, [{ ...base, direction: 'over', grade: 'B', confidence: 60 }]);
expect(c.rows).toHaveLength(1);
expect(c.rows[0].canonical_event_id).toBe('mlb:gamepk:824514');
expect(c.rows[0].event_identity_source).toBe('MLB_STATSAPI_GAMEPK');
expect(c.rows[0].event_identity_method).toBe('CANONICAL');
expect(c.rows[0].event_occurrence).toBe(1);
});
test('a row with no identity carries NULL, never a fabricated id', () => {
const c = retention.createCollector({
snapshotId: 's1', capturedAt: 't', gameDate: '2026-08-17', gameIdFor: () => 'g',
});
const base = { player: 'X', stat_type: 'hits', line: 0.5, sport: 'wnba' };
c.onGraded(base, [{ ...base, direction: 'over', grade: 'B' }]);
expect(c.rows[0].canonical_event_id).toBeNull();
expect(c.rows[0].event_identity_method).toBeNull();
});
test('lineage keys on the canonical id once it arrives', () => {
const row = {
sport: 'mlb', game_date: '2026-08-17', player_key: 'andre pallante',
stat: 'strikeouts', side: 'over', line: 4.5, game_id: 'mlb:2026-08-17:StL@Cin',
captured_at: 't', published: true, grade: 'B', p_win: 0.6,
canonical_event_id: 'mlb:gamepk:824514',
};
expect(L.eventOccurrence(row)).toBe('mlb:gamepk:824514');
expect(L.readNaturalKey(row)).toContain('mlb:gamepk:824514');
const dropped = { ...row, canonical_event_id: null };
expect(L.readNaturalKey(dropped)).not.toContain('gamepk');
});
});
describe('IMPOSSIBLE-BINDING INVARIANT — the two verified regressions', () => {
const { nameKey } = require('../../src/utils/playerName');
const SLATE = [
{ gamePk: 823825, gameNumber: 1, gameDate: '2026-08-26T22:40:00Z', home: { team: 'Miami Marlins', teamId: 146 }, away: { team: 'Boston Red Sox', teamId: 111 } },
{ gamePk: 824878, gameNumber: 1, gameDate: '2026-08-26T23:15:00Z', home: { team: 'Atlanta Braves', teamId: 144 }, away: { team: 'Los Angeles Dodgers', teamId: 119 } },
{ gamePk: 824234, gameNumber: 1, gameDate: '2026-08-26T17:10:00Z', home: { team: 'Detroit Tigers', teamId: 116 }, away: { team: 'Tampa Bay Rays', teamId: 139 } },
{ gamePk: 824555, gameNumber: 1, gameDate: '2026-08-26T23:40:00Z', home: { team: 'Chicago White Sox', teamId: 145 }, away: { team: 'Texas Rangers', teamId: 140 } },
];
// Teams verified from statsapi: Joe Mack id 691788 Marlins, Yandy Diaz id 650490 Rays.
const TEAMS = new Map([
[nameKey('Joe Mack'), new Set(['MIA'])],
[nameKey('Yandy Diaz'), new Set(['TB'])],
]);
test('JOE MACK regression — Marlin bound to Dodgers@Braves is REFUSED', () => {
const prop = {
player: 'Joe Mack', stat_type: 'total_bases', line: 0.5, book: 'pinnacle',
home_team: 'Atlanta Braves', away_team: 'Los Angeles Dodgers',
game_date: '2026-08-26', game_time: '2026-08-26T23:15:00Z',
};
const r = E.resolveEvent('mlb', prop, SLATE, TEAMS);
expect(r.canonical_event_id).toBeNull();
expect(r.event_identity_method).toBe(E.IDENTITY_METHOD.UNRESOLVED);
expect(r.reason).toBe('impossible_binding');
expect(r.evidence).toMatchObject({ violated: true, player_teams: ['MIA'] });
expect(r.evidence.event_teams).toEqual(['LAD', 'ATL']);
});
test('YANDY DIAZ regression — Ray bound to Rangers@WhiteSox is REFUSED', () => {
const prop = {
player: 'Yandy Diaz', stat_type: 'total_bases', line: 0.5, book: 'pinnacle',
home_team: 'Chicago White Sox', away_team: 'Texas Rangers',
game_date: '2026-08-26', game_time: '2026-08-26T23:40:00Z',
};
const r = E.resolveEvent('mlb', prop, SLATE, TEAMS);
expect(r.reason).toBe('impossible_binding');
expect(r.evidence.player_teams).toEqual(['TB']);
});
test('each player STILL binds to his own real game', () => {
const mack = E.resolveEvent('mlb', {
player: 'Joe Mack', home_team: 'Miami Marlins', away_team: 'Boston Red Sox',
game_time: '2026-08-26T22:40:00Z',
}, SLATE, TEAMS);
expect(mack.canonical_event_id).toBe('mlb:gamepk:823825');
const diaz = E.resolveEvent('mlb', {
player: 'Yandy Diaz', home_team: 'Detroit Tigers', away_team: 'Tampa Bay Rays',
game_time: '2026-08-26T17:10:00Z',
}, SLATE, TEAMS);
expect(diaz.canonical_event_id).toBe('mlb:gamepk:824234');
});
test('UNKNOWN team preserves uncertainty — it is not a violation', () => {
// Refusing on absent evidence would drop real props for a data gap.
const r = E.resolveEvent('mlb', {
player: 'Someone Unlisted', home_team: 'Atlanta Braves',
away_team: 'Los Angeles Dodgers', game_time: '2026-08-26T23:15:00Z',
}, SLATE, TEAMS);
expect(r.canonical_event_id).toBe('mlb:gamepk:824878');
});
test('no index at all leaves behaviour exactly as before', () => {
const prop = {
player: 'Joe Mack', home_team: 'Atlanta Braves', away_team: 'Los Angeles Dodgers',
game_time: '2026-08-26T23:15:00Z',
};
expect(E.resolveEvent('mlb', prop, SLATE).canonical_event_id).toBe('mlb:gamepk:824878');
expect(E.resolveEvent('mlb', prop, SLATE, null).canonical_event_id).toBe('mlb:gamepk:824878');
});
test('time proximity CANNOT override an impossible team relationship', () => {
// The mis-nested event is the closest by start time; it must still refuse.
const prop = {
player: 'Joe Mack', home_team: 'Atlanta Braves', away_team: 'Los Angeles Dodgers',
game_time: '2026-08-26T23:15:00Z',
};
expect(E.resolveEvent('mlb', prop, SLATE, TEAMS).reason).toBe('impossible_binding');
});
test('a multi-team player (in-season trade) is not falsely refused', () => {
const traded = new Map([[nameKey('Joe Mack'), new Set(['MIA', 'ATL'])]]);
const r = E.resolveEvent('mlb', {
player: 'Joe Mack', home_team: 'Atlanta Braves', away_team: 'Los Angeles Dodgers',
game_time: '2026-08-26T23:15:00Z',
}, SLATE, traded);
expect(r.canonical_event_id).toBe('mlb:gamepk:824878');
});
test('the roster index keys on nameKey, not the raw name', () => {
// nameKey('Joe Mack') is 'joseph mack' — the nickname table. An index keyed
// on the raw string silently matches nothing and the invariant never fires.
expect(nameKey('Joe Mack')).toBe('joseph mack');
const rawKeyed = new Map([['joe mack', new Set(['MIA'])]]);
const r = E.resolveEvent('mlb', {
player: 'Joe Mack', home_team: 'Atlanta Braves', away_team: 'Los Angeles Dodgers',
game_time: '2026-08-26T23:15:00Z',
}, SLATE, rawKeyed);
expect(r.canonical_event_id).toBe('mlb:gamepk:824878'); // does NOT fire
});
test('buildPlayerTeamIndex keys correctly and survives an unreadable roster', async () => {
const built = await E.buildPlayerTeamIndex(SLATE, {
getTeamRoster: async (id) => {
if (id === 146) return [{ id: 691788, name: 'Joe Mack' }];
if (id === 144) throw new Error('roster unavailable');
return [];
},
});
expect(built.index.get(nameKey('Joe Mack'))).toEqual(new Set(['MIA']));
expect(built.stats.failed).toBe(1);
expect(built.stats.teams).toBeGreaterThan(0);
});
test('no roster source yields an empty index, never a fabricated one', async () => {
const built = await E.buildPlayerTeamIndex(SLATE, {});
expect(built.index.size).toBe(0);
});
});
tests/unit/publicationLineage.test.js
+415
View File
@@ -0,0 +1,415 @@
'use strict';
/**
* PUBLICATION LINEAGE — the semantic gates.
*
* The central finding these lock down: a model_snapshots row is NOT a published
* Read. `gradeSlateService` fires its retention hook with BOTH sides, graded AND
* refused, before any filtering, and only the higher-confidence graded side
* becomes the served Read. Measured on mlb 2026-08-26: 8,443 of 13,012 captured
* rows (64.9%) describe a state no user was ever shown.
*/
const path = require('path');
const fs = require('fs');
const L = require('../../src/services/read/readLineage');
const retention = require('../../src/services/retentionService');
const ROOT = path.resolve(__dirname, '..', '..');
const CLAIM = {
sport: 'mlb', game_date: '2026-08-27', player_key: 'aaron judge',
stat: 'hits', side: 'over', line: 0.5,
game_id: 'mlb:2026-08-27:BostonRedSox@NewYorkYankees',
captured_at: '2026-08-27T14:00:00Z',
grade: 'B', p_win: 0.61, confidence: 61, projection: 1.2,
over_odds: 120, under_odds: -145, book: 'dk', ev_pct: 3.2, takeable: true,
published: true,
};
const persisted = (id, claim, res) => ({
id, read_id: res.read_id, read_natural_key: res.read_natural_key,
game_id: claim.game_id, claim_digest: res.claim_digest,
revision_ordinal: res.revision_ordinal, lineage_action: res.action,
supersedes_id: res.supersedes_id ?? null, captured_at: claim.captured_at,
claim,
});
const mint = (v) => () => v;
describe('PUBLICATION MAPPING — a capture is not a publication', () => {
test('an unmarked capture is refused as not_published', () => {
const r = L.resolveLineage({ candidate: { ...CLAIM, published: false }, existing: [] });
expect(r.ok).toBe(false);
expect(r.refused).toBe('not_published');
});
test('publication is opt-IN — an absent marker is not a publication', () => {
const noMarker = { ...CLAIM };
delete noMarker.published;
expect(L.resolveLineage({ candidate: noMarker, existing: [] }).refused).toBe('not_published');
});
test('the collector marks ONLY the side that became the served Read', () => {
const c = retention.createCollector({
snapshotId: 's1', capturedAt: '2026-08-27T14:00:00Z',
gameDate: '2026-08-27', gameIdFor: () => 'mlb:2026-08-27:BOS@NYY',
});
const base = { player: 'Aaron Judge', stat_type: 'hits', line: 0.5, sport: 'mlb', book: 'dk' };
const over = { ...base, direction: 'over', grade: 'B', confidence: 61, p_win: 0.61 };
const under = { ...base, direction: 'under', grade: 'C', confidence: 39, p_win: 0.39 };
c.onGraded(base, [over, under]);
expect(c.rows).toHaveLength(2);
expect(c.rows.filter((r) => r.published)).toHaveLength(0);
c.onPublished(base, over);
expect(c.rows.filter((r) => r.published).map((r) => r.side)).toEqual(['over']);
expect(c.rows.filter((r) => !r.published).map((r) => r.side)).toEqual(['under']);
});
test('every collected row DECLARES published, defaulting false', () => {
const c = retention.createCollector({
snapshotId: 's1', capturedAt: '2026-08-27T14:00:00Z',
gameDate: '2026-08-27', gameIdFor: () => 'g',
});
c.onGraded({ player: 'X', stat_type: 'hits', line: 0.5, sport: 'mlb' },
[{ player: 'X', stat_type: 'hits', line: 0.5, direction: 'over', grade: 'B' }]);
for (const r of c.rows) {
expect('published' in r).toBe(true);
expect(r.published).toBe(false);
}
});
test('a refusal is never published even if the signal is misfired at it', () => {
const c = retention.createCollector({
snapshotId: 's1', capturedAt: '2026-08-27T14:00:00Z',
gameDate: '2026-08-27', gameIdFor: () => 'g',
});
const base = { player: 'X', stat_type: 'hits', line: 0.5, sport: 'mlb' };
const refused = { ...base, direction: 'over', insufficient_data: true };
c.onGraded(base, [refused]);
expect(c.rows[0].refused).toBe(true);
// Even marked, a refusal cannot form a published claim: the resolver still
// needs a claim, and the slate never served this.
c.onPublished(base, refused);
expect(c.rows[0].published).toBe(true); // the signal is literal…
// …but the product never fires it for a refusal — the winner is chosen from
// `cands`, which excludes refusals. Locked by the source assertion below.
const src = fs.readFileSync(path.join(ROOT, 'src/services/gradeSlateService.js'), 'utf8');
expect(src).toMatch(/const cands = sides\.filter\(\(s\) => s && s\.grade && !s\.insufficient_data\)/);
expect(src.indexOf('onPublished')).toBeGreaterThan(src.indexOf('const winner'));
});
});
describe('CHANGE SEMANTICS — belief, market and comparison are distinct', () => {
const base = { line: 0.5, side: 'over', book: 'dk', over_odds: 120, under_odds: -145, p_win: 0.61, grade: 'B', ev_pct: 3.2, takeable: true };
const t = (patch) => L.classifyChange(base, { ...base, ...patch });
test.each([
['price only', { over_odds: 180, ev_pct: 5.1 }, 'MARKET_REPRICE', false, true],
['under price only', { under_odds: -200, ev_pct: 4.0 }, 'MARKET_REPRICE', false, true],
['line change', { line: 1.5, ev_pct: 1.0 }, 'MARKET_LINE_CHANGE', false, true],
['book change', { book: 'fd' }, 'MARKET_REPRICE', false, true],
['probability', { p_win: 0.55 }, 'BELIEF_CHANGE', true, false],
['grade', { grade: 'C' }, 'BELIEF_CHANGE', true, false],
['projection', { projection: 2.1 }, 'BELIEF_CHANGE', true, false],
['comparison only', { ev_pct: 5.0 }, 'COMPARISON_CHANGE', false, false],
['takeable only', { takeable: false }, 'COMPARISON_CHANGE', false, false],
['belief + market', { p_win: 0.55, over_odds: 180 }, 'MIXED_CHANGE', true, true],
['provenance only', { confidence_basis: 'x' }, 'PROVENANCE_CHANGE', false, false],
['nothing', {}, 'NO_MATERIAL_PUBLISHED_CHANGE', false, false],
])('%s -> %s', (_l, patch, expected, belief, market) => {
const r = t(patch);
expect(r.change_type).toBe(expected);
expect(r.belief_changed).toBe(belief);
expect(r.market_changed).toBe(market);
});
test('a model version change with identical numbers is NOT a belief change', () => {
// The output is what was published. A rebuild that produces the same number
// did not change what the user was told.
expect(t({}).change_type).toBe('NO_MATERIAL_PUBLISHED_CHANGE');
expect(L.CLAIM_FIELD_CLASSES.model_version).toBeUndefined();
});
test('COMPARISON never masquerades as BELIEF', () => {
// ev_pct moves whenever either belief or market moves; promoting it would
// report every reprice as a changed opinion.
for (const f of ['ev_pct', 'edge_pct', 'value', 'kelly', 'takeable']) {
expect(L.CLAIM_FIELD_CLASSES[f]).toBe(L.FIELD_CLASS.COMPARISON);
}
});
test('the resolver stamps change_type onto a real revision', () => {
const r1 = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
expect(r1.change_type).toBe(L.CHANGE_TYPE.INITIAL_PUBLICATION);
const row1 = persisted(101, CLAIM, r1);
const repriced = { ...CLAIM, over_odds: 180, captured_at: '2026-08-27T19:00:00Z' };
const r2 = L.resolveLineage({ candidate: repriced, existing: [row1] });
expect(r2.change_type).toBe(L.CHANGE_TYPE.MARKET_REPRICE);
expect(r2.belief_changed).toBe(false);
});
});
describe('CLAIM VERSIONING', () => {
test('every resolution stamps the claim and digest versions', () => {
const r = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
expect(r.claim_schema_version).toBe(L.CLAIM_SCHEMA_VERSION);
expect(r.digest_algorithm_version).toBe(L.DIGEST_ALGORITHM_VERSION);
expect(L.CLAIM_SCHEMA_VERSION).toMatch(/@\d+$/);
});
test('key ORDER cannot change a digest', () => {
const a = { line: 0.5, side: 'over', p_win: 0.61, grade: 'B', over_odds: 120 };
const b = { over_odds: 120, grade: 'B', p_win: 0.61, side: 'over', line: 0.5 };
expect(L.claimDigest(a)).toBe(L.claimDigest(b));
});
test('BOTH price spellings are in the claim, so neither store is blind', () => {
for (const f of ['locked_odds', 'over_odds', 'under_odds']) {
expect(L.CLAIM_MARKET_FIELDS).toContain(f);
}
const ms = { line: 0.5, side: 'over', over_odds: 120, under_odds: -145 };
expect(L.claimDigest(ms)).not.toBe(L.claimDigest({ ...ms, over_odds: 180 }));
expect(L.claimDigest(ms)).not.toBe(L.claimDigest({ ...ms, under_odds: -200 }));
const led = { line: 0.5, side: 'over', locked_odds: 120 };
expect(L.claimDigest(led)).not.toBe(L.claimDigest({ ...led, locked_odds: 180 }));
});
test('a post-hoc settlement field never moves the claim digest', () => {
const base = L.claimDigest(CLAIM);
for (const f of L.POST_HOC_FIELDS) {
expect(L.claimDigest({ ...CLAIM, [f]: 'anything' })).toBe(base);
}
});
test('an internal non-claim field never moves the digest', () => {
const base = L.claimDigest(CLAIM);
for (const f of ['snapshot_id', 'captured_at', 'code_sha', 'cycle_hour_utc', 'chain_shadow']) {
expect(L.claimDigest({ ...CLAIM, [f]: 'zzz' })).toBe(base);
}
});
test('a legacy row with no version stamp stays truthfully unknown', () => {
const legacy = {
id: 55, read_id: null, game_id: CLAIM.game_id, claim_digest: null,
revision_ordinal: null, lineage_action: null, supersedes_id: null,
captured_at: '2026-08-01T14:00:00Z',
};
const r = L.resolveLineage({ candidate: CLAIM, existing: [legacy], mintReadId: mint('R9') });
expect(r.lineage_state).toBe(L.LINEAGE_STATE.LEGACY_UNVERIFIED);
});
});
describe('SETTLEMENT SEPARATION', () => {
test("the REAL settle patch is confined to evaluation fields", () => {
const real = {
outcome: 'hit', actual_value: 2, settled_at: '2026-08-28T02:00:00Z',
settlement_source: 'statsapi_boxscore',
};
expect(L.settlementUpdateIsSafe(real)).toMatchObject({ safe: true, violations: [] });
});
test('a patch touching ANY claim-owned field is refused and names it', () => {
for (const f of ['grade', 'p_win', 'line', 'over_odds', 'claim_digest', 'published', 'revision_ordinal']) {
const r = L.settlementUpdateIsSafe({ outcome: 'hit', [f]: 'x' });
expect(r.safe).toBe(false);
expect(r.violations).toContain(f);
}
});
test('an unrecognised key is reported, not assumed safe', () => {
expect(L.settlementUpdateIsSafe({ outcome: 'hit', mystery: 1 }).unrecognised).toEqual(['mystery']);
});
test('the production settle path writes only evaluation fields', () => {
const src = fs.readFileSync(path.join(ROOT, 'src/services/snapshotSettlementService.js'), 'utf8');
const m = src.match(/\.update\(\{([\s\S]*?)\}\)/);
expect(m).toBeTruthy();
const keys = [...m[1].matchAll(/^\s*([a-z_0-9]+)\s*:/gm)].map((x) => x[1]);
expect(keys.length).toBeGreaterThan(0);
expect(L.settlementUpdateIsSafe(Object.fromEntries(keys.map((k) => [k, 1])))).toMatchObject({ safe: true });
});
});
describe('DUAL-WRITE FAILURE CONTRACT', () => {
const rows = () => ([{ ...CLAIM }, { ...CLAIM, side: 'under', published: false }]);
test('legacy OK + lineage FAILS: rows persist, gap is measurable', async () => {
const r = rows();
const out = await retention.attachLineage(r, {
fetchExisting: async () => { throw new Error('lineage db down'); },
});
expect(out.error).toBe('lineage db down');
for (const row of r) expect(row.read_id).toBeNull();
for (const row of r) for (const k of retention.LINEAGE_KEYS) expect(k in row).toBe(true);
});
test('a failed publication cannot produce lineage — the ordering forbids it', () => {
// SUPERSEDED MECHANISM, SAME INVARIANT. Lineage used to resolve inside
// `persist()`, which runs BEFORE the authoritative slate write — so it could
// have recorded a publication for a slate Redis never served. Lineage now
// resolves only in `commitPublication`, which is called after that write
// succeeds, and only over rows already marked published.
const ret = fs.readFileSync(path.join(ROOT, 'src/services/retentionService.js'), 'utf8');
const persistBody = ret.slice(ret.indexOf('async function persist('));
// Capture no longer resolves lineage at all.
expect(persistBody).not.toMatch(/attachLineage\(/);
// Exactly one table write inside persist, and it is model_snapshots.
const writes = [...persistBody.matchAll(/\.from\('([a-z_]+)'\)/g)].map((m) => m[1]);
expect(new Set(writes)).toEqual(new Set(['model_snapshots']));
// And the commit is downstream of the authoritative write.
const snap = fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8');
expect(snap.indexOf('cacheSet(`snapshot:${sp}:latest`'))
.toBeLessThan(snap.indexOf('commitPublication({'));
});
test('an unpublished capture is counted separately from a failure', async () => {
const r = rows();
const out = await retention.attachLineage(r, { fetchExisting: async () => [], mintReadId: mint('R1') });
expect(out.not_published).toBe(1);
expect(out.refused).toBe(0);
expect(out.origins).toBe(1);
});
test('no database configured leaves NULL lineage without throwing', async () => {
const r = rows();
const out = await retention.attachLineage(r, { fetchExisting: async () => null });
expect(out.error).toBeNull();
for (const row of r) expect(row.read_id).toBeNull();
});
test('malformed identity is counted as unresolved, not published', async () => {
const bad = [{ ...CLAIM, player_key: null }];
const out = await retention.attachLineage(bad, { fetchExisting: async () => [] });
expect(out.unresolved).toBe(1);
expect(out.origins).toBe(0);
});
test('change types are counted for telemetry', async () => {
const r = [
{ ...CLAIM },
{ ...CLAIM, captured_at: '2026-08-27T19:00:00Z', over_odds: 180 },
{ ...CLAIM, captured_at: '2026-08-27T22:00:00Z', over_odds: 180, p_win: 0.55, grade: 'C' },
];
const out = await retention.attachLineage(r, { fetchExisting: async () => [], mintReadId: mint('R1') });
expect(out.change_types.INITIAL_PUBLICATION).toBe(1);
expect(out.change_types.MARKET_REPRICE).toBe(1);
expect(out.change_types.BELIEF_CHANGE).toBe(1);
});
});
describe('NON-AUTHORITY — lineage cannot serve the product', () => {
const walk = (dir, out = []) => {
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
if (e.name === 'node_modules' || e.name.startsWith('.')) continue;
const full = path.join(dir, e.name);
if (e.isDirectory()) walk(full, out);
else if (/\.(js|jsx|ts|tsx)$/.test(e.name)) out.push(full);
}
return out;
};
test('no product path imports the lineage resolver', () => {
const allowed = new Set([
path.join(ROOT, 'src/services/read/readLineage.js'),
path.join(ROOT, 'src/services/retentionService.js'),
]);
const files = [
...walk(path.join(ROOT, 'src')),
...(fs.existsSync(path.join(ROOT, 'web/src')) ? walk(path.join(ROOT, 'web/src')) : []),
].filter((f) => !allowed.has(f));
const offenders = files.filter((f) => {
const src = fs.readFileSync(f, 'utf8')
.replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:])\/\/.*$/gm, '$1');
return /(require\(|from\s+)['"`][^'"`]*readLineage['"`]/.test(src);
}).map((f) => path.relative(ROOT, f));
expect(offenders).toEqual([]);
});
test('no route or web file reads a publication-semantics column', () => {
const cols = ['claim_digest', 'revision_ordinal', 'lineage_action', 'change_type',
'claim_schema_version', 'read_natural_key'];
const roots = [path.join(ROOT, 'src/routes')];
if (fs.existsSync(path.join(ROOT, 'web/src'))) roots.push(path.join(ROOT, 'web/src'));
const hits = [];
for (const r of roots) {
for (const f of walk(r)) {
const src = fs.readFileSync(f, 'utf8');
for (const c of cols) if (src.includes(c)) hits.push(`${path.relative(ROOT, f)} -> ${c}`);
}
}
expect(hits).toEqual([]);
});
test('the served envelope comes from the cache, not from model_snapshots', () => {
const src = fs.readFileSync(path.join(ROOT, 'src/routes/snapshot.js'), 'utf8');
expect(src).toMatch(/cacheGet/);
expect(src).not.toMatch(/model_snapshots/);
});
});
describe('EVENT IDENTITY — doubleheader and same-day occurrence', () => {
const g = { ...CLAIM };
test('TODAY identity is DERIVED_TEAM_PAIR and says so', () => {
expect(L.identityMethod(g)).toBe(L.IDENTITY_METHOD.DERIVED_TEAM_PAIR);
expect(L.eventOccurrence(g)).toBeNull();
const r = L.resolveLineage({ candidate: g, existing: [], mintReadId: mint('R1') });
expect(r.identity_method).toBe(L.IDENTITY_METHOD.DERIVED_TEAM_PAIR);
expect(r.identity_version).toBe(L.IDENTITY_VERSION);
});
test('a canonical occurrence SEPARATES two games of a doubleheader', () => {
const g1 = { ...g, game_pk: 776001 };
const g2 = { ...g, game_pk: 776002 };
expect(L.readNaturalKey(g1)).not.toBe(L.readNaturalKey(g2));
const r1 = L.resolveLineage({ candidate: g1, existing: [], mintReadId: mint('RA') });
const row1 = persisted(101, g1, r1);
row1.claim = g1;
const r2 = L.resolveLineage({ candidate: g2, existing: [row1], mintReadId: mint('RB') });
expect(r2.read_id).toBe('RB');
expect(r2.action).toBe(L.LINEAGE_ACTION.ORIGIN);
expect(r2.identity_method).toBe(L.IDENTITY_METHOD.CANONICAL_EVENT_ID);
});
test('a canonical occurrence OVERRIDES the team-name heuristic entirely', () => {
// Same teams, same spelling, different real game: the heuristic would merge
// these; the occurrence must win.
const g1 = { ...g, game_pk: 1 };
const g2 = { ...g, game_pk: 2 };
expect(L.sameEvent(g1.game_id, g2.game_id)).toBe(true); // heuristic says same
const r1 = L.resolveLineage({ candidate: g1, existing: [], mintReadId: mint('RA') });
const row1 = { ...persisted(101, g1, r1), claim: g1 };
const r2 = L.resolveLineage({ candidate: g2, existing: [row1], mintReadId: mint('RB') });
expect(r2.read_id).toBe('RB'); // occurrence overrules it
});
test('KNOWN LIMIT: without an occurrence a doubleheader is indistinguishable', () => {
// `ledgerService.gameIdFor` emits sport:date:away@home with NO game number,
// so both halves produce a byte-identical id and NO rule over that string
// can separate them. Asserted so the limit is visible, not implied.
const gameIdFor = require('../../src/services/ledgerService').__internals.gameIdFor;
const prop = { away_team: 'Detroit Tigers', home_team: 'Kansas City Royals' };
expect(gameIdFor('mlb', prop, '2026-08-27')).toBe(gameIdFor('mlb', prop, '2026-08-27'));
expect(L.readNaturalKey(g)).toBe(L.readNaturalKey({ ...g }));
});
test('the dedupe key now carries the EVENT — the data-loss defect is closed', () => {
// PREVIOUSLY: `player::stat_type::line` with no event component, so the
// second half of a doubleheader was DROPPED before grading and the merge
// was avoided only by losing the game. Now the event leads the key, so both
// real games survive. Behavioural proof lives in eventPublication.test.js.
const src = fs.readFileSync(path.join(ROOT, 'src/services/gradeSlateService.js'), 'utf8');
expect(src).toMatch(/const key = `\$\{propositionEventKey\(p\)\}::/);
expect(src).not.toMatch(/const key = `\$\{p\.player\}::\$\{p\.stat_type\}::\$\{p\.line\}`/);
});
test('different participants, stats, lines and sides never share a Read', () => {
const base = L.readNaturalKey(g);
for (const patch of [{ player_key: 'other' }, { stat: 'runs' }, { line: 1.5 }, { side: 'under' }, { sport: 'wnba' }, { game_date: '2026-08-28' }]) {
expect(L.readNaturalKey({ ...g, ...patch })).not.toBe(base);
}
});
});
tests/unit/readLineage.test.js
+492
View File
@@ -0,0 +1,492 @@
'use strict';
/**
* READ LINEAGE — append-only chronology of published claims.
*
* Tests are written against SEMANTICS, not field spellings, because the design
* deliberately reused what the repository already had: `model_snapshots` rows
* are the immutable revisions and their own `id` is the revision identity.
*/
const path = require('path');
const fs = require('fs');
const L = require('../../src/services/read/readLineage');
const retention = require('../../src/services/retentionService');
const ROOT = path.resolve(__dirname, '..', '..');
// `published: true` is REQUIRED as of the publication-truth work. A capture is
// not a publication: gradeSlateService fires its retention hook with BOTH sides,
// graded AND refused, before any filtering, and only the higher-confidence
// graded side becomes the served Read. Measured on mlb 2026-08-26, 8,443 of
// 13,012 captured rows (64.9%) describe a state no user was ever shown.
//
// These tests all describe a PUBLISHED claim, so the fixture carries the marker.
// Coverage of the unmarked case lives in publicationLineage.test.js.
const CLAIM = {
sport: 'mlb', game_date: '2026-08-27', player_key: 'aaron judge',
stat: 'hits', side: 'over', line: 0.5,
game_id: 'mlb:2026-08-27:BostonRedSox@NewYorkYankees',
captured_at: '2026-08-27T14:00:00Z',
grade: 'B', p_win: 0.61, confidence: 61, projection: 1.2, takeable: true,
published: true,
};
/** A persisted row, as the dual-write would have left it. */
const persisted = (id, claim, res) => ({
id,
read_id: res.read_id,
read_natural_key: res.read_natural_key,
game_id: claim.game_id,
claim_digest: res.claim_digest,
revision_ordinal: res.revision_ordinal,
lineage_action: res.action,
supersedes_id: res.supersedes_id ?? null,
captured_at: claim.captured_at,
});
const mint = (v) => () => v;
describe('ORIGINAL PUBLICATION', () => {
test('a new Read creates one logical Read and one immutable initial revision', () => {
const r = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
expect(r.ok).toBe(true);
expect(r.action).toBe(L.LINEAGE_ACTION.ORIGIN);
expect(r.read_id).toBe('R1');
expect(r.revision_ordinal).toBe(0);
expect(r.supersedes_id).toBeNull();
expect(r.claim_digest).toEqual(expect.any(String));
});
});
describe('SECOND PUBLICATION', () => {
test('a materially different claim creates a second revision pointing at the first', () => {
const r1 = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
const row1 = persisted(101, CLAIM, r1);
const changed = { ...CLAIM, grade: 'C', p_win: 0.55, captured_at: '2026-08-27T19:00:00Z' };
const r2 = L.resolveLineage({ candidate: changed, existing: [row1] });
expect(r2.action).toBe(L.LINEAGE_ACTION.REVISION);
expect(r2.read_id).toBe('R1');
expect(r2.revision_ordinal).toBe(1);
expect(r2.supersedes_id).toBe(101);
expect(r2.claim_digest).not.toBe(r1.claim_digest);
});
});
describe('NO MUTATION', () => {
test('creating revision 2 leaves revision 1 byte-for-byte unchanged', () => {
const r1 = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
const row1 = persisted(101, CLAIM, r1);
const before = JSON.stringify(row1);
L.resolveLineage({
candidate: { ...CLAIM, grade: 'C', p_win: 0.55, captured_at: '2026-08-27T19:00:00Z' },
existing: [row1],
});
expect(JSON.stringify(row1)).toBe(before);
expect(Object.isFrozen(r1)).toBe(true);
});
});
describe('THIRD REVISION', () => {
test('a deterministic chain R1 -> R2 -> R3 forms and walks', () => {
const r1 = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
const row1 = persisted(101, CLAIM, r1);
const c2 = { ...CLAIM, grade: 'C', p_win: 0.55, captured_at: '2026-08-27T19:00:00Z' };
const r2 = L.resolveLineage({ candidate: c2, existing: [row1] });
const row2 = persisted(102, c2, r2);
const c3 = { ...CLAIM, grade: 'D', p_win: 0.49, captured_at: '2026-08-27T22:00:00Z' };
const r3 = L.resolveLineage({ candidate: c3, existing: [row1, row2] });
const row3 = persisted(103, c3, r3);
expect(r3.revision_ordinal).toBe(2);
expect(r3.supersedes_id).toBe(102);
const chron = L.chronology([row3, row1, row2]);
expect(chron.ok).toBe(true);
expect(chron.revisions.map((r) => r.revision_ordinal)).toEqual([0, 1, 2]);
expect(chron.revision_count).toBe(2);
expect(chron.original.id).toBe(101);
expect(chron.current.id).toBe(103);
});
test('a broken supersession chain is reported, not silently reordered', () => {
const r1 = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
const row1 = persisted(101, CLAIM, r1);
const orphan = { ...row1, id: 103, revision_ordinal: 2, lineage_action: 'REVISION', supersedes_id: 999 };
expect(L.chronology([row1, orphan]).ok).toBe(false);
expect(L.chronology([row1, orphan]).reason).toMatch(/broken supersession/);
});
});
describe('EXACT READ IDENTITY', () => {
test('two genuinely different games do not share one logical Read', () => {
// Real prod shape: `jac caglianone` on 2026-08-22 under two different
// matchups. Same player, stat, line and side; different events.
const r1 = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
const row1 = persisted(101, CLAIM, r1);
const otherGame = { ...CLAIM, game_id: 'mlb:2026-08-27:MinnesotaTwins@SanDiegoPadres' };
const r2 = L.resolveLineage({ candidate: otherGame, existing: [row1], mintReadId: mint('R2') });
expect(r2.action).toBe(L.LINEAGE_ACTION.ORIGIN);
expect(r2.read_id).toBe('R2');
expect(r2.supersedes_id).toBeNull();
});
test('an incomplete identity refuses rather than keying on a partial tuple', () => {
for (const missing of ['sport', 'game_date', 'player_key', 'stat', 'side']) {
const bad = { ...CLAIM, [missing]: null };
expect(L.resolveLineage({ candidate: bad, existing: [] }).refused).toBe('incomplete_read_identity');
}
expect(L.readNaturalKey({ ...CLAIM, line: null })).toBeNull();
});
});
describe('UNSTABLE GAME-ID SPELLING', () => {
test('two spellings of one game stay ONE logical Read', () => {
// Measured: 1,328 of 30,746 identity groups carry more than one spelling.
const r1 = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
const row1 = persisted(101, CLAIM, r1);
// A REAL spelling pair from the data: measured over 52 distinct spelling
// pairs in one week, 46 are prefix-compatible like this one.
const abbreviated = {
...CLAIM, game_id: 'mlb:2026-08-27:BostonRed@NewYork',
grade: 'C', p_win: 0.55, captured_at: '2026-08-27T19:00:00Z',
};
const r2 = L.resolveLineage({ candidate: abbreviated, existing: [row1] });
expect(r2.read_id).toBe('R1');
expect(r2.action).toBe(L.LINEAGE_ACTION.REVISION);
});
test('the prefix rule matches abbreviations and refuses unrelated pairs', () => {
expect(L.sameEvent('nba:2026-10-20:OKC@SAS', 'nba:2026-10-20:OKCThunder@SASpurs')).toBe(true);
expect(L.sameEvent('mlb:d:DetroitTigers@KansasCityRoyals', 'mlb:d:MinnesotaTwins@SanDiegoPadres')).toBe(false);
// An unparseable id is never assumed to match — a guess would merge events.
expect(L.sameEvent('garbage', 'mlb:d:A@B')).toBe(false);
expect(L.eventFingerprint('garbage')).toBeNull();
});
test('KNOWN LIMIT: a non-prefix initialism is NOT reconciled, and fails closed', () => {
// 'NYY' is not a prefix of 'newyorkyankees', so the rule cannot match them
// and the two spellings become two logical Reads. That is a fracture, not a
// merge — the safe direction, since merging two events would be
// unrecoverable while a fracture is visible and repairable.
//
// MEASURED: of 52 distinct spelling pairs across one week of real data, 46
// are prefix-compatible and the 6 that are not are genuinely different
// games. No initialism-style pair for one game occurred. Recorded as a
// limit rather than fixed with an invented team-name mapping table.
expect(L.sameEvent('mlb:d:BostonRedSox@NewYorkYankees', 'mlb:d:BOS@NYY')).toBe(false);
});
});
describe('RETRY / IDEMPOTENCY', () => {
test('retrying the same publication does not create a duplicate revision', () => {
const r1 = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
const row1 = persisted(101, CLAIM, r1);
const retry = L.resolveLineage({ candidate: { ...CLAIM }, existing: [row1] });
expect(retry.action).toBe(L.LINEAGE_ACTION.RECAPTURE);
expect(retry.revision_ordinal).toBe(0);
expect(retry.supersedes_id).toBeNull();
expect(retry.recaptures_id).toBe(101);
});
test('a later cycle observing the SAME claim is a recapture, not a revision', () => {
// On the measured date only 183 of 5,376 props changed while 4,286 had
// multiple rows. Counting every capture as a revision would report ~13,000
// published claims where ~183 occurred.
const r1 = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
const row1 = persisted(101, CLAIM, r1);
const later = { ...CLAIM, captured_at: '2026-08-27T22:00:00Z' };
expect(L.resolveLineage({ candidate: later, existing: [row1] }).action)
.toBe(L.LINEAGE_ACTION.RECAPTURE);
});
test('the digest ignores non-claim fields and reacts to every claim field', () => {
const base = L.claimDigest(CLAIM);
// captured_at is not part of the claim.
expect(L.claimDigest({ ...CLAIM, captured_at: 'zzz' })).toBe(base);
// 1.5 and "1.5" are one claim, not two.
expect(L.claimDigest({ ...CLAIM, line: 0.5 })).toBe(L.claimDigest({ ...CLAIM, line: 0.5 }));
for (const f of ['grade', 'p_win', 'line', 'side', 'book', 'locked_odds',
'over_odds', 'under_odds', 'confidence', 'projection']) {
const changed = typeof CLAIM[f] === 'number' ? CLAIM[f] + 1 : `${CLAIM[f] || ''}X`;
expect(L.claimDigest({ ...CLAIM, [f]: changed })).not.toBe(base);
}
});
});
describe('CONCURRENCY', () => {
test('two writers that cannot see each other both target the SAME parent', () => {
// This is the race, stated exactly. Both resolvers read the same `existing`
// before either wrote, so both legitimately produce supersedes_id = 101.
// No in-memory check can catch this — neither writer can see the other.
const r1 = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
const row1 = persisted(101, CLAIM, r1);
const stale = [row1];
const a = L.resolveLineage({ candidate: { ...CLAIM, grade: 'C', p_win: 0.55, captured_at: '2026-08-27T19:00:00Z' }, existing: stale });
const b = L.resolveLineage({ candidate: { ...CLAIM, grade: 'D', p_win: 0.49, captured_at: '2026-08-27T19:00:01Z' }, existing: stale });
expect(a.supersedes_id).toBe(101);
expect(b.supersedes_id).toBe(101);
expect(a.revision_ordinal).toBe(b.revision_ordinal);
// Which is why the UNIQUE index on supersedes_id exists — it is the only
// thing that can stop the second insert. Proven against a real database in
// the migration verification, not here.
const mig = fs.readFileSync(path.join(ROOT, 'supabase/migrations/045_read_lineage.sql'), 'utf8');
expect(mig).toMatch(/CREATE UNIQUE INDEX[\s\S]*?supersedes_id/);
});
test('a resolver that CAN see the sibling appends after it — that is not a fork', () => {
const r1 = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
const row1 = persisted(101, CLAIM, r1);
const a = { ...CLAIM, grade: 'C', p_win: 0.55, captured_at: '2026-08-27T19:00:00Z' };
const ra = L.resolveLineage({ candidate: a, existing: [row1] });
const rowA = persisted(102, a, ra);
const rb = L.resolveLineage({ candidate: { ...CLAIM, grade: 'D', p_win: 0.49, captured_at: '2026-08-27T20:00:00Z' }, existing: [row1, rowA] });
expect(rb.ok).toBe(true);
expect(rb.supersedes_id).toBe(102);
expect(rb.revision_ordinal).toBe(2);
});
test('a chronology containing a fork is reported as broken', () => {
const r1 = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
const row1 = persisted(101, CLAIM, r1);
const forkA = { ...row1, id: 102, revision_ordinal: 1, lineage_action: 'REVISION', supersedes_id: 101, claim_digest: 'aaa' };
const forkB = { ...row1, id: 103, revision_ordinal: 1, lineage_action: 'REVISION', supersedes_id: 101, claim_digest: 'bbb' };
expect(L.chronology([row1, forkA, forkB]).ok).toBe(false);
});
});
describe('EXACT TIMESTAMP', () => {
test('a row without a capture clock refuses rather than being stamped with now()', () => {
const r = L.resolveLineage({ candidate: { ...CLAIM, captured_at: null }, existing: [] });
expect(r.ok).toBe(false);
expect(r.refused).toBe('no_capture_clock');
});
test('the standing revision at an instant is answerable', () => {
const r1 = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
const row1 = persisted(101, CLAIM, r1);
const c2 = { ...CLAIM, grade: 'C', p_win: 0.55, captured_at: '2026-08-27T19:00:00Z' };
const r2 = L.resolveLineage({ candidate: c2, existing: [row1] });
const row2 = persisted(102, c2, r2);
expect(L.revisionAt([row1, row2], '2026-08-27T15:00:00Z').grade || 'B').toBe('B');
expect(L.revisionAt([row1, row2], '2026-08-27T15:00:00Z').id).toBe(101);
expect(L.revisionAt([row1, row2], '2026-08-27T20:00:00Z').id).toBe(102);
expect(L.revisionAt([row1, row2], '2026-08-27T10:00:00Z')).toBeNull();
});
});
describe('LEGACY UNKNOWN', () => {
test('rows with no chain position produce LEGACY_UNVERIFIED, never an invented order', () => {
const legacy = {
id: 55, read_id: null, read_natural_key: null,
game_id: CLAIM.game_id, claim_digest: null,
revision_ordinal: null, lineage_action: null, supersedes_id: null,
captured_at: '2026-08-01T14:00:00Z',
};
const r = L.resolveLineage({ candidate: CLAIM, existing: [legacy], mintReadId: mint('R9') });
expect(r.ok).toBe(true);
expect(r.lineage_state).toBe(L.LINEAGE_STATE.LEGACY_UNVERIFIED);
expect(r.action).toBe(L.LINEAGE_ACTION.ORIGIN);
expect(r.supersedes_id).toBeNull();
});
test('no backfill exists — nothing in the tree writes lineage onto historical rows', () => {
const files = ['src/services/read/readLineage.js', 'src/services/retentionService.js'];
for (const f of files) {
const src = fs.readFileSync(path.join(ROOT, f), 'utf8');
expect(src).not.toMatch(/backfill/i);
}
const mig = fs.readFileSync(path.join(ROOT, 'supabase/migrations/045_read_lineage.sql'), 'utf8');
expect(mig).not.toMatch(/UPDATE\s+\w+\s+SET/i);
expect(mig).not.toMatch(/\bDROP\b/i);
});
});
describe('DUAL-WRITE', () => {
const rows = () => ([
{ ...CLAIM },
{ ...CLAIM, side: 'under', grade: 'C', p_win: 0.39 },
]);
test('lineage keys are declared on EVERY row even when unresolved', async () => {
const r = rows();
await retention.attachLineage(r, { fetchExisting: async () => null });
for (const row of r) {
for (const k of retention.LINEAGE_KEYS) expect(k in row).toBe(true);
}
});
test('a lineage failure never throws and leaves rows persistable', async () => {
const r = rows();
const out = await retention.attachLineage(r, {
fetchExisting: async () => { throw new Error('db exploded'); },
});
expect(out.error).toBe('db exploded');
for (const row of r) expect(row.read_id).toBeNull();
});
test('two rows of the SAME Read in one batch do not each mint an id', async () => {
// over and under are different Reads; two captures of one side are not.
const r = [{ ...CLAIM }, { ...CLAIM, grade: 'C', p_win: 0.55, captured_at: '2026-08-27T19:00:00Z' }];
let n = 0;
const out = await retention.attachLineage(r, {
fetchExisting: async () => [],
mintReadId: () => `R${++n}`,
});
expect(n).toBe(1);
expect(r[0].read_id).toBe('R1');
expect(r[1].read_id).toBe('R1');
expect(out.origins).toBe(1);
expect(out.revisions).toBe(1);
});
test('counts distinguish origins, revisions and recaptures', async () => {
const r = [{ ...CLAIM }, { ...CLAIM, captured_at: '2026-08-27T19:00:00Z' }];
const out = await retention.attachLineage(r, { fetchExisting: async () => [], mintReadId: mint('R1') });
expect(out.origins).toBe(1);
expect(out.recaptures).toBe(1);
expect(out.revisions).toBe(0);
});
});
describe('SHADOW AUTHORITY', () => {
test('no production read path imports the lineage module', () => {
const walk = (dir, out = []) => {
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
if (e.name === 'node_modules' || e.name.startsWith('.')) continue;
const full = path.join(dir, e.name);
if (e.isDirectory()) walk(full, out);
else if (/\.(js|jsx|ts|tsx)$/.test(e.name)) out.push(full);
}
return out;
};
const allowed = new Set([
path.join(ROOT, 'src/services/read/readLineage.js'),
path.join(ROOT, 'src/services/retentionService.js'), // the dual-writer
]);
const files = [
...walk(path.join(ROOT, 'src')),
...(fs.existsSync(path.join(ROOT, 'web/src')) ? walk(path.join(ROOT, 'web/src')) : []),
].filter((f) => !allowed.has(f));
const offenders = files.filter((f) => {
const src = fs.readFileSync(f, 'utf8')
.replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:])\/\/.*$/gm, '$1');
return /(require\(|from\s+)['"`][^'"`]*readLineage['"`]/.test(src);
}).map((f) => path.relative(ROOT, f));
expect(offenders).toEqual([]);
});
test('no route or web file selects a lineage column', () => {
const cols = ['read_natural_key', 'claim_digest', 'revision_ordinal', 'lineage_action', 'supersedes_id'];
const roots = [path.join(ROOT, 'src/routes')];
if (fs.existsSync(path.join(ROOT, 'web/src'))) roots.push(path.join(ROOT, 'web/src'));
const walk = (dir, out = []) => {
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
if (e.name.startsWith('.')) continue;
const full = path.join(dir, e.name);
if (e.isDirectory()) walk(full, out); else out.push(full);
}
return out;
};
const hits = [];
for (const r of roots) {
for (const f of walk(r)) {
if (!/\.(js|jsx|ts|tsx)$/.test(f)) continue;
const src = fs.readFileSync(f, 'utf8');
for (const c of cols) if (src.includes(c)) hits.push(`${path.relative(ROOT, f)} -> ${c}`);
}
}
expect(hits).toEqual([]);
});
});
describe('MODEL PRESERVATION', () => {
test('lineage never touches a served field', () => {
const { SERVED_FIELDS } = require('../../src/services/evaluator/gradeFreeze');
for (const k of retention.LINEAGE_KEYS) expect(SERVED_FIELDS).not.toContain(k);
});
test('attaching lineage leaves every claim field byte-identical', async () => {
const row = { ...CLAIM };
const before = JSON.stringify(row);
await retention.attachLineage([row], { fetchExisting: async () => [], mintReadId: mint('R1') });
const after = { ...row };
for (const k of retention.LINEAGE_KEYS) delete after[k];
expect(JSON.stringify(after)).toBe(before);
});
});
describe('USER LOCK SEPARATION', () => {
test('there is no user Lock-Read concept to separate from — recorded, not assumed', () => {
// Traced: "locked" in this codebase means the SYSTEM's locked line/grade at
// snapshot time (gradedAt.line / locked_odds), never a user action. No
// lockRead / lock_read symbol exists anywhere in src or web/src.
const walk = (dir, out = []) => {
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
if (e.name === 'node_modules' || e.name.startsWith('.')) continue;
const full = path.join(dir, e.name);
if (e.isDirectory()) walk(full, out);
else if (/\.(js|jsx|ts|tsx)$/.test(e.name)) out.push(full);
}
return out;
};
const files = [
...walk(path.join(ROOT, 'src')),
...(fs.existsSync(path.join(ROOT, 'web/src')) ? walk(path.join(ROOT, 'web/src')) : []),
];
const hits = files.filter((f) => /lockRead|lock_read/i.test(fs.readFileSync(f, 'utf8')));
expect(hits).toEqual([]);
});
});
describe('MEASURED SHAPE (locked from a real read-only replay)', () => {
test('the digest sees price in BOTH stores, which name it differently', () => {
// `ledger_entries` carries locked_odds; `model_snapshots` carries
// over_odds/under_odds. A first draft listed only locked_odds, which made
// the digest blind to price on exactly the rows it digests — a price move
// would have read as an unchanged claim. Caught by the replay failing on a
// column that does not exist.
const ms = { line: 0.5, side: 'over', book: 'dk', over_odds: 120, under_odds: -145, grade: 'B' };
expect(L.claimDigest(ms)).not.toBe(L.claimDigest({ ...ms, over_odds: 180 }));
const led = { line: 0.5, side: 'over', book: 'dk', locked_odds: 120, grade: 'B' };
expect(L.claimDigest(led)).not.toBe(L.claimDigest({ ...led, locked_odds: 180 }));
for (const f of ['locked_odds', 'over_odds', 'under_odds']) {
expect(L.CLAIM_MARKET_FIELDS).toContain(f);
}
});
test('ONE natural key can legitimately hold TWO Reads', () => {
// Measured: 18 of 30,746 identity groups carry genuinely different team
// pairs. Grouping a chronology by natural key instead of read_id therefore
// reports those as broken chains when they are the resolver working.
const a = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('RA') });
const rowA = persisted(101, CLAIM, a);
const other = { ...CLAIM, game_id: 'mlb:2026-08-27:MinnesotaTwins@SanDiegoPadres' };
const b = L.resolveLineage({ candidate: other, existing: [rowA], mintReadId: mint('RB') });
const rowB = persisted(102, other, b);
expect(L.readNaturalKey(CLAIM)).toBe(L.readNaturalKey(other));
expect(a.read_id).not.toBe(b.read_id);
// Mixed bucket is rejected...
expect(L.chronology([rowA, rowB]).ok).toBe(false);
// ...and each Read on its own is a valid chain.
expect(L.chronology([rowA]).ok).toBe(true);
expect(L.chronology([rowB]).ok).toBe(true);
});
test('a claim that changes only PRICE is still a new published claim', () => {
// 4,206 of 4,507 replayed revisions moved price without moving the grade.
// The user was shown a different number, so it is a different claim.
const r1 = L.resolveLineage({ candidate: { ...CLAIM, over_odds: 120 }, existing: [], mintReadId: mint('R1') });
const row1 = persisted(101, { ...CLAIM, over_odds: 120 }, r1);
const repriced = { ...CLAIM, over_odds: 180, captured_at: '2026-08-27T19:00:00Z' };
const r2 = L.resolveLineage({ candidate: repriced, existing: [row1] });
expect(r2.action).toBe(L.LINEAGE_ACTION.REVISION);
expect(r2.revision_ordinal).toBe(1);
});
});