Persist lock-time multi-book lines to lock_lines (unblocks the staleness audit)
The over-side skew audit's confirming check — was our locked line stale-high vs consensus AT LOCK — was BLOCKED because multi-book lines at lock were never persisted (bookprices is Redis current-only). This persists them. - migration 033: lock_lines table (tracked + applied to prod). One row per (graded prop × book) with both odds + a lock timestamp. RLS enabled, NO policies -> service-role only (fence). UNIQUE key -> idempotent re-runs. - lockLineCapture.js: buildLockRows (pure, graded-props only, honest-absent single-book) + idempotent upsert persist. Built from the in-memory props at the LOCK moment (ts) -> no Redis re-read, no TTL race. - snapshotService: persist right after `enriched` (the lock moment; gradedAt uses the same ts). Best-effort + fenced. FENCE (measurement-only): lock_lines is read by NOTHING on the grade path (gradeSlateService, snapshot dedup/indexOdds, challengers, selector, ledger) — a grep test asserts it, and RLS locks it to the service role. Grade byte- identical proven: runSnapshot grades are identical with persist on/off (test). Volume ~1.5-3k rows/day (graded props x books x 5 snapshots); weeks retained, no pruning needed short-term. Does NOT retroactively fix the existing 62 rows — future accrual only; confirmation still needs weeks of settled rows. Full suite 3842 green, web build exit 0. No grade/locked_odds/outcome/served surface changed. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VsztNChZ7vEvSR61AuMhD1
This commit is contained in:
@@ -0,0 +1,132 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* LOCK-LINE CAPTURE — the lock-time analog of `closingCapture` (measurement-only).
|
||||
*
|
||||
* WHY: the over-side skew audit (2026-07-29) returned SURVIVES-BASELINE for the
|
||||
* champion's takeable-MLB-over edge, but the confirming check — was our locked line
|
||||
* stale-high vs consensus AT LOCK — is BLOCKED because multi-book lines at lock were
|
||||
* never persisted (`bookprices` is Redis, current-only). This persists them: as a grade
|
||||
* LOCKS, each book's line + both-side prices for that graded prop is written to
|
||||
* `lock_lines`, timestamped at the lock moment. A future audit joins it to
|
||||
* `closing_captures` to run the blocked staleness check.
|
||||
*
|
||||
* STRUCTURAL FENCE (enforced, not a comment):
|
||||
* 1. This module only READS the `props` array + a set of graded keys and RETURNS rows /
|
||||
* writes its OWN table (`lock_lines`). It never mutates `props`.
|
||||
* 2. `lock_lines` is read by NOTHING on the grade path — not gradeSlateService,
|
||||
* snapshotService's dedup/indexOdds, any challenger, the selector, or the ledger.
|
||||
* A test greps the grade-path files and asserts zero `lock_lines`/`lockLineCapture`
|
||||
* references; migration 033 puts RLS-with-no-policies on the table (service-role
|
||||
* only).
|
||||
* 3. In runSnapshot the persist is a best-effort leaf whose result is used by nothing
|
||||
* downstream — the graded slate is byte-identical whether or not it runs (locked by
|
||||
* a runSnapshot grade-identity test).
|
||||
*
|
||||
* NO TTL RACE: rows are built from the in-memory `props` array synchronously at the lock
|
||||
* moment (`ts`), never by re-reading the Redis `bookprices` key later — so the snapshot
|
||||
* we persist is exactly the one that locked, not a later/expired one.
|
||||
*
|
||||
* HONEST-ABSENT: a graded prop with only one priced book persists as ONE row. Never a
|
||||
* fabricated second book. A book row with neither price is not a line and is skipped.
|
||||
*/
|
||||
|
||||
const { nameKey, normalizeName } = require('../utils/playerName');
|
||||
|
||||
// Sharp/no-vig reference book (mirrors closingCapture). Tagged so the audit can weight
|
||||
// pinnacle as the sharp consensus vs the retail books.
|
||||
const SHARP_BOOKS = new Set(['pinnacle']);
|
||||
|
||||
function etDate(iso) {
|
||||
if (!iso) return null;
|
||||
const t = new Date(iso);
|
||||
if (Number.isNaN(t.getTime())) return null;
|
||||
return new Intl.DateTimeFormat('en-CA', {
|
||||
timeZone: 'America/New_York', year: 'numeric', month: '2-digit', day: '2-digit',
|
||||
}).format(t);
|
||||
}
|
||||
|
||||
function numOrNull(v) {
|
||||
if (v == null || v === '') return null;
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
}
|
||||
|
||||
/** Canonical graded-prop key (player + stat), matching how the ledger/snapshot key. */
|
||||
function lockKey(player, stat) {
|
||||
return `${nameKey(player)}|${String(stat || '').toLowerCase()}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build flat lock-line rows for the GRADED props only (targeted → low volume). One row
|
||||
* per (prop × book) carrying both over/under prices. PURE — no I/O; never mutates props.
|
||||
*
|
||||
* @param {string} sport
|
||||
* @param {Array} props multi-book flat props (oddsNormalizer shape)
|
||||
* @param {Set} gradedKeys Set of `${nameKey}|${stat}` for props that locked a grade
|
||||
* @param {Object} [opts] { lockedAt } — the exact lock timestamp (snapshot ts)
|
||||
*/
|
||||
function buildLockRows(sport, props, gradedKeys, opts = {}) {
|
||||
const lockedAt = opts.lockedAt || new Date().toISOString();
|
||||
const seen = new Set(); // dedupe (prop × book) within this lock moment
|
||||
const out = [];
|
||||
for (const p of props || []) {
|
||||
if (!p || !p.player || !p.stat_type || !p.book) continue;
|
||||
const stat = String(p.stat_type).toLowerCase();
|
||||
const k = lockKey(p.player, stat);
|
||||
if (gradedKeys && !gradedKeys.has(k)) continue; // graded props only
|
||||
|
||||
const over = numOrNull(p.over_odds);
|
||||
const under = numOrNull(p.under_odds);
|
||||
if (over == null && under == null) continue; // not a real line — honest-absent, skip
|
||||
|
||||
const dedupe = `${k}|${p.book}`;
|
||||
if (seen.has(dedupe)) continue;
|
||||
seen.add(dedupe);
|
||||
|
||||
out.push({
|
||||
sport,
|
||||
player_key: nameKey(p.player),
|
||||
player_name: normalizeName(p.player).display || p.player,
|
||||
stat,
|
||||
game_date: etDate(p.game_time),
|
||||
game_time: p.game_time || null,
|
||||
book: p.book,
|
||||
line_type: SHARP_BOOKS.has(String(p.book || '').toLowerCase()) ? 'sharp' : 'book',
|
||||
line: numOrNull(p.line),
|
||||
over_odds: over,
|
||||
under_odds: under,
|
||||
locked_at: lockedAt,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist append-only. Idempotent via the UNIQUE key (a snapshot retry never
|
||||
* double-inserts). Best-effort — a persistence failure never breaks the snapshot.
|
||||
*/
|
||||
async function persist(rows, deps = {}) {
|
||||
const out = { attempted: rows ? rows.length : 0, written: 0, skipped: false, error: null };
|
||||
if (!out.attempted) return out;
|
||||
try {
|
||||
const getClient = deps.getClient || require('../utils/supabase').getSupabaseServiceClient;
|
||||
const sb = getClient();
|
||||
if (!sb) { out.skipped = true; return out; }
|
||||
const CHUNK = 250;
|
||||
for (let i = 0; i < rows.length; i += CHUNK) {
|
||||
const chunk = rows.slice(i, i + CHUNK);
|
||||
const { error } = await sb.from('lock_lines').upsert(chunk, {
|
||||
onConflict: 'sport,player_key,stat,game_date,book,locked_at',
|
||||
ignoreDuplicates: true,
|
||||
});
|
||||
if (error) { out.error = error.message; break; }
|
||||
out.written += chunk.length;
|
||||
}
|
||||
} catch (e) {
|
||||
out.error = e && e.message ? e.message : String(e);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
module.exports = { buildLockRows, persist, lockKey, SHARP_BOOKS };
|
||||
@@ -288,6 +288,10 @@ async function runSnapshot(sport, opts = {}) {
|
||||
// Fenced: written to its own `bookprices:{sport}` key, read by nothing on
|
||||
// the grade path. Injectable; a failure never touches grading.
|
||||
captureBookPrices: opts.captureBookPrices || require('./bookPriceStore').captureBookPrices,
|
||||
// Lock-line persistence (measurement-only). Persists multi-book lines to the DB at
|
||||
// the lock moment so a future audit can run the currently-BLOCKED staleness check.
|
||||
// Fenced: writes its own `lock_lines` table, read by nothing on the grade path.
|
||||
lockLineCapture: opts.lockLineCapture || require('./lockLineCapture'),
|
||||
refreshTeamStats: opts.refreshTeamStats
|
||||
|| (process.env.NODE_ENV === 'test'
|
||||
? async () => null
|
||||
@@ -557,6 +561,22 @@ async function runSnapshot(sport, opts = {}) {
|
||||
};
|
||||
});
|
||||
|
||||
// LOCK-LINE PERSISTENCE (measurement-only). At THIS moment the grades have locked to
|
||||
// their lines (`gradedAt` uses `ts`), so `props` is the multi-book snapshot AS IT
|
||||
// EXISTED AT LOCK. Persist each graded prop's per-book lines to `lock_lines`,
|
||||
// timestamped `ts`, so a future audit can check whether our locked line was stale-high
|
||||
// vs consensus at lock. Built from the in-memory `props` (no Redis re-read → no TTL
|
||||
// race). Best-effort + structurally fenced: nothing on the grade path reads lock_lines,
|
||||
// and the graded slate is byte-identical whether or not this runs.
|
||||
try {
|
||||
const gradedKeys = new Set(enriched.map((g) => `${norm(g.player || g.player_name)}|${String(g.stat_type || g.stat || '').toLowerCase()}`));
|
||||
const lockRows = deps.lockLineCapture.buildLockRows(sp, props, gradedKeys, { lockedAt: ts });
|
||||
const lr = await deps.lockLineCapture.persist(lockRows);
|
||||
console.log(`[lock-lines] ${sp}: ${lr.written}/${lr.attempted} multi-book rows persisted at lock${lr.skipped ? ' (skipped — no supabase env)' : ''}${lr.error ? ` ERROR: ${lr.error}` : ''}`);
|
||||
} catch (e) {
|
||||
console.warn(`[lock-lines] ${sp} persist failed (measurement-only, snapshot continues):`, e.message);
|
||||
}
|
||||
|
||||
// Session 64 — retention persists HERE, after enrichment, so archetype/team/
|
||||
// opponent are populated. Feature values were captured at grade time and are
|
||||
// NOT touched by the merge (mergeEnrichment only fills the three null fields).
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
-- Migration 033: lock_lines — multi-book lines at the LOCK moment (measurement-only).
|
||||
--
|
||||
-- The lock-time analog of `closing_captures`. As a grade LOCKS to a line, this persists
|
||||
-- each book's line + both-side prices for that (graded) prop, timestamped at the lock
|
||||
-- moment. A future audit joins lock_lines (lock) to closing_captures (close) to answer
|
||||
-- the currently-BLOCKED question: was our locked line stale-high vs consensus AT LOCK?
|
||||
--
|
||||
-- FENCE: measurement/display-only. It must NOT feed the consensus-line selector, the
|
||||
-- champion, any challenger, the graded line, or the ledger. Enforced by (a) a separate
|
||||
-- table nothing on the grade path reads, and (b) RLS enabled with NO policies, so only
|
||||
-- the service role (which bypasses RLS) can read/write it — no client, no anon.
|
||||
--
|
||||
-- One row per (prop × book) carrying BOTH over_odds + under_odds (the de-vig needs both;
|
||||
-- one row halves volume vs closing_captures' per-side rows). Append-only; the UNIQUE key
|
||||
-- makes a re-run (snapshot retry) idempotent. Honest-absent: single-book props persist
|
||||
-- as one row — never a fabricated second book.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS lock_lines (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
sport text NOT NULL,
|
||||
player_key text NOT NULL,
|
||||
player_name text,
|
||||
stat text NOT NULL,
|
||||
game_date date,
|
||||
game_time timestamptz,
|
||||
book text,
|
||||
line_type text, -- 'sharp' (pinnacle) | 'book'
|
||||
line numeric,
|
||||
over_odds integer,
|
||||
under_odds integer,
|
||||
locked_at timestamptz NOT NULL, -- the LOCK moment (snapshot gradedAt ts)
|
||||
missed_reason text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT lock_lines_uniq UNIQUE NULLS NOT DISTINCT (sport, player_key, stat, game_date, book, locked_at)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS lock_lines_join_idx ON lock_lines (sport, player_key, stat, game_date);
|
||||
CREATE INDEX IF NOT EXISTS lock_lines_locked_at_idx ON lock_lines (locked_at);
|
||||
|
||||
-- FENCE: RLS on, no policies → service-role only. Never reachable by a client/anon,
|
||||
-- and never by the grade path (which does not query this table).
|
||||
ALTER TABLE lock_lines ENABLE ROW LEVEL SECURITY;
|
||||
@@ -0,0 +1,124 @@
|
||||
// Lock-line capture — persist multi-book lines at the lock moment (measurement-only).
|
||||
// Locks: build correctness, honest-absent single-book, the STRUCTURAL FENCE (no mutation,
|
||||
// no grade-path read), and grade byte-identical through runSnapshot.
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const llc = require('../../src/services/lockLineCapture');
|
||||
const snapshot = require('../../src/services/snapshotService');
|
||||
const { nameKey } = require('../../src/utils/playerName');
|
||||
|
||||
// Judge TB across 3 books + Betts hits (1 book) + a NON-graded prop.
|
||||
const props = [
|
||||
{ player: 'Aaron Judge', stat_type: 'total_bases', line: 1.5, over_odds: -115, under_odds: -105, book: 'draftkings', game_time: '2026-07-29T23:00:00Z' },
|
||||
{ player: 'Aaron Judge', stat_type: 'total_bases', line: 1.5, over_odds: -110, under_odds: -110, book: 'betmgm', game_time: '2026-07-29T23:00:00Z' },
|
||||
{ player: 'Aaron Judge', stat_type: 'total_bases', line: 1.5, over_odds: -120, under_odds: 100, book: 'pinnacle', game_time: '2026-07-29T23:00:00Z' },
|
||||
{ player: 'Mookie Betts', stat_type: 'hits', line: 1.5, over_odds: 120, under_odds: -150, book: 'draftkings', game_time: '2026-07-29T23:00:00Z' },
|
||||
{ player: 'Not Graded', stat_type: 'hits', line: 0.5, over_odds: -110, under_odds: -110, book: 'draftkings', game_time: '2026-07-29T23:00:00Z' },
|
||||
];
|
||||
const gradedKeys = new Set([`${nameKey('Aaron Judge')}|total_bases`, `${nameKey('Mookie Betts')}|hits`]);
|
||||
|
||||
describe('buildLockRows', () => {
|
||||
it('emits one row per graded prop×book, both odds, sharp tag on pinnacle', () => {
|
||||
const rows = llc.buildLockRows('mlb', props, gradedKeys, { lockedAt: 'T0' });
|
||||
const judge = rows.filter((r) => r.stat === 'total_bases');
|
||||
expect(judge).toHaveLength(3);
|
||||
expect(judge.map((r) => r.book).sort()).toEqual(['betmgm', 'draftkings', 'pinnacle']);
|
||||
expect(judge.every((r) => r.over_odds != null && r.under_odds != null)).toBe(true);
|
||||
expect(judge.every((r) => r.locked_at === 'T0')).toBe(true);
|
||||
expect(judge.find((r) => r.book === 'pinnacle').line_type).toBe('sharp');
|
||||
expect(judge.find((r) => r.book === 'draftkings').line_type).toBe('book');
|
||||
expect(judge[0].player_key).toBe(nameKey('Aaron Judge'));
|
||||
});
|
||||
|
||||
it('HONEST-ABSENT: a single-book graded prop persists as ONE row, never a fake second', () => {
|
||||
const rows = llc.buildLockRows('mlb', props, gradedKeys, { lockedAt: 'T0' });
|
||||
const betts = rows.filter((r) => r.stat === 'hits' && r.player_key === nameKey('Mookie Betts'));
|
||||
expect(betts).toHaveLength(1);
|
||||
expect(betts[0].book).toBe('draftkings');
|
||||
});
|
||||
|
||||
it('excludes NON-graded props (only the locked props are persisted)', () => {
|
||||
const rows = llc.buildLockRows('mlb', props, gradedKeys, { lockedAt: 'T0' });
|
||||
expect(rows.some((r) => r.player_key === nameKey('Not Graded'))).toBe(false);
|
||||
});
|
||||
|
||||
it('drops a book row with neither price (not a real line)', () => {
|
||||
const rows = llc.buildLockRows('mlb', [
|
||||
{ player: 'X', stat_type: 'hits', line: 0.5, over_odds: null, under_odds: null, book: 'draftkings' },
|
||||
], new Set([`${nameKey('X')}|hits`]), { lockedAt: 'T0' });
|
||||
expect(rows).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('STRUCTURAL FENCE', () => {
|
||||
it('never mutates the props it reads (deep-frozen input)', () => {
|
||||
const frozen = Object.freeze(props.map((p) => Object.freeze({ ...p })));
|
||||
expect(() => llc.buildLockRows('mlb', frozen, gradedKeys, { lockedAt: 'T0' })).not.toThrow();
|
||||
});
|
||||
|
||||
it('no grade-path module reads lock_lines / lockLineCapture', () => {
|
||||
const gradePathFiles = [
|
||||
'src/services/gradeSlateService.js',
|
||||
'src/services/ledgerService.js',
|
||||
'src/services/challengerProjection.js',
|
||||
'src/services/contactChallenger.js',
|
||||
'src/services/projectionChallenger.js',
|
||||
'src/services/intelligence/analyzeViaEngine1.js',
|
||||
];
|
||||
for (const rel of gradePathFiles) {
|
||||
const src = fs.readFileSync(path.join(__dirname, '../../', rel), 'utf8');
|
||||
expect(src).not.toMatch(/lock_lines/);
|
||||
expect(src).not.toMatch(/lockLineCapture/);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('GRADE BYTE-IDENTICAL through runSnapshot', () => {
|
||||
function memCache() {
|
||||
const store = {};
|
||||
return { store, cacheGet: async (k) => (k in store ? store[k] : null), cacheSet: async (k, v) => { store[k] = v; } };
|
||||
}
|
||||
const grades = [
|
||||
{ player: 'Aaron Judge', stat_type: 'total_bases', line: 1.5, direction: 'over', grade: 'A', confidence: 71, edge_pct: 4.2 },
|
||||
{ player: 'Mookie Betts', stat_type: 'hits', line: 1.5, direction: 'under', grade: 'B', confidence: 60, edge_pct: 1.1 },
|
||||
];
|
||||
const fakeGrade = () => async (_s, _p, opts) => {
|
||||
await opts.cacheSet('grades:x', { grades, updated_at: opts.now(), source: 'test' });
|
||||
return { written: true, count: grades.length };
|
||||
};
|
||||
const baseOpts = (cache, lockDep) => ({
|
||||
getOdds: async () => ({ props, provider: 'test' }),
|
||||
gradeAndCacheSlate: fakeGrade(),
|
||||
resolveStats: async () => ({ found: false }),
|
||||
classify: () => ({ primary: null }),
|
||||
cacheGet: cache.cacheGet,
|
||||
cacheSet: cache.cacheSet,
|
||||
now: () => '2026-07-29T00:00:00.000Z',
|
||||
nowMs: () => 1000,
|
||||
notify: async () => {},
|
||||
retention: null,
|
||||
ledger: { recordPipelineGrades: async () => ({ written: 0 }), captureClosing: async () => {}, __internals: require('../../src/services/ledgerService').__internals },
|
||||
refreshTeamStats: async () => null,
|
||||
buildEspnIndex: async () => ({}),
|
||||
gameBinder: { attachGameTimes: async () => ({ bound: 2, alreadyHad: 0, unresolved: 0, ambiguous: 0 }) },
|
||||
lockLineCapture: lockDep,
|
||||
});
|
||||
|
||||
it('grades identical whether persist runs or is a no-op, and lock rows target graded props at ts', async () => {
|
||||
const captured = [];
|
||||
const withPersist = { buildLockRows: llc.buildLockRows, persist: async (rows) => { captured.push(...rows); return { written: rows.length, attempted: rows.length }; } };
|
||||
const noop = { buildLockRows: () => [], persist: async () => ({ written: 0, attempted: 0 }) };
|
||||
|
||||
const c1 = memCache(); await snapshot.runSnapshot('mlb', baseOpts(c1, withPersist));
|
||||
const c2 = memCache(); await snapshot.runSnapshot('mlb', baseOpts(c2, noop));
|
||||
|
||||
const strip = (snap) => JSON.stringify((snap.grades || []).map((g) => ({ player: g.player, stat_type: g.stat_type, line: g.line, direction: g.direction, grade: g.grade, gradedAt: g.gradedAt })));
|
||||
expect(strip(c1.store['snapshot:mlb:latest'])).toBe(strip(c2.store['snapshot:mlb:latest']));
|
||||
|
||||
// lock rows were built for the graded props, timestamped at the lock moment (ts).
|
||||
expect(captured.length).toBeGreaterThan(0);
|
||||
expect(captured.every((r) => r.locked_at === '2026-07-29T00:00:00.000Z')).toBe(true);
|
||||
expect(new Set(captured.map((r) => r.player_key))).toEqual(new Set([nameKey('Aaron Judge'), nameKey('Mookie Betts')]));
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user