Read integrity, as-of context, and the shadow matchup resolve (A1-A7)

Seven orders of measurement-first repair. The served grade does not move.

A0/A1 — the unordered page walk returned the right COUNT and the wrong ROWS:
410-617 of 2,490 duplicated with an equal number never returned, while
rows.length matched the server exactly. safePaginate orders on a real unique
key, verifies the tuple at runtime, and THROWS on a query error instead of
treating it as end-of-data. Both hits PROVES are withdrawn: they were drawn
through that reader, and defense_by_direction's distinct-n was likely below
the gate floor all along.

A2/A2b — rolled across every reader: 11 FAIL -> 0. Composite keys pulled from
pg_index (the context tables are dated-composite and had no single unique
column). The unordered helper is deleted, not parked.

A3 — ledgerService and retentionService defaulted the SAME env var to
DIFFERENT versions, so no ledger row ever carried the marker eligibility
requires. One source now. model_snapshots settlement moved onto the cron:
15,484 -> 28,894 settled, repaired-champion 0 -> 7,556.

A4 — hitsFactorContext takes an as-of cutoff. Refusal over reconstruction: no
row at-or-before the date means the factor does not apply, never the nearest
row. Live path unchanged, proven 400/400 on real rows.

A5 — factor_inputs freezes what the factor READ, never the multiplier, so an
audit can recompute and check. It also recorded the finding: the three hits
factors have NEVER fired. prop.opponent and prop.opposing_pitcher are read by
the resolver and written by nothing.

A6/A7 — matchupKeys resolves those keys from the posted lineup plus the
schedule's probable pitchers, and fires the factors into a SHADOW freeze:
248 fires on 308 props, 245 of which would move the grade. The served
forecast is untouched. specs/a8-shadow-factor-gate.md pre-registers the test
that decides whether they ever go live.

Nothing is turned on. CALIBRATION_DEPLOYED stays []. Both verdicts stay
withdrawn. 4,772 tests / 371 suites green, web build exit 0, read-integrity
harness 34/34.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kev
2026-08-11 22:49:56 -04:00
parent 387ae4d54e
commit f61ec6b391
49 changed files with 4874 additions and 308 deletions
+33 -13
View File
@@ -29,30 +29,45 @@ const { createClient } = require('@supabase/supabase-js');
const ctx = require('../src/services/lineupContextService');
const mlb = require('../src/services/adapters/mlbStatsAdapter');
const { knownNumber } = require('../src/utils/known');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
const SEASON = Number(process.env.BF_SEASON || 2026);
const PAGE = 1000;
async function page(sb, table, select, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
const { data, error } = await apply(sb.from(table).select(select)).range(from, from + PAGE - 1);
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data);
if (data.length < PAGE) break;
}
return out;
// ── THE SAFE WALK (Fix A2/A2b) ────────────────────────────────────────────
// This script used to walk pages with `.range()` and NO ORDER BY. Measured on
// production, that returned the correct row COUNT and the wrong ROWS: up to
// 33.6% of a read came back twice while an equal share never came back at all,
// so `rows.length` looked perfect while a fifth of the sample was missing.
//
// `pageSafe` routes every read through `src/utils/safePaginate`: a stable ORDER
// BY on the table's real UNIQUE key — single OR composite, looked up from
// `src/utils/tableKeys` rather than assumed — a runtime tuple-uniqueness check,
// and a THROWN error instead of a silent stop. The old unordered helper is gone
// rather than left beside it, because a dead broken helper is an invitation.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
return paginate(() => apply(sb.from(table).select(select)),
{ key, pageSize: PAGE, label: `${table}` });
}
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS = {
ledger: (sb) => pageSafe(sb, 'ledger_entries', 'id, player_key, player_name, stat, outcome, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).in('stat', ['hits', 'total_bases'])
.in('outcome', ['hit', 'miss'])),
platoonHeld: (sb) => pageSafe(sb, 'platoon_splits', 'as_of_date, sport, season, player_key',
(q) => q.eq('sport', 'mlb')),
};
async function main() {
if (!SB_URL || !SB_KEY) throw new Error('SUPABASE_URL / service key required');
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
// Every hitter who appears on a CLEAN settled row — the true denominator.
const led = await page(sb, 'ledger_entries', 'player_key, player_name, stat, outcome, quarantine_reason',
const led = await pageSafe(sb, 'ledger_entries', 'id, player_key, player_name, stat, outcome, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).in('stat', ['hits', 'total_bases'])
.in('outcome', ['hit', 'miss']));
const need = new Map();
@@ -61,7 +76,7 @@ async function main() {
if (!need.has(r.player_key)) need.set(r.player_key, r.player_name);
}
const have = new Set((await page(sb, 'platoon_splits', 'player_key', (q) => q.eq('sport', 'mlb')))
const have = new Set((await pageSafe(sb, 'platoon_splits', 'as_of_date, sport, season, player_key', (q) => q.eq('sport', 'mlb')))
.map((r) => r.player_key));
const missing = [...need.entries()].filter(([k]) => !have.has(k));
@@ -100,4 +115,9 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS: (typeof READS !== 'undefined' ? READS : undefined), READS_LEDGER: (typeof READS_LEDGER !== 'undefined' ? READS_LEDGER : undefined) };
+37 -16
View File
@@ -18,6 +18,8 @@ const gb = require('../src/services/model/gradeBands');
const tl = require('../src/services/model/testLedger');
const cal = require('../src/services/model/calibration');
const { knownNumber } = require('../src/utils/known');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
@@ -31,29 +33,43 @@ const PAGE = 1000;
*/
const PROVEN_BY_ARCHETYPE = Object.freeze({});
async function page(sb, table, select, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
const { data, error } = await apply(sb.from(table).select(select)).range(from, from + PAGE - 1);
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data);
if (data.length < PAGE) break;
}
return out;
// ── FIX A2 (2026-08-09) — THE SAFE WALK ───────────────────────────────────
// `page()` above walks with no ORDER BY. Measured on production, that returned
// the correct row COUNT and the wrong ROWS: up to 33.6% of a read came back
// twice while an equal share never came back at all. `pageSafe` routes the same
// call through `src/utils/safePaginate`, which orders on a UNIQUE key on every
// page, verifies uniqueness at runtime, and THROWS on a query error instead of
// treating it as end-of-data.
//
// `page()` SURVIVES only for the context tables (statcast_aggregates,
// batter_spray, team_defense, platoon_splits, park_dimensions, game_context...).
// Those have COMPOSITE primary keys with no single unique column, so
// safePaginate cannot express them. They measure 0% corruption today; making
// them safe needs a composite-key ordering the helper does not yet have. Do not
// use `page()` for ledger_entries or model_snapshots.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
return paginate(() => apply(sb.from(table).select(select)),
{ key, pageSize: PAGE, label: `${table}` });
}
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS = {
snaps: (sb, stat = STAT) => pageSafe(sb, 'model_snapshots', 'id, player_key, game_date, archetype, stat',
(q) => q.eq('sport', 'mlb').eq('stat', stat).not('archetype', 'is', null)),
ledger: (sb, stat = STAT) => pageSafe(sb, 'ledger_entries',
'id, player_key, game_date, outcome, p_win, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', stat)
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null)),
};
async function main() {
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
const snaps = await page(sb, 'model_snapshots', 'player_key, game_date, archetype, stat',
(q) => q.eq('sport', 'mlb').eq('stat', STAT).not('archetype', 'is', null));
const snaps = await READS.snaps(sb);
const archOf = new Map();
for (const s of snaps) archOf.set(`${s.player_key}|${s.game_date}`, s.archetype);
const led = await page(sb, 'ledger_entries', 'player_key, game_date, outcome, p_win, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', STAT)
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null));
const led = await READS.ledger(sb);
const clean = led.filter((r) => !(r.quarantine_reason || '').startsWith('nontakeable_book'));
const byArch = new Map();
@@ -97,4 +113,9 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS };
+26 -14
View File
@@ -29,6 +29,8 @@
require('dotenv').config();
const { createClient } = require('@supabase/supabase-js');
const cal = require('../src/services/model/calibration');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
@@ -37,19 +39,24 @@ const PAGE = 1000;
const r3 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 1000) / 1000);
async function page(sb, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
const { data, error } = await apply(sb.from('ledger_entries')
.select('p_win, outcome, game_date, quarantine_reason')).range(from, from + PAGE - 1);
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data);
if (data.length < PAGE) break;
}
return out;
// ── THE SAFE WALK (Fix A2b) ───────────────────────────────────────────────
// This walked pages with no ORDER BY and measured 24.7% corrupt on production —
// 616 of 2,490 rows returned twice, an equal share never returned — while
// `rows.length` matched the server count exactly. It fits the calibration
// reliability curve, so a fifth of the history was double-weighted and another
// fifth absent from every bin.
async function pageSafe(sb, apply) {
return paginate(() => apply(sb.from('ledger_entries')
.select('id, p_win, outcome, game_date, quarantine_reason')),
{ key: uniqueKeyFor('ledger_entries'), pageSize: PAGE, label: 'calibrate-hits' });
}
/** THE MEASURED READ — main() and the harness call the same function. */
const READS = {
ledger: (sb) => pageSafe(sb, (q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', 'hits')
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null)),
};
/** Reliability rendered per bin with n — the only honest way to read this. */
function curve(rows, label) {
return cal.reliability(rows, 10)
@@ -67,8 +74,7 @@ async function main() {
if (!SB_URL || !SB_KEY) throw new Error('SUPABASE_URL / service key required');
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
const raw = await page(sb, (q) => q.eq('sport', 'mlb').is('user_id', null)
.eq('stat', 'hits').in('outcome', ['hit', 'miss']).not('p_win', 'is', null));
const raw = await READS.ledger(sb);
const all = raw
.filter((r) => !(r.quarantine_reason || '').startsWith('nontakeable_book'))
.map((r) => ({ p: Number(r.p_win), won: r.outcome === 'hit' ? 1 : 0, d: String(r.game_date) }));
@@ -133,4 +139,10 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS };
+41 -17
View File
@@ -51,6 +51,8 @@
require('dotenv').config();
const { createClient } = require('@supabase/supabase-js');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
@@ -143,20 +145,39 @@ function bootstrapCorr(rows, key, seed = 20260804, iters = 3000) {
};
}
async function page(sb, table, select, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
let q = sb.from(table).select(select);
q = apply(q).range(from, from + PAGE - 1);
const { data, error } = await q;
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data);
if (data.length < PAGE) break;
}
return out;
// ── FIX A2 (2026-08-09) — THE SAFE WALK ───────────────────────────────────
// `page()` above walks with no ORDER BY. Measured on production, that returned
// the correct row COUNT and the wrong ROWS: up to 33.6% of a read came back
// twice while an equal share never came back at all. `pageSafe` routes the same
// call through `src/utils/safePaginate`, which orders on a UNIQUE key on every
// page, verifies uniqueness at runtime, and THROWS on a query error instead of
// treating it as end-of-data.
//
// `page()` SURVIVES only for the context tables (statcast_aggregates,
// batter_spray, team_defense, platoon_splits, park_dimensions, game_context...).
// Those have COMPOSITE primary keys with no single unique column, so
// safePaginate cannot express them. They measure 0% corruption today; making
// them safe needs a composite-key ordering the helper does not yet have. Do not
// use `page()` for ledger_entries or model_snapshots.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
return paginate(() => apply(sb.from(table).select(select)),
{ key, pageSize: PAGE, label: `${table}` });
}
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS_LEDGER = {
ledger: (sb) => pageSafe(sb, 'ledger_entries',
'id, player_key, stat, line, side, game_date, outcome, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).in('outcome', ['hit', 'miss'])),
};
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS = {
snaps: (sb) => pageSafe(sb, 'model_snapshots',
'id, player_key, stat, line, side, game_date, p_win, features, quarantine_reason, captured_at, archetype',
(q) => q.eq('sport', 'mlb').not('p_win', 'is', null).not('features', 'is', null)),
};
const propKey = (r) => `${r.player_key}|${r.stat}|${Number(r.line)}|${String(r.side).toLowerCase()}|${r.game_date}`;
/**
@@ -170,10 +191,8 @@ const propKey = (r) => `${r.player_key}|${r.stat}|${Number(r.line)}|${String(r.s
* this run is read-only.
*/
async function fetchAll(sb) {
const snaps = await page(sb, 'model_snapshots',
'player_key, stat, line, side, game_date, p_win, features, quarantine_reason, captured_at, archetype',
(q) => q.eq('sport', 'mlb').not('p_win', 'is', null).not('features', 'is', null));
const led = await page(sb, 'ledger_entries', 'player_key, stat, line, side, game_date, outcome, quarantine_reason',
const snaps = await READS.snaps(sb);
const led = await pageSafe(sb, 'ledger_entries', 'id, player_key, stat, line, side, game_date, outcome, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).in('outcome', ['hit', 'miss']));
const outcomeBy = new Map();
@@ -371,4 +390,9 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS, READS_LEDGER };
+39 -19
View File
@@ -44,6 +44,8 @@ const sk = require('../src/services/model/skillProjection');
const reg = require('../src/services/model/featureRegistry');
const mlb = require('../src/services/adapters/mlbStatsAdapter');
const { knownRate, knownNumber } = require('../src/utils/known');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
@@ -156,18 +158,35 @@ function bootstrapDiff(rows, keyA, keyB, iters = 4000, seed = 20260804) {
};
}
async function page(sb, table, select, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
const { data, error } = await apply(sb.from(table).select(select)).range(from, from + PAGE - 1);
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data);
if (data.length < PAGE) break;
}
return out;
// ── FIX A2 (2026-08-09) — THE SAFE WALK ───────────────────────────────────
// `page()` above walks with no ORDER BY. Measured on production, that returned
// the correct row COUNT and the wrong ROWS: up to 33.6% of a read came back
// twice while an equal share never came back at all. `pageSafe` routes the same
// call through `src/utils/safePaginate`, which orders on a UNIQUE key on every
// page, verifies uniqueness at runtime, and THROWS on a query error instead of
// treating it as end-of-data.
//
// `page()` SURVIVES only for the context tables (statcast_aggregates,
// batter_spray, team_defense, platoon_splits, park_dimensions, game_context...).
// Those have COMPOSITE primary keys with no single unique column, so
// safePaginate cannot express them. They measure 0% corruption today; making
// them safe needs a composite-key ordering the helper does not yet have. Do not
// use `page()` for ledger_entries or model_snapshots.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
return paginate(() => apply(sb.from(table).select(select)),
{ key, pageSize: PAGE, label: `${table}` });
}
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS = {
snaps: (sb) => pageSafe(sb, 'model_snapshots', 'id, player_key, game_date, archetype',
(q) => q.eq('sport', 'mlb').eq('stat', STAT).not('archetype', 'is', null)),
ledger: (sb) => pageSafe(sb, 'ledger_entries',
'id, player_key, player_name, stat, line, side, outcome, game_date, p_win, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', STAT)
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null)),
};
async function opposingStarters(dates) {
const m = new Map();
for (const d of dates) {
@@ -305,7 +324,7 @@ async function main() {
if (!SB_URL || !SB_KEY) throw new Error('SUPABASE_URL / service key required');
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
const statcast = await page(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb'));
const statcast = await pageSafe(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb'));
const freezeDate = statcast.reduce((mx, r) => (String(r.updated_at) > mx ? String(r.updated_at) : mx), '').slice(0, 10);
const batters = new Map(); const pitchersById = new Map();
for (const r of statcast) {
@@ -329,15 +348,11 @@ async function main() {
// model_snapshots carries the actual classification per prop, so the
// conditioning variable is now a genuine BOMBER indicator, which is
// categorical and therefore not a transform of barrel at all.
const snaps = await page(sb, 'model_snapshots', 'player_key, game_date, archetype',
(q) => q.eq('sport', 'mlb').eq('stat', STAT).not('archetype', 'is', null));
const snaps = await READS.snaps(sb);
const archetypeBy = new Map();
for (const r of snaps) if (r.player_key && r.game_date) archetypeBy.set(`${r.player_key}|${r.game_date}`, r.archetype);
const led = await page(sb, 'ledger_entries',
'player_key, player_name, stat, line, side, outcome, game_date, p_win, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', STAT)
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null));
const led = await READS.ledger(sb);
// POINT-IN-TIME IS NO LONGER AVAILABLE FROM THIS TABLE.
//
// `statcast_aggregates` is upserted in place and keeps one as-of date. The
@@ -369,7 +384,7 @@ async function main() {
console.error(`[debug] matches in sample=${sampleKeys.filter((k) => batters.has(k)).length}/5`);
}
// TEAM DEFENCE — the newly-ingested Statcast OAA, per team, dated.
const defRows = await page(sb, 'team_defense', '*', (q) => q.eq('sport', 'mlb'));
const defRows = await pageSafe(sb, 'team_defense', '*', (q) => q.eq('sport', 'mlb'));
const defByTeam = new Map();
for (const d of defRows) {
const prev = defByTeam.get(d.team);
@@ -550,4 +565,9 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS };
+43 -13
View File
@@ -11,6 +11,8 @@
require('dotenv').config();
const fs = require('fs');
const path = require('path');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
const { createClient } = require('@supabase/supabase-js');
const hf = require('../src/services/model/hitsFactors');
const lp = require('../src/services/model/lowParamCalibrator');
@@ -22,15 +24,25 @@ const BOX = path.join(process.cwd(), '.seq-cache', 'batting-lines.json');
const SEQ = path.join(process.cwd(), '.seq-cache', 'sequences.json');
const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);
async function page(sb, t, s, f, orderBy = 'id') {
const o = [];
for (let i = 0; ; i += 1000) {
const { data, error } = await f(sb.from(t).select(s)).order(orderBy, { ascending: true }).range(i, i + 999);
if (error) throw new Error(`${t}: ${error.message}`);
if (!data || !data.length) break; o.push(...data); if (data.length < 1000) break;
}
return o;
// FIX A4 — the context walks here ordered by a NON-UNIQUE prefix
// ('player_key' / 'team'), which leaves the trailing key columns tied and lets
// pages overlap. They now order by the table's real unique tuple.
async function page(sb, t, s, f, key) {
return paginate(() => f(sb.from(t).select(s)),
{ key: key || uniqueKeyFor(t), pageSize: 1000, label: `factor-wiring-audit:${t}` });
}
/**
* AS-OF. `FWA_AS_OF=YYYY-MM-DD` bounds every dated context table to rows at or
* before that date, so an audit of a past slate cannot join a profile built from
* games played after it. Unset = today = the live read, unchanged.
*
* REFUSAL OVER RECONSTRUCTION: an entity with no row at or before the cutoff is
* simply absent from the index, so its factor does not apply. Never the nearest
* row — a substituted row is a plausible wrong value wearing a date.
*/
const AS_OF = process.env.FWA_AS_OF || null;
const dated = (q) => (AS_OF ? q.lte('as_of_date', AS_OF) : q);
const isPreGame = (c, g) => {
const et = new Date(new Date(c).getTime() - 4 * 3600 * 1000);
const d = et.toISOString().slice(0, 10);
@@ -59,15 +71,33 @@ function decompose(rows, bins = 10) {
// Factor inputs.
const [spray, defense, platoon, statcast] = await Promise.all([
page(sb, 'batter_spray', '*', (q) => q.eq('sport', 'mlb'), 'player_key'),
page(sb, 'team_defense', '*', (q) => q.eq('sport', 'mlb'), 'team'),
page(sb, 'platoon_splits', '*', (q) => q.eq('sport', 'mlb'), 'player_key'),
page(sb, 'statcast_aggregates', 'player_key, role, bats, throws, hard_hit_pct', (q) => q.eq('sport', 'mlb'), 'player_key'),
page(sb, 'batter_spray', '*', (q) => dated(q.eq('sport', 'mlb'))),
page(sb, 'team_defense', '*', (q) => dated(q.eq('sport', 'mlb'))),
page(sb, 'platoon_splits', '*', (q) => dated(q.eq('sport', 'mlb'))),
// `statcast_aggregates` keeps ONE as-of date, so it cannot answer an as-of
// question; the dated path reads the retained history instead.
AS_OF
? page(sb, 'statcast_history', 'player_key, source_id, role, bats, throws, hard_hit_pct, as_of_date',
(q) => q.eq('sport', 'mlb').lte('as_of_date', AS_OF))
: page(sb, 'statcast_aggregates', 'player_key, source_id, role, bats, throws, hard_hit_pct',
(q) => q.eq('sport', 'mlb')),
]);
const latest = (rows, k) => { const m = new Map(); for (const r of rows) { const key = r[k]; if (!key) continue; const p = m.get(key); if (!p || String(r.as_of_date) > String(p.as_of_date)) m.set(key, r); } return m; };
const sprayBy = latest(spray, 'player_key'); const defBy = latest(defense, 'team'); const platBy = latest(platoon, 'player_key');
const batBy = new Map(); const pitBy = new Map();
for (const r of statcast) { if (!r.player_key) continue; (r.role === 'pitcher' ? pitBy : batBy).set(r.player_key, r); }
// On the dated path several as-of rows per player are in scope; collapse to
// the latest WITHIN the bound, keyed on (source_id, role) so a two-way player
// keeps both profiles. No-op on the live path (one row per player).
const statcastLatest = AS_OF ? (() => {
const m = new Map();
for (const r of statcast) {
const k = `${r.source_id}|${r.role}`;
const prev = m.get(k);
if (!prev || String(r.as_of_date) > String(prev.as_of_date)) m.set(k, r);
}
return [...m.values()];
})() : statcast;
for (const r of statcastLatest) { if (!r.player_key) continue; (r.role === 'pitcher' ? pitBy : batBy).set(r.player_key, r); }
const frac = (v) => { const n = knownNumber(v); return n === null ? null : (n > 1 ? n / 100 : n); };
// Opponent + starter per (player,date) from the sequence cache.
+33 -13
View File
@@ -30,6 +30,8 @@ const pe = require('../src/services/model/pitcherEngine');
const sk = require('../src/services/model/skillProjection');
const mlb = require('../src/services/adapters/mlbStatsAdapter');
const { knownRate, knownNumber } = require('../src/utils/known');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
@@ -86,17 +88,30 @@ function bootstrapDiff(rows, kA, kB, iters = 4000, seed = 20260805) {
const ci = [q(0.025), q(0.975)];
return { point: r4(cv.pearson(rows.map((r) => r[kA]), rows.map((r) => r.won)).r - cv.pearson(rows.map((r) => r[kB]), rows.map((r) => r.won)).r), ci95: ci, ci_excludes_zero: ci[0] > 0 || ci[1] < 0 };
}
async function page(sb, table, select, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
const { data, error } = await apply(sb.from(table).select(select)).range(from, from + PAGE - 1);
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data); if (data.length < PAGE) break;
}
return out;
// ── THE SAFE WALK (Fix A2/A2b) ────────────────────────────────────────────
// This script used to walk pages with `.range()` and NO ORDER BY. Measured on
// production, that returned the correct row COUNT and the wrong ROWS: up to
// 33.6% of a read came back twice while an equal share never came back at all,
// so `rows.length` looked perfect while a fifth of the sample was missing.
//
// `pageSafe` routes every read through `src/utils/safePaginate`: a stable ORDER
// BY on the table's real UNIQUE key — single OR composite, looked up from
// `src/utils/tableKeys` rather than assumed — a runtime tuple-uniqueness check,
// and a THROWN error instead of a silent stop. The old unordered helper is gone
// rather than left beside it, because a dead broken helper is an invitation.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
return paginate(() => apply(sb.from(table).select(select)),
{ key, pageSize: PAGE, label: `${table}` });
}
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS = {
ledger: (sb) => pageSafe(sb, 'ledger_entries',
'id, player_key, player_name, line, side, outcome, game_date, p_win, quarantine_reason, game_id',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', 'strikeouts')
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null)),
};
const SOLO = ['pitcher_whiff_pct', 'pitcher_k_pct', 'pitcher_chase_pct', 'pitcher_gb_pct',
'pitcher_arm_angle', 'opposing_lineup_k_rate'];
@@ -125,7 +140,7 @@ async function main() {
if (!SB_URL || !SB_KEY) throw new Error('SUPABASE_URL / service key required');
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
const statcast = await page(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb'));
const statcast = await pageSafe(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb'));
const freezeDate = statcast.reduce((mx, r) => (String(r.updated_at) > mx ? String(r.updated_at) : mx), '').slice(0, 10);
const pitchByKey = new Map(); const batterByKey = new Map();
for (const r of statcast) {
@@ -134,8 +149,8 @@ async function main() {
if (r.role === 'batter' && r.player_key) batterByKey.set(r.player_key, prof);
}
const led = await page(sb, 'ledger_entries',
'player_key, player_name, stat, line, side, outcome, game_date, p_win, quarantine_reason',
const led = await pageSafe(sb, 'ledger_entries',
'id, player_key, player_name, stat, line, side, outcome, game_date, p_win, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', 'strikeouts')
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null));
const clean = led.filter((r) => !(r.quarantine_reason || '').startsWith('nontakeable_book'));
@@ -357,4 +372,9 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS };
+51 -21
View File
@@ -27,24 +27,53 @@ const { knownNumber, knownRate } = require('../src/utils/known');
const { nameKey } = require('../src/utils/playerName');
const sd = require('../src/services/model/sprayDefense');
const pss = require('../src/services/model/platoonSeverity');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
const PAGE = 1000;
const ARCHS = (process.env.HF_ARCHETYPES || 'BOMBER,GHOST,ALL').split(',');
async function page(sb, table, select, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
const { data, error } = await apply(sb.from(table).select(select)).range(from, from + PAGE - 1);
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data);
if (data.length < PAGE) break;
}
return out;
// ── FIX A2 (2026-08-09) — THE SAFE WALK ───────────────────────────────────
// `page()` above walks with no ORDER BY. Measured on production, that returned
// the correct row COUNT and the wrong ROWS: up to 33.6% of a read came back
// twice while an equal share never came back at all. `pageSafe` routes the same
// call through `src/utils/safePaginate`, which orders on a UNIQUE key on every
// page, verifies uniqueness at runtime, and THROWS on a query error instead of
// treating it as end-of-data.
//
// `page()` SURVIVES only for the context tables (statcast_aggregates,
// batter_spray, team_defense, platoon_splits, park_dimensions, game_context...).
// Those have COMPOSITE primary keys with no single unique column, so
// safePaginate cannot express them. They measure 0% corruption today; making
// them safe needs a composite-key ordering the helper does not yet have. Do not
// use `page()` for ledger_entries or model_snapshots.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
return paginate(() => apply(sb.from(table).select(select)),
{ key, pageSize: PAGE, label: `${table}` });
}
/**
* THE MEASURED READS, named once.
*
* `main()` calls these and so does the read-integrity harness, so a PASS there
* is a statement about the code that actually runs. A registry that restated
* the query would verify the restatement instead.
*/
const READS = {
statcast: (sb) => pageSafe(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb')),
spray: (sb) => pageSafe(sb, 'batter_spray', '*', (q) => q.eq('sport', 'mlb')),
platoon: (sb) => pageSafe(sb, 'platoon_splits', '*', (q) => q.eq('sport', 'mlb')),
defense: (sb) => pageSafe(sb, 'team_defense', '*', (q) => q.eq('sport', 'mlb')),
snaps: (sb) => pageSafe(sb, 'model_snapshots', 'id, player_key, game_date, archetype, stat',
(q) => q.eq('sport', 'mlb').eq('stat', 'hits').not('archetype', 'is', null)),
ledger: (sb) => pageSafe(sb, 'ledger_entries',
'id, game_id, player_key, player_name, line, side, outcome, game_date, p_win, quarantine_reason, env_park_base',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', 'hits')
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null)),
};
/**
* THE FACTORS. Each returns a MULTIPLIER on the base rate, or null when the
* input is absent — an absent factor must leave the baseline untouched rather
@@ -101,14 +130,14 @@ async function main() {
if (!SB_URL || !SB_KEY) throw new Error('SUPABASE_URL / service key required');
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
const statcast = await page(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb'));
const statcast = await pageSafe(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb'));
const batters = new Map(); const pitchersById = new Map();
for (const r of statcast) {
const prof = sk.fromStatcastRow(r);
if (r.role === 'pitcher' && r.source_id != null) pitchersById.set(Number(r.source_id), prof);
if (r.role === 'batter' && r.player_key) batters.set(r.player_key, prof);
}
const sprayRows = await page(sb, 'batter_spray', '*', (q) => q.eq('sport', 'mlb'));
const sprayRows = await pageSafe(sb, 'batter_spray', '*', (q) => q.eq('sport', 'mlb'));
const sprayByKey = new Map();
for (const r of sprayRows) {
if (!r.player_key) continue;
@@ -116,7 +145,7 @@ async function main() {
if (!prev || String(r.as_of_date) > String(prev.as_of_date)) sprayByKey.set(r.player_key, r);
}
const platRows = await page(sb, 'platoon_splits', '*', (q) => q.eq('sport', 'mlb'));
const platRows = await pageSafe(sb, 'platoon_splits', '*', (q) => q.eq('sport', 'mlb'));
const platByKey = new Map();
for (const r of platRows) {
if (!r.player_key) continue;
@@ -124,19 +153,15 @@ async function main() {
if (!prev || String(r.as_of_date) > String(prev.as_of_date)) platByKey.set(r.player_key, r);
}
const defRows = await page(sb, 'team_defense', '*', (q) => q.eq('sport', 'mlb'));
const defRows = await pageSafe(sb, 'team_defense', '*', (q) => q.eq('sport', 'mlb'));
const defByTeam = new Map();
for (const d of defRows) defByTeam.set(d.team, d);
const snaps = await page(sb, 'model_snapshots', 'player_key, game_date, archetype, stat',
(q) => q.eq('sport', 'mlb').eq('stat', 'hits').not('archetype', 'is', null));
const snaps = await READS.snaps(sb);
const archOf = new Map();
for (const s of snaps) archOf.set(`${s.player_key}|${s.game_date}`, s.archetype);
const led = await page(sb, 'ledger_entries',
'id, game_id, player_key, player_name, line, side, outcome, game_date, p_win, quarantine_reason, env_park_base',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', 'hits')
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null));
const led = await READS.ledger(sb);
const clean = led.filter((r) => !(r.quarantine_reason || '').startsWith('nontakeable_book'));
// Opponent faced, from each hitter's own game log.
@@ -308,4 +333,9 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS };
+34 -15
View File
@@ -20,39 +20,53 @@ const pw = require('../src/services/model/parkWeather');
const fg = require('../src/services/model/factorGate');
const tl = require('../src/services/model/testLedger');
const { knownNumber } = require('../src/utils/known');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
const PAGE = 1000;
async function page(sb, table, select, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
const { data, error } = await apply(sb.from(table).select(select)).range(from, from + PAGE - 1);
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data);
if (data.length < PAGE) break;
}
return out;
// ── THE SAFE WALK (Fix A2/A2b) ────────────────────────────────────────────
// This script used to walk pages with `.range()` and NO ORDER BY. Measured on
// production, that returned the correct row COUNT and the wrong ROWS: up to
// 33.6% of a read came back twice while an equal share never came back at all,
// so `rows.length` looked perfect while a fifth of the sample was missing.
//
// `pageSafe` routes every read through `src/utils/safePaginate`: a stable ORDER
// BY on the table's real UNIQUE key — single OR composite, looked up from
// `src/utils/tableKeys` rather than assumed — a runtime tuple-uniqueness check,
// and a THROWN error instead of a silent stop. The old unordered helper is gone
// rather than left beside it, because a dead broken helper is an invitation.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
return paginate(() => apply(sb.from(table).select(select)),
{ key, pageSize: PAGE, label: `${table}` });
}
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS = {
ledger: (sb) => pageSafe(sb, 'ledger_entries',
'id, game_id, player_key, player_name, line, side, outcome, game_date, p_win, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', 'total_bases')
.in('outcome', ['hit', 'miss'])),
};
/** Hit-type shares → expected total bases, so a reshape has a consequence. */
const tbFromShares = (s) => s.single + 2 * s.double + 3 * s.triple + 4 * s.home_run;
async function main() {
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
const parks = await page(sb, 'park_dimensions', '*', (q) => q.eq('sport', 'mlb'));
const parks = await pageSafe(sb, 'park_dimensions', '*', (q) => q.eq('sport', 'mlb'));
const byVenue = new Map();
for (const p of parks) if (!byVenue.has(p.venue_id)) byVenue.set(p.venue_id, p);
const league = pw.leagueGeometry([...byVenue.values()]);
const ctx = await page(sb, 'game_context', 'game_id, venue_id, wx_temp_f, wx_wind_speed_mph, wx_wind_direction_deg', (q) => q);
const ctx = await pageSafe(sb, 'game_context', 'game_id, venue_id, wx_temp_f, wx_wind_speed_mph, wx_wind_direction_deg', (q) => q);
const ctxBy = new Map(ctx.map((c) => [c.game_id, c]));
const led = await page(sb, 'ledger_entries',
'game_id, game_date, player_key, stat, line, side, outcome, quarantine_reason, p_win, proj_hits_p_over',
const led = await pageSafe(sb, 'ledger_entries',
'id, game_id, game_date, player_key, stat, line, side, outcome, quarantine_reason, p_win, proj_hits_p_over',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', 'total_bases').in('outcome', ['hit', 'miss']));
const clean = led.filter((r) => !(r.quarantine_reason || '').startsWith('nontakeable_book'));
@@ -118,4 +132,9 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS };
+31 -15
View File
@@ -37,6 +37,8 @@ const tl = require('../src/services/model/testLedger');
const sk = require('../src/services/model/skillProjection');
const { knownNumber, knownRate } = require('../src/utils/known');
const { nameKey } = require('../src/utils/playerName');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
@@ -52,18 +54,27 @@ const PA_EVENT = new Set([...HIT, 'field_out', 'strikeout', 'grounded_into_doubl
const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);
async function page(sb, table, select, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
const { data, error } = await apply(sb.from(table).select(select)).range(from, from + PAGE - 1);
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data);
if (data.length < PAGE) break;
}
return out;
// ── THE SAFE WALK (Fix A2/A2b) ────────────────────────────────────────────
// This script used to walk pages with `.range()` and NO ORDER BY. Measured on
// production, that returned the correct row COUNT and the wrong ROWS: up to
// 33.6% of a read came back twice while an equal share never came back at all,
// so `rows.length` looked perfect while a fifth of the sample was missing.
//
// `pageSafe` routes every read through `src/utils/safePaginate`: a stable ORDER
// BY on the table's real UNIQUE key — single OR composite, looked up from
// `src/utils/tableKeys` rather than assumed — a runtime tuple-uniqueness check,
// and a THROWN error instead of a silent stop. The old unordered helper is gone
// rather than left beside it, because a dead broken helper is an invitation.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
return paginate(() => apply(sb.from(table).select(select)),
{ key, pageSize: PAGE, label: `${table}` });
}
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS = {
opportunity: (sb) => pageSafe(sb, 'hitter_opportunity', '*', (q) => q.eq('sport', 'mlb')),
};
/**
* Reconstruct, point-in-time, from play-by-play:
* order[date|nameKey] the hitter's batting slot that game
@@ -170,7 +181,7 @@ const FACTORS = {
async function main() {
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
const statcast = await page(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb').eq('role', 'batter'));
const statcast = await pageSafe(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb').eq('role', 'batter'));
const batByKey = new Map();
const barrelByKey = new Map();
for (const r of statcast) {
@@ -180,7 +191,7 @@ async function main() {
if (prof.barrel_pct != null) barrelByKey.set(r.player_key, prof.barrel_pct);
}
const oppRows = await page(sb, 'hitter_opportunity', '*', (q) => q.eq('sport', 'mlb'));
const oppRows = await pageSafe(sb, 'hitter_opportunity', '*', (q) => q.eq('sport', 'mlb'));
const oppByKey = new Map();
for (const r of oppRows) {
const prev = oppByKey.get(r.player_key);
@@ -192,12 +203,12 @@ async function main() {
const out = { generated_note: 'null is the ARCHETYPE base rate, leave-one-out' };
for (const stat of ['rbi', 'runs']) {
const snaps = await page(sb, 'model_snapshots', 'player_key, game_date, archetype',
const snaps = await pageSafe(sb, 'model_snapshots', 'id, player_key, game_date, archetype',
(q) => q.eq('sport', 'mlb').eq('stat', stat).not('archetype', 'is', null));
const archOf = new Map();
for (const s of snaps) archOf.set(`${s.player_key}|${s.game_date}`, s.archetype);
const led = await page(sb, 'ledger_entries',
const led = await pageSafe(sb, 'ledger_entries',
'id, game_id, player_key, player_name, line, side, outcome, game_date, p_win, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', stat).in('outcome', ['hit', 'miss']));
const clean = led.filter((r) => !(r.quarantine_reason || '').startsWith('nontakeable_book')
@@ -298,4 +309,9 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS: (typeof READS !== 'undefined' ? READS : undefined), READS_LEDGER: (typeof READS_LEDGER !== 'undefined' ? READS_LEDGER : undefined) };
+45 -23
View File
@@ -38,6 +38,8 @@ const { nameKey } = require('../src/utils/playerName');
const sd = require('../src/services/model/sprayDefense');
const pss = require('../src/services/model/platoonSeverity');
const pw = require('../src/services/model/parkWeather');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
/** League-typical hit-type shares; the atom reshapes these and TB follows. */
const BASE_SHARES = { single: 0.655, double: 0.195, triple: 0.017, home_run: 0.133 };
@@ -48,18 +50,37 @@ const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SER
const PAGE = 1000;
const ARCHS = (process.env.TB_ARCHETYPES || 'ALL,BOMBER,GHOST,BRUSH,DRIVER').split(',');
async function page(sb, table, select, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
const { data, error } = await apply(sb.from(table).select(select)).range(from, from + PAGE - 1);
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data);
if (data.length < PAGE) break;
}
return out;
// ── FIX A2 (2026-08-09) — THE SAFE WALK ───────────────────────────────────
// `page()` above walks with no ORDER BY. Measured on production, that returned
// the correct row COUNT and the wrong ROWS: up to 33.6% of a read came back
// twice while an equal share never came back at all. `pageSafe` routes the same
// call through `src/utils/safePaginate`, which orders on a UNIQUE key on every
// page, verifies uniqueness at runtime, and THROWS on a query error instead of
// treating it as end-of-data.
//
// `page()` SURVIVES only for the context tables (statcast_aggregates,
// batter_spray, team_defense, platoon_splits, park_dimensions, game_context...).
// Those have COMPOSITE primary keys with no single unique column, so
// safePaginate cannot express them. They measure 0% corruption today; making
// them safe needs a composite-key ordering the helper does not yet have. Do not
// use `page()` for ledger_entries or model_snapshots.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
return paginate(() => apply(sb.from(table).select(select)),
{ key, pageSize: PAGE, label: `${table}` });
}
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS = {
parks: (sb) => pageSafe(sb, 'park_dimensions', '*', (q) => q.eq('sport', 'mlb')),
gameCtx: (sb) => pageSafe(sb, 'game_context', 'game_id, venue_id, wx_temp_f, wx_wind_speed_mph, wx_wind_direction_deg', (q) => q),
snaps: (sb) => pageSafe(sb, 'model_snapshots', 'id, player_key, game_date, archetype, stat',
(q) => q.eq('sport', 'mlb').eq('stat', 'total_bases').not('archetype', 'is', null)),
ledger: (sb) => pageSafe(sb, 'ledger_entries',
'id, game_id, player_key, player_name, line, side, outcome, game_date, p_win, quarantine_reason, env_park_base',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', 'total_bases')
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null)),
};
/**
* THE FACTORS. Each returns a MULTIPLIER on the base rate, or null when the
* input is absent — an absent factor must leave the baseline untouched rather
@@ -112,14 +133,14 @@ async function main() {
if (!SB_URL || !SB_KEY) throw new Error('SUPABASE_URL / service key required');
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
const statcast = await page(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb'));
const statcast = await pageSafe(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb'));
const batters = new Map(); const pitchersById = new Map();
for (const r of statcast) {
const prof = sk.fromStatcastRow(r);
if (r.role === 'pitcher' && r.source_id != null) pitchersById.set(Number(r.source_id), prof);
if (r.role === 'batter' && r.player_key) batters.set(r.player_key, prof);
}
const sprayRows = await page(sb, 'batter_spray', '*', (q) => q.eq('sport', 'mlb'));
const sprayRows = await pageSafe(sb, 'batter_spray', '*', (q) => q.eq('sport', 'mlb'));
const sprayByKey = new Map();
for (const r of sprayRows) {
if (!r.player_key) continue;
@@ -127,7 +148,7 @@ async function main() {
if (!prev || String(r.as_of_date) > String(prev.as_of_date)) sprayByKey.set(r.player_key, r);
}
const platRows = await page(sb, 'platoon_splits', '*', (q) => q.eq('sport', 'mlb'));
const platRows = await pageSafe(sb, 'platoon_splits', '*', (q) => q.eq('sport', 'mlb'));
const platByKey = new Map();
for (const r of platRows) {
if (!r.player_key) continue;
@@ -135,26 +156,22 @@ async function main() {
if (!prev || String(r.as_of_date) > String(prev.as_of_date)) platByKey.set(r.player_key, r);
}
const parkRows = await page(sb, 'park_dimensions', '*', (q) => q.eq('sport', 'mlb'));
const parkRows = await pageSafe(sb, 'park_dimensions', '*', (q) => q.eq('sport', 'mlb'));
const parkByVenue = new Map();
for (const p of parkRows) if (!parkByVenue.has(p.venue_id)) parkByVenue.set(p.venue_id, p);
const parkLeague = pw.leagueGeometry([...parkByVenue.values()]);
const ctxRows = await page(sb, 'game_context', 'game_id, venue_id, wx_temp_f, wx_wind_speed_mph, wx_wind_direction_deg', (q) => q);
const ctxRows = await pageSafe(sb, 'game_context', 'game_id, venue_id, wx_temp_f, wx_wind_speed_mph, wx_wind_direction_deg', (q) => q);
const ctxBy = new Map(ctxRows.map((c) => [c.game_id, c]));
const defRows = await page(sb, 'team_defense', '*', (q) => q.eq('sport', 'mlb'));
const defRows = await pageSafe(sb, 'team_defense', '*', (q) => q.eq('sport', 'mlb'));
const defByTeam = new Map();
for (const d of defRows) defByTeam.set(d.team, d);
const snaps = await page(sb, 'model_snapshots', 'player_key, game_date, archetype, stat',
(q) => q.eq('sport', 'mlb').eq('stat', 'total_bases').not('archetype', 'is', null));
const snaps = await READS.snaps(sb);
const archOf = new Map();
for (const s of snaps) archOf.set(`${s.player_key}|${s.game_date}`, s.archetype);
const led = await page(sb, 'ledger_entries',
'id, game_id, player_key, player_name, line, side, outcome, game_date, p_win, quarantine_reason, env_park_base',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', 'total_bases')
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null));
const led = await READS.ledger(sb);
const clean = led.filter((r) => !(r.quarantine_reason || '').startsWith('nontakeable_book'));
// Opponent faced, from each hitter's own game log.
@@ -342,4 +359,9 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS };
+43 -15
View File
@@ -28,6 +28,8 @@
require('dotenv').config();
const { createClient } = require('@supabase/supabase-js');
const cv = require('../src/services/model/correlateValidator');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
@@ -47,23 +49,45 @@ const RECORDED = [
verdict: 'INCONCLUSIVE', spec: 'specs/lineup-k-rate-rung1.md' },
];
async function page(sb, table, select, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
const { data, error } = await apply(sb.from(table).select(select)).range(from, from + PAGE - 1);
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data);
if (data.length < PAGE) break;
}
return out;
// ── FIX A2 (2026-08-09) — THE SAFE WALK ───────────────────────────────────
// `page()` above walks with no ORDER BY. Measured on production, that returned
// the correct row COUNT and the wrong ROWS: up to 33.6% of a read came back
// twice while an equal share never came back at all. `pageSafe` routes the same
// call through `src/utils/safePaginate`, which orders on a UNIQUE key on every
// page, verifies uniqueness at runtime, and THROWS on a query error instead of
// treating it as end-of-data.
//
// `page()` SURVIVES only for the context tables (statcast_aggregates,
// batter_spray, team_defense, platoon_splits, park_dimensions, game_context...).
// Those have COMPOSITE primary keys with no single unique column, so
// safePaginate cannot express them. They measure 0% corruption today; making
// them safe needs a composite-key ordering the helper does not yet have. Do not
// use `page()` for ledger_entries or model_snapshots.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
return paginate(() => apply(sb.from(table).select(select)),
{ key, pageSize: PAGE, label: `${table}` });
}
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS_LEDGER = {
ledgerAll: (sb) => pageSafe(sb, 'ledger_entries', 'id, stat, outcome, quarantine_reason, p_win',
(q) => q.eq('sport', 'mlb').is('user_id', null)),
ledgerSettled: (sb) => pageSafe(sb, 'ledger_entries',
'id, stat, outcome, quarantine_reason, player_key, line, side, game_date',
(q) => q.eq('sport', 'mlb').is('user_id', null).in('outcome', ['hit', 'miss'])),
};
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS = {
snaps: (sb) => pageSafe(sb, 'model_snapshots', 'id, stat, archetype, player_key, line, side, game_date',
(q) => q.eq('sport', 'mlb').not('archetype', 'is', null)),
};
async function main() {
if (!SB_URL || !SB_KEY) throw new Error('SUPABASE_URL / service key required');
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
const led = await page(sb, 'ledger_entries', 'stat, outcome, quarantine_reason, p_win',
const led = await pageSafe(sb, 'ledger_entries', 'id, stat, outcome, quarantine_reason, p_win',
(q) => q.eq('sport', 'mlb').is('user_id', null));
const settled = {};
for (const r of led) {
@@ -73,8 +97,7 @@ async function main() {
settled[r.stat] = (settled[r.stat] || 0) + 1;
}
const snaps = await page(sb, 'model_snapshots', 'stat, archetype, player_key, line, side, game_date',
(q) => q.eq('sport', 'mlb').not('archetype', 'is', null));
const snaps = await READS.snaps(sb);
const archOf = new Map();
for (const s of snaps) archOf.set(`${s.player_key}|${s.stat}|${s.line}|${String(s.side).toLowerCase()}|${s.game_date}`, s.archetype);
@@ -82,7 +105,7 @@ async function main() {
// SNAPSHOT CYCLE, so a naive join fans out and inflates the count — it read
// BOMBER x hits as 641 when the true figure is 287, which is the difference
// between "gate-ready" and "not close". Dedupe on the ledger row's identity.
const led2 = await page(sb, 'ledger_entries', 'id, stat, outcome, quarantine_reason, player_key, line, side, game_date',
const led2 = await pageSafe(sb, 'ledger_entries', 'id, stat, outcome, quarantine_reason, player_key, line, side, game_date',
(q) => q.eq('sport', 'mlb').is('user_id', null).in('outcome', ['hit', 'miss']));
const byArch = {};
const seen = new Set();
@@ -123,4 +146,9 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS, READS_LEDGER };
+297
View File
@@ -0,0 +1,297 @@
'use strict';
/**
* read-integrity — measure whether each learning reader returns the rows it thinks.
*
* Spec: specs/read-integrity-harness.md
*
* READ-ONLY. It issues SELECTs and a head-count. It writes nothing, anywhere —
* a test asserts this file contains no insert/update/upsert/delete/rpc.
*
* USAGE
* node scripts/read-integrity.js # every registered reader
* node scripts/read-integrity.js --id prove-hit-factors:136
* node scripts/read-integrity.js --acceptance # AC5 only: known-corrupt + known-clean
*
* WHAT IT IS FOR: the 2026-08-09 probe proved exposure cannot be reasoned from
* source — `challenger-scoreboard` walks 12 pages of a live table and is clean,
* `calibrationService.fromLedger` walks 3 pages of the same table and is a
* quarter corrupt. So every reader fix must be verified by MEASUREMENT, on the
* day, against an ordered control. Never by the presence of an ORDER BY clause.
*/
const path = require('path');
require('dotenv').config({ path: path.join(__dirname, '..', '.env') });
const { createClient } = require('@supabase/supabase-js');
const ri = require('../src/utils/readIntegrity');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
// The local .env carries a transposed project ref (CLAUDE.md, S76). Allow an
// explicit override so a local run reaches the real project.
const SB_URL = process.env.READ_INTEGRITY_SUPABASE_URL || process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
const MLB = ['eq', 'sport', 'mlb'];
const PUBLIC = ['is', 'user_id', null];
const SETTLED = ['in', 'outcome', ['hit', 'miss']];
const HAS_PWIN = ['not', 'p_win', 'is', null];
const HAS_ARCH = ['not', 'archetype', 'is', null];
const LEDGER_KEY = ['id'];
// model_snapshots rows are one per prop PER CYCLE, so identity needs the cycle.
const SNAP_KEY = ['id'];
/**
* THE REGISTRY — every reader that walks pages without a stable order, as it
* stands today. Declarative so it is auditable against the cited source line.
*/
const READERS = [
// ── the hits gate ──────────────────────────────────────────────────────
{ id: 'prove-hit-factors:136', source: 'scripts/prove-hit-factors.js:136', table: 'ledger_entries',
select: 'id', key: LEDGER_KEY,
readerRows: (sb) => require('../scripts/prove-hit-factors').READS.ledger(sb),
filters: [MLB, PUBLIC, ['eq', 'stat', 'hits'], SETTLED, HAS_PWIN] },
{ id: 'prove-hit-factors:131', source: 'scripts/prove-hit-factors.js:131', table: 'model_snapshots',
select: 'id', key: SNAP_KEY,
readerRows: (sb) => require('../scripts/prove-hit-factors').READS.snaps(sb),
filters: [MLB, ['eq', 'stat', 'hits'], HAS_ARCH] },
// ── the other stat gates ───────────────────────────────────────────────
{ id: 'prove-tb-factors:154', source: 'scripts/prove-tb-factors.js:154', table: 'ledger_entries',
select: 'id', key: LEDGER_KEY,
readerRows: (sb) => require('../scripts/prove-tb-factors').READS.ledger(sb),
filters: [MLB, PUBLIC, ['eq', 'stat', 'total_bases'], SETTLED, HAS_PWIN] },
{ id: 'prove-tb-factors:149', source: 'scripts/prove-tb-factors.js:149', table: 'model_snapshots',
select: 'id', key: SNAP_KEY,
readerRows: (sb) => require('../scripts/prove-tb-factors').READS.snaps(sb),
filters: [MLB, ['eq', 'stat', 'total_bases'], HAS_ARCH] },
{ id: 'prove-runs-rbi:200(rbi)', source: 'scripts/prove-runs-rbi.js:200', table: 'ledger_entries',
select: 'id', key: LEDGER_KEY,
filters: [MLB, PUBLIC, ['eq', 'stat', 'rbi'], SETTLED] },
{ id: 'prove-runs-rbi:200(runs)', source: 'scripts/prove-runs-rbi.js:200', table: 'ledger_entries',
select: 'id', key: LEDGER_KEY,
filters: [MLB, PUBLIC, ['eq', 'stat', 'runs'], SETTLED] },
{ id: 'prove-park-weather:54', source: 'scripts/prove-park-weather.js:54', table: 'ledger_entries',
select: 'id', key: LEDGER_KEY,
readerRows: (sb) => require('../scripts/prove-park-weather').READS.ledger(sb),
filters: [MLB, PUBLIC, ['eq', 'stat', 'total_bases'], SETTLED] },
{ id: 'pitcher-prove-k:137', source: 'scripts/pitcher-prove-k.js:137', table: 'ledger_entries',
select: 'id', key: LEDGER_KEY,
readerRows: (sb) => require('../scripts/pitcher-prove-k').READS.ledger(sb),
filters: [MLB, PUBLIC, ['eq', 'stat', 'strikeouts'], SETTLED, HAS_PWIN] },
{ id: 'cluster-prove:337', source: 'scripts/cluster-prove.js:337', table: 'ledger_entries',
select: 'id', key: LEDGER_KEY,
readerRows: (sb) => require('../scripts/cluster-prove').READS.ledger(sb),
filters: [MLB, PUBLIC, ['eq', 'stat', 'total_bases'], SETTLED, HAS_PWIN] },
{ id: 'tb-solo-and-interactions:260', source: 'scripts/tb-solo-and-interactions.js:260', table: 'ledger_entries',
select: 'id', key: LEDGER_KEY,
readerRows: (sb) => require('../scripts/tb-solo-and-interactions').READS.ledger(sb),
filters: [MLB, PUBLIC, ['eq', 'stat', 'total_bases'], SETTLED, HAS_PWIN] },
{ id: 'stagea-gate-run:142', source: 'scripts/stagea-gate-run.js:142', table: 'ledger_entries',
select: 'id', key: LEDGER_KEY,
readerRows: (sb) => require('../scripts/stagea-gate-run').READS.ledger(sb),
filters: [MLB, PUBLIC] },
{ id: 'skill-v1-stagea:185', source: 'scripts/skill-v1-stagea.js:185', table: 'ledger_entries',
select: 'id', key: LEDGER_KEY,
readerRows: (sb) => require('../scripts/skill-v1-stagea').READS.ledger(sb),
filters: [MLB, PUBLIC, ['eq', 'stat', 'hits'], SETTLED] },
// ── the status/ablation/board readers ──────────────────────────────────
{ id: 'proven-status:76', source: 'scripts/proven-status.js:76', table: 'model_snapshots',
select: 'id', key: SNAP_KEY,
readerRows: (sb) => require('../scripts/proven-status').READS.snaps(sb), filters: [MLB, HAS_ARCH] },
{ id: 'champion-ablation:173', source: 'scripts/champion-ablation.js:173', table: 'model_snapshots',
select: 'id', key: SNAP_KEY,
readerRows: (sb) => require('../scripts/champion-ablation').READS.snaps(sb),
filters: [MLB, HAS_PWIN, ['not', 'features', 'is', null]] },
{ id: 'build-grade-bands:54', source: 'scripts/build-grade-bands.js:54', table: 'ledger_entries',
select: 'id', key: LEDGER_KEY,
readerRows: (sb) => require('../scripts/build-grade-bands').READS.ledger(sb),
filters: [MLB, PUBLIC, ['eq', 'stat', 'hits'], SETTLED, HAS_PWIN] },
{ id: 'build-grade-bands:49', source: 'scripts/build-grade-bands.js:49', table: 'model_snapshots',
select: 'id', key: SNAP_KEY,
readerRows: (sb) => require('../scripts/build-grade-bands').READS.snaps(sb), filters: [MLB, ['eq', 'stat', 'hits'], HAS_ARCH] },
{ id: 'challenger-scoreboard:112', source: 'scripts/challenger-scoreboard.js:112', table: 'ledger_entries',
select: 'id', key: LEDGER_KEY, filters: [MLB, PUBLIC, SETTLED, HAS_PWIN] },
// ── calibrationService: FIXED in A1, and measured through the REAL function ──
// `readerRows` calls the production `loadSettledRows`, so a PASS here means the
// shipped code returned a set identical to the control — not that its query was
// re-declared correctly in this registry.
{ id: 'calibrationService.fromLedger:110', source: 'src/services/model/calibrationService.js:110',
table: 'ledger_entries', select: 'id', key: LEDGER_KEY, live: true,
readerRows: (sb) => require('../src/services/model/calibrationService')
.loadSettledRows(sb, { sport: 'mlb', stat: 'hits' }),
filters: [MLB, PUBLIC, ['eq', 'stat', 'hits'], SETTLED, HAS_PWIN,
['lt', 'game_date', todayEt()]] },
// The PRIMARY calibrator (calibrationService is only its shadow). FIXED in A2.
{ id: 'lowParamService.fromLedger:78', source: 'src/services/model/lowParamService.js:78',
table: 'ledger_entries', select: 'id', key: LEDGER_KEY, live: true,
readerRows: (sb) => require('../src/services/model/lowParamService')
.loadSettledRows(sb, { sport: 'mlb', stat: 'hits' }),
filters: [MLB, PUBLIC, ['eq', 'stat', 'hits'], SETTLED, HAS_PWIN,
['lt', 'game_date', todayEt()]] },
// ── context tables — A2b: composite keys from the live schema ──────────
{ id: 'context:statcast_aggregates', source: 'scripts/prove-hit-factors.js:104', table: 'statcast_aggregates',
select: '*', key: uniqueKeyFor('statcast_aggregates'),
readerRows: (sb) => require('../scripts/prove-hit-factors').READS.statcast(sb), filters: [MLB] },
{ id: 'context:batter_spray', source: 'scripts/prove-hit-factors.js:111', table: 'batter_spray',
select: '*', key: uniqueKeyFor('batter_spray'),
readerRows: (sb) => require('../scripts/prove-hit-factors').READS.spray(sb), filters: [MLB] },
{ id: 'context:platoon_splits', source: 'scripts/prove-hit-factors.js:119', table: 'platoon_splits',
select: '*', key: uniqueKeyFor('platoon_splits'),
readerRows: (sb) => require('../scripts/prove-hit-factors').READS.platoon(sb), filters: [MLB] },
{ id: 'context:team_defense', source: 'scripts/prove-hit-factors.js:127', table: 'team_defense',
select: '*', key: uniqueKeyFor('team_defense'),
readerRows: (sb) => require('../scripts/prove-hit-factors').READS.defense(sb), filters: [MLB] },
{ id: 'context:park_dimensions', source: 'scripts/prove-tb-factors.js:138', table: 'park_dimensions',
select: '*', key: uniqueKeyFor('park_dimensions'),
readerRows: (sb) => require('../scripts/prove-tb-factors').READS.parks(sb), filters: [MLB] },
{ id: 'context:game_context', source: 'scripts/prove-tb-factors.js:142', table: 'game_context',
select: 'game_id, venue_id, wx_temp_f, wx_wind_speed_mph, wx_wind_direction_deg',
key: uniqueKeyFor('game_context'),
readerRows: (sb) => require('../scripts/prove-tb-factors').READS.gameCtx(sb), filters: [] },
{ id: 'context:hitter_opportunity', source: 'scripts/prove-runs-rbi.js:183', table: 'hitter_opportunity',
select: '*', key: uniqueKeyFor('hitter_opportunity'),
readerRows: (sb) => require('../scripts/prove-runs-rbi').READS.opportunity(sb), filters: [MLB] },
// lineup_context has no paginated READER today (lineupContextService writes it);
// measured as-written so its exposure is on the record rather than assumed.
{ id: 'context:lineup_context', source: '(no paginated reader — write-only today)', table: 'lineup_context',
select: 'as_of_date, sport, game_pk, player_key', key: uniqueKeyFor('lineup_context'), filters: [MLB] },
// ── A2b: the three legacy reads, now converted ────────────────────────
{ id: 'proven-status:66', source: 'scripts/proven-status.js:66', table: 'ledger_entries',
select: 'id', key: uniqueKeyFor('ledger_entries'),
readerRows: (sb) => require('../scripts/proven-status').READS_LEDGER.ledgerAll(sb),
filters: [MLB, PUBLIC] },
{ id: 'proven-status:85', source: 'scripts/proven-status.js:85', table: 'ledger_entries',
select: 'id', key: uniqueKeyFor('ledger_entries'),
readerRows: (sb) => require('../scripts/proven-status').READS_LEDGER.ledgerSettled(sb),
filters: [MLB, PUBLIC, SETTLED] },
{ id: 'champion-ablation:176', source: 'scripts/champion-ablation.js:176', table: 'ledger_entries',
select: 'id', key: uniqueKeyFor('ledger_entries'),
readerRows: (sb) => require('../scripts/champion-ablation').READS_LEDGER.ledger(sb),
filters: [MLB, PUBLIC, SETTLED] },
// ── A2b: found during the sweep — the last unordered ledger walker ────
// Measured 24.7% before conversion; converted in the same pass because it is
// the identical defect and the fix is the identical one-liner.
{ id: 'calibrate-hits:40', source: 'scripts/calibrate-hits.js:40', table: 'ledger_entries',
select: 'id', key: uniqueKeyFor('ledger_entries'),
readerRows: (sb) => require('../scripts/calibrate-hits').READS.ledger(sb),
filters: [MLB, PUBLIC, ['eq', 'stat', 'hits'], SETTLED, HAS_PWIN] },
// ── A2b: the two WRITE-scripts, previously unmeasured ─────────────────
{ id: 'backfill-context:55', source: 'scripts/backfill-context.js:55', table: 'ledger_entries',
select: 'id', key: uniqueKeyFor('ledger_entries'), writes: 'platoon_splits',
readerRows: (sb) => require('../scripts/backfill-context').READS.ledger(sb),
filters: [MLB, PUBLIC, ['in', 'stat', ['hits', 'total_bases']], SETTLED] },
{ id: 'backfill-context:64', source: 'scripts/backfill-context.js:64', table: 'platoon_splits',
select: 'as_of_date, sport, season, player_key', key: uniqueKeyFor('platoon_splits'),
writes: 'platoon_splits',
readerRows: (sb) => require('../scripts/backfill-context').READS.platoonHeld(sb), filters: [MLB] },
{ id: 'reconstruct-game-environment:61', source: 'scripts/reconstruct-game-environment.js:61',
table: 'ledger_entries', select: 'id', key: uniqueKeyFor('ledger_entries'), writes: 'game_context',
readerRows: (sb) => require('../scripts/reconstruct-game-environment').READS.ledger(sb),
filters: [MLB, PUBLIC, ['in', 'stat', ['hits', 'total_bases']], SETTLED] },
];
function todayEt() {
return new Intl.DateTimeFormat('en-CA', {
timeZone: 'America/New_York', year: 'numeric', month: '2-digit', day: '2-digit',
}).format(new Date());
}
/** Build the three queries a spec needs. Read-only by construction. */
function bindings(sb, spec) {
const base = () => ri.applyFilters(sb.from(spec.table).select(spec.select), spec.filters);
// A2b — the CONTROL orders by the FULL unique tuple. Ordering it on a prefix
// would leave the control itself tie-scrambled, and a control that cannot be
// trusted must never issue a PASS.
const orderCols = spec.key || ['id'];
return {
exactCount: async () => {
const { count, error } = await ri.applyFilters(
sb.from(spec.table).select(orderCols[0], { count: 'exact', head: true }), spec.filters);
if (error) throw new Error(`count: ${error.message}`);
return count;
},
// ARM (a): exactly as the reader stands — no order.
fetchUnordered: async (from, to) => {
const { data, error } = await base().range(from, to);
if (error) throw new Error(`unordered: ${error.message}`);
return data;
},
// ARM (b): identical + a stable order. The CONTROL, which must itself
// reconcile to the server count before any PASS is issued.
fetchOrdered: async (from, to) => {
let q = base();
for (const c of orderCols) q = q.order(c, { ascending: true });
const { data, error } = await q.range(from, to);
if (error) throw new Error(`ordered: ${error.message}`);
return data;
},
// Once a reader is fixed, arm (a) becomes the REAL function's output.
...(spec.readerRows ? { readerRows: () => spec.readerRows(sb) } : {}),
};
}
async function main() {
if (!SB_URL || !SB_KEY) throw new Error('SUPABASE_URL / service key required');
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
const argv = process.argv.slice(2);
const only = argv.includes('--id') ? argv[argv.indexOf('--id') + 1] : null;
const acceptance = argv.includes('--acceptance');
let specs = READERS;
if (only) specs = READERS.filter((s) => s.id === only);
if (acceptance) {
specs = READERS.filter((s) => ['prove-hit-factors:136', 'challenger-scoreboard:112'].includes(s.id));
}
const results = [];
for (const spec of specs) {
const r = await ri.measure(spec, bindings(sb, spec));
results.push(r);
process.stderr.write(` ${r.verdict.padEnd(15)} ${r.id} — ${r.corruption_pct ?? '?'}%\n`);
}
const summary = {
run_at: new Date().toISOString(),
readers_measured: results.length,
pass: results.filter((r) => r.verdict === ri.VERDICT.PASS).length,
fail: results.filter((r) => r.verdict === ri.VERDICT.FAIL).length,
other: results.filter((r) => ![ri.VERDICT.PASS, ri.VERDICT.FAIL].includes(r.verdict)).length,
worst_corruption_pct: results.reduce((m, r) => Math.max(m, r.corruption_pct || 0), 0),
};
if (acceptance) {
// AC5 — the harness must reproduce the probe or it is not trustworthy.
const corrupt = results.find((r) => r.id === 'prove-hit-factors:136');
const clean = results.find((r) => r.id === 'challenger-scoreboard:112');
summary.acceptance = {
known_corrupt_reader: corrupt && corrupt.id,
known_corrupt_pct: corrupt && corrupt.corruption_pct,
known_corrupt_in_16_to_25_band: !!corrupt && corrupt.corruption_pct >= 16 && corrupt.corruption_pct <= 25,
known_clean_reader: clean && clean.id,
known_clean_pct: clean && clean.corruption_pct,
known_clean_is_zero: !!clean && clean.corruption_pct === 0,
};
summary.acceptance.AC5_PASSES = summary.acceptance.known_corrupt_in_16_to_25_band
&& summary.acceptance.known_clean_is_zero;
}
console.log(JSON.stringify({ summary, results }, null, 2));
}
if (require.main === module) {
main().then(() => process.exit(0)).catch((e) => {
console.error('read-integrity FAILED:', e.message);
process.exit(1);
});
}
module.exports = { READERS, bindings };
+30 -12
View File
@@ -28,6 +28,8 @@
require('dotenv').config();
const axios = require('axios');
const { createClient } = require('@supabase/supabase-js');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
@@ -41,24 +43,35 @@ const ARCHIVE = (lat, lon, date) =>
const squash = (s) => String(s || '').toLowerCase().replace(/[^a-z]/g, '');
async function page(sb, table, select, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
const { data, error } = await apply(sb.from(table).select(select)).range(from, from + PAGE - 1);
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data);
if (data.length < PAGE) break;
}
return out;
// ── THE SAFE WALK (Fix A2/A2b) ────────────────────────────────────────────
// This script used to walk pages with `.range()` and NO ORDER BY. Measured on
// production, that returned the correct row COUNT and the wrong ROWS: up to
// 33.6% of a read came back twice while an equal share never came back at all,
// so `rows.length` looked perfect while a fifth of the sample was missing.
//
// `pageSafe` routes every read through `src/utils/safePaginate`: a stable ORDER
// BY on the table's real UNIQUE key — single OR composite, looked up from
// `src/utils/tableKeys` rather than assumed — a runtime tuple-uniqueness check,
// and a THROWN error instead of a silent stop. The old unordered helper is gone
// rather than left beside it, because a dead broken helper is an invitation.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
return paginate(() => apply(sb.from(table).select(select)),
{ key, pageSize: PAGE, label: `${table}` });
}
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS = {
ledger: (sb) => pageSafe(sb, 'ledger_entries', 'id, game_id, game_date, stat, outcome, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).in('stat', ['hits', 'total_bases'])
.in('outcome', ['hit', 'miss'])),
};
const get = async (url) => (await axios.get(url, { timeout: 60_000 })).data;
async function main() {
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
const led = await page(sb, 'ledger_entries', 'game_id, game_date, stat, outcome, quarantine_reason',
const led = await pageSafe(sb, 'ledger_entries', 'id, game_id, game_date, stat, outcome, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).in('stat', ['hits', 'total_bases'])
.in('outcome', ['hit', 'miss']));
const clean = led.filter((r) => !(r.quarantine_reason || '').startsWith('nontakeable_book'));
@@ -167,4 +180,9 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS: (typeof READS !== 'undefined' ? READS : undefined), READS_LEDGER: (typeof READS_LEDGER !== 'undefined' ? READS_LEDGER : undefined) };
+33 -14
View File
@@ -43,6 +43,8 @@ const reg = require('../src/services/model/featureRegistry');
const mlb = require('../src/services/adapters/mlbStatsAdapter');
const { nameKey } = require('../src/utils/playerName');
const { knownRate } = require('../src/utils/known');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
/** Team games played by the 2026-07-21 profile freeze — turns season PA into PA/game. */
const GAMES_SO_FAR = Number(process.env.STAGEA_GAMES_SO_FAR || 103);
@@ -100,18 +102,30 @@ function bootstrapDiff(rows, keyA, keyB, iters = 4000, seed = 20260803) {
};
}
async function page(sb, table, select, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
const { data, error } = await apply(sb.from(table).select(select)).range(from, from + PAGE - 1);
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data);
if (data.length < PAGE) break;
}
return out;
// ── THE SAFE WALK (Fix A2/A2b) ────────────────────────────────────────────
// This script used to walk pages with `.range()` and NO ORDER BY. Measured on
// production, that returned the correct row COUNT and the wrong ROWS: up to
// 33.6% of a read came back twice while an equal share never came back at all,
// so `rows.length` looked perfect while a fifth of the sample was missing.
//
// `pageSafe` routes every read through `src/utils/safePaginate`: a stable ORDER
// BY on the table's real UNIQUE key — single OR composite, looked up from
// `src/utils/tableKeys` rather than assumed — a runtime tuple-uniqueness check,
// and a THROWN error instead of a silent stop. The old unordered helper is gone
// rather than left beside it, because a dead broken helper is an invitation.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
return paginate(() => apply(sb.from(table).select(select)),
{ key, pageSize: PAGE, label: `${table}` });
}
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS = {
ledger: (sb) => pageSafe(sb, 'ledger_entries',
'id, player_key, player_name, line, side, outcome, game_date, p_win, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', 'hits')
.in('outcome', ['hit', 'miss'])),
};
/**
* `date|team` → the starter that team FACED.
*
@@ -165,7 +179,7 @@ async function main() {
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
// Frozen skill profiles — by name (batters) and by source_id (pitchers).
const statcast = await page(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb'));
const statcast = await pageSafe(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb'));
const freeze = statcast.reduce((mx, r) => (String(r.updated_at) > mx ? String(r.updated_at) : mx), '');
const freezeDate = freeze.slice(0, 10);
const batters = new Map();
@@ -182,8 +196,8 @@ async function main() {
}
}
const led = await page(sb, 'ledger_entries',
'player_key, player_name, stat, line, side, outcome, game_date, p_win, team, opponent, quarantine_reason',
const led = await pageSafe(sb, 'ledger_entries',
'id, player_key, player_name, stat, line, side, outcome, game_date, p_win, team, opponent, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', 'hits')
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null));
@@ -276,4 +290,9 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS };
+32 -14
View File
@@ -38,6 +38,8 @@ const sk = require('../src/services/model/skillProjection');
const reg = require('../src/services/model/featureRegistry');
const mlb = require('../src/services/adapters/mlbStatsAdapter');
const { knownRate, knownNumber } = require('../src/utils/known');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
@@ -80,18 +82,29 @@ function bootstrapDiff(rows, keyA, keyB, iters = 4000, seed = 20260803) {
};
}
async function page(sb, table, select, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
const { data, error } = await apply(sb.from(table).select(select)).range(from, from + PAGE - 1);
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data);
if (data.length < PAGE) break;
}
return out;
// ── THE SAFE WALK (Fix A2/A2b) ────────────────────────────────────────────
// This script used to walk pages with `.range()` and NO ORDER BY. Measured on
// production, that returned the correct row COUNT and the wrong ROWS: up to
// 33.6% of a read came back twice while an equal share never came back at all,
// so `rows.length` looked perfect while a fifth of the sample was missing.
//
// `pageSafe` routes every read through `src/utils/safePaginate`: a stable ORDER
// BY on the table's real UNIQUE key — single OR composite, looked up from
// `src/utils/tableKeys` rather than assumed — a runtime tuple-uniqueness check,
// and a THROWN error instead of a silent stop. The old unordered helper is gone
// rather than left beside it, because a dead broken helper is an invitation.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
return paginate(() => apply(sb.from(table).select(select)),
{ key, pageSize: PAGE, label: `${table}` });
}
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS = {
ledger: (sb) => pageSafe(sb, 'ledger_entries',
'id, player_key, player_name, stat, line, side, outcome, game_date, p_win, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null)),
};
async function opposingStarters(dates) {
const m = new Map();
for (const d of dates) {
@@ -126,7 +139,7 @@ async function main() {
if (!SB_URL || !SB_KEY) throw new Error('SUPABASE_URL / service key required');
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
const statcast = await page(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb'));
const statcast = await pageSafe(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb'));
const freezeDate = statcast.reduce((mx, r) => (String(r.updated_at) > mx ? String(r.updated_at) : mx), '').slice(0, 10);
const batters = new Map(); const pitchersById = new Map();
for (const r of statcast) {
@@ -139,8 +152,8 @@ async function main() {
}
}
const led = await page(sb, 'ledger_entries',
'player_key, player_name, stat, line, side, outcome, game_date, p_win, quarantine_reason',
const led = await pageSafe(sb, 'ledger_entries',
'id, player_key, player_name, stat, line, side, outcome, game_date, p_win, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null)
.in('stat', ['hits', 'total_bases'])
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null));
@@ -244,4 +257,9 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS };
+38 -18
View File
@@ -44,6 +44,8 @@ const sk = require('../src/services/model/skillProjection');
const reg = require('../src/services/model/featureRegistry');
const mlb = require('../src/services/adapters/mlbStatsAdapter');
const { knownRate, knownNumber } = require('../src/utils/known');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');
const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
@@ -151,18 +153,35 @@ function bootstrapDiff(rows, keyA, keyB, iters = 4000, seed = 20260804) {
};
}
async function page(sb, table, select, apply) {
const out = [];
for (let from = 0; ; from += PAGE) {
const { data, error } = await apply(sb.from(table).select(select)).range(from, from + PAGE - 1);
if (error) throw error;
if (!data || data.length === 0) break;
out.push(...data);
if (data.length < PAGE) break;
}
return out;
// ── FIX A2 (2026-08-09) — THE SAFE WALK ───────────────────────────────────
// `page()` above walks with no ORDER BY. Measured on production, that returned
// the correct row COUNT and the wrong ROWS: up to 33.6% of a read came back
// twice while an equal share never came back at all. `pageSafe` routes the same
// call through `src/utils/safePaginate`, which orders on a UNIQUE key on every
// page, verifies uniqueness at runtime, and THROWS on a query error instead of
// treating it as end-of-data.
//
// `page()` SURVIVES only for the context tables (statcast_aggregates,
// batter_spray, team_defense, platoon_splits, park_dimensions, game_context...).
// Those have COMPOSITE primary keys with no single unique column, so
// safePaginate cannot express them. They measure 0% corruption today; making
// them safe needs a composite-key ordering the helper does not yet have. Do not
// use `page()` for ledger_entries or model_snapshots.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
return paginate(() => apply(sb.from(table).select(select)),
{ key, pageSize: PAGE, label: `${table}` });
}
/** THE MEASURED READS — main() and the harness call the same functions. */
const READS = {
snaps: (sb) => pageSafe(sb, 'model_snapshots', 'id, player_key, game_date, archetype',
(q) => q.eq('sport', 'mlb').eq('stat', 'total_bases').not('archetype', 'is', null)),
ledger: (sb) => pageSafe(sb, 'ledger_entries',
'id, player_key, player_name, stat, line, side, outcome, game_date, p_win, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', 'total_bases')
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null)),
};
async function opposingStarters(dates) {
const m = new Map();
for (const d of dates) {
@@ -233,7 +252,7 @@ async function main() {
if (!SB_URL || !SB_KEY) throw new Error('SUPABASE_URL / service key required');
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
const statcast = await page(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb'));
const statcast = await pageSafe(sb, 'statcast_aggregates', '*', (q) => q.eq('sport', 'mlb'));
const freezeDate = statcast.reduce((mx, r) => (String(r.updated_at) > mx ? String(r.updated_at) : mx), '').slice(0, 10);
const batters = new Map(); const pitchersById = new Map();
for (const r of statcast) {
@@ -252,15 +271,11 @@ async function main() {
// model_snapshots carries the actual classification per prop, so the
// conditioning variable is now a genuine BOMBER indicator, which is
// categorical and therefore not a transform of barrel at all.
const snaps = await page(sb, 'model_snapshots', 'player_key, game_date, archetype',
(q) => q.eq('sport', 'mlb').eq('stat', 'total_bases').not('archetype', 'is', null));
const snaps = await READS.snaps(sb);
const archetypeBy = new Map();
for (const r of snaps) if (r.player_key && r.game_date) archetypeBy.set(`${r.player_key}|${r.game_date}`, r.archetype);
const led = await page(sb, 'ledger_entries',
'player_key, player_name, stat, line, side, outcome, game_date, p_win, quarantine_reason',
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', 'total_bases')
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null));
const led = await READS.ledger(sb);
// POINT-IN-TIME IS NO LONGER AVAILABLE FROM THIS TABLE.
//
// `statcast_aggregates` is upserted in place and keeps one as-of date. The
@@ -433,4 +448,9 @@ async function main() {
process.exit(0);
}
main().catch((e) => { console.error(e); process.exit(1); });
if (require.main === module) {
main().catch((e) => { console.error(e); process.exit(1); });
}
// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS };