Wire the three proven hits factors pre-grade: transmission proven, gain
inconclusive THE BUG THIS NEARLY SHIPPED AS A FINDING. The first audit reported 0 factors fired on all 1,140 rows. Not a result -- my paging helper ordered by `id`, and batter_spray, team_defense, platoon_splits and statcast_aggregates have composite primary keys with NO id column. The query errored, the loop broke on error, and four fully-populated tables read as empty. hitsFactorContext.js -- the PRODUCTION loader -- had the identical defect, so live wiring would have loaded nothing and served unadjusted while logging success. Third occurrence of this class in one session. Both loaders now order by a real column and THROW rather than degrade. The Phase 2 gate is what caught it: no resolution number was quoted until transmission was proved. PHASE 1 — pipeline is now base -> FACTORS -> CALIBRATE -> GRADE. Context built in snapshotService BEFORE gradeAndCacheSlate (was line 640+, grade at 454), threaded per prop, applied to p_over before p_win is set with p_win_prefactor and a full trace retained. Hits only. Coverage 859/1140 rows (75%): 474 with all three factors, 256 two, 129 one, 281 none. PHASE 2 — TRANSMISSION PROVEN, 12/12 sign-correct, 4/4 per factor, each applied IN ISOLATION. My first table compared each factor's expected sign against the COMPOSITE change and showed 3 false failures -- with three factors firing the net can oppose any single member; that was a flaw in the test, not the wiring. Two under-side rows confirm the flip is handled: a factor raising p(over) correctly lowers p_win. Switch hitters (Bailey, Bell, Rocchio) took no spray adjustment while their other factors fired normally -- the refusal is selective, not a blanket skip. PHASE 3/4 — both maps refit on the factor-adjusted forecast; the shadow-duel baseline is VOID and restarts, since it accumulated against a different forecast. Point-in-time, 765 held-out rows: reliability 0.00795 -> 0.00828 RESOLUTION 0.00229 -> 0.00345 (variance explained 0.93% -> 1.39%) Brier 0.25398 -> 0.25305 delta -0.00093 CI [-0.00225,+0.00002] Resolution rose 51% relative. The CI TOUCHES ZERO on 4 eval dates, so the composition does NOT earn a proven keep -- three isolated passes did not grant a composed pass. INCONCLUSIVE, reported as such. The gain is far below the sum of the isolated effects, which is expected: all three run through the same pitcher-batter confrontation and share signal. PHASE 5 — 1.39% of variance is still far below what band separation needs. The pivot was correct and incomplete: the plumbing defect was real and is fixed, three proven factors reach the served number for the first time, and transmission alone did not buy grade separation. Next arc is factor STRENGTH and BREADTH, not more plumbing. PHASE 6 — rbi anomaly logged, not chased: 14.51% variance explained vs hits 1.03%, on the stat we do not serve corrected and which has no proven factors. Either the biggest lever on the board or a mirage; it deserves its own order. The byte-identical invariant INVERTED for hits by design. All 13 frozen non-hits modules verified unchanged, probabilityEstimator included -- the factors ride outside it. No new Bonferroni slot; the composed OOS claim is reported with its CI and not claimed as a pass. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W1sivYNqY2TS5ftykmHBU9
This commit is contained in:
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Phases 0, 2, 3 and 4 — replay the factor wiring on settled hits rows.
|
||||
*
|
||||
* Transmission is proved MECHANICALLY before any resolution number is quoted,
|
||||
* because "resolution went up" is exactly what a subtle bug also prints.
|
||||
*/
|
||||
|
||||
require('dotenv').config();
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { createClient } = require('@supabase/supabase-js');
|
||||
const hf = require('../src/services/model/hitsFactors');
|
||||
const lp = require('../src/services/model/lowParamCalibrator');
|
||||
const guards = require('../src/services/model/calibrationGuards');
|
||||
const { knownNumber } = require('../src/utils/known');
|
||||
const { nameKey } = require('../src/utils/playerName');
|
||||
|
||||
const BOX = path.join(process.cwd(), '.seq-cache', 'batting-lines.json');
|
||||
const SEQ = path.join(process.cwd(), '.seq-cache', 'sequences.json');
|
||||
const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);
|
||||
|
||||
async function page(sb, t, s, f, orderBy = 'id') {
|
||||
const o = [];
|
||||
for (let i = 0; ; i += 1000) {
|
||||
const { data, error } = await f(sb.from(t).select(s)).order(orderBy, { ascending: true }).range(i, i + 999);
|
||||
if (error) throw new Error(`${t}: ${error.message}`);
|
||||
if (!data || !data.length) break; o.push(...data); if (data.length < 1000) break;
|
||||
}
|
||||
return o;
|
||||
}
|
||||
const isPreGame = (c, g) => {
|
||||
const et = new Date(new Date(c).getTime() - 4 * 3600 * 1000);
|
||||
const d = et.toISOString().slice(0, 10);
|
||||
return d < g || (d === g && et.getUTCHours() < 19);
|
||||
};
|
||||
function makeRnd(seed) { let s = seed >>> 0; return () => { s ^= s << 13; s >>>= 0; s ^= s >>> 17; s ^= s << 5; s >>>= 0; return s / 4294967296; }; }
|
||||
|
||||
function decompose(rows, bins = 10) {
|
||||
const base = mean(rows.map((r) => r.won));
|
||||
const unc = base * (1 - base);
|
||||
let rel = 0; let res = 0;
|
||||
for (let k = 0; k < bins; k += 1) {
|
||||
const lo = k / bins; const hi = (k + 1) / bins;
|
||||
const sl = rows.filter((r) => r.p >= lo && (hi >= 1 ? r.p <= 1 : r.p < hi));
|
||||
if (!sl.length) continue;
|
||||
const w = sl.length / rows.length;
|
||||
rel += w * (mean(sl.map((r) => r.p)) - mean(sl.map((r) => r.won))) ** 2;
|
||||
res += w * (mean(sl.map((r) => r.won)) - base) ** 2;
|
||||
}
|
||||
return { base_rate: r4(base), reliability: r5(rel), resolution: r5(res), uncertainty: r5(unc), share: r4(res / unc) };
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const sb = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY, { auth: { persistSession: false } });
|
||||
const lines = JSON.parse(fs.readFileSync(BOX, 'utf8')).lines;
|
||||
|
||||
// Factor inputs.
|
||||
const [spray, defense, platoon, statcast] = await Promise.all([
|
||||
page(sb, 'batter_spray', '*', (q) => q.eq('sport', 'mlb'), 'player_key'),
|
||||
page(sb, 'team_defense', '*', (q) => q.eq('sport', 'mlb'), 'team'),
|
||||
page(sb, 'platoon_splits', '*', (q) => q.eq('sport', 'mlb'), 'player_key'),
|
||||
page(sb, 'statcast_aggregates', 'player_key, role, bats, throws, hard_hit_pct', (q) => q.eq('sport', 'mlb'), 'player_key'),
|
||||
]);
|
||||
const latest = (rows, k) => { const m = new Map(); for (const r of rows) { const key = r[k]; if (!key) continue; const p = m.get(key); if (!p || String(r.as_of_date) > String(p.as_of_date)) m.set(key, r); } return m; };
|
||||
const sprayBy = latest(spray, 'player_key'); const defBy = latest(defense, 'team'); const platBy = latest(platoon, 'player_key');
|
||||
const batBy = new Map(); const pitBy = new Map();
|
||||
for (const r of statcast) { if (!r.player_key) continue; (r.role === 'pitcher' ? pitBy : batBy).set(r.player_key, r); }
|
||||
const frac = (v) => { const n = knownNumber(v); return n === null ? null : (n > 1 ? n / 100 : n); };
|
||||
|
||||
// Opponent + starter per (player,date) from the sequence cache.
|
||||
const { games } = JSON.parse(fs.readFileSync(SEQ, 'utf8'));
|
||||
const oppOf = new Map(); const spOf = new Map();
|
||||
for (const g of games) {
|
||||
for (const side of ['home', 'away']) {
|
||||
const opp = g[side === 'home' ? 'away' : 'home'];
|
||||
const st = (g[side].arms || []).find((a) => a.started);
|
||||
const half = side === 'home' ? 'top' : 'bottom';
|
||||
for (const pa of g.pas.filter((p) => p.half === half)) {
|
||||
const k = `${g.date}|${nameKey(pa.batter_name || '')}`;
|
||||
if (!oppOf.has(k)) { oppOf.set(k, g[side].team); if (st) spOf.set(k, st.name); }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const snaps = await page(sb, 'model_snapshots', 'id, game_date, captured_at, stat, player_key, player_name, line, side, p_win, refused',
|
||||
(q) => q.eq('sport', 'mlb').eq('stat', 'hits'));
|
||||
const picked = new Map();
|
||||
for (const r of snaps) {
|
||||
if (!isPreGame(r.captured_at, r.game_date) || r.refused || knownNumber(r.p_win) === null) continue;
|
||||
const k = [r.game_date, r.player_key, r.line].join('|');
|
||||
const prev = picked.get(k);
|
||||
if (!prev || knownNumber(r.p_win) > knownNumber(prev.p_win)) picked.set(k, r);
|
||||
}
|
||||
guards.assertPickedSideDedup([...picked.values()].map((r) => ({ propKey: [r.game_date, r.player_key, r.line].join('|'), side: r.side, p: knownNumber(r.p_win) })));
|
||||
|
||||
const rows = []; const transmission = []; const unreadable = [];
|
||||
for (const r of picked.values()) {
|
||||
const b = lines[`${r.game_date}|${r.player_key}`]; const L = knownNumber(r.line);
|
||||
if (!b || L === null || !r.side) continue;
|
||||
const v = knownNumber(b.hits); if (v === null) continue;
|
||||
const over = v > L;
|
||||
const won = (String(r.side).toLowerCase() === 'under' ? !over : over) ? 1 : 0;
|
||||
|
||||
const key = r.player_key;
|
||||
const bat = batBy.get(key);
|
||||
const oppTeam = oppOf.get(`${r.game_date}|${key}`);
|
||||
const def = oppTeam ? (defBy.get(oppTeam) || defBy.get(String(oppTeam).split(' ').pop())) : null;
|
||||
const spName = spOf.get(`${r.game_date}|${key}`);
|
||||
const pit = spName ? pitBy.get(nameKey(spName)) : null;
|
||||
const sp = platBy.get(key);
|
||||
const ctx = {
|
||||
spray: sprayBy.get(key) || null,
|
||||
positionOaa: def && def.position_oaa ? def.position_oaa : null,
|
||||
bats: bat && bat.bats ? String(bat.bats)[0] : null,
|
||||
throws: pit && pit.throws ? String(pit.throws)[0] : null,
|
||||
pitcherHardHit: pit ? frac(pit.hard_hit_pct) : null,
|
||||
platoonSplits: sp ? { vl: { pa: sp.vl_pa, atBats: sp.vl_ab, hits: sp.vl_hits }, vr: { pa: sp.vr_pa, atBats: sp.vr_ab, hits: sp.vr_hits } } : null,
|
||||
};
|
||||
// The engine adjusts p_over then flips for unders; replay that exactly.
|
||||
const pOverRaw = String(r.side).toLowerCase() === 'under' ? 1 - knownNumber(r.p_win) : knownNumber(r.p_win);
|
||||
const adj = hf.adjustProbability(pOverRaw, ctx);
|
||||
const pAfter = adj.factors_fired > 0
|
||||
? (String(r.side).toLowerCase() === 'under' ? 1 - adj.p_adjusted : adj.p_adjusted)
|
||||
: knownNumber(r.p_win);
|
||||
|
||||
rows.push({ date: r.game_date, p_before: knownNumber(r.p_win), p: pAfter, won, fired: adj.factors_fired, applied: adj.applied });
|
||||
|
||||
// TRANSMISSION IS TESTED PER FACTOR, IN ISOLATION.
|
||||
// Comparing one factor's expected sign against the COMPOSITE p_win change is
|
||||
// wrong: with three factors firing, two pulling down and one up, the net can
|
||||
// oppose any single member and look like a defect when nothing is broken.
|
||||
// So each factor is applied ALONE to the same base and its own sign checked.
|
||||
const isUnder = String(r.side).toLowerCase() === 'under';
|
||||
for (const a of adj.applied) {
|
||||
if (transmission.filter((t) => t.factor === a.factor).length >= 4) continue;
|
||||
if (Math.abs(a.multiplier - 1) < 0.03) continue;
|
||||
const solo = { spray: null, positionOaa: null, bats: ctx.bats, throws: null, pitcherHardHit: null, platoonSplits: null };
|
||||
if (a.factor === 'defense_by_direction') { solo.spray = ctx.spray; solo.positionOaa = ctx.positionOaa; }
|
||||
if (a.factor === 'pitcher_contact_profile') solo.pitcherHardHit = ctx.pitcherHardHit;
|
||||
if (a.factor === 'platoon_severity') { solo.platoonSplits = ctx.platoonSplits; solo.throws = ctx.throws; }
|
||||
const one = hf.adjustProbability(pOverRaw, solo);
|
||||
if (one.factors_fired !== 1) continue;
|
||||
const soloWin = isUnder ? 1 - one.p_adjusted : one.p_adjusted;
|
||||
transmission.push({
|
||||
factor: a.factor, player: r.player_name, date: r.game_date,
|
||||
expected: a.multiplier > 1 ? 'raise p(over)' : 'lower p(over)', multiplier: a.multiplier,
|
||||
side: r.side, p_before: knownNumber(r.p_win), p_after_solo: r4(soloWin),
|
||||
sign_correct: isUnder
|
||||
? ((a.multiplier > 1) === (soloWin < knownNumber(r.p_win)))
|
||||
: ((a.multiplier > 1) === (soloWin > knownNumber(r.p_win))),
|
||||
});
|
||||
}
|
||||
if (adj.skipped.some((s) => /switch hitter/.test(s.reason || '')) && unreadable.length < 4) {
|
||||
unreadable.push({ player: r.player_name, reason: 'switch hitter — spray side unreadable', p_before: knownNumber(r.p_win), p_after: pAfter, moved_by_spray: false });
|
||||
}
|
||||
}
|
||||
|
||||
// ── PHASE 4: OOS, point-in-time ──
|
||||
rows.sort((a, b) => a.date.localeCompare(b.date));
|
||||
const dates = [...new Set(rows.map((r) => r.date))].sort();
|
||||
const perDate = new Map(); for (const r of rows) perDate.set(r.date, (perDate.get(r.date) || 0) + 1);
|
||||
let acc = 0; let cut = dates[dates.length - 1];
|
||||
for (const d of dates) { acc += perDate.get(d); if (acc >= rows.length * 0.45) { cut = d; break; } }
|
||||
const fit = rows.filter((r) => r.date < cut); const ev = rows.filter((r) => r.date >= cut);
|
||||
|
||||
const mBefore = lp.fitPlatt(fit.map((r) => ({ p: r.p_before, won: r.won, date: r.date })));
|
||||
const mAfter = lp.fitPlatt(fit.map((r) => ({ p: r.p, won: r.won, date: r.date })));
|
||||
const evBefore = ev.map((r) => ({ ...r, p: mBefore && !mBefore.refused ? lp.applyPlatt(mBefore, r.p_before) : r.p_before })).filter((r) => r.p != null);
|
||||
const evAfter = ev.map((r) => ({ ...r, p: mAfter && !mAfter.refused ? lp.applyPlatt(mAfter, r.p) : r.p })).filter((r) => r.p != null);
|
||||
|
||||
const bBefore = guards.safeBrier(evBefore.map((r) => r.p), evBefore.map((r) => r.won));
|
||||
const bAfter = guards.safeBrier(evAfter.map((r) => r.p), evAfter.map((r) => r.won));
|
||||
|
||||
const byDate = new Map();
|
||||
for (let i = 0; i < evAfter.length; i += 1) { const d = evAfter[i].date; if (!byDate.has(d)) byDate.set(d, []); byDate.get(d).push({ a: evAfter[i].p, b: evBefore[i] ? evBefore[i].p : null, won: evAfter[i].won }); }
|
||||
const keys = [...byDate.keys()]; const rnd = makeRnd(20260807); const diffs = [];
|
||||
for (let it = 0; it < 3000; it += 1) {
|
||||
const s = [];
|
||||
for (let i = 0; i < keys.length; i += 1) s.push(...byDate.get(keys[Math.floor(rnd() * keys.length)]));
|
||||
const u = s.filter((x) => x.b != null);
|
||||
if (!u.length) continue;
|
||||
diffs.push(guards.safeBrier(u.map((x) => x.a), u.map((x) => x.won)) - guards.safeBrier(u.map((x) => x.b), u.map((x) => x.won)));
|
||||
}
|
||||
diffs.sort((a, b) => a - b);
|
||||
|
||||
console.log(JSON.stringify({
|
||||
coverage: { rows: rows.length, any_factor_fired: rows.filter((r) => r.fired > 0).length,
|
||||
by_count: [0, 1, 2, 3].map((k) => ({ factors: k, n: rows.filter((r) => r.fired === k).length })) },
|
||||
PHASE_2_transmission: transmission,
|
||||
PHASE_2_unreadable_static: unreadable,
|
||||
PHASE_4: {
|
||||
split_at: cut, fit_n: fit.length, eval_n: ev.length, eval_dates: keys.length,
|
||||
resolution_before: decompose(evBefore), resolution_after: decompose(evAfter),
|
||||
brier_before: r5(bBefore), brier_after: r5(bAfter), brier_delta: r5(bAfter - bBefore),
|
||||
brier_ci_date_block: diffs.length ? [r5(diffs[Math.floor(diffs.length * 0.025)]), r5(diffs[Math.floor(diffs.length * 0.975)])] : null,
|
||||
},
|
||||
}, null, 2));
|
||||
process.exit(0);
|
||||
})().catch((e) => { console.error(e); process.exit(1); });
|
||||
|
||||
const r5 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 100000) / 100000);
|
||||
const r4 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 10000) / 10000);
|
||||
Reference in New Issue
Block a user