/* ============================================================ Session 68 — LAYER 1: mechanism-data ingestion. Pure derivation + the honesty rules. No network, no database: the adapter's fetch and the Supabase client are both injected. ============================================================ */ const svc = require('../../src/services/statcastAggregateService'); const adapter = require('../../src/services/adapters/statcastAdapter'); const { flipName, indexBy, indexPitchMix, BATTER_DISCIPLINE } = adapter.__internals; /** Minimal feed indexes, shaped exactly like adapter.fetchSeason() output. */ function feeds({ batter = [], batterBB = [], pitcher = [], pitcherBB = [], mix = [], arsenal = [] } = {}) { return { batterDiscipline: indexBy(batter, BATTER_DISCIPLINE), batterBattedBall: indexBy(batterBB, adapter.__internals.BATTED_BALL), pitcherDiscipline: indexBy(pitcher, adapter.__internals.PITCHER_DISCIPLINE), pitcherBattedBall: indexBy(pitcherBB, adapter.__internals.BATTED_BALL), pitchMix: adapter.__internals.indexArsenal(arsenal, indexPitchMix(mix)), counts: {}, }; } const BELL = { 'last_name, first_name': 'Bell, Josh', player_id: '605137', pa: '387', k_percent: '21.7', bb_percent: '7.5', whiff_percent: '24.8', swing_percent: '51', oz_swing_percent: '30.6', barrel_batted_rate: '10.3', hard_hit_percent: '43.4', }; const SKUBAL = { 'last_name, first_name': 'Skubal, Tarik', player_id: '669373', p_formatted_ip: '82.2', k_percent: '30.5', whiff_percent: '32.1', groundballs_percent: '49', arm_angle: '46.9', }; const SKUBAL_MIX = { 'last_name, first_name': 'Skubal, Tarik', pitcher_id: '669373', pitch_hand: 'L', pitch_type: 'FF', pitch_type_name: '4-Seam Fastball', pitch_per: '0.371', avg_speed: '96.7', pitcher_break_z_induced: '17.1', pitcher_break_x: '3.6', pitches_thrown: '463', }; describe('adapter normalisation', () => { it('flips Savant "Last, First" into a real name', () => { expect(flipName('Bell, Josh')).toBe('Josh Bell'); expect(flipName('Guerrero Jr., Vladimir')).toBe('Vladimir Guerrero Jr.'); expect(flipName('Ohtani')).toBe('Ohtani'); expect(flipName('')).toBeNull(); }); it('takes the FULL arsenal from the per-pitch feed, not the primary-only one', () => { // The movement endpoint returns ONE row per pitcher (their primary pitch), // so using it alone silently recorded a five-pitch arsenal as one pitch. const ARS = ['FF', 'CH', 'SI', 'SL', 'CU'].map((t, i) => ({ 'last_name, first_name': 'Skubal, Tarik', player_id: '669373', pitch_type: t, pitch_name: t, pitch_usage: String(40 - i * 7), whiff_percent: '25', })); const idx = adapter.__internals.indexArsenal(ARS, indexPitchMix([SKUBAL_MIX])); const e = idx.get(669373); expect(e.pitches).toHaveLength(5); expect(e.throws).toBe('L'); // handedness from movement expect(e.pitches[0].type).toBe('FF'); // sorted by usage expect(e.pitches[0].velo).toBe(96.7); // velo folded in expect(e.pitches[1].velo).toBeNull(); // absent, not guessed }); it('normalises pitch usage from a FRACTION to a percentage', () => { const [entry] = [...indexPitchMix([SKUBAL_MIX]).values()]; expect(entry.pitches[0].usage_pct).toBe(37.1); // not 0.371 expect(entry.throws).toBe('L'); }); it('a missing metric is ABSENT, never 0', () => { const [entry] = [...indexBy([{ 'last_name, first_name': 'X, Y', player_id: '1' }], BATTER_DISCIPLINE).values()]; expect(entry.metrics.k_pct).toBeUndefined(); expect(entry.metrics.whiff_pct).toBeUndefined(); }); }); describe('buildRows — derivation', () => { const rows = () => svc.buildRows(2026, feeds({ batter: [BELL], pitcher: [SKUBAL], mix: [SKUBAL_MIX], }), { now: '2026-07-21T00:00:00.000Z' }); it('produces one row per player per role, keyed by the source id', () => { const r = rows(); expect(r).toHaveLength(2); expect(r.find((x) => x.role === 'batter').source_id).toBe(605137); expect(r.find((x) => x.role === 'pitcher').source_id).toBe(669373); }); it('joins to our canonical player_key', () => { const b = rows().find((x) => x.role === 'batter'); expect(b.player_name).toBe('Josh Bell'); expect(b.player_key).toBe(require('../../src/utils/playerName').nameKey('Josh Bell')); }); it('carries pitcher handedness from the movement feed', () => { expect(rows().find((x) => x.role === 'pitcher').throws).toBe('L'); }); it('leaves batter handedness ABSENT (roster join, never guessed)', () => { expect(rows().find((x) => x.role === 'batter').bats).toBeNull(); }); it('flags thin samples rather than dropping or inflating them', () => { const thin = svc.buildRows(2026, feeds({ batter: [{ ...BELL, pa: '12' }], pitcher: [{ ...SKUBAL, p_formatted_ip: '2.1' }], }), {}); expect(thin.every((r) => r._sufficient === false)).toBe(true); // Thin is STORED, not dropped — "thin" and "missing" are different claims. expect(thin).toHaveLength(2); expect(thin[0].sample_pa).toBe(12); }); it('a player with no sample at all is insufficient, not zero', () => { const [r] = svc.buildRows(2026, feeds({ batter: [{ 'last_name, first_name': 'A, B', player_id: '9' }] }), {}); expect(r.sample_pa).toBeNull(); // NOT 0 expect(r._sufficient).toBe(false); expect(r.k_pct).toBeNull(); }); it('keeps every raw metric in `metrics` so Layer 2/3 never needs a re-ingest', () => { const b = rows().find((x) => x.role === 'batter'); expect(b.metrics.chase_pct).toBe(30.6); expect(b.chase_pct).toBe(30.6); }); }); describe('refreshSeason — the job', () => { const okClient = () => { const calls = []; return { calls, from: () => ({ upsert: async (batch) => { calls.push(batch); return { error: null }; } }), }; }; it('backfill and refresh are the SAME idempotent call', async () => { const fetchSeason = async () => feeds({ batter: [BELL], pitcher: [SKUBAL], mix: [SKUBAL_MIX] }); const a = okClient(); const b = okClient(); const r1 = await svc.refreshSeason({ supabase: a, fetchSeason, now: '2026-07-21T00:00:00.000Z' }); const r2 = await svc.refreshSeason({ supabase: b, fetchSeason, now: '2026-07-21T00:00:00.000Z' }); expect(r1.ok).toBe(true); expect(r1.written).toBe(2); expect(JSON.stringify(a.calls)).toBe(JSON.stringify(b.calls)); // identical → idempotent }); it('upserts on the natural key so a re-run never duplicates', async () => { // Track PER TABLE — the refresh now writes the aggregate AND its dated // history snapshot, and the two have deliberately different natural keys. const seen = {}; const sb = { from: (t) => ({ upsert: async (_b, o) => { seen[t] = o; return { error: null }; } }) }; await svc.refreshSeason({ supabase: sb, fetchSeason: async () => feeds({ batter: [BELL] }) }); expect(seen.statcast_aggregates.onConflict).toBe('sport,season,source_id,role'); }); it('RETAINS a dated point-in-time snapshot alongside the live aggregate', async () => { // statcast_aggregates is upserted IN PLACE, so it holds one as-of date and // destroys every earlier version — which silently makes any backtest score a // game with a profile that already contains it. The history table is the // only thing that makes point-in-time validation possible at all. const byTable = {}; const sb = { from: (t) => ({ upsert: async (b, o) => { (byTable[t] = byTable[t] || []).push({ b, o }); return { error: null }; } }) }; const out = await svc.refreshSeason({ supabase: sb, fetchSeason: async () => feeds({ batter: [BELL] }), now: '2026-08-03T11:00:00.000Z', }); expect(out.history_retained).toBe(1); expect(out.history_as_of).toBe('2026-08-03'); expect(byTable.statcast_history[0].o.onConflict).toBe('as_of_date,sport,season,source_id,role'); expect(byTable.statcast_history[0].b[0].as_of_date).toBe('2026-08-03'); }); it('a retention failure NEVER fails the refresh — stale-but-current beats nothing', async () => { const sb = { from: (t) => ({ upsert: async () => (t === 'statcast_history' ? { error: { message: 'history table missing' } } : { error: null }), }) }; const out = await svc.refreshSeason({ supabase: sb, fetchSeason: async () => feeds({ batter: [BELL] }) }); expect(out.ok).toBe(true); // the refresh still succeeded expect(out.written).toBe(1); expect(out.history_error).toMatch(/history table missing/); }); it('REFUSES to write when every feed is empty — a bad night cannot blank a good table', async () => { const sb = okClient(); const out = await svc.refreshSeason({ supabase: sb, fetchSeason: async () => feeds({}) }); expect(out.ok).toBe(false); expect(out.reason).toMatch(/refusing to write/); expect(sb.calls).toHaveLength(0); }); it('strips the internal flag before it reaches Postgres', async () => { const sb = okClient(); await svc.refreshSeason({ supabase: sb, fetchSeason: async () => feeds({ batter: [BELL] }) }); expect(sb.calls[0][0]._sufficient).toBeUndefined(); }); it('keys two-way players by ROLE — one player, two real profiles', async () => { // Ohtani appears in both the batter and the pitcher feeds. Without role in // the key, one upsert batch hits the same row twice and Postgres refuses // the whole chunk — found by inducing the real job, not by review. const OHTANI_B = { 'last_name, first_name': 'Ohtani, Shohei', player_id: '660271', pa: '400' }; const OHTANI_P = { 'last_name, first_name': 'Ohtani, Shohei', player_id: '660271', p_formatted_ip: '40' }; const rows = svc.buildRows(2026, feeds({ batter: [OHTANI_B], pitcher: [OHTANI_P] }), {}); expect(rows).toHaveLength(2); expect(new Set(rows.map((r) => r.source_id)).size).toBe(1); expect(new Set(rows.map((r) => r.role))).toEqual(new Set(['batter', 'pitcher'])); }); it('reports the join rate — the honest-absent rate for the mechanism tier', async () => { const out = await svc.refreshSeason({ supabase: okClient(), fetchSeason: async () => feeds({ batter: [BELL, { player_id: '77', pa: '100' }] }), }); expect(out.joined).toBe(1); expect(out.unjoined).toBe(1); // no name → no key → stored, joins later }); }); describe('staleness — never serve stale as fresh', () => { it('fires past the max age', () => { expect(svc.isStale({ updated_at: '2026-07-01T00:00:00Z', age_hours: 72 })).toBe(true); expect(svc.isStale({ updated_at: '2026-07-20T00:00:00Z', age_hours: 6 })).toBe(false); }); it('NEVER-BUILT is not STALE — different condition, different fix', () => { // Paging on a fresh install teaches the operator to ignore the alarm. expect(svc.isStale({ updated_at: null, age_hours: null })).toBe(false); expect(svc.isStale(null)).toBe(false); }); it('respects an explicit threshold', () => { expect(svc.isStale({ updated_at: 'x', age_hours: 30 }, 24)).toBe(true); expect(svc.isStale({ updated_at: 'x', age_hours: 30 }, 48)).toBe(false); }); }); describe('scheduler + route wiring', () => { const fs = require('fs'); const path = require('path'); const read = (r) => fs.readFileSync(path.join(__dirname, '..', '..', r), 'utf8'); it('is server-scheduled with a kill switch', () => { const s = read('src/snapshotScheduler.js'); expect(s).toContain('STATCAST_HOUR_UTC'); expect(s).toMatch(/process\.env\.STATCAST === '0'/); // kill switch, early-return form expect(s).toContain('refreshSeason'); }); it('RUNS ON ITS OWN TICK — not behind the snapshot-hours guard', () => { // THE REGRESSION THIS LOCKS (found 2026-08-03): the refresh used to live // inside `tick()`, BELOW `if (!HOURS_UTC.includes(h)) return`. HOURS_UTC is // 14,19,22,1,3 and the block tests h === STATCAST_HOUR_UTC (default 11), so // the guard could never admit the hour it waited for. It was unreachable // code that had never run once, and the aggregates sat 13 days stale while // every consumer served them as current. The previous assertion here passed // the entire time, because it only checked that the STRING existed. const s = read('src/snapshotScheduler.js'); const statcastTick = s.slice(s.indexOf('const statcastTick')); expect(statcastTick.length).toBeGreaterThan(0); // Its own tick, registered on the interval alongside the others. expect(s).toMatch(/void statcastTick\(\)/); // And it must NOT be reachable only via the snapshot-hours guard. // Scope to the snapshot tick's OWN body (it ends where refreshTick begins), // so the statcastTick doc comment above it cannot satisfy this by accident. const tickBody = s.slice(s.indexOf('const tick = async'), s.indexOf('const refreshTick')); expect(tickBody).toContain('HOURS_UTC.includes(h)'); // the guard is still there expect(tickBody).not.toContain('STATCAST_HOUR_UTC'); // and statcast is NOT behind it }); it('pages on BOTH a failed run and silent staleness', () => { const s = read('src/snapshotScheduler.js'); expect(s).toMatch(/Statcast refresh failed/); expect(s).toMatch(/Statcast aggregates stale/); }); it('is induce-able on demand (never wait on the cron)', () => { const s = read('src/routes/internal.js'); expect(s).toContain("router.post('/statcast/refresh'"); expect(s).toContain("router.get('/statcast/status'"); }); });