Layer 1: Statcast mechanism-data ingestion (backfill + nightly refresh)

The data foundation for the archetype and projection layers, built as the pattern every sport inherits. Layers 2 and 3 are not touched. PHASE 0 GATE — both match rates measured live, both 100%. Batters 40/40; PITCHERS 66/66 across five real rosters (CLE, DET, MIN, NYY, LAD) joined by MLBAM id against the 713-pitcher Savant feed. Zero honest-absent on identity, because the join is an integer both systems use natively — and the snapshot pipeline already stores it per graded row. SOURCE — five Baseball Savant CSV leaderboards, free and public, pulled with axios and the CSV parser savantAdapter already runs in prod. pybaseball is deliberately NOT used: it is an MIT wrapper over these same URLs, and adding it would reintroduce a Python runtime in a stack where the existing Python service is already offline. min=1 on every feed, not Savant's default min=q, so the long tail arrives and OUR minimum-sample gate decides what is thin — explicit and testable rather than silently dropped upstream. Measured: 1,354 rows per season (604 batters, 750 pitchers), all five feeds in about five seconds. Pitcher mechanism includes arm angle, GB/FB/LD, chase and whiff; batters get exit velo, launch angle, barrel and hard-hit, chase and z-swing. Handedness rides in free on the movement feed (677 pitchers); batter handedness stays absent pending a roster join rather than being guessed. BACKFILL AND REFRESH ARE THE SAME CALL — a full re-pull upserted on (sport, season, source_id). Idempotent and self-healing: a missed night self-corrects on the next run, with no incremental who-played bookkeeping to drift out of sync. At 1,354 rows the simple thing is also the robust one. HONESTY RULES, each with a test: a metric the feed did not carry is null and never 0; a thin sample is STORED and flagged rather than dropped or inflated, because thin and missing are different claims; an unjoined player is stored with a null player_key and joins later; and if every feed comes back empty the job REFUSES to write, so a bad night can never blank a good table. Freshness is treated as a truth property. updated_at on every row, and the scheduler pages on a failed run AND on silent staleness — a job that stops being scheduled never produces a failure, so staleness has to alarm on its own. Never-built is deliberately not stale: different condition, different fix, and paging on a fresh install teaches the operator to ignore the alarm. Nightly at STATCAST_HOUR_UTC (default 11 UTC, after every game is final), kill switch STATCAST=0, and induce-able at POST /api/internal/statcast/refresh with a freshness probe at /statcast/status — we verify a refresh by running it, not by waiting for the slot. Migration 030 applied. Promoted columns for the classification-critical metrics plus a metrics JSONB carrying every raw field, so Layer 2 can reach something we did not promote without a re-ingest. Raw per-pitch stays out of Postgres on purpose: one season is ~0.85 GB against a 500 MB plan ceiling, and it is re-pullable from the free source if Layer 3 ever needs it. Pattern documented in docs/MECHANISM-DATA.md for NBA tracking and NFL Next Gen. Tests 3581 passed / 292 suites, web build exit 0. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VCNgGSt5qvcLxaeQqa7Zpj
2026-07-20 21:37:30 -04:00
parent 0264486bf9
commit 528cb1a6d0
8 changed files with 935 additions and 2 deletions
@@ -0,0 +1,202 @@
+/* ============================================================
+   Session 68 — LAYER 1: mechanism-data ingestion.
+
+   Pure derivation + the honesty rules. No network, no database: the adapter's
+   fetch and the Supabase client are both injected.
+   ============================================================ */
+
+const svc = require('../../src/services/statcastAggregateService');
+const adapter = require('../../src/services/adapters/statcastAdapter');
+
+const { flipName, indexBy, indexPitchMix, BATTER_DISCIPLINE } = adapter.__internals;
+
+/** Minimal feed indexes, shaped exactly like adapter.fetchSeason() output. */
+function feeds({ batter = [], batterBB = [], pitcher = [], pitcherBB = [], mix = [] } = {}) {
+  return {
+    batterDiscipline: indexBy(batter, BATTER_DISCIPLINE),
+    batterBattedBall: indexBy(batterBB, adapter.__internals.BATTED_BALL),
+    pitcherDiscipline: indexBy(pitcher, adapter.__internals.PITCHER_DISCIPLINE),
+    pitcherBattedBall: indexBy(pitcherBB, adapter.__internals.BATTED_BALL),
+    pitchMix: indexPitchMix(mix),
+    counts: {},
+  };
+}
+
+const BELL = {
+  'last_name, first_name': 'Bell, Josh', player_id: '605137', pa: '387',
+  k_percent: '21.7', bb_percent: '7.5', whiff_percent: '24.8', swing_percent: '51',
+  oz_swing_percent: '30.6', barrel_batted_rate: '10.3', hard_hit_percent: '43.4',
+};
+const SKUBAL = {
+  'last_name, first_name': 'Skubal, Tarik', player_id: '669373', p_formatted_ip: '82.2',
+  k_percent: '30.5', whiff_percent: '32.1', groundballs_percent: '49', arm_angle: '46.9',
+};
+const SKUBAL_MIX = {
+  'last_name, first_name': 'Skubal, Tarik', pitcher_id: '669373', pitch_hand: 'L',
+  pitch_type: 'FF', pitch_type_name: '4-Seam Fastball', pitch_per: '0.371',
+  avg_speed: '96.7', pitcher_break_z_induced: '17.1', pitcher_break_x: '3.6', pitches_thrown: '463',
+};
+
+describe('adapter normalisation', () => {
+  it('flips Savant "Last, First" into a real name', () => {
+    expect(flipName('Bell, Josh')).toBe('Josh Bell');
+    expect(flipName('Guerrero Jr., Vladimir')).toBe('Vladimir Guerrero Jr.');
+    expect(flipName('Ohtani')).toBe('Ohtani');
+    expect(flipName('')).toBeNull();
+  });
+
+  it('normalises pitch usage from a FRACTION to a percentage', () => {
+    const [entry] = [...indexPitchMix([SKUBAL_MIX]).values()];
+    expect(entry.pitches[0].usage_pct).toBe(37.1); // not 0.371
+    expect(entry.throws).toBe('L');
+  });
+
+  it('a missing metric is ABSENT, never 0', () => {
+    const [entry] = [...indexBy([{ 'last_name, first_name': 'X, Y', player_id: '1' }], BATTER_DISCIPLINE).values()];
+    expect(entry.metrics.k_pct).toBeUndefined();
+    expect(entry.metrics.whiff_pct).toBeUndefined();
+  });
+});
+
+describe('buildRows — derivation', () => {
+  const rows = () => svc.buildRows(2026, feeds({
+    batter: [BELL], pitcher: [SKUBAL], mix: [SKUBAL_MIX],
+  }), { now: '2026-07-21T00:00:00.000Z' });
+
+  it('produces one row per player per role, keyed by the source id', () => {
+    const r = rows();
+    expect(r).toHaveLength(2);
+    expect(r.find((x) => x.role === 'batter').source_id).toBe(605137);
+    expect(r.find((x) => x.role === 'pitcher').source_id).toBe(669373);
+  });
+
+  it('joins to our canonical player_key', () => {
+    const b = rows().find((x) => x.role === 'batter');
+    expect(b.player_name).toBe('Josh Bell');
+    expect(b.player_key).toBe(require('../../src/utils/playerName').nameKey('Josh Bell'));
+  });
+
+  it('carries pitcher handedness from the movement feed', () => {
+    expect(rows().find((x) => x.role === 'pitcher').throws).toBe('L');
+  });
+
+  it('leaves batter handedness ABSENT (roster join, never guessed)', () => {
+    expect(rows().find((x) => x.role === 'batter').bats).toBeNull();
+  });
+
+  it('flags thin samples rather than dropping or inflating them', () => {
+    const thin = svc.buildRows(2026, feeds({
+      batter: [{ ...BELL, pa: '12' }], pitcher: [{ ...SKUBAL, p_formatted_ip: '2.1' }],
+    }), {});
+    expect(thin.every((r) => r._sufficient === false)).toBe(true);
+    // Thin is STORED, not dropped — "thin" and "missing" are different claims.
+    expect(thin).toHaveLength(2);
+    expect(thin[0].sample_pa).toBe(12);
+  });
+
+  it('a player with no sample at all is insufficient, not zero', () => {
+    const [r] = svc.buildRows(2026, feeds({ batter: [{ 'last_name, first_name': 'A, B', player_id: '9' }] }), {});
+    expect(r.sample_pa).toBeNull(); // NOT 0
+    expect(r._sufficient).toBe(false);
+    expect(r.k_pct).toBeNull();
+  });
+
+  it('keeps every raw metric in `metrics` so Layer 2/3 never needs a re-ingest', () => {
+    const b = rows().find((x) => x.role === 'batter');
+    expect(b.metrics.chase_pct).toBe(30.6);
+    expect(b.chase_pct).toBe(30.6);
+  });
+});
+
+describe('refreshSeason — the job', () => {
+  const okClient = () => {
+    const calls = [];
+    return {
+      calls,
+      from: () => ({ upsert: async (batch) => { calls.push(batch); return { error: null }; } }),
+    };
+  };
+
+  it('backfill and refresh are the SAME idempotent call', async () => {
+    const fetchSeason = async () => feeds({ batter: [BELL], pitcher: [SKUBAL], mix: [SKUBAL_MIX] });
+    const a = okClient(); const b = okClient();
+    const r1 = await svc.refreshSeason({ supabase: a, fetchSeason, now: '2026-07-21T00:00:00.000Z' });
+    const r2 = await svc.refreshSeason({ supabase: b, fetchSeason, now: '2026-07-21T00:00:00.000Z' });
+    expect(r1.ok).toBe(true);
+    expect(r1.written).toBe(2);
+    expect(JSON.stringify(a.calls)).toBe(JSON.stringify(b.calls)); // identical → idempotent
+  });
+
+  it('upserts on the natural key so a re-run never duplicates', async () => {
+    let opts = null;
+    const sb = { from: () => ({ upsert: async (_b, o) => { opts = o; return { error: null }; } }) };
+    await svc.refreshSeason({ supabase: sb, fetchSeason: async () => feeds({ batter: [BELL] }) });
+    expect(opts.onConflict).toBe('sport,season,source_id');
+  });
+
+  it('REFUSES to write when every feed is empty — a bad night cannot blank a good table', async () => {
+    const sb = okClient();
+    const out = await svc.refreshSeason({ supabase: sb, fetchSeason: async () => feeds({}) });
+    expect(out.ok).toBe(false);
+    expect(out.reason).toMatch(/refusing to write/);
+    expect(sb.calls).toHaveLength(0);
+  });
+
+  it('strips the internal flag before it reaches Postgres', async () => {
+    const sb = okClient();
+    await svc.refreshSeason({ supabase: sb, fetchSeason: async () => feeds({ batter: [BELL] }) });
+    expect(sb.calls[0][0]._sufficient).toBeUndefined();
+  });
+
+  it('reports the join rate — the honest-absent rate for the mechanism tier', async () => {
+    const out = await svc.refreshSeason({
+      supabase: okClient(),
+      fetchSeason: async () => feeds({ batter: [BELL, { player_id: '77', pa: '100' }] }),
+    });
+    expect(out.joined).toBe(1);
+    expect(out.unjoined).toBe(1); // no name → no key → stored, joins later
+  });
+});
+
+describe('staleness — never serve stale as fresh', () => {
+  it('fires past the max age', () => {
+    expect(svc.isStale({ updated_at: '2026-07-01T00:00:00Z', age_hours: 72 })).toBe(true);
+    expect(svc.isStale({ updated_at: '2026-07-20T00:00:00Z', age_hours: 6 })).toBe(false);
+  });
+
+  it('NEVER-BUILT is not STALE — different condition, different fix', () => {
+    // Paging on a fresh install teaches the operator to ignore the alarm.
+    expect(svc.isStale({ updated_at: null, age_hours: null })).toBe(false);
+    expect(svc.isStale(null)).toBe(false);
+  });
+
+  it('respects an explicit threshold', () => {
+    expect(svc.isStale({ updated_at: 'x', age_hours: 30 }, 24)).toBe(true);
+    expect(svc.isStale({ updated_at: 'x', age_hours: 30 }, 48)).toBe(false);
+  });
+});
+
+describe('scheduler + route wiring', () => {
+  const fs = require('fs');
+  const path = require('path');
+  const read = (r) => fs.readFileSync(path.join(__dirname, '..', '..', r), 'utf8');
+
+  it('is server-scheduled with a kill switch', () => {
+    const s = read('src/snapshotScheduler.js');
+    expect(s).toContain('STATCAST_HOUR_UTC');
+    expect(s).toMatch(/process\.env\.STATCAST !== '0'/);
+    expect(s).toContain('refreshSeason');
+  });
+
+  it('pages on BOTH a failed run and silent staleness', () => {
+    const s = read('src/snapshotScheduler.js');
+    expect(s).toMatch(/Statcast refresh failed/);
+    expect(s).toMatch(/Statcast aggregates stale/);
+  });
+
+  it('is induce-able on demand (never wait on the cron)', () => {
+    const s = read('src/routes/internal.js');
+    expect(s).toContain("router.post('/statcast/refresh'");
+    expect(s).toContain("router.get('/statcast/status'");
+  });
+});