From c2d6e8ee7d95adc0f69207f33483f2d1071f6ed9 Mon Sep 17 00:00:00 2001 From: Kev Date: Mon, 3 Aug 2026 16:49:15 -0400 Subject: [PATCH] Mirror statcast_history on its source so retention cannot drift The first production run of the point-in-time retention failed with "Could not find the 'swing_pct' column of 'statcast_history'" -- the hand-enumerated column list had already drifted from the table it was copying. The refresh itself still succeeded and wrote all 1,387 aggregate rows, which verified the best-effort guard in prod: a retention failure does not fail the refresh. The table is now created with LIKE statcast_aggregates, and the writer passes the row through whole instead of hand-stripping columns, so there is no drift surface left. Recreating was safe -- nothing had been retained. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01W1sivYNqY2TS5ftykmHBU9 --- src/services/statcastAggregateService.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/services/statcastAggregateService.js b/src/services/statcastAggregateService.js index afa98d3..9b84727 100644 --- a/src/services/statcastAggregateService.js +++ b/src/services/statcastAggregateService.js @@ -272,14 +272,13 @@ async function refreshSeason(opts = {}) { const asOf = (opts.asOfDate || started).slice(0, 10); let retained = 0; for (let i = 0; i < rows.length; i += chunk) { - const batch = rows.slice(i, i + chunk).map((r) => { - const row = toDbRow(r); - delete row.updated_at; delete row.source; - delete row.position; delete row.role_detail; - delete row.games_started; delete row.games_pitched; - delete row.saves; delete row.holds; - return { ...row, as_of_date: asOf }; - }); + // The history table MIRRORS statcast_aggregates (created via LIKE), so the + // row goes in whole. The first prod attempt hand-stripped columns against a + // hand-enumerated schema and failed on the first one that had drifted + // ('swing_pct'); passing the same shape through removes the drift surface + // entirely. `updated_at` rides along as the refresh timestamp; `as_of_date` + // is the day the snapshot describes and is what point-in-time queries use. + const batch = rows.slice(i, i + chunk).map((r) => ({ ...toDbRow(r), as_of_date: asOf })); const { error } = await sb.from('statcast_history') .upsert(batch, { onConflict: 'as_of_date,sport,season,source_id,role' }); if (error) { summary.history_error = error.message; break; }