vyndr/scripts/reconstruct-game-environment.js

#!/usr/bin/env node
'use strict';

/**
 * reconstruct-game-environment — GIVE THE PAST GAMES THEIR REAL CONDITIONS.
 *
 * `game_context` has never held a single weather reading. The reason is not the
 * fetcher, which is correct and points at Open-Meteo's ARCHIVE endpoint; it is
 * that nothing ever joined. The ledger keys a game as
 * `mlb:2026-08-03:WashingtonNationals@PhiladelphiaPhillies` and game_context
 * keys it as `mlb:823437`, so every lookup missed and the columns stayed NULL —
 * which reads exactly like "the weather was unavailable" rather than "the two
 * tables have never been introduced." The same class of failure as the doubled
 * /leaderboard path: graceful degradation wearing the mask of honest absence.
 *
 * This walks the dates in the settled ledger, resolves each slug to the real
 * statsapi game and venue, and writes a game_context row keyed by the LEDGER's
 * slug so the join exists. Then it pulls the actual archived weather for that
 * date and location.
 *
 * ARCHIVE, NOT FORECAST — asking the forecast endpoint about a past date returns
 * a re-forecast, which is a model's opinion about the past, not the past. Absent
 * stays NULL; nothing here is imputed.
 *
 *   SUPABASE_URL=... node scripts/reconstruct-game-environment.js
 */

require('dotenv').config();
const axios = require('axios');
const { createClient } = require('@supabase/supabase-js');
const { paginate } = require('../src/utils/safePaginate');
const { uniqueKeyFor } = require('../src/utils/tableKeys');

const SB_URL = process.env.SUPABASE_URL;
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
const PAGE = 1000;
const SCHEDULE = (d) => `https://statsapi.mlb.com/api/v1/schedule?sportId=1&date=${d}&hydrate=venue(location)`;
const ARCHIVE = (lat, lon, date) =>
  `https://archive-api.open-meteo.com/v1/archive?latitude=${lat}&longitude=${lon}`
  + `&start_date=${date}&end_date=${date}`
  + '&hourly=temperature_2m,wind_speed_10m,wind_direction_10m,precipitation'
  + '&temperature_unit=fahrenheit&wind_speed_unit=mph';

const squash = (s) => String(s || '').toLowerCase().replace(/[^a-z]/g, '');

// ── THE SAFE WALK (Fix A2/A2b) ────────────────────────────────────────────
// This script used to walk pages with `.range()` and NO ORDER BY. Measured on
// production, that returned the correct row COUNT and the wrong ROWS: up to
// 33.6% of a read came back twice while an equal share never came back at all,
// so `rows.length` looked perfect while a fifth of the sample was missing.
//
// `pageSafe` routes every read through `src/utils/safePaginate`: a stable ORDER
// BY on the table's real UNIQUE key — single OR composite, looked up from
// `src/utils/tableKeys` rather than assumed — a runtime tuple-uniqueness check,
// and a THROWN error instead of a silent stop. The old unordered helper is gone
// rather than left beside it, because a dead broken helper is an invitation.
async function pageSafe(sb, table, select, apply, key = uniqueKeyFor(table)) {
  return paginate(() => apply(sb.from(table).select(select)),
    { key, pageSize: PAGE, label: `${table}` });
}

/** THE MEASURED READS — main() and the harness call the same functions. */
const READS = {
  ledger: (sb) => pageSafe(sb, 'ledger_entries', 'id, game_id, game_date, stat, outcome, quarantine_reason',
    (q) => q.eq('sport', 'mlb').is('user_id', null).in('stat', ['hits', 'total_bases'])
      .in('outcome', ['hit', 'miss'])),
};

const get = async (url) => (await axios.get(url, { timeout: 60_000 })).data;

async function main() {
  const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });

  const led = await pageSafe(sb, 'ledger_entries', 'id, game_id, game_date, stat, outcome, quarantine_reason',
    (q) => q.eq('sport', 'mlb').is('user_id', null).in('stat', ['hits', 'total_bases'])
      .in('outcome', ['hit', 'miss']));
  const clean = led.filter((r) => !(r.quarantine_reason || '').startsWith('nontakeable_book'));

  // Distinct games, as the LEDGER names them.
  const games = new Map();
  for (const r of clean) if (r.game_id && !games.has(r.game_id)) games.set(r.game_id, r.game_date);
  const dates = [...new Set([...games.values()])].sort();
  console.error(`[env] ${games.size} distinct settled games across ${dates.length} dates`);

  // date -> statsapi games, indexed by the same squashed away@home the slug uses.
  const resolved = new Map();
  const venues = new Map();
  for (const d of dates) {
    let sched = null;
    try { sched = await get(SCHEDULE(d)); } catch { sched = null; }
    for (const day of (sched && sched.dates) || []) {
      for (const g of day.games || []) {
        const away = squash(g.teams?.away?.team?.name);
        const home = squash(g.teams?.home?.team?.name);
        resolved.set(`${d}|${away}@${home}`, g);
        const v = g.venue || {};
        if (v.id && !venues.has(v.id)) {
          const loc = v.location || {};
          venues.set(v.id, {
            venue_id: v.id,
            venue_name: v.name || null,
            lat: loc.defaultCoordinates?.latitude ?? null,
            lon: loc.defaultCoordinates?.longitude ?? null,
          });
        }
      }
    }
  }

  // Join each ledger slug to its real game + venue.
  const rows = []; let unmatched = 0;
  for (const [gid, date] of games) {
    const m = /^mlb:(\d{4}-\d{2}-\d{2}):(.+?)@(.+)$/.exec(gid);
    if (!m) { unmatched += 1; continue; }
    const g = resolved.get(`${m[1]}|${squash(m[2])}@${squash(m[3])}`);
    if (!g) { unmatched += 1; continue; }
    rows.push({
      game_id: gid,                       // the LEDGER's key — this is the whole fix
      game_date: date,
      venue_id: g.venue?.id ?? null,
      source_game_pk: g.gamePk ?? null,
    });
  }
  console.error(`[env] matched ${rows.length}, unmatched ${unmatched}, venues seen ${venues.size}`);

  for (let i = 0; i < rows.length; i += 200) {
    const { error } = await sb.from('game_context')
      .upsert(rows.slice(i, i + 200), { onConflict: 'game_id' });
    if (error) console.error('[env] context write failed:', error.message);
  }

  // ACTUAL archived weather, one call per (venue, date) that we need.
  const need = new Map();
  for (const r of rows) {
    const v = venues.get(r.venue_id);
    if (!v || v.lat == null || v.lon == null) continue;
    need.set(`${r.venue_id}|${r.game_date}`, { v, date: r.game_date });
  }
  console.error(`[env] fetching ${need.size} venue-days of archived weather`);

  const wx = new Map();
  for (const [k, { v, date }] of need) {
    try {
      const p = await get(ARCHIVE(v.lat, v.lon, date));
      const h = p && p.hourly;
      if (h && Array.isArray(h.time) && h.time.length) {
        const i = Math.min(h.time.length - 1, 19);   // ~7pm local, typical first pitch
        wx.set(k, {
          wx_temp_f: h.temperature_2m?.[i] ?? null,
          wx_wind_speed_mph: h.wind_speed_10m?.[i] ?? null,
          wx_wind_direction_deg: h.wind_direction_10m?.[i] ?? null,
          wx_precip_mm: h.precipitation?.[i] ?? null,
          wx_source: 'open_meteo_archive',
        });
      }
    } catch { /* absent stays absent */ }
  }

  let withWx = 0;
  for (let i = 0; i < rows.length; i += 200) {
    const batch = rows.slice(i, i + 200).map((r) => {
      const w = wx.get(`${r.venue_id}|${r.game_date}`);
      if (w) withWx += 1;
      return w ? { ...r, ...w } : r;
    });
    const { error } = await sb.from('game_context').upsert(batch, { onConflict: 'game_id' });
    if (error) console.error('[env] weather write failed:', error.message);
  }

  console.log(JSON.stringify({
    settled_games: games.size,
    matched_to_statsapi: rows.length,
    unmatched,
    distinct_venues: venues.size,
    venue_days_requested: need.size,
    venue_days_returned: wx.size,
    game_rows_with_actual_weather: withWx,
    source: 'open_meteo_archive (actual, not re-forecast)',
  }, null, 2));
  process.exit(0);
}

if (require.main === module) {
  main().catch((e) => { console.error(e); process.exit(1); });
}

// Exported so the read-integrity harness measures THE REAL FUNCTION.
module.exports = { READS: (typeof READS !== 'undefined' ? READS : undefined), READS_LEDGER: (typeof READS_LEDGER !== 'undefined' ? READS_LEDGER : undefined) };