Session 58: Phase 1 — Truth Infrastructure (2327 tests)

ledger_entries is live (migration 019 applied to prod, RLS + NULLS NOT DISTINCT dedupe verified against the real database). Every grade now persists, settles against the real result, and carries closing-line value. - ledgerService: pipeline pre-grade upserts (public model record, user_id null, idempotent), closing capture on every snapshot (last write before game start = the close), settlement with SIGNED CLV (over = locked - closing; beat/faded/flat), 30d model aggregate with the hard n>=20 rule. - Write paths: snapshotService -> ledger (priority path); Next /api/scan -> ledger for authenticated users only (anon never touches the public record). Refused reads write nothing and don't burn a scan. - Honest refusal (work-order 1.5): no projection => insufficient_data, grade null, "INSUFFICIENT DATA - no read" UI. The web gradeAdapter no longer displays the line as the model projection (the audit's model==line / +0% edge degenerate); the card renders absent states. projectionFor is sport-aware (l5 -> l20 -> {stat}_per_90 -> xG). - /ledger: MY READS | MODEL tabs; model header shows hit% + beat-close% only at n>=20, else RECORD BUILDING + live pending count. ModelRecord deferred-render strip on landing + player hero. CLV + outcome chips, revised_from_grade strikethrough (Phase 2.5 ready). - SYNC (Task 5): thresholds vs SNAPSHOT_EXPECTED_INTERVAL (normal <1.5x, amber >=1.5x, STALE red >=3x) via /api/snapshot/summary. - Phase 2.5 logged in specs/vyndr-roadmap.md (build after Phase 3). - Data-semantics hardening: strict null-safe numeric parsing everywhere a market value is handled (Number(null)===0 would have fabricated lines). Backend 2309 -> 2327 tests (201 suites), web build exit 0. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-10 21:34:26 -04:00
parent 2c79373a3b
commit d296e40cb6
29 changed files with 1578 additions and 223 deletions
@@ -0,0 +1,379 @@
+'use strict';
+
+/**
+ * ledgerService — the truth infrastructure (Session 58, work-order Phase 1).
+ *
+ * Persists every grade to Supabase `ledger_entries`:
+ *   - Pipeline pre-grades (user_id NULL) — the PUBLIC model record. Written by
+ *     snapshotService after each snapshot locks. This is the priority path:
+ *     hundreds of settles per night vs user scans trickling in.
+ *   - User scans are written by the web /api/scan route (it owns the user
+ *     identity); this service owns settlement + closing capture for BOTH.
+ *
+ * DATA SEMANTICS: `line` / `locked_odds` / `book` / `closing_line` /
+ * `closing_odds` are REAL book values captured at grade / refresh time —
+ * never computed. `model_value` is VYNDR's projection. A grade with no
+ * projection (insufficient_data) is never written — no hollow rows.
+ *
+ * Closing-line value: captureClosing runs on EVERY snapshot and overwrites
+ * closing_line/closing_odds for today's unsettled rows with the CURRENT feed
+ * values — the last write before the game starts is the closing line (once a
+ * game starts its props leave the feed, so updates stop naturally).
+ * settleLedger then computes `clv` SIGNED BY SIDE: for an OVER, a closing
+ * line BELOW the locked line = the market moved toward the graded side =
+ * positive = 'beat'. For an UNDER, the inverse.
+ *
+ * Everything is injectable; without SUPABASE env the service no-ops
+ * gracefully (tests / local dev without a database).
+ */
+
+const { nameKey, normalizeName } = require('../utils/playerName');
+const { settleResult, statValue } = require('./outcomeService');
+
+const CONFLICT = 'user_id,player_key,stat,line,side,game_id';
+const UPSERT_CHUNK = 200;
+const SETTLE_FETCH_LIMIT = 500;
+const AGG_WINDOW_DAYS = 30;
+const AGG_FETCH_LIMIT = 5000;
+/** Below this many settled rows, callers must not render a percentage. */
+const MIN_AGG_SAMPLE = 20;
+
+function isConfigured() {
+  return Boolean(process.env.SUPABASE_URL
+    && (process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY));
+}
+
+function defaultClient() {
+  return require('../utils/supabase').getSupabaseServiceClient();
+}
+
+/** ET calendar date (YYYY-MM-DD) of an ISO timestamp; null when unparseable. */
+function dateET(iso) {
+  if (!iso) return null;
+  const d = new Date(iso);
+  if (Number.isNaN(d.getTime())) return null;
+  return new Intl.DateTimeFormat('en-CA', {
+    timeZone: 'America/New_York', year: 'numeric', month: '2-digit', day: '2-digit',
+  }).format(d);
+}
+
+const todayET = () => dateET(new Date().toISOString());
+
+/** Derived game id when the feed carries no event id: sport:date:AWAY@HOME. */
+function gameIdFor(sport, prop, gameDate) {
+  const away = String((prop && prop.away_team) || 'UNK').replace(/\s+/g, '');
+  const home = String((prop && prop.home_team) || 'UNK').replace(/\s+/g, '');
+  return `${sport}:${gameDate}:${away}@${home}`;
+}
+
+const sideOf = (direction) =>
+  (String(direction || 'over').toLowerCase() === 'under' ? 'under' : 'over');
+
+// Strict numeric parse: null/undefined stay null (Number(null) is 0 — a
+// fabricated zero line/odds is exactly what the data-semantics rule forbids).
+const numOrNull = (v) => (v == null || !Number.isFinite(Number(v)) ? null : Number(v));
+
+/** Index odds props by nameKey|stat for lock/closing lookups. */
+function indexProps(props) {
+  const map = {};
+  for (const p of props || []) {
+    if (!p || !p.player || !p.stat_type) continue;
+    const k = `${nameKey(p.player)}|${String(p.stat_type).toLowerCase()}`;
+    if (!map[k]) map[k] = p;
+  }
+  return map;
+}
+
+function oddsForSide(prop, side) {
+  if (!prop) return null;
+  const v = side === 'under'
+    ? (prop.under_odds ?? prop.under ?? null)
+    : (prop.over_odds ?? prop.over ?? null);
+  return v == null ? null : String(v);
+}
+
+/**
+ * Build ledger rows from a snapshot's enriched grades + the raw odds props.
+ * Skips anything without a real grade or without a captured line — the
+ * ledger never holds a fabricated market value or a refused read.
+ */
+function rowsFromSnapshot(sport, grades, oddsProps, nowIso) {
+  const sp = String(sport || '').toLowerCase();
+  const byKey = indexProps(oddsProps);
+  const rows = [];
+  for (const g of grades || []) {
+    if (!g || !g.grade || g.insufficient_data) continue;
+    const player = g.player || g.player_name;
+    if (!player) continue;
+    const stat = String(g.stat_type || g.stat || '').toLowerCase();
+    if (!stat) continue;
+    const side = sideOf(g.direction);
+    const locked = g.gradedAt || {};
+    const line = numOrNull(locked.line) ?? numOrNull(g.line);
+    if (line == null) continue; // no real captured line → no row
+    const prop = byKey[`${nameKey(player)}|${stat}`] || null;
+    const gradedTs = locked.timestamp || nowIso;
+    const gameDate = dateET(prop && prop.game_time) || dateET(gradedTs) || todayET();
+    rows.push({
+      user_id: null,
+      player_key: nameKey(player),
+      player_name: normalizeName(player).display || player,
+      sport: sp,
+      stat,
+      line,
+      side,
+      locked_odds: locked.odds != null ? String(locked.odds) : oddsForSide(prop, side),
+      book: (prop && prop.book) || g.book || null,
+      grade: g.grade,
+      edge: numOrNull(g.edge_pct),
+      confidence: numOrNull(g.confidence),
+      model_value: numOrNull(g.projection),
+      graded_at: gradedTs,
+      game_id: gameIdFor(sp, prop, gameDate),
+      game_date: gameDate,
+    });
+  }
+  return rows;
+}
+
+/**
+ * Upsert the pipeline's pre-grades (user_id NULL — the public model record).
+ * Idempotent: re-runs hit the dedupe constraint and are IGNORED, so the
+ * original locked line/odds are never overwritten by a later run.
+ */
+async function recordPipelineGrades(sport, grades, oddsProps, opts = {}) {
+  if (!opts.sb && !isConfigured()) return { skipped: 'supabase not configured', written: 0 };
+  const sb = opts.sb || defaultClient();
+  const nowIso = (opts.now || (() => new Date().toISOString()))();
+  const rows = rowsFromSnapshot(sport, grades, oddsProps, nowIso);
+  if (rows.length === 0) return { written: 0 };
+  let written = 0;
+  for (let i = 0; i < rows.length; i += UPSERT_CHUNK) {
+    const chunk = rows.slice(i, i + UPSERT_CHUNK);
+    const { error } = await sb.from('ledger_entries')
+      .upsert(chunk, { onConflict: CONFLICT, ignoreDuplicates: true });
+    if (error) return { written, error: error.message };
+    written += chunk.length;
+  }
+  return { written };
+}
+
+/**
+ * Overwrite closing_line/closing_odds on today's UNSETTLED rows from the
+ * current (real) odds feed. Runs on every snapshot; the last capture before
+ * game start is the closing line. Matches by player_key+stat — the closing
+ * line may legitimately differ from the locked line (that's CLV).
+ */
+async function captureClosing(sport, oddsProps, opts = {}) {
+  if (!opts.sb && !isConfigured()) return { skipped: 'supabase not configured', updated: 0 };
+  const sb = opts.sb || defaultClient();
+  const sp = String(sport || '').toLowerCase();
+  const gameDate = opts.gameDate || todayET();
+  const byKey = indexProps(oddsProps);
+  if (Object.keys(byKey).length === 0) return { updated: 0 };
+
+  const { data: open, error } = await sb.from('ledger_entries')
+    .select('id, player_key, stat, side')
+    .eq('sport', sp)
+    .eq('game_date', gameDate)
+    .is('outcome', null)
+    .limit(SETTLE_FETCH_LIMIT);
+  if (error) return { updated: 0, error: error.message };
+
+  let updated = 0;
+  // Group row ids by identical closing values → one UPDATE per prop.
+  const groups = new Map();
+  for (const row of open || []) {
+    const prop = byKey[`${row.player_key}|${row.stat}`];
+    const closingLine = prop ? numOrNull(prop.line) : null;
+    if (closingLine == null) continue;
+    const closingOdds = oddsForSide(prop, row.side);
+    const gk = `${closingLine}|${closingOdds ?? ''}`;
+    if (!groups.has(gk)) groups.set(gk, { line: closingLine, odds: closingOdds, ids: [] });
+    groups.get(gk).ids.push(row.id);
+  }
+  for (const g of groups.values()) {
+    const { error: upErr } = await sb.from('ledger_entries')
+      .update({ closing_line: g.line, closing_odds: g.odds })
+      .in('id', g.ids);
+    if (!upErr) updated += g.ids.length;
+  }
+  return { updated };
+}
+
+/**
+ * Signed CLV per the Phase 1 amendment: positive = the market moved TOWARD
+ * the graded side. OVER: locked − closing (closing dropped ⇒ positive).
+ * UNDER: closing − locked.
+ */
+function computeClv(side, lockedLine, closingLine) {
+  const locked = numOrNull(lockedLine);
+  const closing = numOrNull(closingLine);
+  if (locked == null || closing == null) return null;
+  const raw = sideOf(side) === 'over' ? locked - closing : closing - locked;
+  return Math.round(raw * 100) / 100;
+}
+
+function clvResultOf(clv) {
+  if (clv == null) return null;
+  if (clv > 0) return 'beat';
+  if (clv < 0) return 'faded';
+  return 'flat';
+}
+
+/**
+ * Settle unsettled ledger rows with game_date <= yesterday against the real
+ * stat result (same free MLB game-log source outcomeService uses; other
+ * sports stay pending until they have a settled-result feed). Also computes
+ * CLV from the captured closing line. Idempotent: only rows with
+ * outcome IS NULL are fetched, and a row is written at most once.
+ */
+async function settleLedger(sport, opts = {}) {
+  if (!opts.sb && !isConfigured()) return { skipped: 'supabase not configured', settled: 0, pending: 0 };
+  const sb = opts.sb || defaultClient();
+  const sp = String(sport || '').toLowerCase();
+  const nowIso = (opts.now || (() => new Date().toISOString()))();
+  const getPlayerStats = opts.getPlayerStats || defaultGetPlayerStats;
+  const cutoff = opts.beforeDate || todayET(); // settle strictly-before today
+
+  const { data: open, error } = await sb.from('ledger_entries')
+    .select('id, player_key, player_name, stat, line, side, closing_line')
+    .eq('sport', sp)
+    .is('outcome', null)
+    .lt('game_date', cutoff)
+    .order('game_date', { ascending: true })
+    .limit(SETTLE_FETCH_LIMIT);
+  if (error) return { settled: 0, pending: 0, error: error.message };
+  if (!open || open.length === 0) return { settled: 0, pending: 0 };
+
+  // We need each row's game_date for the log match — refetch with it included.
+  const { data: rows } = await sb.from('ledger_entries')
+    .select('id, player_name, stat, line, side, closing_line, game_date')
+    .in('id', open.map((r) => r.id));
+
+  // One game-log fetch per unique player.
+  const players = [...new Set((rows || []).map((r) => r.player_name))];
+  const logByPlayer = {};
+  for (const player of players) {
+    try {
+      const stats = await getPlayerStats(player, sp);
+      logByPlayer[player] = stats && stats.found && Array.isArray(stats.last10) ? stats.last10 : [];
+    } catch { logByPlayer[player] = []; }
+  }
+
+  let settled = 0;
+  let pending = 0;
+  for (const row of rows || []) {
+    const log = logByPlayer[row.player_name] || [];
+    const gameRow = log.find((r) => r && r.date === row.game_date);
+    if (!gameRow) { pending += 1; continue; }
+    const actual = statValue(gameRow.stat, row.stat);
+    if (actual == null) { pending += 1; continue; }
+    const outcome = settleResult(row.side, actual, row.line);
+    if (!outcome) { pending += 1; continue; }
+    const clv = computeClv(row.side, row.line, row.closing_line);
+    const { error: upErr } = await sb.from('ledger_entries')
+      .update({
+        outcome,
+        actual_value: actual,
+        settled_at: nowIso,
+        clv,
+        clv_result: clvResultOf(clv),
+      })
+      .eq('id', row.id)
+      .is('outcome', null); // double-settle guard even across concurrent runs
+    if (upErr) { pending += 1; continue; }
+    settled += 1;
+  }
+  return { settled, pending };
+}
+
+async function settleAllLedgers(opts = {}) {
+  const sports = opts.sports || ['mlb', 'nba', 'wnba', 'soccer'];
+  const results = [];
+  for (const sp of sports) {
+    try { results.push({ sport: sp, ...(await settleLedger(sp, opts)) }); }
+    catch (e) { results.push({ sport: sp, settled: 0, pending: 0, error: e.message }); }
+  }
+  return results;
+}
+
+async function defaultGetPlayerStats(name, sport) {
+  if (String(sport).toLowerCase() === 'mlb') {
+    return require('./adapters/mlbStatsAdapter').getPlayerStats(name);
+  }
+  return { found: false }; // no free settled-result feed yet → pending
+}
+
+/**
+ * 30-day aggregate over the PUBLIC model record (user_id NULL): hit rate,
+ * beat-the-close rate, pending count. Percentages are null below
+ * MIN_AGG_SAMPLE — the UI must show "record building" instead.
+ */
+async function getModelAggregate(opts = {}) {
+  const empty = {
+    window_days: AGG_WINDOW_DAYS, min_sample: MIN_AGG_SAMPLE,
+    settled: 0, hits: 0, misses: 0, pushes: 0, hit_pct: null,
+    clv_sample: 0, clv_beat: 0, clv_faded: 0, clv_flat: 0, beat_close_pct: null,
+    pending: 0,
+  };
+  if (!opts.sb && !isConfigured()) return empty;
+  const sb = opts.sb || defaultClient();
+  const nowMs = (opts.nowMs || (() => Date.now()))();
+  const since = new Date(nowMs - AGG_WINDOW_DAYS * 24 * 3600 * 1000).toISOString().slice(0, 10);
+
+  let settledQ = sb.from('ledger_entries')
+    .select('outcome, clv_result, player_key')
+    .is('user_id', null)
+    .not('outcome', 'is', null)
+    .gte('game_date', since)
+    .limit(AGG_FETCH_LIMIT);
+  if (opts.sport) settledQ = settledQ.eq('sport', String(opts.sport).toLowerCase());
+  if (opts.playerKey) settledQ = settledQ.eq('player_key', opts.playerKey);
+  const { data: settledRows, error } = await settledQ;
+  if (error) return { ...empty, error: error.message };
+
+  let pendingQ = sb.from('ledger_entries')
+    .select('id', { count: 'exact', head: true })
+    .is('user_id', null)
+    .is('outcome', null);
+  if (opts.sport) pendingQ = pendingQ.eq('sport', String(opts.sport).toLowerCase());
+  if (opts.playerKey) pendingQ = pendingQ.eq('player_key', opts.playerKey);
+  const { count: pending } = await pendingQ;
+
+  const agg = { ...empty, pending: pending || 0 };
+  for (const r of settledRows || []) {
+    agg.settled += 1;
+    if (r.outcome === 'hit') agg.hits += 1;
+    else if (r.outcome === 'miss') agg.misses += 1;
+    else if (r.outcome === 'push') agg.pushes += 1;
+    if (r.clv_result) {
+      agg.clv_sample += 1;
+      if (r.clv_result === 'beat') agg.clv_beat += 1;
+      else if (r.clv_result === 'faded') agg.clv_faded += 1;
+      else agg.clv_flat += 1;
+    }
+  }
+  const decided = agg.hits + agg.misses;
+  // n<20 → null: never render a percentage on a small sample.
+  if (agg.settled >= MIN_AGG_SAMPLE && decided > 0) {
+    agg.hit_pct = Math.round((agg.hits / decided) * 100);
+  }
+  if (agg.settled >= MIN_AGG_SAMPLE && agg.clv_sample > 0) {
+    agg.beat_close_pct = Math.round((agg.clv_beat / agg.clv_sample) * 100);
+  }
+  return agg;
+}
+
+module.exports = {
+  recordPipelineGrades,
+  captureClosing,
+  settleLedger,
+  settleAllLedgers,
+  getModelAggregate,
+  MIN_AGG_SAMPLE,
+  __internals: {
+    rowsFromSnapshot, computeClv, clvResultOf, indexProps, gameIdFor,
+    dateET, sideOf, oddsForSide, isConfigured, CONFLICT,
+  },
+};