vyndr/src/services/retentionService.js

'use strict';

/**
 * MODEL SNAPSHOT RETENTION (Session 64, Phase 2 — priority zero).
 *
 * As of 2026-07-20 the only store of model history was `ledger_entries`: 640
 * rows, 6 game days, and NO MODEL INPUTS. Every other warehouse table in the
 * schema was empty. That meant we could score the grades we emitted but could
 * not replay a different model against the same conditions — the only question
 * a backtest exists to answer, and the gate the metrics-engine north star
 * requires ("does this metric predict better than without it?").
 *
 * This writes `model_snapshots`: one APPEND-ONLY row per graded prop per
 * snapshot cycle, carrying the market values, the model output, the refusal
 * (if any), and the FEATURE VECTOR that produced it.
 *
 * CONTRACT: best-effort, exactly like the ledger write. A retention failure
 * must NEVER break a snapshot. Every path returns a summary; none throw.
 *
 * Two deliberate choices:
 *  - REFUSALS ARE STORED. The ledger drops them, so a gate that refuses props
 *    which would have won is invisible. That is lost edge we cannot measure.
 *  - EVERY ROW IS STAMPED with model_version + code_sha. A backtest that mixes
 *    model eras is worthless, and `ledger_entries` is already permanently
 *    contaminated across the 2026-07-19 fix boundary with no way to separate.
 */

const crypto = require('crypto');
const { normalizeName, nameKey } = require('../utils/playerName');

/**
 * Bump when the grading model changes in a way that makes rows non-comparable.
 * This is the marker `ledger_entries` never had.
 */
const MODEL_VERSION = process.env.MODEL_VERSION || 'engine1@2026-07-20';

function codeSha() {
  return process.env.SOURCE_COMMIT || process.env.GIT_SHA || process.env.COOLIFY_GIT_COMMIT_SHA || null;
}

/** Strict numeric parse — `Number(null) === 0` is this codebase's classic
 *  fabrication bug, so absent must stay absent. */
function numOrNull(v) {
  if (v == null || v === '') return null;
  const n = Number(v);
  return Number.isFinite(n) ? n : null;
}

function intOrNull(v) {
  const n = numOrNull(typeof v === 'string' ? v.replace('+', '') : v);
  return n == null ? null : Math.round(n);
}

function boolOrNull(v) {
  return typeof v === 'boolean' ? v : null;
}

/**
 * Build retention rows from ONE prop's graded sides (both over and under,
 * graded or refused). `base` is the prop the grader was called with.
 */
function rowsFromSides(base, sides, ctx = {}) {
  const rows = [];
  const list = Array.isArray(sides) ? sides : [];
  for (const s of list) {
    if (!s) continue;
    const player = s.player || base.player;
    if (!player) continue;
    const stat = s.stat_type || base.stat_type;
    const line = numOrNull(s.line != null ? s.line : base.line);
    const side = String(s.direction || '').toLowerCase();
    if (!stat || line == null || (side !== 'over' && side !== 'under')) continue;

    const refused = !!(s.insufficient_data || !s.grade);

    rows.push({
      snapshot_id: ctx.snapshotId,
      captured_at: ctx.capturedAt,
      cycle_hour_utc: ctx.cycleHourUtc ?? null,
      model_version: MODEL_VERSION,
      code_sha: codeSha(),

      sport: ctx.sport,
      game_id: ctx.gameIdFor ? ctx.gameIdFor(base, s) : (base.game_id || `${ctx.sport}:${ctx.gameDate}`),
      game_date: ctx.gameDate,
      player_key: nameKey(player),
      player_name: normalizeName(player).display || player,
      team: s.team || base.team || null,
      opponent: s.opponent || base.opponent || null,
      stat,
      line,
      side,

      book: s.book || base.book || null,
      book_odds: intOrNull(s.book_odds),
      over_odds: intOrNull(base.over_odds),
      under_odds: intOrNull(base.under_odds),
      fair_odds: intOrNull(s.fair_odds),
      fair_prob: numOrNull(s.fair_prob),
      overround: numOrNull(s.overround),
      devig_method: s.devig_method || null,

      grade: s.grade || null,
      grade_11: s._grade_11 || null,
      confidence: numOrNull(s.confidence),
      confidence_basis: s.confidence_basis || null,
      p_win: numOrNull(s.p_win),
      ev_pct: numOrNull(s.ev_pct),
      projection: numOrNull(s.projection),
      edge_pct: numOrNull(s.edge_pct),
      takeable: boolOrNull(s.takeable),
      value: boolOrNull(s.value),
      archetype: s.archetype || null,

      refused,
      refusal_reason: refused
        ? (s.suppressed_reason || (s.insufficient_data ? 'insufficient_data' : 'no_grade'))
        : null,

      // The counterfactual enabler. Absent on pre-feature refusals (the juice
      // gate runs before features are computed) — honestly null, never faked.
      features: s._features && Object.keys(s._features).length ? s._features : null,
    });
  }
  return rows;
}

/** A collector to hand to gradeAndCacheSlate's `onGraded` hook. */
function createCollector(ctx) {
  const rows = [];
  return {
    rows,
    onGraded(base, sides) {
      try {
        rows.push(...rowsFromSides(base, sides, ctx));
      } catch { /* collection never affects grading */ }
    },
  };
}

/**
 * Persist rows. Chunked, upsert-on-conflict-ignore so a retried cycle can never
 * duplicate. No-ops (successfully) without Supabase env, so tests and local dev
 * never touch a database.
 */
async function persist(rows, deps = {}) {
  const out = { attempted: Array.isArray(rows) ? rows.length : 0, written: 0, skipped: false, error: null };
  if (!out.attempted) return out;
  try {
    const getClient = deps.getClient || require('../utils/supabase').getSupabaseServiceClient;
    const supabase = getClient();
    if (!supabase) { out.skipped = true; return out; }
    const CHUNK = 250;
    for (let i = 0; i < rows.length; i += CHUNK) {
      const chunk = rows.slice(i, i + CHUNK);
      const { error } = await supabase
        .from('model_snapshots')
        .upsert(chunk, {
          onConflict: 'snapshot_id,player_key,stat,line,side',
          ignoreDuplicates: true,
        });
      if (error) { out.error = error.message; break; }
      out.written += chunk.length;
    }
  } catch (e) {
    out.error = e && e.message ? e.message : String(e);
  }
  return out;
}

/**
 * Fill archetype / team / opponent onto collected rows from the ENRICHED grades.
 *
 * Retention collects at GRADE time, which is the only moment the feature vector
 * exists — but archetype/team/opponent are attached later, during snapshot
 * enrichment. Capturing at grade time alone left all three permanently null,
 * which specifically blocks the archetype-baselined metrics work.
 *
 * CONTRACT: this ONLY fills those three fields. It must never touch `features`
 * or any model output — grade-time values are the record, and enrichment must
 * not rewrite history. Unmatched rows (e.g. refusals, which never reach the
 * enriched slate) pass through untouched with the fields left null: honestly
 * absent, not guessed.
 */
function mergeEnrichment(rows, enrichedGrades) {
  if (!Array.isArray(rows) || !rows.length) return rows || [];
  const byPlayer = new Map();
  for (const g of enrichedGrades || []) {
    const raw = g && (g.player || g.player_name);
    if (!raw) continue;
    const k = nameKey(raw);
    // First enriched grade per player wins; archetype/team are player-level.
    if (!byPlayer.has(k)) {
      byPlayer.set(k, {
        archetype: g.archetype ?? null,
        team: g.team ?? null,
        opponent: g.opponent ?? null,
      });
    }
  }
  return rows.map((r) => {
    const e = byPlayer.get(r.player_key);
    if (!e) return r;
    return {
      ...r,
      archetype: r.archetype ?? e.archetype ?? null,
      team: r.team ?? e.team ?? null,
      opponent: r.opponent ?? e.opponent ?? null,
    };
  });
}

function newSnapshotId() {
  return crypto.randomUUID();
}

module.exports = {
  MODEL_VERSION,
  codeSha,
  rowsFromSides,
  createCollector,
  mergeEnrichment,
  persist,
  newSnapshotId,
  __internals: { numOrNull, intOrNull, boolOrNull },
};