vyndr/scripts/verify-grade-range.js

#!/usr/bin/env node
/**
 * Session 63 — GRADE-RANGE VERIFICATION (on merit, not by rescaling).
 *
 * Replays REAL props from the live board through the REAL engine path that
 * Session 63 repaired, and reports the resulting grade distribution.
 *
 * What is real here:
 *   - the props (player / stat / line / side) come from the live API board
 *   - the game logs come from statsapi.mlb.com + ESPN (free, no quota)
 *   - the consistency factor + L5/L20 features are computed from those logs
 *   - the grade comes from engine1.gradeProp, unmodified
 *
 * What is NOT covered (documented, not hidden):
 *   - `opp_rank_stat` needs `team_stats:{sport}:{abbr}` in Redis, which only
 *     the production snapshot populates. Locally it stays null, so this run
 *     UNDERSTATES the restored range — it omits a ±1.0 factor. Any A/D seen
 *     here is therefore a floor, not a ceiling.
 *
 * Usage: node scripts/verify-grade-range.js [sport] [limit]
 */

const engine1 = require('../src/services/intelligence/engine1');
const featureCache = require('../src/services/intelligence/featureCache');
const consistencyScore = require('../src/services/intelligence/consistencyScore');
const { estimateProbability } = require('../src/services/intelligence/probabilityEstimator');
const { fourLetterGrade } = require('../src/utils/gradeAdapter').__internals;

const API = process.env.VERIFY_API || 'https://api.vyndr.app';
const SPORT = process.argv[2] || 'mlb';
const LIMIT = Number(process.argv[3] || 40);

async function board(sport) {
  const res = await fetch(`${API}/api/snapshot/${sport}`);
  const json = await res.json();
  const grades = Array.isArray(json.grades) ? json.grades : [];
  return grades.map((g) => ({
    player: g.player,
    stat: g.stat_type,
    line: Number(g.line),
    direction: String(g.direction || 'over').toLowerCase(),
    oldGrade: g.grade,
    oldConfidence: g.confidence,
  })).filter((p) => p.player && p.stat && Number.isFinite(p.line));
}

async function gradeOne(p, sport) {
  const rows = await featureCache.getStatRows(p.player, sport, p.stat);
  const features = await featureCache.__internals.gameLogFeatures(p.player, sport, p.stat);
  const consistency = await consistencyScore.getConsistency({
    playerName: p.player, sport, statType: p.stat, gameLogs: rows,
  });
  const prop = { line: p.line, direction: p.direction };
  const res = engine1.gradeProp({ features, trap: {}, consistency, prop });
  const est = estimateProbability({ gameLogs: rows, line: p.line, statType: p.stat, features });
  const pWin = Number.isFinite(est.p_over)
    ? (p.direction === 'under' ? 1 - est.p_over : est.p_over)
    : null;
  return {
    ...p,
    rows: rows.length,
    consistency: consistency.consistency,
    newGrade11: res.grade,
    newGrade: fourLetterGrade(res.grade),
    p_win: pWin == null ? null : Math.round(pWin * 1000) / 1000,
  };
}

(async () => {
  const props = (await board(SPORT)).slice(0, LIMIT);
  if (!props.length) { console.log(`no live props for ${SPORT}`); return; }
  console.log(`Replaying ${props.length} REAL ${SPORT.toUpperCase()} props through the repaired engine\n`);

  const out = [];
  for (const p of props) {
    try { out.push(await gradeOne(p, SPORT)); }
    catch (e) { console.warn(`  ! ${p.player} ${p.stat}: ${e.message}`); }
  }

  const tally = (arr, key) => arr.reduce((m, r) => { const k = r[key] ?? 'null'; m[k] = (m[k] || 0) + 1; return m; }, {});
  const pct = (n) => `${Math.round((n / out.length) * 1000) / 10}%`;

  console.log('--- 4-LETTER DISTRIBUTION ---');
  console.log('BEFORE (live board):', tally(out, 'oldGrade'));
  const after = tally(out, 'newGrade');
  console.log('AFTER  (repaired)  :', after);
  for (const g of ['A', 'B', 'C', 'D', 'F']) if (after[g]) console.log(`   ${g}: ${after[g]} (${pct(after[g])})`);

  console.log('\n--- 11-STEP DISTRIBUTION (pre-collapse) ---');
  console.log(tally(out, 'newGrade11'));

  console.log('\n--- REVIVED SIGNALS ---');
  const withRows = out.filter((r) => r.rows > 0).length;
  const withP = out.filter((r) => r.p_win != null).length;
  const withCons = out.filter((r) => r.consistency && r.consistency !== 'unknown').length;
  console.log(`game-log rows present : ${withRows}/${out.length}`);
  console.log(`p_win computed        : ${withP}/${out.length}   (was 0 in prod)`);
  console.log(`consistency known     : ${withCons}/${out.length}   (was 0 for MLB)`);

  console.log('\n--- MOVERS (grade changed) ---');
  for (const r of out.filter((r) => r.oldGrade !== r.newGrade).slice(0, 15)) {
    console.log(`  ${r.oldGrade} → ${r.newGrade.padEnd(2)} (${r.newGrade11.padEnd(2)})  ${r.player} ${r.stat} ${r.direction} ${r.line}  n=${r.rows} cons=${r.consistency} p=${r.p_win}`);
  }

  // Redis runs in degraded mode locally and keeps a reconnect timer alive, so
  // the process would never exit on its own — flush and leave deliberately.
  await new Promise((r) => process.stdout.write('', r));
  process.exit(0);
})().catch((e) => { console.error('verify failed:', e.message); process.exit(1); });