vyndr/src/services/gradeSlateService.js

'use strict';

/**
 * Grade-slate writer (Session 32).
 *
 * Closes the content pipeline. The grading engine (engine1 via
 * analyzeViaEngine1) grades props on demand, but nothing ever persisted a
 * sport's graded slate. `contentTemplateService.collectSlateData` reads a
 * `grades:{sport}` cache "when present" — without a writer it was always
 * empty, so slate/POTD content degraded to lines/schedule and never reached
 * `dataLevel: 'full'`.
 *
 * This service grades a sport's freshly-fetched props and writes the
 * `grades:{sport}` cache in the exact shape contentTemplateService expects.
 * It is wired fire-and-forget into `oddsService.recordDownstream` so it runs
 * on a cache MISS (≈hourly) WITHOUT holding the odds HTTP response — content
 * endpoints read the grades cache independently and asynchronously.
 *
 * The legacy grade shape (player_name?/player, stat_type, line, direction,
 * grade, confidence, edge_pct, reasoning.summary) is already what
 * `contentTemplateService.normalizeGrade` reads — so no field remapping is
 * needed at the write boundary.
 */

// Cost bounds: the odds slate carries one row per player+stat+line+book.
// We dedupe to unique player+stat+line and cap how many we grade, because
// each grade fans out to feature computation. Grading runs at most once per
// cache-miss per sport, but we still bound the herd.
const { isModelBook } = require('../config/bookRoles');

// RAISED 25 -> 500 on 2026-08-01, on measured cost, not taste.
//
// The 25 dated from Session 32 and was a herd guard written before anyone had
// measured what a grade costs. Measured on a real prod slate (n=80):
//   mean 721ms/prop, median 666ms, p90 1024ms
//   -> ~72s for 500 props at concurrency 5.
// Both callers tolerate that: the snapshot cron runs 5x/day, and
// oddsService.recordDownstream is fire-and-forget and never holds an HTTP
// response.
//
// What it was costing: the live MLB slate carries 585 unique gradeable props.
// The cap graded 25 of them and silently discarded 560 — 95.7% of the product.
//
// RAISED 500 -> 1500 on 2026-08-03, and this one is about SAMPLE, not display.
//
// Measured on the live board (internal/diagnose-refusals, n=300 sample):
//   unique gradeable props 1,244 · graded after suppression/refusal ~70%
// So a 500 cap grades ~334 and discards ~744 — and because dedupeProps takes
// FIRST-ROW-WINS IN FEED ORDER, what survives is decided by feed position, not
// by value. Pitcher props are ~2.6% of the feed, so the cap was handing us SIX
// strikeout props a slate against 32 available. At six a slate, the gate's
// n>=500 is three months away for every pitcher stat, and the entire
// prove-it programme is blocked on an arbitrary truncation.
//
// Cost, measured not guessed: 721ms/prop at concurrency 5 -> ~179s for the full
// 1,244. Both callers tolerate it (the snapshot cron runs 5x/day;
// recordDownstream is fire-and-forget and never holds an HTTP response), and
// statsapi is free and unlimited. Concurrency stays at 5 — one variable at a time.
//
// Env-tunable so the ceiling can move without a deploy: GRADE_SLATE_LIMIT.
const DEFAULT_LIMIT = Number(process.env.GRADE_SLATE_LIMIT) > 0
  ? Number(process.env.GRADE_SLATE_LIMIT)
  : 1500;
// Unchanged at 5 deliberately: raising the cap already multiplies total load by
// 20x, and concurrency is the knob that decides how hard we hit statsapi at
// once. One variable at a time.
const DEFAULT_CONCURRENCY = 5;
const DEFAULT_TTL = 7200; // 2 hours — matches the spec's grades-cache TTL.

// Collapse the multi-book prop rows to one entry per gradeable prop.
//
// ORDER ZERO GATE (2026-08-01). `normalizeProps` now emits every DISPLAY book so
// the surfaces can shop lines, which means DFS pick'em and exchange rows arrive
// here for the first time. The MODEL must not eat them: it has never been
// measured against those books, and a fixed-payout DFS number is not a market
// price at all. So we re-filter to MODEL_BOOKS BEFORE the first-row-wins pick
// and before the limit — which makes the graded set byte-identical to what it
// was before the widening. This gate lifts only when the MLB calibration is
// re-run on the consensus ruler and v2 is promoted.
/**
 * WHAT DUPLICATES IS THIS FUNCTION SUPPOSED TO REMOVE?
 *
 * ONE: the same proposition offered by SEVERAL BOOKS. Provider is collapsed on
 * purpose — grading the same player/stat/line once per book would multiply the
 * work with no new information, and the price anchor is chosen later.
 *
 * WHAT IT WAS ALSO REMOVING, WRONGLY
 *
 * The key was `player::stat_type::line` with NO event component, so the SAME
 * proposition in TWO DIFFERENT GAMES also collapsed — and first-row-wins
 * silently discarded the second real game.
 *
 * VERIFIED on production: 2026-08-17 St. Louis @ Cincinnati was a doubleheader
 * (gamePk 824514 / 824478). VYNDR recorded ONE derived id holding FOUR distinct
 * starting pitchers, i.e. both games merged. 19 doubleheaders / 38 games are
 * affected across the 2026 season to date.
 *
 * The key therefore adds the EVENT, and nothing else. Provider stays collapsed
 * because collapsing books is the function's actual job.
 *
 * `canonical_event_id` is used when identity resolved. When it did not, the
 * fallback is the legacy derived label, which reproduces the previous behaviour
 * exactly — it does not pretend to distinguish what it cannot.
 */
let fractureSeq = 0;

/**
 * The EVENT component of the dedupe key.
 *
 * -- FAIL CLOSED WHERE CANONICAL IDENTITY IS EXPECTED --------------------
 * A first version fell back to `legacy:{date}:{away}@{home}` whenever identity
 * did not resolve. That is precisely the collision-prone key this whole change
 * exists to remove: on a doubleheader it is byte-identical for two real games,
 * so an unresolved MLB prop could still merge two events.
 *
 * So for a sport where canonical identity is EXPECTED (MLB), an unresolved prop
 * FRACTURES: it gets a key unique to that row, which cannot merge with anything.
 * The cost is that books stop collapsing for that prop, so it may be graded more
 * than once. That is a coverage/efficiency cost, and it is the right direction:
 *
 *   UNKNOWN EVENT IDENTITY MAY LOSE COVERAGE.
 *   IT MAY NOT MERGE TWO REAL EVENTS.
 *
 * MEASURED: over MLB 2026-08-20..27, 101 of 101 team-pairs resolve from teams
 * alone; only a doubleheader pair needs the start time to disambiguate. So the
 * fracture path is rare in normal operation and bites only where identity is
 * genuinely unknown.
 *
 * -- SPORTS WITHOUT A RESOLVER ARE UNCHANGED ----------------------------
 * WNBA, NBA and the rest have no canonical adapter and never did. Fracturing
 * them would multiply their grading load for no safety gain, because their
 * identity was never canonical to begin with. They keep the legacy label, which
 * is the exact behaviour that existed before this change, and their lineage is
 * excluded from the canary for the same reason.
 */
function propositionEventKey(p) {
  if (p && p.canonical_event_id) return p.canonical_event_id;

  const method = p && p.event_identity_method;
  const expectsCanonical = method && method !== 'UNSUPPORTED_SPORT';
  if (expectsCanonical) {
    // Unique per row: this prop can never share an event key with another.
    fractureSeq += 1;
    return `unresolved-event:${method}:${fractureSeq}`;
  }

  const away = String((p && p.away_team) || '').replace(/\s+/g, '');
  const home = String((p && p.home_team) || '').replace(/\s+/g, '');
  const date = (p && p.game_date) || '';
  return `legacy:${date}:${away}@${home}`;
}

function dedupeProps(props, limit) {
  const seen = new Set();
  const out = [];
  for (const p of props || []) {
    if (!p || !p.player || !p.stat_type || p.line == null) continue;
    if (!isModelBook(p.book)) continue;
    const key = `${propositionEventKey(p)}::${p.player}::${p.stat_type}::${p.line}`;
    if (seen.has(key)) continue;
    seen.add(key);
    out.push(p);
    if (out.length >= limit) break;
  }
  return out;
}

// engine1 is direction-aware, so a prop grades differently over vs under.
// Grade both sides and keep the higher-confidence verdict — that's the
// side the engine actually favors.
async function gradeBestSide(grade, prop, sport, opts = {}) {
  // PIPELINE ORDER: the factor context must reach the engine BEFORE it grades,
  // because factors adjust the forecast the grade is read from. It was
  // previously computed downstream of the grade it should inform.
  const factorContext = typeof opts.factorContext === 'function'
    ? opts.factorContext(prop, sport) : null;
  // FIX A6 — the SHADOW pair. `factor_context` (above) is the LIVE context and
  // is unchanged; these two let the engine compute what the factors WOULD do
  // with the join keys, and freeze it. Neither reaches the served forecast.
  const matchupKeys = typeof opts.matchupKeys === 'function'
    ? opts.matchupKeys(prop, sport) : null;
  const base = {
    factor_context: factorContext,
    factor_context_resolver: typeof opts.factorContext === 'function' ? opts.factorContext : null,
    matchup_keys: matchupKeys,
    player: prop.player,
    stat_type: prop.stat_type,
    line: prop.line,
    sport,
    book: prop.book,
    // Session 62 (A1-S1) — real book odds ride into the engine so the
    // quarter-Kelly sizing can compute from actual prices (or not at all).
    over_odds: prop.over_odds ?? null,
    under_odds: prop.under_odds ?? null,
    // Session 64 (Order 1.6) — carry the BOUND game through to grading so
    // opponent/home-away features resolve against the RIGHT game. Without this
    // the grader fell back to ESPN's dateless "today", which at the late slots
    // is the previous day's card — binding yesterday's opponent into
    // opp_rank_stat and the home/away factor.
    game_date: prop.game_date ?? null,
    game_time: prop.game_time ?? null,
    home_team: prop.home_team ?? null,
    away_team: prop.away_team ?? null,
    // CANONICAL EVENT IDENTITY rides with the grade so retention and lineage
    // key on the real event rather than re-deriving a label from team names.
    canonical_event_id: prop.canonical_event_id ?? null,
    event_identity_source: prop.event_identity_source ?? null,
    event_identity_method: prop.event_identity_method ?? null,
    event_identity_version: prop.event_identity_version ?? null,
    event_occurrence: prop.event_occurrence ?? null,
  };
  const sides = await Promise.all([
    Promise.resolve()
      .then(() => grade({ ...base, direction: 'over' }))
      .catch(() => null),
    Promise.resolve()
      .then(() => grade({ ...base, direction: 'under' }))
      .catch(() => null),
  ]);
  // Session 64 — RETENTION HOOK. Fires with BOTH sides, graded AND refused,
  // before any filtering. Refusals never reach the slate or the ledger, so this
  // is the only point where "the gate refused this prop" is observable — and a
  // gate that refuses winners is invisible without it. Best-effort: a retention
  // collector must never affect grading.
  if (typeof opts.onGraded === 'function') {
    try { opts.onGraded(base, sides); } catch { /* never breaks the slate */ }
  }

  // Session 58 (work-order 1.5) — a refused read (insufficient_data /
  // no grade) never enters the graded slate: no hollow rows in the grades
  // cache, the snapshot, or the ledger.
  const cands = sides.filter((s) => s && s.grade && !s.insufficient_data);
  if (cands.length === 0) return null;
  const winner = cands.reduce((a, b) => ((Number(b.confidence) || 0) > (Number(a.confidence) || 0) ? b : a));
  // Strip the internal retention fields so they never reach a cache or payload.
  delete winner._features;
  delete winner._grade_11;
  // PUBLICATION SIGNAL. `onGraded` above fired with both sides before any
  // filtering; this fires ONLY for the side that becomes the served Read, so
  // retention can tell a published claim from a captured model state. Measured:
  // 64.9% of captured rows describe a state no user ever saw.
  if (typeof opts.onPublished === 'function') {
    try { opts.onPublished(base, winner); } catch { /* never breaks the slate */ }
  }

  // CARRY THE GAME (2026-08-01). The legacy grade shape drops home/away, so by
  // the time the challenger runs, nothing on the grade says WHICH GAME it is —
  // measured: `team` was null on 416/416 stored grades, so the park/weather
  // resolver could never find a venue and the environment axis fired on ZERO
  // rows while all 14 weather forecasts sat resolved and unused.
  //
  // The park depends on the GAME, not on the player's roster team, so binding
  // the game directly is both the fix and the more correct join: it does not
  // depend on a stats-resolve that can legitimately fail.
  if (winner.home_team == null && prop.home_team != null) winner.home_team = prop.home_team;
  if (winner.away_team == null && prop.away_team != null) winner.away_team = prop.away_team;
  return winner;
}

// Run an async mapper over items with a bounded concurrency.
async function mapLimit(items, concurrency, fn) {
  const results = new Array(items.length);
  let cursor = 0;
  async function worker() {
    while (cursor < items.length) {
      const i = cursor++;
      results[i] = await fn(items[i], i);
    }
  }
  const pool = Array.from({ length: Math.min(concurrency, items.length) }, () => worker());
  await Promise.all(pool);
  return results;
}

/**
 * Grade a sport's props and write the `grades:{sport}` cache.
 *
 * @param {string} sport
 * @param {Array}  props        normalized odds props (oddsNormalizer shape)
 * @param {Object} [opts]
 * @param {Function} [opts.grade]    grader (default analyzeViaEngine1)
 * @param {Function} [opts.cacheSet] cache writer (default redis.cacheSet)
 * @param {string}   [opts.source]   provider tag ('propline' | 'odds-api')
 * @param {number}   [opts.limit]    max unique props graded
 * @param {number}   [opts.ttl]      cache TTL seconds
 * @param {Function} [opts.now]      timestamp source (testable)
 * @returns {Promise<{written:boolean,count:number,error?:string}>}
 */
async function gradeAndCacheSlate(sport, props, opts = {}) {
  const grade = opts.grade || require('./intelligence/analyzeViaEngine1').analyzeViaEngine1;
  const cacheSet = opts.cacheSet || require('../utils/redis').cacheSet;
  const source = opts.source || 'odds-api';
  const limit = opts.limit || DEFAULT_LIMIT;
  const ttl = opts.ttl || DEFAULT_TTL;
  const concurrency = opts.concurrency || DEFAULT_CONCURRENCY;
  const now = opts.now || (() => new Date().toISOString());

  if (!Array.isArray(props) || props.length === 0) {
    return { written: false, count: 0 };
  }

  try {
    const unique = dedupeProps(props, limit);
    if (unique.length === 0) return { written: false, count: 0 };

    const graded = (await mapLimit(unique, concurrency, (p) => gradeBestSide(grade, p, sport, opts)))
      .filter(Boolean)
      .sort((a, b) => (Number(b.confidence) || 0) - (Number(a.confidence) || 0));

    if (graded.length === 0) return { written: false, count: 0 };

    const envelope = { grades: graded, updated_at: now(), source };
    await cacheSet(`grades:${sport}`, envelope, ttl);
    return { written: true, count: graded.length };
  } catch (e) {
    // Best-effort — slate grading must never break odds delivery.
    console.warn('[gradeSlateService] grade slate failed:', e.message);
    return { written: false, count: 0, error: e.message };
  }
}

module.exports = {
  gradeAndCacheSlate,
  __internals: { dedupeProps, gradeBestSide, mapLimit, DEFAULT_LIMIT, DEFAULT_TTL, DEFAULT_CONCURRENCY },
};