vyndr/src/services/model/factorGate.js

'use strict';

/**
 * factorGate — DOES THIS FACTOR READ TONIGHT'S GAME, OR JUST MOVE THE NUMBER?
 *
 * Every gate in this codebase so far asks one question: is there a correlation.
 * That is necessary and it is not sufficient, because it cannot tell apart the
 * two ways a factor can look alive:
 *
 *   PROVES   the factor moves the prediction off the player's base rate AND the
 *            moved prediction is MORE ACCURATE out-of-sample. It is reading the
 *            game.
 *   THEATER  the factor moves the prediction — sometimes a lot — and accuracy
 *            does not improve, or gets worse. The number looks responsive. It is
 *            responding to nothing.
 *
 * THEATER IS THE DANGEROUS ONE, and it is what a product ships by accident. A
 * grade that swings on park and platoon LOOKS like it read tonight's matchup;
 * a user cannot tell the difference from the outside, and neither can a
 * correlation test. arch-v1 was exactly this: it moved 76% of rows by 2.5 points
 * and changed resolution by 0.0000. It was live for months.
 *
 * So a factor must clear BOTH:
 *
 *   (a) movement    mean |Δp| against the base-rate baseline is real
 *   (b) improvement paired bootstrap on Brier score, CI excluding zero
 *
 * (a) alone is rejected BY NAME as THEATER rather than filed as "inconclusive",
 * because the distinction is the whole point: an inconclusive factor might work
 * with more data, and a theatrical one is actively misleading the user now.
 *
 * ── WHY BRIER AND NOT CORRELATION ────────────────────────────────────────
 * Correlation asks whether the ORDERING improved. This asks whether the NUMBER
 * got closer to what happened, which is what a probability claims. A factor can
 * improve ordering while degrading the number, and for a graded probability the
 * number is the product.
 */

const { knownNumber } = require('../../utils/known');

/** Minimum mean |Δp| for a factor to count as having moved anything at all. */
const MIN_MOVEMENT = 0.01;
/** Observations needed for the effect ESTIMATE to be stable. */
const MIN_N = 500;
/**
 * Clusters needed for the cluster-robust INTERVAL to be trustworthy.
 *
 * These two floors answer different questions and must not be collapsed. Rows
 * govern whether the point estimate is stable; clusters govern whether the
 * interval around it means anything. Transplanting the 500-row bar onto clusters
 * refuses a factor measured over 1,059 rows and 85 games — which has ample
 * observations AND ample clusters — while telling us nothing about either.
 *
 * 40 is the conventional floor below which cluster-robust inference is known to
 * under-cover regardless of how many rows sit inside the clusters. It is a
 * statement about when the bootstrap can be believed, not a bar tuned to let
 * anything through: a venue-constant factor still caps at 30 ballparks and is
 * still refused, permanently.
 */
const MIN_CLUSTERS = 40;

const brier = (ps, ys) => (ps.length
  ? ps.reduce((s, p, i) => s + (p - ys[i]) ** 2, 0) / ps.length : null);

function makeRnd(seed) {
  let s = seed >>> 0;
  return () => { s ^= s << 13; s >>>= 0; s ^= s >>> 17; s ^= s << 5; s >>>= 0; return s / 4294967296; };
}

/**
 * How far does the factor move the prediction off the baseline?
 *
 * Reported as the MEAN ABSOLUTE shift and its spread. A factor that shifts every
 * prediction by the same amount is not reading the game either — it is a
 * constant — so the spread matters as much as the mean.
 */
function movement(rows) {
  const deltas = [];
  for (const r of rows || []) {
    const b = knownNumber(r && r.baseline);
    const c = knownNumber(r && r.conditioned);
    if (b === null || c === null) continue;    // absent, never assumed equal
    deltas.push(c - b);
  }
  if (deltas.length === 0) return { n: 0, mean_abs_shift: null, sd_shift: null, max_abs_shift: null };
  const meanAbs = deltas.reduce((s, d) => s + Math.abs(d), 0) / deltas.length;
  const mean = deltas.reduce((s, d) => s + d, 0) / deltas.length;
  const sd = deltas.length > 1
    ? Math.sqrt(deltas.reduce((s, d) => s + (d - mean) ** 2, 0) / (deltas.length - 1)) : 0;
  return {
    n: deltas.length,
    mean_abs_shift: round4(meanAbs),
    mean_signed_shift: round4(mean),
    sd_shift: round4(sd),
    max_abs_shift: round4(Math.max(...deltas.map(Math.abs))),
  };
}

/**
 * Did the moved prediction get CLOSER to what happened?
 *
 * Paired bootstrap on the Brier difference — the same rows score both models, so
 * treating their errors as independent would overstate certainty. NEGATIVE delta
 * means the conditioned model has lower Brier, i.e. it improved.
 */
function improvement(rows, iters = 4000, seed = 20260805, cumulativeTests = 1) {
  const usable = (rows || []).filter((r) =>
    knownNumber(r.baseline) !== null && knownNumber(r.conditioned) !== null && knownNumber(r.won) !== null);
  if (usable.length < 30) return null;
  const rnd = makeRnd(seed);
  const diffs = [];

  // ── PSEUDO-REPLICATION ────────────────────────────────────────────────────
  // A factor that assigns ONE value per game (park, weather, opposing starter)
  // gives every prop row in that game the identical treatment. Resampling ROWS
  // then treats 18 hitters in one ballpark as 18 independent readings of that
  // ballpark, and the interval collapses to a width the evidence never earned —
  // so the gate PASSES a factor on sample it does not have. Measured here: 928
  // total_bases rows carry only 53 distinct games.
  //
  // When rows carry a `cluster`, resample whole clusters. The interval then
  // reflects the unit the treatment actually varies over. Rows without a
  // cluster keep the original row-resampling path byte-for-byte.
  const clustered = usable.some((r) => r.cluster != null);
  const groups = new Map();
  if (clustered) {
    for (const r of usable) {
      const k = String(r.cluster);
      if (!groups.has(k)) groups.set(k, []);
      groups.get(k).push(r);
    }
  }
  const keys = clustered ? [...groups.keys()] : null;

  for (let it = 0; it < iters; it += 1) {
    const b = []; const c = []; const y = [];
    if (clustered) {
      for (let i = 0; i < keys.length; i += 1) {
        const g = groups.get(keys[Math.floor(rnd() * keys.length)]);
        for (const r of g) { b.push(r.baseline); c.push(r.conditioned); y.push(r.won > 0 ? 1 : 0); }
      }
    } else {
      for (let i = 0; i < usable.length; i += 1) {
        const r = usable[Math.floor(rnd() * usable.length)];
        b.push(r.baseline); c.push(r.conditioned); y.push(r.won > 0 ? 1 : 0);
      }
    }
    diffs.push(brier(c, y) - brier(b, y));
  }
  diffs.sort((x, y) => x - y);
  // CUMULATIVE CORRECTION APPLIED TO THE INTERVAL ITSELF. A plain 95% CI is the
  // right bar for ONE test and far too lenient for a programme that has run
  // dozens: at 50 cumulative tests, roughly two or three 95% intervals exclude
  // zero by chance alone. So the interval widens to 1 − 0.05/tests, which is the
  // same discipline the p-value gate applies, expressed as an interval.
  const tests = Math.max(1, Math.round(knownNumber(cumulativeTests) ?? 1));
  const alpha = 0.05 / tests;
  const q = (p) => round4(diffs[Math.floor(Math.min(diffs.length - 1, Math.max(0, p * (diffs.length - 1))))]);
  const ci = [q(alpha / 2), q(1 - alpha / 2)];
  const ys = usable.map((r) => (r.won > 0 ? 1 : 0));
  return {
    n: usable.length,
    // The number the gate must actually judge sample against.
    effective_n: clustered ? keys.length : usable.length,
    cluster_unit: clustered ? 'cluster' : 'row',
    brier_baseline: round4(brier(usable.map((r) => r.baseline), ys)),
    brier_conditioned: round4(brier(usable.map((r) => r.conditioned), ys)),
    brier_delta: round4(brier(usable.map((r) => r.conditioned), ys) - brier(usable.map((r) => r.baseline), ys)),
    ci: ci,
    ci_level: round4(1 - alpha),
    bonferroni_tests: tests,
    improves: ci[1] < 0,       // whole interval below zero = genuinely better
    degrades: ci[0] > 0,
  };
}

/**
 * THE VERDICT. Both conditions, named outcomes.
 *
 * `cumulativeTests` is the programme-lifetime Bonferroni denominator; it tightens
 * the improvement requirement the same way it does everywhere else.
 */
function adjudicate(rows, opts = {}) {
  const minN = opts.minN ?? MIN_N;
  const minMove = opts.minMovement ?? MIN_MOVEMENT;
  const mv = movement(rows);
  const imp = improvement(rows, opts.iters, opts.seed, opts.cumulativeTests);

  const base = { factor: opts.factor || null, archetype: opts.archetype || null, stat: opts.stat || 'hits', movement: mv, improvement: imp };

  // TWO FLOORS, because they answer different questions. Rows decide whether the
  // point estimate is stable; clusters decide whether the interval around it can
  // be believed. A factor needs both.
  if (mv.n < minN) {
    return { ...base, verdict: 'CANDIDATE_PENDING_SAMPLE', reason: `n ${mv.n} < ${minN}`, rows_needed: minN - mv.n };
  }
  const minClusters = opts.minClusters ?? MIN_CLUSTERS;
  if (imp && imp.cluster_unit === 'cluster' && imp.effective_n < minClusters) {
    return {
      ...base,
      verdict: 'CANDIDATE_PENDING_SAMPLE',
      reason: `${mv.n} rows but only ${imp.effective_n} independent clusters < ${minClusters}`
        + ' — the rows are not independent readings and the interval cannot be trusted at this cluster count',
      clusters_needed: minClusters - imp.effective_n,
    };
  }
  if (mv.mean_abs_shift === null || mv.mean_abs_shift < minMove) {
    // It never moved the number, so it cannot be reading anything.
    return { ...base, verdict: 'INERT', reason: `mean |shift| ${mv.mean_abs_shift} < ${minMove}` };
  }
  if (!imp) return { ...base, verdict: 'CANDIDATE_PENDING_SAMPLE', reason: 'too few paired rows to bootstrap' };

  // NOT PROVEN is not the same as THEATER, and collapsing them would repeat the
  // error this codebase keeps having to correct: insufficient evidence is not
  // evidence of absence. A factor whose POINT ESTIMATE improves accuracy but
  // whose corrected interval still spans zero has not earned its place — and it
  // is not decorative either. It is a real candidate held to a bar that rises
  // with every hypothesis the programme tests. Saying so keeps THEATER meaning
  // the one thing it must mean: moves the number, reads nothing.
  if (!imp.improves && imp.brier_delta < 0) {
    return {
      ...base,
      verdict: 'NOT_PROVEN_AT_CORRECTED_BAR',
      reason: `moves ${mv.mean_abs_shift} and the point estimate improves Brier by ${-imp.brier_delta}, but the interval corrected for ${imp.bonferroni_tests} tests still spans zero (${JSON.stringify(imp.ci)} at level ${imp.ci_level})`,
      note: 'a real candidate, not theatre — it improves on the point estimate and needs more sample, or a tighter bar than the programme can currently afford it',
    };
  }
  if (imp.improves) {
    return { ...base, verdict: 'PROVES', reason: `moves ${mv.mean_abs_shift} and improves Brier by ${-imp.brier_delta} (CI ${JSON.stringify(imp.ci)} at level ${imp.ci_level}, corrected for ${imp.bonferroni_tests} tests)` };
  }
  // MOVED BUT DID NOT IMPROVE. Named, not softened.
  return {
    ...base,
    verdict: 'THEATER',
    reason: `moves the prediction by ${mv.mean_abs_shift} on average (max ${mv.max_abs_shift}) while accuracy does NOT improve (Brier delta ${imp.brier_delta}, CI ${JSON.stringify(imp.ci)} at level ${imp.ci_level})`,
    consequence: "wiring this would make the grade LOOK like it read tonight's game while reading nothing — the failure mode a user cannot detect from outside",
  };
}

const round4 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 10000) / 10000);

module.exports = {
  MIN_CLUSTERS, movement, improvement, adjudicate, MIN_MOVEMENT, MIN_N };