vyndr/tests/unit/backtestHarness.test.js

/**
 * Session 64 — BACKTEST HARNESS.
 *
 * The harness is the validator every future metric family is gated on. Its
 * FIRST proven property is not the math — it is the REFUSAL. A harness that
 * emits a confident number on 4 rows is worse than no harness, because it
 * launders noise into a claim. These tests are written first, deliberately.
 */

const h = require('../../src/services/backtestHarness');

const row = (o = {}) => ({
  sport: 'mlb', model_version: 'engine1@2026-07-20',
  grade: 'B', grade_11: 'B-', p_win: 0.6,
  outcome: 'hit', quarantine_reason: null, snap_quarantine: null,
  features: { l5_avg: 1.2 },
  ...o,
});

describe('THE REFUSAL — sample-size honesty is the first property', () => {
  test('a tiny sample reports INSUFFICIENT, never a point estimate', () => {
    const rows = [row(), row({ outcome: 'miss' }), row()];
    const out = h.runBacktest(rows, { minSample: 20 });
    const b = out.grade_buckets.find((x) => x.bucket === 'B');
    expect(b.status).toBe('INSUFFICIENT');
    expect(b.n).toBe(3);
    expect(b.need).toBe(20);
    expect(b.hit_rate).toBeNull();          // no number, at all
  });

  test('the top-line verdict itself refuses on thin data', () => {
    const out = h.runBacktest([row(), row()], { minSample: 20 });
    expect(out.verdict).toBe('INSUFFICIENT_HISTORY');
    expect(out.can_validate).toBe(false);
  });

  test('an INSUFFICIENT bucket carries no rate anywhere in its payload', () => {
    const out = h.runBacktest([row()], { minSample: 20 });
    const json = JSON.stringify(out.grade_buckets);
    expect(json).not.toMatch(/"hit_rate":\s*[0-9]/);
  });

  test('at/above threshold it DOES report a rate with an interval', () => {
    const rows = Array.from({ length: 25 }, (_, i) => row({ outcome: i < 15 ? 'hit' : 'miss' }));
    const out = h.runBacktest(rows, { minSample: 20 });
    const b = out.grade_buckets.find((x) => x.bucket === 'B');
    expect(b.status).toBe('OK');
    expect(b.hit_rate).toBeCloseTo(0.6, 2);
    expect(b.ci_low).toBeLessThan(0.6);
    expect(b.ci_high).toBeGreaterThan(0.6);
  });
});

describe('DENOMINATOR — the excluded states', () => {
  test('quarantined, void, unrecoverable, pending and push never count', () => {
    const rows = [
      row({ outcome: 'hit' }), row({ outcome: 'miss' }),
      row({ outcome: 'void' }), row({ outcome: 'unrecoverable' }),
      row({ outcome: null }), row({ outcome: 'push' }),
      row({ outcome: 'hit', quarantine_reason: 'wrong_opponent_grade' }),
      row({ outcome: 'hit', snap_quarantine: 'wrong_opponent_grade' }),
    ];
    const out = h.runBacktest(rows, { minSample: 1 });
    expect(out.counts.scored).toBe(2);          // only the real hit + miss
    expect(out.counts.excluded_quarantine).toBe(2);
    expect(out.counts.excluded_terminal).toBe(2);
    expect(out.counts.excluded_pending).toBe(1);
    expect(out.counts.excluded_push).toBe(1);
  });
});

describe('NEVER MIX sports or model versions', () => {
  test('buckets are split per sport and per version', () => {
    const rows = [
      ...Array.from({ length: 21 }, () => row({ sport: 'mlb' })),
      ...Array.from({ length: 21 }, () => row({ sport: 'wnba' })),
      ...Array.from({ length: 21 }, () => row({ model_version: 'engine2@x' })),
    ];
    const out = h.runBacktest(rows, { minSample: 20 });
    const keys = out.strata.map((s) => `${s.sport}|${s.model_version}`);
    expect(new Set(keys).size).toBe(3);
  });
});

describe('MONOTONICITY — and refusing to rank overlapping buckets', () => {
  test('reports overlapping intervals as not distinguishable, not as a ranking', () => {
    const mk = (grade, hits, n) => Array.from({ length: n }, (_, i) => row({ grade, outcome: i < hits ? 'hit' : 'miss' }));
    // A: 13/25 (52%), B: 14/25 (56%) — intervals overlap heavily.
    const out = h.runBacktest([...mk('A', 13, 25), ...mk('B', 14, 25)], { minSample: 20 });
    const cmp = out.monotonicity.comparisons.find((c) => c.pair === 'A>B' || c.pair === 'B>A');
    expect(cmp.distinguishable).toBe(false);
    expect(out.monotonicity.verdict).toMatch(/NOT_DISTINGUISHABLE|INSUFFICIENT/);
  });
});

describe('PROBABILITY calibration', () => {
  test('a thin sample reports INSUFFICIENT and NO Brier score', () => {
    const rows = [row({ p_win: 0.8 }), row({ p_win: 0.2, outcome: 'miss' })];
    const out = h.runBacktest(rows, { minSample: 20 });
    expect(out.probability.status).toBe('INSUFFICIENT');
    expect(out.probability.brier).toBeNull();
  });

  test('Brier score is computed from p_win vs realized outcome', () => {
    const rows = [row({ p_win: 1, outcome: 'hit' }), row({ p_win: 0, outcome: 'miss' })];
    const out = h.runBacktest(rows, { minSample: 1 });
    expect(out.probability.status).toBe('OK');
    expect(out.probability.brier).toBeCloseTo(0, 6);   // perfect prediction
  });

  test('rows without p_win are excluded from Brier, not defaulted to 0.5', () => {
    const rows = [row({ p_win: null, outcome: 'hit' }), row({ p_win: 1, outcome: 'hit' })];
    const out = h.runBacktest(rows, { minSample: 1 });
    expect(out.probability.n).toBe(1);
  });
});

describe('ALTERNATIVE-MODEL REPLAY SEAM', () => {
  test('identity replay reproduces the live prediction exactly', () => {
    const rows = [row({ p_win: 0.7 }), row({ p_win: 0.3, outcome: 'miss' })];
    const res = h.replayAlternative(rows, (r) => ({ p_win: r.p_win }), { minSample: 1 });
    expect(res.identical).toBe(true);
    expect(res.candidate.brier).toBeCloseTo(res.live.brier, 9);
  });

  test('a row whose needed input was never retained is UN-BACKTESTABLE, not guessed', () => {
    const rows = [row({ features: {} })];
    const res = h.replayAlternative(rows, (r) => (
      r.features && r.features.opp_rank_stat != null
        ? { p_win: 0.9 }
        : { unavailable: 'opp_rank_stat_not_retained' }
    ), { minSample: 1 });
    expect(res.un_backtestable).toBe(1);
    expect(res.candidate.n).toBe(0);
  });

  test('the seam never refetches — it only sees the stored feature vector', () => {
    let sawOnlyStored = true;
    h.replayAlternative([row()], (r) => {
      if (Object.keys(r).some((k) => /fetch|client|http/i.test(k))) sawOnlyStored = false;
      return { p_win: 0.5 };
    }, { minSample: 1 });
    expect(sawOnlyStored).toBe(true);
  });
});