vyndr/tests/unit/factorGate.test.js

'use strict';

/**
 * The two-part factor gate.
 *
 * The case these tests exist for is THEATER: a factor that moves the number
 * convincingly and improves nothing. A correlation test cannot see it, and
 * neither can a user — arch-v1 moved 76% of rows by 2.5 points, changed
 * resolution by 0.0000, and stayed live for months.
 */

const fg = require('../../src/services/model/factorGate');

/** n rows at a fixed baseline, with the conditioned value shifted by `shift(i)`
 *  and the outcome determined by `trueP(i)` — so a factor can be made genuinely
 *  informative or purely decorative on demand. */
function rows(n, baseline, shift, trueP) {
  const out = [];
  for (let i = 0; i < n; i += 1) {
    const p = typeof trueP === 'function' ? trueP(i) : trueP;
    // Interleaved outcomes, never front-loaded — front-loading correlates the
    // outcome with position and quietly rigs any split.
    const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0;
    out.push({
      baseline,
      conditioned: Math.min(0.99, Math.max(0.01, baseline + (typeof shift === 'function' ? shift(i) : shift))),
      won,
    });
  }
  return out;
}

describe('THEATER — moves the number, reads nothing', () => {
  it('is REJECTED BY NAME, not filed as inconclusive', () => {
    // Truth is a flat 0.5. The factor swings the prediction ±0.15 at random
    // relative to the outcome, so it looks responsive and knows nothing.
    const r = rows(800, 0.5, (i) => (i % 2 === 0 ? 0.15 : -0.15), 0.5);
    const v = fg.adjudicate(r, { factor: 'decorative' });
    expect(v.verdict).toBe('THEATER');
    expect(v.movement.mean_abs_shift).toBeCloseTo(0.15, 2);
    expect(v.improvement.improves).toBe(false);
    expect(v.consequence).toMatch(/LOOK like it read/);
  });

  it('an unproven-but-favourable factor is NOT called theatre', () => {
    // Point estimate improves, corrected interval spans zero. That is a real
    // candidate held to a rising bar — collapsing it into THEATER would repeat
    // the "insufficient evidence = evidence of absence" error.
    const r = [];
    for (let i = 0; i < 800; i += 1) {
      const hot = i % 2 === 0; const p = hot ? 0.56 : 0.44;
      const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0;
      r.push({ baseline: 0.5, conditioned: hot ? 0.53 : 0.47, won });
    }
    const v = fg.adjudicate(r, { factor: 'weak-but-real', cumulativeTests: 200 });
    expect(['NOT_PROVEN_AT_CORRECTED_BAR', 'PROVES']).toContain(v.verdict);
    if (v.verdict === 'NOT_PROVEN_AT_CORRECTED_BAR') {
      expect(v.improvement.brier_delta).toBeLessThan(0);
      expect(v.note).toMatch(/not theatre/);
    }
  });

  it('a factor that moves a LOT is not thereby better — that is the trap', () => {
    const big = fg.adjudicate(rows(800, 0.5, (i) => (i % 2 === 0 ? 0.3 : -0.3), 0.5), { factor: 'loud' });
    const small = fg.adjudicate(rows(800, 0.5, (i) => (i % 2 === 0 ? 0.02 : -0.02), 0.5), { factor: 'quiet' });
    expect(big.verdict).toBe('THEATER');
    expect(small.verdict).toBe('THEATER');
    expect(big.movement.mean_abs_shift).toBeGreaterThan(small.movement.mean_abs_shift * 5);
  });
});

describe('PROVES — moves the number AND gets closer to the truth', () => {
  it('passes a factor that genuinely splits the population', () => {
    // Truth alternates 0.8 / 0.2; the factor moves the prediction the right way.
    const r = [];
    for (let i = 0; i < 800; i += 1) {
      const hot = i % 2 === 0;
      const p = hot ? 0.8 : 0.2;
      const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0;
      r.push({ baseline: 0.5, conditioned: hot ? 0.78 : 0.22, won });
    }
    const v = fg.adjudicate(r, { factor: 'real' });
    expect(v.verdict).toBe('PROVES');
    expect(v.improvement.brier_delta).toBeLessThan(0);
    expect(v.improvement.ci[1]).toBeLessThan(0);
  });
});

describe('INERT and PENDING are distinct from THEATER', () => {
  it('a factor that never moves the number is INERT, not theatre', () => {
    const v = fg.adjudicate(rows(800, 0.5, 0.0005, 0.5), { factor: 'flat' });
    expect(v.verdict).toBe('INERT');
    // Nothing was claimed, so nothing is misleading — a different problem.
  });

  it('thin sample is CANDIDATE_PENDING_SAMPLE with the rows still needed', () => {
    const v = fg.adjudicate(rows(120, 0.5, 0.1, 0.5), { factor: 'thin' });
    expect(v.verdict).toBe('CANDIDATE_PENDING_SAMPLE');
    expect(v.rows_needed).toBe(380);
    // Crucially NOT 'THEATER' — it might work; we simply cannot tell yet.
  });
});

describe('the measurements themselves', () => {
  it('movement reports spread, because a constant shift reads nothing either', () => {
    const constant = fg.movement(rows(200, 0.5, 0.1, 0.5));
    const varied = fg.movement(rows(200, 0.5, (i) => (i % 2 ? 0.1 : -0.1), 0.5));
    expect(constant.sd_shift).toBeCloseTo(0, 6);
    expect(varied.sd_shift).toBeGreaterThan(0.09);
  });

  it('an unreadable side is DROPPED, never treated as no-change', () => {
    const m = fg.movement([
      { baseline: 0.5, conditioned: 0.6 },
      { baseline: null, conditioned: 0.9 },
      { baseline: 0.5, conditioned: null },
    ]);
    expect(m.n).toBe(1);
  });

  it('improvement uses a PAIRED bootstrap — same rows score both models', () => {
    const r = rows(400, 0.5, 0.0, 0.5);
    const imp = fg.improvement(r);
    // Identical models must show no difference and a CI spanning zero.
    expect(imp.brier_delta).toBeCloseTo(0, 6);
    expect(imp.improves).toBe(false);
    expect(imp.degrades).toBe(false);
  });

  it('the interval WIDENS with the cumulative test count — the bar rises', () => {
    const r = [];
    for (let i = 0; i < 800; i += 1) {
      const hot = i % 2 === 0; const p = hot ? 0.8 : 0.2;
      const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0;
      r.push({ baseline: 0.5, conditioned: hot ? 0.78 : 0.22, won });
    }
    const one = fg.improvement(r, 3000, 1, 1);
    const fifty = fg.improvement(r, 3000, 1, 50);
    expect(fifty.ci_level).toBeGreaterThan(one.ci_level);
    // A wider interval can only ever make PROVES harder, never easier.
    expect(fifty.ci[1]).toBeGreaterThanOrEqual(one.ci[1]);
  });

  it('a factor that makes the number WORSE is flagged degrading', () => {
    const r = [];
    for (let i = 0; i < 600; i += 1) {
      const p = 0.8;
      const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0;
      r.push({ baseline: 0.8, conditioned: 0.2, won });   // confidently backwards
    }
    const imp = fg.improvement(r);
    expect(imp.degrades).toBe(true);
    expect(fg.adjudicate(r, { factor: 'backwards' }).verdict).toBe('THEATER');
  });
});

describe('pseudo-replication — sample counted in the unit the factor varies over', () => {
  // A game-level factor (park, weather, opposing starter) hands every prop row
  // in a game the identical treatment. Eighteen hitters in one ballpark are one
  // reading of that ballpark, not eighteen.
  const build = (games, perGame, seed = 1) => {
    let s = seed;
    const rnd = () => (s = (s * 1103515245 + 12345) % 2147483648) / 2147483648;
    const rows = [];
    for (let g = 0; g < games; g += 1) {
      const shift = (rnd() - 0.5) * 0.06;      // the game's treatment
      // Outcomes are correlated WITHIN a game — a high-scoring night lifts every
      // hitter in it. That shared component is exactly what row-resampling
      // cannot see and what makes 18 rows worth far less than 18 readings.
      const gameLevel = (rnd() - 0.5) * 0.5;
      for (let i = 0; i < perGame; i += 1) {
        const base = 0.3 + rnd() * 0.4;
        const p = Math.max(0.02, Math.min(0.98, base + gameLevel));
        rows.push({ cluster: `g${g}`, baseline: base, conditioned: base + shift, won: rnd() < p ? 1 : 0 });
      }
    }
    return rows;
  };

  it('judges sample by CLUSTERS, so 900 rows over 50 games is 50 readings', () => {
    const rows = build(50, 18);
    // 900 rows clears the row floor; 50 clusters is what actually decides it.
    const v = fg.adjudicate(rows, { factor: 'park', minN: 500, minClusters: 100 });
    expect(rows.length).toBeGreaterThan(500);          // looks like plenty
    expect(v.verdict).toBe('CANDIDATE_PENDING_SAMPLE'); // and is not
    expect(v.improvement.effective_n).toBe(50);
    expect(v.improvement.cluster_unit).toBe('cluster');
    expect(v.reason).toMatch(/not independent readings/);
  });

  it('the clustered interval is WIDER than the row interval on the same rows', () => {
    // This is the whole hazard: resampling rows would have manufactured a
    // confidence the evidence never supported.
    const rows = build(40, 20, 7);
    const clustered = fg.adjudicate(rows, { factor: 'park', minN: 10, minClusters: 5 });
    const flat = fg.adjudicate(rows.map(({ cluster, ...r }) => r), { factor: 'park', minN: 10 });
    const width = (v) => v.improvement.ci[1] - v.improvement.ci[0];
    expect(width(clustered)).toBeGreaterThan(width(flat));
  });

  it('rows with no cluster keep the original row-resampling behaviour', () => {
    const rows = build(40, 20, 3).map(({ cluster, ...r }) => r);
    const v = fg.adjudicate(rows, { factor: 'x', minN: 10, minClusters: 5 });
    expect(v.improvement.cluster_unit).toBe('row');
    expect(v.improvement.effective_n).toBe(v.improvement.n);
  });
});

describe('the two floors answer different questions', () => {
  const rows = [];
  let s2 = 11;
  const rnd = () => (s2 = (s2 * 1103515245 + 12345) % 2147483648) / 2147483648;
  for (let g = 0; g < 60; g += 1) {
    for (let i = 0; i < 20; i += 1) {
      const base = 0.35 + rnd() * 0.3;
      rows.push({ cluster: `g${g}`, baseline: base, conditioned: base - 0.02, won: rnd() < base ? 1 : 0 });
    }
  }

  it('ample rows with too FEW clusters is refused on the cluster floor', () => {
    const v = fg.adjudicate(rows, { minN: 500, minClusters: 200 });
    expect(v.verdict).toBe('CANDIDATE_PENDING_SAMPLE');
    expect(v.reason).toMatch(/independent clusters/);
    expect(v.clusters_needed).toBe(140);
  });

  it('too few ROWS is refused on the row floor even with many clusters', () => {
    const thin = rows.filter((_, i) => i % 20 === 0);   // 60 rows, 60 clusters
    const v = fg.adjudicate(thin, { minN: 500, minClusters: 40 });
    expect(v.verdict).toBe('CANDIDATE_PENDING_SAMPLE');
    expect(v.reason).toMatch(/^n 60 < 500/);
  });

  it('a venue-constant factor stays refused however many rows accrue', () => {
    // 30 ballparks is the whole universe; rows can grow forever and the
    // interval never becomes trustworthy.
    const venues = [];
    for (let v = 0; v < 30; v += 1) {
      for (let i = 0; i < 200; i += 1) {
        const base = 0.4 + rnd() * 0.2;
        venues.push({ cluster: `v${v}`, baseline: base, conditioned: base - 0.03, won: rnd() < base ? 1 : 0 });
      }
    }
    const v = fg.adjudicate(venues, { minN: 500 });
    expect(venues.length).toBe(6000);
    expect(v.verdict).toBe('CANDIDATE_PENDING_SAMPLE');
    expect(v.improvement.effective_n).toBe(30);
  });
});