vyndr/tests/unit/factorGate.test.js

'use strict';

/**
 * The two-part factor gate.
 *
 * The case these tests exist for is THEATER: a factor that moves the number
 * convincingly and improves nothing. A correlation test cannot see it, and
 * neither can a user — arch-v1 moved 76% of rows by 2.5 points, changed
 * resolution by 0.0000, and stayed live for months.
 */

const fg = require('../../src/services/model/factorGate');

/** n rows at a fixed baseline, with the conditioned value shifted by `shift(i)`
 *  and the outcome determined by `trueP(i)` — so a factor can be made genuinely
 *  informative or purely decorative on demand. */
function rows(n, baseline, shift, trueP) {
  const out = [];
  for (let i = 0; i < n; i += 1) {
    const p = typeof trueP === 'function' ? trueP(i) : trueP;
    // Interleaved outcomes, never front-loaded — front-loading correlates the
    // outcome with position and quietly rigs any split.
    const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0;
    out.push({
      baseline,
      conditioned: Math.min(0.99, Math.max(0.01, baseline + (typeof shift === 'function' ? shift(i) : shift))),
      won,
    });
  }
  return out;
}

describe('THEATER — moves the number, reads nothing', () => {
  it('is REJECTED BY NAME, not filed as inconclusive', () => {
    // Truth is a flat 0.5. The factor swings the prediction ±0.15 at random
    // relative to the outcome, so it looks responsive and knows nothing.
    const r = rows(800, 0.5, (i) => (i % 2 === 0 ? 0.15 : -0.15), 0.5);
    const v = fg.adjudicate(r, { factor: 'decorative' });
    expect(v.verdict).toBe('THEATER');
    expect(v.movement.mean_abs_shift).toBeCloseTo(0.15, 2);
    expect(v.improvement.improves).toBe(false);
    expect(v.consequence).toMatch(/LOOK like it read/);
  });

  it('an unproven-but-favourable factor is NOT called theatre', () => {
    // Point estimate improves, corrected interval spans zero. That is a real
    // candidate held to a rising bar — collapsing it into THEATER would repeat
    // the "insufficient evidence = evidence of absence" error.
    const r = [];
    for (let i = 0; i < 800; i += 1) {
      const hot = i % 2 === 0; const p = hot ? 0.56 : 0.44;
      const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0;
      r.push({ baseline: 0.5, conditioned: hot ? 0.53 : 0.47, won });
    }
    const v = fg.adjudicate(r, { factor: 'weak-but-real', cumulativeTests: 200 });
    expect(['NOT_PROVEN_AT_CORRECTED_BAR', 'PROVES']).toContain(v.verdict);
    if (v.verdict === 'NOT_PROVEN_AT_CORRECTED_BAR') {
      expect(v.improvement.brier_delta).toBeLessThan(0);
      expect(v.note).toMatch(/not theatre/);
    }
  });

  it('a factor that moves a LOT is not thereby better — that is the trap', () => {
    const big = fg.adjudicate(rows(800, 0.5, (i) => (i % 2 === 0 ? 0.3 : -0.3), 0.5), { factor: 'loud' });
    const small = fg.adjudicate(rows(800, 0.5, (i) => (i % 2 === 0 ? 0.02 : -0.02), 0.5), { factor: 'quiet' });
    expect(big.verdict).toBe('THEATER');
    expect(small.verdict).toBe('THEATER');
    expect(big.movement.mean_abs_shift).toBeGreaterThan(small.movement.mean_abs_shift * 5);
  });
});

describe('PROVES — moves the number AND gets closer to the truth', () => {
  it('passes a factor that genuinely splits the population', () => {
    // Truth alternates 0.8 / 0.2; the factor moves the prediction the right way.
    const r = [];
    for (let i = 0; i < 800; i += 1) {
      const hot = i % 2 === 0;
      const p = hot ? 0.8 : 0.2;
      const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0;
      r.push({ baseline: 0.5, conditioned: hot ? 0.78 : 0.22, won });
    }
    const v = fg.adjudicate(r, { factor: 'real' });
    expect(v.verdict).toBe('PROVES');
    expect(v.improvement.brier_delta).toBeLessThan(0);
    expect(v.improvement.ci[1]).toBeLessThan(0);
  });
});

describe('INERT and PENDING are distinct from THEATER', () => {
  it('a factor that never moves the number is INERT, not theatre', () => {
    const v = fg.adjudicate(rows(800, 0.5, 0.0005, 0.5), { factor: 'flat' });
    expect(v.verdict).toBe('INERT');
    // Nothing was claimed, so nothing is misleading — a different problem.
  });

  it('thin sample is CANDIDATE_PENDING_SAMPLE with the rows still needed', () => {
    const v = fg.adjudicate(rows(120, 0.5, 0.1, 0.5), { factor: 'thin' });
    expect(v.verdict).toBe('CANDIDATE_PENDING_SAMPLE');
    expect(v.rows_needed).toBe(380);
    // Crucially NOT 'THEATER' — it might work; we simply cannot tell yet.
  });
});

describe('the measurements themselves', () => {
  it('movement reports spread, because a constant shift reads nothing either', () => {
    const constant = fg.movement(rows(200, 0.5, 0.1, 0.5));
    const varied = fg.movement(rows(200, 0.5, (i) => (i % 2 ? 0.1 : -0.1), 0.5));
    expect(constant.sd_shift).toBeCloseTo(0, 6);
    expect(varied.sd_shift).toBeGreaterThan(0.09);
  });

  it('an unreadable side is DROPPED, never treated as no-change', () => {
    const m = fg.movement([
      { baseline: 0.5, conditioned: 0.6 },
      { baseline: null, conditioned: 0.9 },
      { baseline: 0.5, conditioned: null },
    ]);
    expect(m.n).toBe(1);
  });

  it('improvement uses a PAIRED bootstrap — same rows score both models', () => {
    const r = rows(400, 0.5, 0.0, 0.5);
    const imp = fg.improvement(r);
    // Identical models must show no difference and a CI spanning zero.
    expect(imp.brier_delta).toBeCloseTo(0, 6);
    expect(imp.improves).toBe(false);
    expect(imp.degrades).toBe(false);
  });

  it('the interval WIDENS with the cumulative test count — the bar rises', () => {
    const r = [];
    for (let i = 0; i < 800; i += 1) {
      const hot = i % 2 === 0; const p = hot ? 0.8 : 0.2;
      const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0;
      r.push({ baseline: 0.5, conditioned: hot ? 0.78 : 0.22, won });
    }
    const one = fg.improvement(r, 3000, 1, 1);
    const fifty = fg.improvement(r, 3000, 1, 50);
    expect(fifty.ci_level).toBeGreaterThan(one.ci_level);
    // A wider interval can only ever make PROVES harder, never easier.
    expect(fifty.ci[1]).toBeGreaterThanOrEqual(one.ci[1]);
  });

  it('a factor that makes the number WORSE is flagged degrading', () => {
    const r = [];
    for (let i = 0; i < 600; i += 1) {
      const p = 0.8;
      const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0;
      r.push({ baseline: 0.8, conditioned: 0.2, won });   // confidently backwards
    }
    const imp = fg.improvement(r);
    expect(imp.degrades).toBe(true);
    expect(fg.adjudicate(r, { factor: 'backwards' }).verdict).toBe('THEATER');
  });
});