'use strict'; /** * The two-part factor gate. * * The case these tests exist for is THEATER: a factor that moves the number * convincingly and improves nothing. A correlation test cannot see it, and * neither can a user — arch-v1 moved 76% of rows by 2.5 points, changed * resolution by 0.0000, and stayed live for months. */ const fg = require('../../src/services/model/factorGate'); /** n rows at a fixed baseline, with the conditioned value shifted by `shift(i)` * and the outcome determined by `trueP(i)` — so a factor can be made genuinely * informative or purely decorative on demand. */ function rows(n, baseline, shift, trueP) { const out = []; for (let i = 0; i < n; i += 1) { const p = typeof trueP === 'function' ? trueP(i) : trueP; // Interleaved outcomes, never front-loaded — front-loading correlates the // outcome with position and quietly rigs any split. const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0; out.push({ baseline, conditioned: Math.min(0.99, Math.max(0.01, baseline + (typeof shift === 'function' ? shift(i) : shift))), won, }); } return out; } describe('THEATER — moves the number, reads nothing', () => { it('is REJECTED BY NAME, not filed as inconclusive', () => { // Truth is a flat 0.5. The factor swings the prediction ±0.15 at random // relative to the outcome, so it looks responsive and knows nothing. const r = rows(800, 0.5, (i) => (i % 2 === 0 ? 0.15 : -0.15), 0.5); const v = fg.adjudicate(r, { factor: 'decorative' }); expect(v.verdict).toBe('THEATER'); expect(v.movement.mean_abs_shift).toBeCloseTo(0.15, 2); expect(v.improvement.improves).toBe(false); expect(v.consequence).toMatch(/LOOK like it read/); }); it('an unproven-but-favourable factor is NOT called theatre', () => { // Point estimate improves, corrected interval spans zero. That is a real // candidate held to a rising bar — collapsing it into THEATER would repeat // the "insufficient evidence = evidence of absence" error. const r = []; for (let i = 0; i < 800; i += 1) { const hot = i % 2 === 0; const p = hot ? 0.56 : 0.44; const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0; r.push({ baseline: 0.5, conditioned: hot ? 0.53 : 0.47, won }); } const v = fg.adjudicate(r, { factor: 'weak-but-real', cumulativeTests: 200 }); expect(['NOT_PROVEN_AT_CORRECTED_BAR', 'PROVES']).toContain(v.verdict); if (v.verdict === 'NOT_PROVEN_AT_CORRECTED_BAR') { expect(v.improvement.brier_delta).toBeLessThan(0); expect(v.note).toMatch(/not theatre/); } }); it('a factor that moves a LOT is not thereby better — that is the trap', () => { const big = fg.adjudicate(rows(800, 0.5, (i) => (i % 2 === 0 ? 0.3 : -0.3), 0.5), { factor: 'loud' }); const small = fg.adjudicate(rows(800, 0.5, (i) => (i % 2 === 0 ? 0.02 : -0.02), 0.5), { factor: 'quiet' }); expect(big.verdict).toBe('THEATER'); expect(small.verdict).toBe('THEATER'); expect(big.movement.mean_abs_shift).toBeGreaterThan(small.movement.mean_abs_shift * 5); }); }); describe('PROVES — moves the number AND gets closer to the truth', () => { it('passes a factor that genuinely splits the population', () => { // Truth alternates 0.8 / 0.2; the factor moves the prediction the right way. const r = []; for (let i = 0; i < 800; i += 1) { const hot = i % 2 === 0; const p = hot ? 0.8 : 0.2; const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0; r.push({ baseline: 0.5, conditioned: hot ? 0.78 : 0.22, won }); } const v = fg.adjudicate(r, { factor: 'real' }); expect(v.verdict).toBe('PROVES'); expect(v.improvement.brier_delta).toBeLessThan(0); expect(v.improvement.ci[1]).toBeLessThan(0); }); }); describe('INERT and PENDING are distinct from THEATER', () => { it('a factor that never moves the number is INERT, not theatre', () => { const v = fg.adjudicate(rows(800, 0.5, 0.0005, 0.5), { factor: 'flat' }); expect(v.verdict).toBe('INERT'); // Nothing was claimed, so nothing is misleading — a different problem. }); it('thin sample is CANDIDATE_PENDING_SAMPLE with the rows still needed', () => { const v = fg.adjudicate(rows(120, 0.5, 0.1, 0.5), { factor: 'thin' }); expect(v.verdict).toBe('CANDIDATE_PENDING_SAMPLE'); expect(v.rows_needed).toBe(380); // Crucially NOT 'THEATER' — it might work; we simply cannot tell yet. }); }); describe('the measurements themselves', () => { it('movement reports spread, because a constant shift reads nothing either', () => { const constant = fg.movement(rows(200, 0.5, 0.1, 0.5)); const varied = fg.movement(rows(200, 0.5, (i) => (i % 2 ? 0.1 : -0.1), 0.5)); expect(constant.sd_shift).toBeCloseTo(0, 6); expect(varied.sd_shift).toBeGreaterThan(0.09); }); it('an unreadable side is DROPPED, never treated as no-change', () => { const m = fg.movement([ { baseline: 0.5, conditioned: 0.6 }, { baseline: null, conditioned: 0.9 }, { baseline: 0.5, conditioned: null }, ]); expect(m.n).toBe(1); }); it('improvement uses a PAIRED bootstrap — same rows score both models', () => { const r = rows(400, 0.5, 0.0, 0.5); const imp = fg.improvement(r); // Identical models must show no difference and a CI spanning zero. expect(imp.brier_delta).toBeCloseTo(0, 6); expect(imp.improves).toBe(false); expect(imp.degrades).toBe(false); }); it('the interval WIDENS with the cumulative test count — the bar rises', () => { const r = []; for (let i = 0; i < 800; i += 1) { const hot = i % 2 === 0; const p = hot ? 0.8 : 0.2; const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0; r.push({ baseline: 0.5, conditioned: hot ? 0.78 : 0.22, won }); } const one = fg.improvement(r, 3000, 1, 1); const fifty = fg.improvement(r, 3000, 1, 50); expect(fifty.ci_level).toBeGreaterThan(one.ci_level); // A wider interval can only ever make PROVES harder, never easier. expect(fifty.ci[1]).toBeGreaterThanOrEqual(one.ci[1]); }); it('a factor that makes the number WORSE is flagged degrading', () => { const r = []; for (let i = 0; i < 600; i += 1) { const p = 0.8; const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0; r.push({ baseline: 0.8, conditioned: 0.2, won }); // confidently backwards } const imp = fg.improvement(r); expect(imp.degrades).toBe(true); expect(fg.adjudicate(r, { factor: 'backwards' }).verdict).toBe('THEATER'); }); });