'use strict'; /** * The two-part factor gate. * * The case these tests exist for is THEATER: a factor that moves the number * convincingly and improves nothing. A correlation test cannot see it, and * neither can a user — arch-v1 moved 76% of rows by 2.5 points, changed * resolution by 0.0000, and stayed live for months. */ const fg = require('../../src/services/model/factorGate'); /** n rows at a fixed baseline, with the conditioned value shifted by `shift(i)` * and the outcome determined by `trueP(i)` — so a factor can be made genuinely * informative or purely decorative on demand. */ function rows(n, baseline, shift, trueP) { const out = []; for (let i = 0; i < n; i += 1) { const p = typeof trueP === 'function' ? trueP(i) : trueP; // Interleaved outcomes, never front-loaded — front-loading correlates the // outcome with position and quietly rigs any split. const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0; out.push({ baseline, conditioned: Math.min(0.99, Math.max(0.01, baseline + (typeof shift === 'function' ? shift(i) : shift))), won, }); } return out; } describe('THEATER — moves the number, reads nothing', () => { it('is REJECTED BY NAME, not filed as inconclusive', () => { // Truth is a flat 0.5. The factor swings the prediction ±0.15 at random // relative to the outcome, so it looks responsive and knows nothing. const r = rows(800, 0.5, (i) => (i % 2 === 0 ? 0.15 : -0.15), 0.5); const v = fg.adjudicate(r, { factor: 'decorative' }); expect(v.verdict).toBe('THEATER'); expect(v.movement.mean_abs_shift).toBeCloseTo(0.15, 2); expect(v.improvement.improves).toBe(false); expect(v.consequence).toMatch(/LOOK like it read/); }); it('an unproven-but-favourable factor is NOT called theatre', () => { // Point estimate improves, corrected interval spans zero. That is a real // candidate held to a rising bar — collapsing it into THEATER would repeat // the "insufficient evidence = evidence of absence" error. const r = []; for (let i = 0; i < 800; i += 1) { const hot = i % 2 === 0; const p = hot ? 0.56 : 0.44; const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0; r.push({ baseline: 0.5, conditioned: hot ? 0.53 : 0.47, won }); } const v = fg.adjudicate(r, { factor: 'weak-but-real', cumulativeTests: 200 }); expect(['NOT_PROVEN_AT_CORRECTED_BAR', 'PROVES']).toContain(v.verdict); if (v.verdict === 'NOT_PROVEN_AT_CORRECTED_BAR') { expect(v.improvement.brier_delta).toBeLessThan(0); expect(v.note).toMatch(/not theatre/); } }); it('a factor that moves a LOT is not thereby better — that is the trap', () => { const big = fg.adjudicate(rows(800, 0.5, (i) => (i % 2 === 0 ? 0.3 : -0.3), 0.5), { factor: 'loud' }); const small = fg.adjudicate(rows(800, 0.5, (i) => (i % 2 === 0 ? 0.02 : -0.02), 0.5), { factor: 'quiet' }); expect(big.verdict).toBe('THEATER'); expect(small.verdict).toBe('THEATER'); expect(big.movement.mean_abs_shift).toBeGreaterThan(small.movement.mean_abs_shift * 5); }); }); describe('PROVES — moves the number AND gets closer to the truth', () => { it('passes a factor that genuinely splits the population', () => { // Truth alternates 0.8 / 0.2; the factor moves the prediction the right way. const r = []; for (let i = 0; i < 800; i += 1) { const hot = i % 2 === 0; const p = hot ? 0.8 : 0.2; const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0; r.push({ baseline: 0.5, conditioned: hot ? 0.78 : 0.22, won }); } const v = fg.adjudicate(r, { factor: 'real' }); expect(v.verdict).toBe('PROVES'); expect(v.improvement.brier_delta).toBeLessThan(0); expect(v.improvement.ci[1]).toBeLessThan(0); }); }); describe('INERT and PENDING are distinct from THEATER', () => { it('a factor that never moves the number is INERT, not theatre', () => { const v = fg.adjudicate(rows(800, 0.5, 0.0005, 0.5), { factor: 'flat' }); expect(v.verdict).toBe('INERT'); // Nothing was claimed, so nothing is misleading — a different problem. }); it('thin sample is CANDIDATE_PENDING_SAMPLE with the rows still needed', () => { const v = fg.adjudicate(rows(120, 0.5, 0.1, 0.5), { factor: 'thin' }); expect(v.verdict).toBe('CANDIDATE_PENDING_SAMPLE'); expect(v.rows_needed).toBe(380); // Crucially NOT 'THEATER' — it might work; we simply cannot tell yet. }); }); describe('the measurements themselves', () => { it('movement reports spread, because a constant shift reads nothing either', () => { const constant = fg.movement(rows(200, 0.5, 0.1, 0.5)); const varied = fg.movement(rows(200, 0.5, (i) => (i % 2 ? 0.1 : -0.1), 0.5)); expect(constant.sd_shift).toBeCloseTo(0, 6); expect(varied.sd_shift).toBeGreaterThan(0.09); }); it('an unreadable side is DROPPED, never treated as no-change', () => { const m = fg.movement([ { baseline: 0.5, conditioned: 0.6 }, { baseline: null, conditioned: 0.9 }, { baseline: 0.5, conditioned: null }, ]); expect(m.n).toBe(1); }); it('improvement uses a PAIRED bootstrap — same rows score both models', () => { const r = rows(400, 0.5, 0.0, 0.5); const imp = fg.improvement(r); // Identical models must show no difference and a CI spanning zero. expect(imp.brier_delta).toBeCloseTo(0, 6); expect(imp.improves).toBe(false); expect(imp.degrades).toBe(false); }); it('the interval WIDENS with the cumulative test count — the bar rises', () => { const r = []; for (let i = 0; i < 800; i += 1) { const hot = i % 2 === 0; const p = hot ? 0.8 : 0.2; const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0; r.push({ baseline: 0.5, conditioned: hot ? 0.78 : 0.22, won }); } const one = fg.improvement(r, 3000, 1, 1); const fifty = fg.improvement(r, 3000, 1, 50); expect(fifty.ci_level).toBeGreaterThan(one.ci_level); // A wider interval can only ever make PROVES harder, never easier. expect(fifty.ci[1]).toBeGreaterThanOrEqual(one.ci[1]); }); it('a factor that makes the number WORSE is flagged degrading', () => { const r = []; for (let i = 0; i < 600; i += 1) { const p = 0.8; const won = Math.floor((i + 1) * p) > Math.floor(i * p) ? 1 : 0; r.push({ baseline: 0.8, conditioned: 0.2, won }); // confidently backwards } const imp = fg.improvement(r); expect(imp.degrades).toBe(true); expect(fg.adjudicate(r, { factor: 'backwards' }).verdict).toBe('THEATER'); }); }); describe('pseudo-replication — sample counted in the unit the factor varies over', () => { // A game-level factor (park, weather, opposing starter) hands every prop row // in a game the identical treatment. Eighteen hitters in one ballpark are one // reading of that ballpark, not eighteen. const build = (games, perGame, seed = 1) => { let s = seed; const rnd = () => (s = (s * 1103515245 + 12345) % 2147483648) / 2147483648; const rows = []; for (let g = 0; g < games; g += 1) { const shift = (rnd() - 0.5) * 0.06; // the game's treatment // Outcomes are correlated WITHIN a game — a high-scoring night lifts every // hitter in it. That shared component is exactly what row-resampling // cannot see and what makes 18 rows worth far less than 18 readings. const gameLevel = (rnd() - 0.5) * 0.5; for (let i = 0; i < perGame; i += 1) { const base = 0.3 + rnd() * 0.4; const p = Math.max(0.02, Math.min(0.98, base + gameLevel)); rows.push({ cluster: `g${g}`, baseline: base, conditioned: base + shift, won: rnd() < p ? 1 : 0 }); } } return rows; }; it('judges sample by CLUSTERS, so 900 rows over 50 games is 50 readings', () => { const rows = build(50, 18); const v = fg.adjudicate(rows, { factor: 'park', minN: 500 }); expect(rows.length).toBeGreaterThan(500); // looks like plenty expect(v.verdict).toBe('CANDIDATE_PENDING_SAMPLE'); // and is not expect(v.improvement.effective_n).toBe(50); expect(v.improvement.cluster_unit).toBe('cluster'); expect(v.reason).toMatch(/not independent readings/); }); it('the clustered interval is WIDER than the row interval on the same rows', () => { // This is the whole hazard: resampling rows would have manufactured a // confidence the evidence never supported. const rows = build(40, 20, 7); const clustered = fg.adjudicate(rows, { factor: 'park', minN: 10 }); const flat = fg.adjudicate(rows.map(({ cluster, ...r }) => r), { factor: 'park', minN: 10 }); const width = (v) => v.improvement.ci[1] - v.improvement.ci[0]; expect(width(clustered)).toBeGreaterThan(width(flat)); }); it('rows with no cluster keep the original row-resampling behaviour', () => { const rows = build(40, 20, 3).map(({ cluster, ...r }) => r); const v = fg.adjudicate(rows, { factor: 'x', minN: 10 }); expect(v.improvement.cluster_unit).toBe('row'); expect(v.improvement.effective_n).toBe(v.improvement.n); }); });