'use strict'; /** * The two ways a calibration measurement lies. * * Both produced a confident, plausible, completely wrong number in the * settlement session, and neither was visible in the output. These lock them out. */ const g = require('../../src/services/model/calibrationGuards'); const cal = require('../../src/services/model/calibration'); /** A population carrying BOTH sides of each prop, as the snapshot table does. */ function bothSides(n) { const rows = []; for (let i = 0; i < n; i += 1) { const p = 0.55 + (i % 7) * 0.05; rows.push({ propKey: `prop${i}`, side: 'over', p }); rows.push({ propKey: `prop${i}`, side: 'under', p: 1 - p }); } return rows; } describe('GUARD 1 — the both-sides tell', () => { it('catches the 0.4998 signature: both sides present AND mean pinned at 0.5', () => { const rows = bothSides(200); const r = g.checkPickedSideDedup(rows); expect(r.violated).toBe(true); expect(r.both_sides_share).toBe(1); expect(Math.abs(r.mean_p - 0.5)).toBeLessThanOrEqual(g.BALANCED_TOLERANCE); expect(r.reason).toMatch(/balanced by construction/); }); it('assert form REFUSES rather than returning a number', () => { expect(() => g.assertPickedSideDedup(bothSides(100))).toThrow(/CALIBRATION GUARD/); }); it('passes once deduped to the model-picked side', () => { // The picked side is the one the model favoured, so the mean sits well // above 0.5 — which is what a real forecaster's book looks like. const picked = bothSides(200).filter((r) => r.p > 0.5); const r = g.checkPickedSideDedup(picked); expect(r.violated).toBe(false); expect(r.mean_p).toBeGreaterThan(0.5 + g.BALANCED_TOLERANCE); }); it('does NOT fire on a genuinely balanced one-sided book', () => { // Either condition alone is unremarkable. A book of one-sided picks that // happens to average 0.5 is honest, and flagging it would be a false alarm. const rows = Array.from({ length: 300 }, (_, i) => ({ propKey: `p${i}`, side: 'over', p: i % 2 ? 0.45 : 0.55, })); const r = g.checkPickedSideDedup(rows); expect(r.both_sides_props).toBe(0); expect(r.violated).toBe(false); }); it('does NOT fire when both sides are present but the mean is skewed', () => { const rows = bothSides(50).concat( Array.from({ length: 400 }, (_, i) => ({ propKey: `x${i}`, side: 'over', p: 0.8 }))); const r = g.checkPickedSideDedup(rows); expect(r.both_sides_props).toBeGreaterThan(0); expect(r.violated).toBe(false); // already deduped elsewhere }); }); describe('GUARD 2 — a null must never score itself', () => { it('(null-1)**2 can no longer pass as a metric', () => { // This is the exact breach: JS scores null as 1 against a win and 0 against // a loss, so the "Brier" silently equals the win rate. const outcomes = [1, 1, 0, 1, 0]; const naive = outcomes.reduce((s, y, i) => s + ((null - y) ** 2), 0) / outcomes.length; const winRate = outcomes.reduce((a, b) => a + b, 0) / outcomes.length; expect(naive).toBeCloseTo(winRate, 10); // the trap, demonstrated expect(g.safeBrier([null, null, null, null, null], outcomes)).toBeNull(); }); it('refuses when ANY single prediction is null', () => { expect(g.safeBrier([0.6, 0.4, null], [1, 0, 1])).toBeNull(); }); it('can be made to hard-fail instead of refusing', () => { expect(() => g.safeBrier([0.6, null], [1, 0], { onNull: 'throw' })) .toThrow(/null prediction reached a Brier term/); }); it('scores normally when every prediction is real', () => { expect(g.safeBrier([1, 0], [1, 0])).toBe(0); expect(g.safeBrier([0.5, 0.5], [1, 0])).toBeCloseTo(0.25, 10); }); it('an unfittable map refuses instead of producing null predictions', () => { // fitIsotonic returns null below its minimum; this is what must happen next. const map = cal.fitIsotonic([{ p: 0.6, won: 1 }, { p: 0.4, won: 0 }]); expect(map).toBeNull(); const out = g.applyOrRefuse(map, [{ p: 0.6 }], cal.applyIsotonic); expect(out.ok).toBe(false); expect(out.reason).toMatch(/no calibration map/); expect(out.rows).toEqual([]); }); it('drops unmappable rows rather than passing nulls downstream', () => { const fit = []; for (let i = 0; i < 400; i += 1) fit.push({ p: 0.3 + (i % 60) / 100, won: i % 3 === 0 ? 1 : 0 }); const map = cal.fitIsotonic(fit); expect(map).not.toBeNull(); const out = g.applyOrRefuse(map, [{ p: 0.5 }, { p: null }], cal.applyIsotonic); expect(out.rows.length).toBe(1); expect(out.dropped).toBe(1); }); });