4d1803f6d7
Fitted the isotonic map on game_date < 2026-08-02 (n=589) and evaluated it on everything from that date forward (n=383). The map never saw the evaluation rows, which is the only thing that makes the result mean anything -- fitting and evaluating on the same rows always looks perfectly calibrated, because the map is reciting the answers it was built from. It works, on most of the distribution. Held-out after correction: 0.477 comes back 0.506, 0.587 comes back 0.580, 0.667 comes back 0.603 -- against raw errors of +0.191, +0.279 and +0.246 in the same bins. Ordering survived, and that was verified pairwise rather than assumed, because a broken map would silently destroy the one thing this model does well. Two findings matter more than the pass. First, the honest ceiling is 0.667. Once the numbers are truthful this model has no 80%-plus hit reads at all -- the top of its range was miscalibration, not confidence. A four-leg ticket at the ceiling is 0.198, where the raw numbers implied 0.686. The high-floor parlay is a two-thirds-per-leg proposition, and that is the number to say out loud. Second, calibration is certified BY BAND rather than by a blanket flag. Held-out error was -0.029 and +0.007 through the middle but -0.167 at the bottom and +0.063 at the top: the model is trustworthy over most of its mass and untrustworthy at both edges. A single true/false would either throw away the 72% that works or ship the edges that do not. Only a probability inside a certified band is marked stackable, and that flag is what chainAcross requires before it will compound anything. The certified band is 0.40 to 0.60, n=276. A methodological catch on the way: my first pass condition demanded honest bins at 0.70 and above -- but honest calibration REMOVES those bins, since the ceiling drops to 0.667. The gate would have failed the repair for succeeding. It now tests the highest remaining band instead of a fixed threshold. Wired forward with the same discipline: calibrationService fits strictly before today, splits by time rather than at random, and returns null on thin history so that "no calibrator" means nothing is stackable rather than "trust the raw numbers". p_win is never mutated -- the calibrated value rides beside it as p_win_calibrated, because a calibration map is a correction to a forecast, not a different forecast, and the counter stays byte-identical. 4,275 tests green (339 suites); web build exit 0. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W1sivYNqY2TS5ftykmHBU9
241 lines
10 KiB
JavaScript
241 lines
10 KiB
JavaScript
'use strict';
|
|
|
|
/**
|
|
* The portable chain + the calibration gate that guards it.
|
|
*
|
|
* The failure these exist to prevent is specific: compounding probabilities that
|
|
* are individually survivable and jointly catastrophic. A 4-leg ticket of "90%"
|
|
* legs whose realized rate is 63% is a 4.4x overstatement, in the direction the
|
|
* user pays for.
|
|
*/
|
|
|
|
const chain = require('../../src/services/model/chain');
|
|
const cal = require('../../src/services/model/calibration');
|
|
|
|
const atom = (id, p, extra = {}) => ({ id, p, calibrated: true, gameId: `g${id}`, ...extra });
|
|
|
|
describe('CHAIN ACROSS — the calibration gate is structural', () => {
|
|
it('REFUSES to compound an uncalibrated atom', () => {
|
|
const out = chain.chainAcross([atom('a', 0.9), { ...atom('b', 0.9), calibrated: false }]);
|
|
expect(out.ok).toBe(false);
|
|
expect(out.reason).toBe('uncalibrated_atoms');
|
|
expect(out.uncalibrated).toEqual(['b']);
|
|
});
|
|
|
|
it('the refusal is the feature — this is the 4.4x case, made unbuildable', () => {
|
|
// Model 0.91^4 = 0.686; realized 0.63^4 = 0.157.
|
|
const legs = ['a', 'b', 'c', 'd'].map((k) => ({ ...atom(k, 0.91), calibrated: false }));
|
|
expect(chain.chainAcross(legs).ok).toBe(false);
|
|
// And the same legs, once genuinely calibrated, DO compound.
|
|
const ok = chain.chainAcross(legs.map((l) => ({ ...l, calibrated: true })));
|
|
expect(ok.ok).toBe(true);
|
|
expect(ok.compound_probability).toBeCloseTo(0.91 ** 4, 3);
|
|
});
|
|
|
|
it('independent cross-game legs multiply', () => {
|
|
const out = chain.chainAcross([atom('a', 0.8), atom('b', 0.5)]);
|
|
expect(out.independent_probability).toBeCloseTo(0.4, 6);
|
|
expect(out.cross_game_legs).toBe(2);
|
|
});
|
|
|
|
it('correlated same-game legs are NOT treated as independent', () => {
|
|
const legs = [atom('a', 0.7, { gameId: 'G1' }), atom('b', 0.7, { gameId: 'G1' })];
|
|
const indep = chain.chainAcross(legs);
|
|
const corr = chain.chainAcross(legs, { correlation: () => 0.6 });
|
|
// Shared pitcher/park/weather makes them land together more often than
|
|
// independence implies — and independence is the FLATTERING error here.
|
|
expect(corr.compound_probability).toBeGreaterThan(indep.compound_probability);
|
|
expect(corr.cross_game_legs).toBe(1);
|
|
});
|
|
|
|
it('an unreadable atom is dropped, never counted as probability zero', () => {
|
|
const out = chain.chainAcross([atom('a', 0.8), { id: 'b', p: null, calibrated: true }]);
|
|
expect(out.ok).toBe(true);
|
|
expect(out.legs).toBe(1); // a p=0 leg would have zeroed the ticket
|
|
expect(out.compound_probability).toBeCloseTo(0.8, 6);
|
|
});
|
|
|
|
it('no usable atoms is an explicit refusal, not a zero', () => {
|
|
expect(chain.chainAcross([]).ok).toBe(false);
|
|
expect(chain.chainAcross([{ id: 'x', p: null }]).reason).toBe('no_usable_atoms');
|
|
});
|
|
});
|
|
|
|
describe('CHAIN UP — same atoms, team reading', () => {
|
|
it('sums atoms into an expected value', () => {
|
|
const out = chain.chainUp([atom('a', 0.3), atom('b', 0.4), atom('c', 0.5)]);
|
|
expect(out.expected_value).toBeCloseTo(1.2, 6);
|
|
expect(out.contributors).toBe(3);
|
|
});
|
|
|
|
it('respects weights, and an absent weight contributes ONCE not zero', () => {
|
|
const out = chain.chainUp([atom('a', 0.5, { weight: 4 }), atom('b', 0.5)]);
|
|
expect(out.expected_value).toBeCloseTo(2.5, 6);
|
|
});
|
|
|
|
it('the archetype-redistribution HOOK is dormant unless supplied', () => {
|
|
const legs = [atom('star', 0.6), atom('bench', 0.1)];
|
|
expect(chain.chainUp(legs).redistributed).toBe(false);
|
|
});
|
|
|
|
it('and LIVE when a sport supplies it — a blowout fades the star, feeds the bench', () => {
|
|
// Dormant in baseball (a nine-run lead does not change who bats next);
|
|
// this is the basketball case the hook exists for.
|
|
const legs = [atom('star', 0.6), atom('bench', 0.1)];
|
|
const out = chain.chainUp(legs, {
|
|
context: { blowout: true },
|
|
redistribute: (as, ctx) => (ctx.blowout
|
|
? as.map((a) => (a.id === 'star' ? { ...a, p: a.p * 0.6 } : { ...a, p: a.p * 2 }))
|
|
: as),
|
|
});
|
|
expect(out.redistributed).toBe(true);
|
|
expect(out.expected_value).toBeCloseTo(0.6 * 0.6 + 0.1 * 2, 6);
|
|
});
|
|
});
|
|
|
|
describe('SELF-CHECK — the model flagging its own suspect calls', () => {
|
|
it('flags an internal inconsistency as LOW CONFIDENCE, without guessing which side is wrong', () => {
|
|
const out = chain.selfCheck({
|
|
perEntity: [{ p: 0.5 }, { p: 0.5 }, { p: 0.5 }], // sums to 1.5
|
|
teamRead: 4.0,
|
|
});
|
|
expect(out.confidence).toBe('LOW');
|
|
expect(out.flags.map((f) => f.flag)).toContain('INTERNAL_INCONSISTENCY');
|
|
expect(out.flags[0].consequence).toMatch(/do not know which/);
|
|
});
|
|
|
|
it('agreement is NORMAL confidence', () => {
|
|
const out = chain.selfCheck({ perEntity: [{ p: 1.0 }, { p: 1.1 }], teamRead: 2.05 });
|
|
expect(out.confidence).toBe('NORMAL');
|
|
expect(out.flags).toEqual([]);
|
|
});
|
|
|
|
it('market divergence FLAGS the script but never claims we are right', () => {
|
|
const out = chain.selfCheck({ perEntity: [{ p: 5.5 }], teamRead: 5.5, marketRead: 3.5 });
|
|
const f = out.flags.find((x) => x.flag === 'SCRIPT_DIVERGES_FROM_MARKET');
|
|
expect(f).toBeTruthy();
|
|
expect(f.consequence).toMatch(/either the best or the worst/);
|
|
// Divergence is not an error signal, so it does not downgrade confidence.
|
|
expect(out.confidence).toBe('NORMAL');
|
|
});
|
|
});
|
|
|
|
describe('PROPAGATION — one settled result improves every reading that shares the atom', () => {
|
|
it('moves a thin atom a lot and a heavy atom barely at all', () => {
|
|
const thin = chain.propagate({ id: 'a', p: 0.5, n: 4 }, { won: 1 });
|
|
const heavy = chain.propagate({ id: 'a', p: 0.5, n: 400 }, { won: 1 });
|
|
expect(thin.p - 0.5).toBeGreaterThan((heavy.p - 0.5) * 10);
|
|
// That gap is the difference between learning and chasing noise.
|
|
expect(heavy.n).toBe(401);
|
|
});
|
|
|
|
it('an unreadable observation changes nothing', () => {
|
|
const before = { id: 'a', p: 0.5, n: 10 };
|
|
expect(chain.propagate(before, { won: null })).toEqual(before);
|
|
expect(chain.propagate(before, null)).toEqual(before);
|
|
});
|
|
});
|
|
|
|
describe('CALIBRATION SERVICE — fit past, apply forward, certify by band', () => {
|
|
const svc = require('../../src/services/model/calibrationService');
|
|
|
|
/** Rows dated so the time-split is meaningful. */
|
|
const hist = (specs) => {
|
|
const out = [];
|
|
let day = 1;
|
|
for (const [p, n, rate] of specs) {
|
|
for (let i = 0; i < n; i += 1) {
|
|
// Wins are INTERLEAVED, not front-loaded. Front-loading makes the
|
|
// outcome correlate with the date, so a time-split would train on the
|
|
// wins and certify on the losses — the generator would be creating the
|
|
// very leakage the split exists to prevent.
|
|
const won = Math.floor((i + 1) * rate) > Math.floor(i * rate) ? 1 : 0;
|
|
out.push({ p, won, date: `2026-07-${String(day).padStart(2, '0')}` });
|
|
if (out.length % 40 === 0) day = Math.min(28, day + 1);
|
|
}
|
|
}
|
|
return out;
|
|
};
|
|
|
|
it('refuses to build on thin history rather than passing raw numbers through', () => {
|
|
// "No calibrator" must mean nothing is stackable, never "trust the model".
|
|
expect(svc.build(hist([[0.6, 50, 0.5]]))).toBeNull();
|
|
expect(svc.build([])).toBeNull();
|
|
});
|
|
|
|
it('corrects an over-confident model and marks the corrected value calibrated', () => {
|
|
// Claims 0.9, realises 0.6 — the shape measured on real hits.
|
|
const c = svc.build(hist([[0.5, 300, 0.5], [0.9, 300, 0.6]]), { minBin: 30 });
|
|
expect(c).not.toBeNull();
|
|
const out = c.calibrate(0.9);
|
|
expect(out.p_calibrated).toBeLessThan(0.75); // the 0.9 claim is corrected down
|
|
expect(out.p_raw).toBe(0.9);
|
|
});
|
|
|
|
it('a probability OUTSIDE a certified band is not stackable', () => {
|
|
const c = svc.build(hist([[0.5, 300, 0.5], [0.9, 300, 0.6]]), { minBin: 30 });
|
|
const far = c.calibrate(0.02);
|
|
// Whatever it maps to, if the band was never certified it cannot compound.
|
|
if (!far.calibrated) expect(far.reason).toBe('outside_certified_band');
|
|
});
|
|
|
|
it('an absent probability is absent, never 0', () => {
|
|
const c = svc.build(hist([[0.5, 300, 0.5], [0.9, 300, 0.6]]), { minBin: 30 });
|
|
const out = c.calibrate(null);
|
|
expect(out.p_calibrated).toBeNull();
|
|
expect(out.calibrated).toBe(false);
|
|
});
|
|
|
|
it('splits by TIME — the certification window is later than the fit window', () => {
|
|
const c = svc.build(hist([[0.5, 300, 0.5], [0.9, 300, 0.6]]), { minBin: 30 });
|
|
expect(c.certified_through >= c.fitted_through).toBe(true);
|
|
expect(c.fit_n).toBeGreaterThan(0);
|
|
expect(c.certify_n).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('END TO END: uncalibrated legs are refused; calibrated ones compound', () => {
|
|
const c = svc.build(hist([[0.5, 300, 0.5], [0.9, 300, 0.6]]), { minBin: 30 });
|
|
const legs = ['a', 'b'].map((id) => {
|
|
const v = c.calibrate(0.5);
|
|
return { id, p: v.p_calibrated, calibrated: v.calibrated, gameId: `g${id}` };
|
|
});
|
|
const out = chain.chainAcross(legs);
|
|
if (legs.every((l) => l.calibrated)) {
|
|
expect(out.ok).toBe(true);
|
|
expect(out.compound_probability).toBeCloseTo(legs[0].p * legs[1].p, 3);
|
|
} else {
|
|
expect(out.reason).toBe('uncalibrated_atoms');
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('CALIBRATION — the gate itself', () => {
|
|
const rows = (spec) => spec.flatMap(([p, n, hitRate]) =>
|
|
Array.from({ length: n }, (_, i) => ({ p, won: i < Math.round(n * hitRate) ? 1 : 0 })));
|
|
|
|
it('passes a model whose stated probabilities are its realized rates', () => {
|
|
const out = cal.isCalibrated(rows([[0.3, 100, 0.30], [0.5, 100, 0.50], [0.9, 100, 0.90]]));
|
|
expect(out.calibrated).toBe(true);
|
|
});
|
|
|
|
it('FAILS the real hits curve — over-confident exactly where a parlay stacks', () => {
|
|
const out = cal.isCalibrated(rows([[0.45, 152, 0.49], [0.75, 152, 0.605], [0.91, 100, 0.63]]));
|
|
expect(out.calibrated).toBe(false);
|
|
expect(out.reason).toBe('bin_error_exceeds_tolerance');
|
|
expect(out.worst_high_confidence_bin.error).toBeGreaterThan(0.15);
|
|
});
|
|
|
|
it('refuses to judge on too little data rather than guessing', () => {
|
|
expect(cal.isCalibrated(rows([[0.5, 20, 0.5]])).reason).toBe('insufficient_sample');
|
|
});
|
|
|
|
it('isotonic fitting corrects the numbers while PRESERVING the ordering', () => {
|
|
const map = cal.fitIsotonic(rows([[0.45, 152, 0.49], [0.75, 152, 0.605], [0.91, 100, 0.63]]));
|
|
expect(map).not.toBeNull();
|
|
const lo = cal.applyIsotonic(map, 0.45);
|
|
const hi = cal.applyIsotonic(map, 0.91);
|
|
expect(hi).toBeGreaterThanOrEqual(lo); // ordering survives
|
|
expect(hi).toBeLessThan(0.75); // the 0.91 claim is corrected down hard
|
|
});
|
|
});
|