Backtest harness — the validator, built refusal-first
Phase 0 gate PASSED: the join is clean. No FK exists; the natural key (sport, player_key, stat, line, side, game_date) yields 283 clean 1:1 joins with ZERO ambiguity. game_id is NOT usable — 400/550 snapshot rows carry UNK@UNK because home/away names weren't threaded into the grader until Order 1.6. Non-joining rows are EXPECTED, not errors: retention stores both sides plus refusals; the ledger keeps only the graded side. Outcomes are NOT denormalized — ledger_entries stays the source of truth. BUILT TEST-FIRST, and the first property proven is the REFUSAL, not the math. Below threshold the harness emits INSUFFICIENT with n and the shortfall and NO rate anywhere in the payload, so a downstream renderer cannot surface one by accident. A test asserts the payload contains no hit_rate number at all. - Wilson intervals (correct at the n we actually have, unlike the normal approximation which emits negative lower bounds). - Strata NEVER mix sport or model_version. - Denominator excludes quarantined, void, unrecoverable, pending, push — asserted by test. - Monotonicity refuses to RANK buckets whose intervals overlap; it reports "not distinguishable on this sample". - Probability calibration (Brier + reliability) also respects the threshold: a thin sample returns status INSUFFICIENT and a NULL score. - Replay seam reads the STORED feature vector only. A row whose input was never retained is UN-BACKTESTABLE, never scored with substituted current data. Identity replay reproduces the live prediction exactly. The tests caught a real bug in my own code: `Number(null) === 0` let a null p_win through as a confident 0% forecast — this codebase's signature fabrication bug, inside the harness whose entire purpose is refusing invented numbers. Fixed with a strict null guard. FIRST LIVE RUN — the correct, passing output: VERDICT: INSUFFICIENT_HISTORY (can_validate=false) 283 joined -> 35 scored (120 quarantined, 124 pending, 4 terminal) C n=18 (short by 2), B n=17 (short by 3) strata: mlb 7, wnba 28 — never mixed migration 028 adds harness_results (append-only trend log; INSUFFICIENT rows are expected and correct) and opsWatch.harnessStaleAlarm pages if the harness stops running — a validator that isn't running looks exactly like one that keeps passing. Suite 283/3403 green, build exit 0. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SmNjJAwEnqHPtXbvSZR8kA
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* RUN THE BACKTEST HARNESS (Session 64).
|
||||
*
|
||||
* Joins model_snapshots (the model's INPUTS + prediction) to ledger_entries
|
||||
* (the single source of truth for OUTCOMES) on the natural key, and runs the
|
||||
* harness. Outcomes are NEVER denormalized onto snapshots.
|
||||
*
|
||||
* Join key: (sport, player_key, stat, line, side, game_date).
|
||||
* Verified empirically: 283 clean 1:1 joins, ZERO ambiguity. `game_id` is NOT
|
||||
* usable — 400/550 snapshot rows carry `UNK@UNK` because home/away team names
|
||||
* weren't threaded into the grader until Session 64 Order 1.6.
|
||||
*
|
||||
* Rows that don't join are EXPECTED, not errors: retention stores BOTH sides
|
||||
* of every prop plus refusals, while the ledger keeps only the graded side of
|
||||
* non-refused props.
|
||||
*
|
||||
* node scripts/run-backtest.js <rows.json> # rows exported via SQL
|
||||
*/
|
||||
|
||||
const harness = require('../src/services/backtestHarness');
|
||||
|
||||
const rows = JSON.parse(require('fs').readFileSync(process.argv[2], 'utf8'));
|
||||
const report = harness.runBacktest(rows, {});
|
||||
|
||||
const pad = (s, n) => String(s).padEnd(n);
|
||||
console.log('══════════ VYNDR BACKTEST HARNESS ══════════');
|
||||
console.log(`generated_at : ${report.generated_at}`);
|
||||
console.log(`min_sample : ${report.min_sample}`);
|
||||
console.log(`VERDICT : ${report.verdict} (can_validate=${report.can_validate})`);
|
||||
console.log('\n--- denominator ---');
|
||||
Object.entries(report.counts).forEach(([k, v]) => console.log(` ${pad(k, 22)} ${v}`));
|
||||
|
||||
console.log('\n--- grade buckets (4-letter) ---');
|
||||
for (const b of report.grade_buckets.sort((a, z) => z.n - a.n)) {
|
||||
console.log(b.status === 'OK'
|
||||
? ` ${pad(b.bucket, 4)} n=${pad(b.n, 5)} hit=${(b.hit_rate * 100).toFixed(1)}% 95% CI [${(b.ci_low * 100).toFixed(1)}, ${(b.ci_high * 100).toFixed(1)}]`
|
||||
: ` ${pad(b.bucket, 4)} n=${pad(b.n, 5)} INSUFFICIENT — need ${b.need} (short by ${b.short_by})`);
|
||||
}
|
||||
|
||||
console.log('\n--- grade buckets (11-step) ---');
|
||||
for (const b of report.grade_11_buckets.sort((a, z) => z.n - a.n)) {
|
||||
console.log(b.status === 'OK'
|
||||
? ` ${pad(b.bucket, 4)} n=${pad(b.n, 5)} hit=${(b.hit_rate * 100).toFixed(1)}%`
|
||||
: ` ${pad(b.bucket, 4)} n=${pad(b.n, 5)} INSUFFICIENT`);
|
||||
}
|
||||
|
||||
console.log(`\n--- monotonicity: ${report.monotonicity.verdict} ---`);
|
||||
report.monotonicity.comparisons.forEach((c) => console.log(
|
||||
` ${c.higher} vs ${c.lower}: ${c.distinguishable ? (c.holds ? 'HOLDS' : 'BROKEN') : 'not distinguishable on this sample'}`,
|
||||
));
|
||||
|
||||
console.log('\n--- probability calibration ---');
|
||||
console.log(report.probability.n
|
||||
? ` n=${report.probability.n} Brier=${report.probability.brier.toFixed(4)}`
|
||||
: ' n=0 — no stored p_win on any joinable settled row');
|
||||
|
||||
console.log('\n--- strata (never mixed) ---');
|
||||
report.strata.forEach((s) => console.log(` ${pad(s.sport, 6)} ${pad(s.model_version, 24)} n=${s.n}`));
|
||||
|
||||
require('fs').writeFileSync(
|
||||
process.argv[3] || '/tmp/backtest-report.json',
|
||||
JSON.stringify(report, null, 2),
|
||||
);
|
||||
console.log(`\nfull report → ${process.argv[3] || '/tmp/backtest-report.json'}`);
|
||||
@@ -0,0 +1,277 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* BACKTEST HARNESS (Session 64) — the validator every metric family is gated on.
|
||||
*
|
||||
* ITS FIRST PROPERTY IS THE REFUSAL. A harness that emits a confident number on
|
||||
* a handful of rows is worse than no harness: it launders noise into a claim,
|
||||
* and claims become marketing. Below the sample threshold this returns
|
||||
* INSUFFICIENT with n and the shortfall — and NO rate anywhere in the payload,
|
||||
* so a downstream renderer cannot accidentally surface one.
|
||||
*
|
||||
* Design rules, each enforced by a test:
|
||||
* - NEVER mix sports or model_versions in a bucket. A backtest across model
|
||||
* eras is meaningless, and `ledger_entries` is already contaminated
|
||||
* pre-2026-07-19 with no per-row version.
|
||||
* - The denominator excludes quarantined, void, unrecoverable, pending AND
|
||||
* push — the same discipline the public record already uses.
|
||||
* - Overlapping confidence intervals are reported as NOT DISTINGUISHABLE
|
||||
* rather than ranked. Two buckets 4 points apart on n=25 are noise.
|
||||
* - The replay seam reads the STORED feature vector only. A backtest that
|
||||
* refetches today's data to score a past night is invalid; a row whose
|
||||
* input was never retained is UN-BACKTESTABLE, and says so.
|
||||
*
|
||||
* Pure functions over rows — no I/O, so it is unit-testable and cannot
|
||||
* accidentally reach the network mid-backtest.
|
||||
*/
|
||||
|
||||
const DEFAULT_MIN_SAMPLE = Number(process.env.BACKTEST_MIN_SAMPLE || 20);
|
||||
const TERMINAL_NON_RESULTS = new Set(['void', 'unrecoverable']);
|
||||
|
||||
/**
|
||||
* Wilson score interval — correct for small n, unlike the normal approximation
|
||||
* (which happily emits negative lower bounds on the sample sizes we actually
|
||||
* have). z=1.96 → 95%.
|
||||
*/
|
||||
function wilsonInterval(hits, n, z = 1.96) {
|
||||
if (!n || n <= 0) return { low: null, high: null };
|
||||
const p = hits / n;
|
||||
const z2 = z * z;
|
||||
const denom = 1 + z2 / n;
|
||||
const centre = p + z2 / (2 * n);
|
||||
const margin = z * Math.sqrt((p * (1 - p) + z2 / (4 * n)) / n);
|
||||
return {
|
||||
low: Math.max(0, (centre - margin) / denom),
|
||||
high: Math.min(1, (centre + margin) / denom),
|
||||
};
|
||||
}
|
||||
|
||||
/** Split rows into scored vs each excluded reason. The counts are reported so
|
||||
* a shrinking denominator is visible rather than silent. */
|
||||
function partition(rows) {
|
||||
const scored = [];
|
||||
const counts = {
|
||||
total: rows.length, scored: 0,
|
||||
excluded_quarantine: 0, excluded_terminal: 0,
|
||||
excluded_pending: 0, excluded_push: 0,
|
||||
};
|
||||
for (const r of rows || []) {
|
||||
if (!r) continue;
|
||||
if (r.quarantine_reason || r.snap_quarantine) { counts.excluded_quarantine += 1; continue; }
|
||||
const o = r.outcome;
|
||||
if (o == null) { counts.excluded_pending += 1; continue; }
|
||||
if (o === 'push') { counts.excluded_push += 1; continue; }
|
||||
if (TERMINAL_NON_RESULTS.has(o)) { counts.excluded_terminal += 1; continue; }
|
||||
if (o !== 'hit' && o !== 'miss') { counts.excluded_terminal += 1; continue; }
|
||||
scored.push(r);
|
||||
}
|
||||
counts.scored = scored.length;
|
||||
return { scored, counts };
|
||||
}
|
||||
|
||||
/** One bucket's calibration. Below `minSample` it reports the refusal and NO rate. */
|
||||
function bucketStats(bucket, rows, minSample) {
|
||||
const n = rows.length;
|
||||
const hits = rows.filter((r) => r.outcome === 'hit').length;
|
||||
if (n < minSample) {
|
||||
return { bucket, n, hits, status: 'INSUFFICIENT', need: minSample, short_by: minSample - n, hit_rate: null, ci_low: null, ci_high: null };
|
||||
}
|
||||
const ci = wilsonInterval(hits, n);
|
||||
return {
|
||||
bucket, n, hits, status: 'OK',
|
||||
hit_rate: hits / n,
|
||||
ci_low: ci.low, ci_high: ci.high,
|
||||
};
|
||||
}
|
||||
|
||||
function groupBy(rows, keyFn) {
|
||||
const m = new Map();
|
||||
for (const r of rows) {
|
||||
const k = keyFn(r);
|
||||
if (!m.has(k)) m.set(k, []);
|
||||
m.get(k).push(r);
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
||||
/**
|
||||
* Monotonicity: does a higher grade actually hit more often? Only buckets that
|
||||
* BOTH cleared the sample gate are compared, and a comparison whose intervals
|
||||
* overlap is reported as not distinguishable rather than ranked.
|
||||
*/
|
||||
function monotonicity(buckets, order) {
|
||||
const ok = buckets.filter((b) => b.status === 'OK');
|
||||
const comparisons = [];
|
||||
for (let i = 0; i < ok.length; i += 1) {
|
||||
for (let j = i + 1; j < ok.length; j += 1) {
|
||||
const a = ok[i];
|
||||
const b = ok[j];
|
||||
const ra = order.indexOf(a.bucket);
|
||||
const rb = order.indexOf(b.bucket);
|
||||
if (ra === -1 || rb === -1) continue;
|
||||
const better = ra < rb ? a : b; // earlier in order = higher grade
|
||||
const worse = ra < rb ? b : a;
|
||||
const overlap = !(better.ci_low > worse.ci_high || worse.ci_low > better.ci_high);
|
||||
comparisons.push({
|
||||
pair: `${a.bucket}>${b.bucket}`,
|
||||
higher: better.bucket, lower: worse.bucket,
|
||||
higher_rate: better.hit_rate, lower_rate: worse.hit_rate,
|
||||
distinguishable: !overlap,
|
||||
holds: overlap ? null : better.hit_rate > worse.hit_rate,
|
||||
});
|
||||
}
|
||||
}
|
||||
let verdict;
|
||||
if (comparisons.length === 0) verdict = 'INSUFFICIENT';
|
||||
else if (comparisons.every((c) => !c.distinguishable)) verdict = 'NOT_DISTINGUISHABLE';
|
||||
else if (comparisons.filter((c) => c.distinguishable).every((c) => c.holds)) verdict = 'MONOTONIC';
|
||||
else verdict = 'BROKEN';
|
||||
return { verdict, comparisons };
|
||||
}
|
||||
|
||||
/** Brier score + reliability curve. Rows without a p_win are EXCLUDED — never
|
||||
* defaulted to 0.5, which would invent a prediction we did not make. */
|
||||
function probabilityCalibration(rows, bins = 10, minSample = null) {
|
||||
// STRICT null guard. `Number(null) === 0` is this codebase's signature
|
||||
// fabrication bug, and a null p_win slipping through would be scored as a
|
||||
// confident 0% prediction — inventing a forecast we never made, inside the
|
||||
// harness whose whole purpose is to refuse invented numbers.
|
||||
const usable = rows.filter((r) => r && r.p_win != null && r.p_win !== '' && Number.isFinite(Number(r.p_win)));
|
||||
if (usable.length === 0) return { n: 0, status: 'INSUFFICIENT', brier: null, reliability: [] };
|
||||
let sum = 0;
|
||||
const buckets = Array.from({ length: bins }, () => ({ n: 0, hits: 0, p_sum: 0 }));
|
||||
for (const r of usable) {
|
||||
const p = Number(r.p_win);
|
||||
const y = r.outcome === 'hit' ? 1 : 0;
|
||||
sum += (p - y) ** 2;
|
||||
const idx = Math.min(bins - 1, Math.max(0, Math.floor(p * bins)));
|
||||
buckets[idx].n += 1;
|
||||
buckets[idx].hits += y;
|
||||
buckets[idx].p_sum += p;
|
||||
}
|
||||
// The refusal applies here too: a Brier score on a handful of rows is noise
|
||||
// wearing a decimal point. Below threshold we report the count and NO score,
|
||||
// so a reader can never mistake it for a validated number.
|
||||
if (minSample != null && usable.length < minSample) {
|
||||
return { n: usable.length, status: 'INSUFFICIENT', need: minSample, brier: null, reliability: [] };
|
||||
}
|
||||
return {
|
||||
n: usable.length,
|
||||
status: 'OK',
|
||||
brier: sum / usable.length,
|
||||
reliability: buckets
|
||||
.map((b, i) => (b.n ? {
|
||||
bin: `${(i / bins).toFixed(1)}-${((i + 1) / bins).toFixed(1)}`,
|
||||
n: b.n, predicted: b.p_sum / b.n, realized: b.hits / b.n,
|
||||
} : null))
|
||||
.filter(Boolean),
|
||||
};
|
||||
}
|
||||
|
||||
const FOUR_ORDER = ['A+', 'A', 'B', 'C', 'D', 'F'];
|
||||
const ELEVEN_ORDER = ['A+', 'A', 'A-', 'B+', 'B', 'B-', 'C+', 'C', 'C-', 'D', 'F'];
|
||||
|
||||
/**
|
||||
* Run the backtest.
|
||||
* @param {Array} rows joined snapshot+outcome rows
|
||||
* @param {Object} opts { minSample }
|
||||
*/
|
||||
function runBacktest(rows, opts = {}) {
|
||||
const minSample = Number.isFinite(opts.minSample) ? opts.minSample : DEFAULT_MIN_SAMPLE;
|
||||
const { scored, counts } = partition(rows || []);
|
||||
|
||||
// Strata: NEVER mix sport or model_version.
|
||||
const strata = [];
|
||||
for (const [key, group] of groupBy(scored, (r) => `${r.sport}|${r.model_version}`)) {
|
||||
const [sport, model_version] = key.split('|');
|
||||
const g4 = [...groupBy(group, (r) => r.grade || '?')]
|
||||
.map(([b, rs]) => bucketStats(b, rs, minSample));
|
||||
const g11 = [...groupBy(group, (r) => r.grade_11 || '?')]
|
||||
.map(([b, rs]) => bucketStats(b, rs, minSample));
|
||||
strata.push({
|
||||
sport, model_version, n: group.length,
|
||||
grade_buckets: g4,
|
||||
grade_11_buckets: g11,
|
||||
monotonicity: monotonicity(g4, FOUR_ORDER),
|
||||
probability: probabilityCalibration(group, 10, minSample),
|
||||
});
|
||||
}
|
||||
|
||||
// Flat views (used by the tests + the log) — computed over ALL scored rows.
|
||||
const grade_buckets = [...groupBy(scored, (r) => r.grade || '?')]
|
||||
.map(([b, rs]) => bucketStats(b, rs, minSample));
|
||||
const grade_11_buckets = [...groupBy(scored, (r) => r.grade_11 || '?')]
|
||||
.map(([b, rs]) => bucketStats(b, rs, minSample));
|
||||
|
||||
const anyOk = grade_buckets.some((b) => b.status === 'OK');
|
||||
return {
|
||||
generated_at: opts.now ? opts.now() : new Date().toISOString(),
|
||||
min_sample: minSample,
|
||||
counts,
|
||||
// THE REFUSAL: with nothing above threshold there is no validation to give.
|
||||
can_validate: anyOk,
|
||||
verdict: anyOk ? 'PARTIAL' : 'INSUFFICIENT_HISTORY',
|
||||
grade_buckets,
|
||||
grade_11_buckets,
|
||||
monotonicity: monotonicity(grade_buckets, FOUR_ORDER),
|
||||
probability: probabilityCalibration(scored, 10, minSample),
|
||||
strata,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* ALTERNATIVE-MODEL REPLAY SEAM.
|
||||
*
|
||||
* `scoreFn(row)` sees ONLY the stored row (including its retained feature
|
||||
* vector) and returns `{ p_win }` — or `{ unavailable: '<reason>' }` when an
|
||||
* input it needs was never retained. Those rows are counted as un-backtestable
|
||||
* instead of being scored with substituted current data, which would make the
|
||||
* result a fiction.
|
||||
*/
|
||||
function replayAlternative(rows, scoreFn, opts = {}) {
|
||||
const minSample = Number.isFinite(opts.minSample) ? opts.minSample : DEFAULT_MIN_SAMPLE;
|
||||
const { scored } = partition(rows || []);
|
||||
const liveRows = [];
|
||||
const candRows = [];
|
||||
let un_backtestable = 0;
|
||||
|
||||
for (const r of scored) {
|
||||
let out;
|
||||
try { out = scoreFn(r); } catch { out = { unavailable: 'scorer_threw' }; }
|
||||
if (!out || out.unavailable || out.p_win == null || !Number.isFinite(Number(out.p_win))) {
|
||||
un_backtestable += 1;
|
||||
continue;
|
||||
}
|
||||
liveRows.push(r);
|
||||
candRows.push({ ...r, p_win: Number(out.p_win) });
|
||||
}
|
||||
|
||||
const live = probabilityCalibration(liveRows);
|
||||
const candidate = probabilityCalibration(candRows);
|
||||
const identical = live.n === candidate.n
|
||||
&& live.n > 0
|
||||
&& Math.abs((live.brier ?? 0) - (candidate.brier ?? 0)) < 1e-9;
|
||||
|
||||
return {
|
||||
n_considered: scored.length,
|
||||
un_backtestable,
|
||||
live,
|
||||
candidate,
|
||||
identical,
|
||||
// Same refusal rule as the main report.
|
||||
verdict: candidate.n >= minSample ? 'COMPARABLE' : 'INSUFFICIENT_HISTORY',
|
||||
improvement: (candidate.n >= minSample && live.brier != null && candidate.brier != null)
|
||||
? live.brier - candidate.brier // positive = candidate is better
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
runBacktest,
|
||||
replayAlternative,
|
||||
wilsonInterval,
|
||||
probabilityCalibration,
|
||||
monotonicity,
|
||||
partition,
|
||||
DEFAULT_MIN_SAMPLE,
|
||||
};
|
||||
@@ -278,8 +278,28 @@ function settlementRateAlarm(results = [], opts = {}) {
|
||||
return { alarm, reason, detail, floor };
|
||||
}
|
||||
|
||||
/**
|
||||
* HARNESS STALENESS ALARM (Session 64). The harness is the gate every future
|
||||
* metric family must pass; if it silently stops running, we lose the ability to
|
||||
* refuse — and a validator that isn't running looks exactly like one that keeps
|
||||
* passing. Same discipline as the retention and settlement alarms.
|
||||
*/
|
||||
function harnessStaleAlarm(lastRanAtIso, now = new Date(), maxAgeHours = 36) {
|
||||
if (!lastRanAtIso) {
|
||||
return { alarm: true, reason: 'the backtest harness has NEVER recorded a run' };
|
||||
}
|
||||
const t = new Date(lastRanAtIso).getTime();
|
||||
if (Number.isNaN(t)) return { alarm: true, reason: 'unparseable last-run timestamp' };
|
||||
const ageH = (now.getTime() - t) / 3_600_000;
|
||||
if (ageH > maxAgeHours) {
|
||||
return { alarm: true, reason: `the backtest harness has not run for ${Math.round(ageH)}h (limit ${maxAgeHours}h)` };
|
||||
}
|
||||
return { alarm: false, reason: null, age_hours: Math.round(ageH) };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
retentionZeroWriteAlarm,
|
||||
harnessStaleAlarm,
|
||||
settlementRateAlarm,
|
||||
SETTLE_RATE_FLOOR,
|
||||
createFailureTracker,
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
/**
|
||||
* Session 64 — BACKTEST HARNESS.
|
||||
*
|
||||
* The harness is the validator every future metric family is gated on. Its
|
||||
* FIRST proven property is not the math — it is the REFUSAL. A harness that
|
||||
* emits a confident number on 4 rows is worse than no harness, because it
|
||||
* launders noise into a claim. These tests are written first, deliberately.
|
||||
*/
|
||||
|
||||
const h = require('../../src/services/backtestHarness');
|
||||
|
||||
const row = (o = {}) => ({
|
||||
sport: 'mlb', model_version: 'engine1@2026-07-20',
|
||||
grade: 'B', grade_11: 'B-', p_win: 0.6,
|
||||
outcome: 'hit', quarantine_reason: null, snap_quarantine: null,
|
||||
features: { l5_avg: 1.2 },
|
||||
...o,
|
||||
});
|
||||
|
||||
describe('THE REFUSAL — sample-size honesty is the first property', () => {
|
||||
test('a tiny sample reports INSUFFICIENT, never a point estimate', () => {
|
||||
const rows = [row(), row({ outcome: 'miss' }), row()];
|
||||
const out = h.runBacktest(rows, { minSample: 20 });
|
||||
const b = out.grade_buckets.find((x) => x.bucket === 'B');
|
||||
expect(b.status).toBe('INSUFFICIENT');
|
||||
expect(b.n).toBe(3);
|
||||
expect(b.need).toBe(20);
|
||||
expect(b.hit_rate).toBeNull(); // no number, at all
|
||||
});
|
||||
|
||||
test('the top-line verdict itself refuses on thin data', () => {
|
||||
const out = h.runBacktest([row(), row()], { minSample: 20 });
|
||||
expect(out.verdict).toBe('INSUFFICIENT_HISTORY');
|
||||
expect(out.can_validate).toBe(false);
|
||||
});
|
||||
|
||||
test('an INSUFFICIENT bucket carries no rate anywhere in its payload', () => {
|
||||
const out = h.runBacktest([row()], { minSample: 20 });
|
||||
const json = JSON.stringify(out.grade_buckets);
|
||||
expect(json).not.toMatch(/"hit_rate":\s*[0-9]/);
|
||||
});
|
||||
|
||||
test('at/above threshold it DOES report a rate with an interval', () => {
|
||||
const rows = Array.from({ length: 25 }, (_, i) => row({ outcome: i < 15 ? 'hit' : 'miss' }));
|
||||
const out = h.runBacktest(rows, { minSample: 20 });
|
||||
const b = out.grade_buckets.find((x) => x.bucket === 'B');
|
||||
expect(b.status).toBe('OK');
|
||||
expect(b.hit_rate).toBeCloseTo(0.6, 2);
|
||||
expect(b.ci_low).toBeLessThan(0.6);
|
||||
expect(b.ci_high).toBeGreaterThan(0.6);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DENOMINATOR — the excluded states', () => {
|
||||
test('quarantined, void, unrecoverable, pending and push never count', () => {
|
||||
const rows = [
|
||||
row({ outcome: 'hit' }), row({ outcome: 'miss' }),
|
||||
row({ outcome: 'void' }), row({ outcome: 'unrecoverable' }),
|
||||
row({ outcome: null }), row({ outcome: 'push' }),
|
||||
row({ outcome: 'hit', quarantine_reason: 'wrong_opponent_grade' }),
|
||||
row({ outcome: 'hit', snap_quarantine: 'wrong_opponent_grade' }),
|
||||
];
|
||||
const out = h.runBacktest(rows, { minSample: 1 });
|
||||
expect(out.counts.scored).toBe(2); // only the real hit + miss
|
||||
expect(out.counts.excluded_quarantine).toBe(2);
|
||||
expect(out.counts.excluded_terminal).toBe(2);
|
||||
expect(out.counts.excluded_pending).toBe(1);
|
||||
expect(out.counts.excluded_push).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('NEVER MIX sports or model versions', () => {
|
||||
test('buckets are split per sport and per version', () => {
|
||||
const rows = [
|
||||
...Array.from({ length: 21 }, () => row({ sport: 'mlb' })),
|
||||
...Array.from({ length: 21 }, () => row({ sport: 'wnba' })),
|
||||
...Array.from({ length: 21 }, () => row({ model_version: 'engine2@x' })),
|
||||
];
|
||||
const out = h.runBacktest(rows, { minSample: 20 });
|
||||
const keys = out.strata.map((s) => `${s.sport}|${s.model_version}`);
|
||||
expect(new Set(keys).size).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MONOTONICITY — and refusing to rank overlapping buckets', () => {
|
||||
test('reports overlapping intervals as not distinguishable, not as a ranking', () => {
|
||||
const mk = (grade, hits, n) => Array.from({ length: n }, (_, i) => row({ grade, outcome: i < hits ? 'hit' : 'miss' }));
|
||||
// A: 13/25 (52%), B: 14/25 (56%) — intervals overlap heavily.
|
||||
const out = h.runBacktest([...mk('A', 13, 25), ...mk('B', 14, 25)], { minSample: 20 });
|
||||
const cmp = out.monotonicity.comparisons.find((c) => c.pair === 'A>B' || c.pair === 'B>A');
|
||||
expect(cmp.distinguishable).toBe(false);
|
||||
expect(out.monotonicity.verdict).toMatch(/NOT_DISTINGUISHABLE|INSUFFICIENT/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('PROBABILITY calibration', () => {
|
||||
test('a thin sample reports INSUFFICIENT and NO Brier score', () => {
|
||||
const rows = [row({ p_win: 0.8 }), row({ p_win: 0.2, outcome: 'miss' })];
|
||||
const out = h.runBacktest(rows, { minSample: 20 });
|
||||
expect(out.probability.status).toBe('INSUFFICIENT');
|
||||
expect(out.probability.brier).toBeNull();
|
||||
});
|
||||
|
||||
test('Brier score is computed from p_win vs realized outcome', () => {
|
||||
const rows = [row({ p_win: 1, outcome: 'hit' }), row({ p_win: 0, outcome: 'miss' })];
|
||||
const out = h.runBacktest(rows, { minSample: 1 });
|
||||
expect(out.probability.status).toBe('OK');
|
||||
expect(out.probability.brier).toBeCloseTo(0, 6); // perfect prediction
|
||||
});
|
||||
|
||||
test('rows without p_win are excluded from Brier, not defaulted to 0.5', () => {
|
||||
const rows = [row({ p_win: null, outcome: 'hit' }), row({ p_win: 1, outcome: 'hit' })];
|
||||
const out = h.runBacktest(rows, { minSample: 1 });
|
||||
expect(out.probability.n).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ALTERNATIVE-MODEL REPLAY SEAM', () => {
|
||||
test('identity replay reproduces the live prediction exactly', () => {
|
||||
const rows = [row({ p_win: 0.7 }), row({ p_win: 0.3, outcome: 'miss' })];
|
||||
const res = h.replayAlternative(rows, (r) => ({ p_win: r.p_win }), { minSample: 1 });
|
||||
expect(res.identical).toBe(true);
|
||||
expect(res.candidate.brier).toBeCloseTo(res.live.brier, 9);
|
||||
});
|
||||
|
||||
test('a row whose needed input was never retained is UN-BACKTESTABLE, not guessed', () => {
|
||||
const rows = [row({ features: {} })];
|
||||
const res = h.replayAlternative(rows, (r) => (
|
||||
r.features && r.features.opp_rank_stat != null
|
||||
? { p_win: 0.9 }
|
||||
: { unavailable: 'opp_rank_stat_not_retained' }
|
||||
), { minSample: 1 });
|
||||
expect(res.un_backtestable).toBe(1);
|
||||
expect(res.candidate.n).toBe(0);
|
||||
});
|
||||
|
||||
test('the seam never refetches — it only sees the stored feature vector', () => {
|
||||
let sawOnlyStored = true;
|
||||
h.replayAlternative([row()], (r) => {
|
||||
if (Object.keys(r).some((k) => /fetch|client|http/i.test(k))) sawOnlyStored = false;
|
||||
return { p_win: 0.5 };
|
||||
}, { minSample: 1 });
|
||||
expect(sawOnlyStored).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -309,3 +309,26 @@ describe('settlementRateAlarm (Session 64)', () => {
|
||||
expect(opsWatch.settlementRateAlarm([{ sport: 'soccer', skipped: 'not configured' }]).alarm).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('harnessStaleAlarm (Session 64)', () => {
|
||||
const opsWatch = require('../../src/services/opsWatch');
|
||||
const NOW = new Date('2026-07-21T12:00:00Z');
|
||||
|
||||
test('never-run is an alarm — a validator that never ran looks like one that passes', () => {
|
||||
const r = opsWatch.harnessStaleAlarm(null, NOW);
|
||||
expect(r.alarm).toBe(true);
|
||||
expect(r.reason).toMatch(/NEVER/);
|
||||
});
|
||||
|
||||
test('a stale run pages', () => {
|
||||
expect(opsWatch.harnessStaleAlarm('2026-07-19T00:00:00Z', NOW).alarm).toBe(true);
|
||||
});
|
||||
|
||||
test('a recent run is quiet', () => {
|
||||
expect(opsWatch.harnessStaleAlarm('2026-07-21T06:00:00Z', NOW).alarm).toBe(false);
|
||||
});
|
||||
|
||||
test('an unparseable timestamp is an alarm, not a silent pass', () => {
|
||||
expect(opsWatch.harnessStaleAlarm('not-a-date', NOW).alarm).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user