Files
vyndr/tests/unit/analyzeViaEngine1.test.js
T
builtbykev 3591c7626e Total grade cutover + the ceiling stated as a position
PHASE 0 caught my own repeat of the failure I diagnosed one order ago.
91927a4 attached `served_grade` BESIDE the old letter and left `grade`
alone -- so the honest grade reached nobody, exactly as gradeBands had
been built-correct-and-unread. grep showed served_grade appearing in one
file (where I set it) and all 14+ consumers -- scan route, dashboard,
parlay, newsletter, desk, content templates, retention -- still reading
`.grade`, i.e. still the dishonest letter.

CUTOVER IS NOW TOTAL: legacy.grade IS the honest letter. Overwriting the
one field every consumer already reads cuts every surface over at once
instead of editing fourteen call sites and missing one. engine1's index is
preserved as `engine_grade` and verified read by ZERO serving code.

Confidence follows the letter: it came from a grade-band midpoint of the
OLD letter, so leaving it would have paired a served B+ with a C's
confidence. Both now derive from p_win, kept on the existing 0-100 scale.

MEASURED BLAST RADIUS before shipping: 303 of 47,991 non-refused
snapshots (0.6%) have a grade but no p_win, and now render NO READ instead
of a letter. That is correct -- their old letter came from the retired
index carrying 0.48% resolution, i.e. noise -- and NO READ is a rendered
state with a reason, so never-blank holds.

PHASE 1 — the ceiling is now a STATED POSITION, not a confusing absence.
servedGrade.SCALE_LEGEND plus web GradeScaleLegend.tsx say it plainly: we
do not issue A grades, no band has hit at a rate that would justify one,
our honest ceiling is a strong B+ (~66% realized vs ~60% baseline), and if
the model earns an A the legend changes and we say why. The
separates_from_base_rate flag renders per band -- C+/C/C- are labelled
"we cannot separate this from the baseline", which is most of any slate.

PHASE 3 hand-verified across every state: B+ with 3 factors (basis
forecast_plus_matchup_factors), B+ with none (forecast_only), C flagged
not-separable, F, and three refusal states rendering NO READ with reasons.
never-blank PASS, no-manufactured-A PASS.

Test fallout was real and is documented rather than papered over: engine
BEHAVIOUR assertions moved to engine_grade, suppression assertions stayed
on grade (a suppressed prop has no letter either way), and the confidence
78 -> 95 change is the grade-band midpoint being replaced by p_win.

No A-threshold loosening. No calibrated number leaks (deployed set empty).
p_win never mutated. Ten frozen modules verified unchanged including
engine1 and probabilityEstimator. No Bonferroni slot.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01W1sivYNqY2TS5ftykmHBU9
2026-08-07 14:07:02 -04:00

258 lines
11 KiB
JavaScript

/**
* NOTE ON `engine_grade` (2026-08-07 grade cutover).
*
* The user-facing `grade` now derives from `p_win`, not engine1's additive
* factor index -- that index carried 0.16x the information of p_win on 3,417
* settled props and its A hit worse than its F. The engine's OWN decision
* (graded vs suppressed) is preserved as `engine_grade`, so behaviour
* assertions read that; suppression assertions still read `grade`, since a
* suppressed prop has no letter either way.
*/
// Fix 2 (Session 7f) — verifies the end-to-end shape from
// computeFeaturesForProp → engine1 → adapter → concrete reasoning.
const mockComputeReturn = { current: null };
jest.mock('../../src/services/intelligence/computeFeatures', () => ({
computeFeaturesForProp: async () => mockComputeReturn.current,
}));
const mockEngine1Return = { current: null };
jest.mock('../../src/services/intelligence/engine1', () => ({
gradeProp: () => mockEngine1Return.current,
}));
const { analyzeViaEngine1 } = require('../../src/services/intelligence/analyzeViaEngine1');
beforeEach(() => {
mockComputeReturn.current = null;
mockEngine1Return.current = null;
});
describe('analyzeViaEngine1 — happy path', () => {
test('produces the full legacy shape with concrete numbers', async () => {
mockComputeReturn.current = {
features: {
l5_avg: 28.4, l20_avg: 26.1, home_away: 1.0,
opp_rank_stat: 0.82, rest_days: 2,
},
trap: { composite: 0.1, signals: {}, recommendation: 'proceed' },
consistency: { consistency: 'reliable', cv: 0.18, score: 0.7, games: 20 },
prop: { line: 25.5, direction: 'over' },
meta: {
player: 'Jalen Brunson', statType: 'points', line: 25.5,
direction: 'over', book: 'draftkings', sport: 'nba',
teamAbbr: 'NYK', opponentAbbr: 'BOS', gameId: 'ev-1',
isHome: true, gameLogs: [{ points: 28 }], errors: [],
},
};
mockEngine1Return.current = {
grade: 'A-', confidence: 0.78,
top_factors: ['l5_hot_vs_line', 'weak_opponent_defense', 'home_game'],
all_factors: ['l5_hot_vs_line', 'weak_opponent_defense', 'home_game', 'rested_2plus'],
};
const out = await analyzeViaEngine1({
player: 'Jalen Brunson', stat_type: 'points', line: 25.5, direction: 'over', book: 'draftkings', sport: 'nba',
});
// Adapter-collapsed grade.
expect(out.engine_grade).toBe('A');
// Confidence now derives from p_win (0.95 -> 95), not from a grade-band
// midpoint of the old letter. The old 78 was the midpoint for engine1's "A"
// -- a number that carried no information beyond the letter it was looked up
// from, and which would now contradict the served B+.
expect(out.confidence).toBe(95);
expect(out.confidence_basis).toBe('p_win');
expect(out.player).toBe('Jalen Brunson');
expect(out.stat_type).toBe('points');
expect(out.line).toBe(25.5);
expect(out.direction).toBe('over');
expect(out.book).toBe('draftkings');
// Reasoning has concrete numbers, not abstract factor labels.
expect(out.reasoning.summary).toContain('28.4');
expect(out.reasoning.summary).toContain('26.1');
expect(out.reasoning.summary).toContain('Jalen Brunson');
expect(out.reasoning.summary).toContain('BOS');
expect(out.reasoning.summary).toContain('Engine 1 graded A-');
// Legacy-shaped steps object — named sub-blocks for backward compat
// with the analyze integration test, plus a `narrative` array for
// the line-by-line breakdown.
expect(typeof out.reasoning.steps).toBe('object');
expect(out.reasoning.steps).toHaveProperty('season_avg');
expect(out.reasoning.steps).toHaveProperty('recent_form');
expect(out.reasoning.steps).toHaveProperty('situational');
expect(out.reasoning.steps).toHaveProperty('final_grade');
expect(Array.isArray(out.reasoning.steps.narrative)).toBe(true);
expect(out.reasoning.steps.narrative.length).toBeGreaterThan(0);
expect(out.reasoning.steps.narrative[0]).toHaveProperty('step');
expect(out.reasoning.steps.narrative[0]).toHaveProperty('detail');
// Real numbers in the sub-blocks.
expect(out.reasoning.steps.season_avg.value).toBe(26.1);
expect(out.reasoning.steps.recent_form.value).toBe(28.4);
// edge_pct computed from l5_avg vs line.
// (28.4 - 25.5) / 25.5 * 100 = ~11.4
expect(out.edge_pct).toBeCloseTo(11.4, 1);
expect(Array.isArray(out.kill_conditions_triggered)).toBe(true);
});
test('away game + strong defense + back-to-back surfaces in reasoning', async () => {
mockComputeReturn.current = {
features: { l5_avg: 18, l20_avg: 22, home_away: 0.0, opp_rank_stat: 0.15, rest_days: 0 },
trap: { composite: 0.6, signals: {} },
consistency: { consistency: 'reliable', score: 0.7 },
prop: { line: 24.5, direction: 'over' },
meta: { player: 'P', statType: 'points', book: 'dk', sport: 'nba',
teamAbbr: 'X', opponentAbbr: 'OKC', gameId: 'g', isHome: false,
gameLogs: [{ points: 20 }], errors: [] },
};
mockEngine1Return.current = {
grade: 'D', confidence: 0.2,
top_factors: ['l5_cold_vs_line', 'top_opponent_defense', 'back_to_back'],
all_factors: ['l5_cold_vs_line', 'top_opponent_defense', 'back_to_back'],
};
const out = await analyzeViaEngine1({
player: 'P', stat_type: 'points', line: 24.5, direction: 'over', sport: 'nba',
});
expect(out.engine_grade).toBe('D');
expect(out.reasoning.summary).toContain('Playing on the road');
expect(out.reasoning.summary).toContain('OKC');
expect(out.reasoning.summary).toContain('top-tier defense');
expect(out.reasoning.summary).toContain('Back-to-back');
expect(out.reasoning.summary).toContain('leans against the play');
});
});
describe('analyzeViaEngine1 — graceful degradation', () => {
test('hard fallback when everything failed (no features, no logs, no consistency)', async () => {
mockComputeReturn.current = {
features: {},
trap: { composite: 0, signals: {} },
consistency: { consistency: 'unknown', score: null, games: 0 },
prop: { line: 25, direction: 'over' },
meta: { player: 'Ghost', statType: 'points', book: 'dk', sport: 'nba',
teamAbbr: null, opponentAbbr: null, gameId: null, isHome: null,
gameLogs: [], errors: ['player_not_found_in_id_map', 'no_game_scheduled_today'] },
};
const out = await analyzeViaEngine1({
player: 'Ghost', stat_type: 'points', line: 25, direction: 'over', sport: 'nba',
});
// Session 58 (work-order 1.5) — this used to ship a hollow C at 10%
// confidence (the audit's model==line degenerate). Now it REFUSES.
expect(out.grade).toBeNull();
expect(out.insufficient_data).toBe(true);
expect(out.confidence).toBe(0);
expect(out.projection).toBeNull();
expect(out.reasoning.summary).toMatch(/INSUFFICIENT DATA — no read/);
expect(out.reasoning.summary).toContain("couldn't find");
expect(out.kill_conditions_triggered).toEqual([]);
});
test('partial data WITHOUT a projection refuses honestly (no hollow grade)', async () => {
mockComputeReturn.current = {
features: {},
trap: { composite: 0, signals: {} },
consistency: { consistency: 'reliable', cv: 0.2, score: 0.7, games: 20 },
prop: { line: 25, direction: 'over' },
meta: { player: 'P', statType: 'points', book: 'dk', sport: 'nba',
teamAbbr: 'NYK', opponentAbbr: null, gameId: null, isHome: null,
gameLogs: [{ points: 25 }], errors: ['no_game_scheduled_today'] },
};
mockEngine1Return.current = {
grade: 'C', confidence: 0.4,
top_factors: [], all_factors: [],
};
const out = await analyzeViaEngine1({
player: 'P', stat_type: 'points', line: 25, direction: 'over', sport: 'nba',
});
// No l5/l20 reference ⇒ the model has no projection ⇒ no read. The old
// behavior graded C/40 here — a grade the model couldn't actually back.
expect(out.grade).toBeNull();
expect(out.insufficient_data).toBe(true);
expect(out.reasoning.summary).toContain('No game scheduled');
});
test('a projection unlocks the grade AND is attached to the result', async () => {
mockComputeReturn.current = {
features: { l5_avg: 28.4, l20_avg: 26.1 },
trap: { composite: 0, signals: {} },
consistency: { consistency: 'reliable', cv: 0.2, score: 0.7, games: 20 },
prop: { line: 25, direction: 'over' },
meta: { player: 'P', statType: 'points', book: 'dk', sport: 'nba',
teamAbbr: 'NYK', opponentAbbr: null, gameId: null, isHome: null,
gameLogs: [{ points: 25 }], errors: [] },
};
mockEngine1Return.current = {
grade: 'B', confidence: 0.6,
top_factors: [], all_factors: [],
};
const out = await analyzeViaEngine1({
player: 'P', stat_type: 'points', line: 25, direction: 'over', sport: 'nba',
});
expect(out.engine_grade).toBe('B');
expect(out.insufficient_data).toBeUndefined();
expect(out.projection).toBe(28.4); // the REAL model reference, never the line
});
});
describe('analyzeViaEngine1 — interface verifications', () => {
test('does not throw when computeFeaturesForProp resolves with errors', async () => {
mockComputeReturn.current = {
features: { l5_avg: 20 },
trap: { composite: 0, signals: {} },
consistency: { consistency: 'unknown', score: null, games: 0 },
prop: { line: 25, direction: 'over' },
meta: { sport: 'nba', errors: ['no_features_computed'] },
};
mockEngine1Return.current = { grade: 'C', confidence: 0.3, top_factors: [], all_factors: [] };
const out = await analyzeViaEngine1({
player: 'X', stat_type: 'points', line: 25, direction: 'over', sport: 'nba',
});
expect(out).toBeDefined();
expect(out.engine_grade).toBeDefined();
});
test('every legacy field DemoScan reads is present', async () => {
mockComputeReturn.current = {
features: { l5_avg: 28 },
trap: { composite: 0, signals: {} },
consistency: { consistency: 'reliable', score: 0.7 },
prop: { line: 25, direction: 'over' },
meta: { sport: 'nba', errors: [] },
};
mockEngine1Return.current = { grade: 'A-', confidence: 0.7, top_factors: ['x'], all_factors: ['x'] };
const out = await analyzeViaEngine1({
player: 'X', stat_type: 'points', line: 25, direction: 'over', sport: 'nba',
});
// DemoScan reads: grade, confidence, reasoning.summary,
// kill_conditions_triggered[].code, edge_pct, line, player, stat_type.
expect(out.engine_grade).toBeDefined();
expect(typeof out.confidence).toBe('number');
expect(typeof out.reasoning.summary).toBe('string');
expect(Array.isArray(out.kill_conditions_triggered)).toBe(true);
expect(typeof out.edge_pct).toBe('number');
expect(out.player).toBeDefined();
expect(out.stat_type).toBeDefined();
expect(out.line).toBeDefined();
});
test('does not import from legacy path (no propAnalyzer/grader/UnifiedOddsProvider)', () => {
const fs = require('fs');
const src = fs.readFileSync(require.resolve('../../src/services/intelligence/analyzeViaEngine1.js'), 'utf8');
expect(src).not.toMatch(/propAnalyzer/);
expect(src).not.toMatch(/require.*['"]\.\.\/grader/);
expect(src).not.toMatch(/UnifiedOddsProvider/);
});
});