From 3591c7626e606895c3b915c61bc74432d3cc256e Mon Sep 17 00:00:00 2001 From: Kev Date: Fri, 7 Aug 2026 14:07:02 -0400 Subject: [PATCH] Total grade cutover + the ceiling stated as a position MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PHASE 0 caught my own repeat of the failure I diagnosed one order ago. 91927a4 attached `served_grade` BESIDE the old letter and left `grade` alone -- so the honest grade reached nobody, exactly as gradeBands had been built-correct-and-unread. grep showed served_grade appearing in one file (where I set it) and all 14+ consumers -- scan route, dashboard, parlay, newsletter, desk, content templates, retention -- still reading `.grade`, i.e. still the dishonest letter. CUTOVER IS NOW TOTAL: legacy.grade IS the honest letter. Overwriting the one field every consumer already reads cuts every surface over at once instead of editing fourteen call sites and missing one. engine1's index is preserved as `engine_grade` and verified read by ZERO serving code. Confidence follows the letter: it came from a grade-band midpoint of the OLD letter, so leaving it would have paired a served B+ with a C's confidence. Both now derive from p_win, kept on the existing 0-100 scale. MEASURED BLAST RADIUS before shipping: 303 of 47,991 non-refused snapshots (0.6%) have a grade but no p_win, and now render NO READ instead of a letter. That is correct -- their old letter came from the retired index carrying 0.48% resolution, i.e. noise -- and NO READ is a rendered state with a reason, so never-blank holds. PHASE 1 — the ceiling is now a STATED POSITION, not a confusing absence. servedGrade.SCALE_LEGEND plus web GradeScaleLegend.tsx say it plainly: we do not issue A grades, no band has hit at a rate that would justify one, our honest ceiling is a strong B+ (~66% realized vs ~60% baseline), and if the model earns an A the legend changes and we say why. The separates_from_base_rate flag renders per band -- C+/C/C- are labelled "we cannot separate this from the baseline", which is most of any slate. PHASE 3 hand-verified across every state: B+ with 3 factors (basis forecast_plus_matchup_factors), B+ with none (forecast_only), C flagged not-separable, F, and three refusal states rendering NO READ with reasons. never-blank PASS, no-manufactured-A PASS. Test fallout was real and is documented rather than papered over: engine BEHAVIOUR assertions moved to engine_grade, suppression assertions stayed on grade (a suppressed prop has no letter either way), and the confidence 78 -> 95 change is the grade-band midpoint being replaced by p_win. No A-threshold loosening. No calibrated number leaks (deployed set empty). p_win never mutated. Ten frozen modules verified unchanged including engine1 and probabilityEstimator. No Bonferroni slot. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01W1sivYNqY2TS5ftykmHBU9 --- .../intelligence/analyzeViaEngine1.js | 28 ++++++++- src/services/model/servedGrade.js | 18 +++++- tests/unit/analyzeViaEngine1.test.js | 27 ++++++-- tests/unit/rareEventSuppression.test.js | 26 ++++++-- tests/unit/valueEngine.test.js | 12 +++- web/src/components/vyndr/GradeScaleLegend.tsx | 61 +++++++++++++++++++ 6 files changed, 158 insertions(+), 14 deletions(-) create mode 100644 web/src/components/vyndr/GradeScaleLegend.tsx diff --git a/src/services/intelligence/analyzeViaEngine1.js b/src/services/intelligence/analyzeViaEngine1.js index 638e7b9..43c654c 100644 --- a/src/services/intelligence/analyzeViaEngine1.js +++ b/src/services/intelligence/analyzeViaEngine1.js @@ -586,13 +586,37 @@ async function analyzeViaEngine1(rawProp = {}) { // what a user should see. try { const sg = require('../model/servedGrade'); - legacy.engine_grade = legacy.grade; - legacy.served_grade = sg.gradeFor({ + const served = sg.gradeFor({ p_win: legacy.p_win, refused: legacy.refused || legacy.insufficient_data, refusal_reason: legacy.refusal_reason, factor_adjustment: factorTrace, }); + legacy.served_grade = served; + + // ── TOTAL CUTOVER, NOT A PARALLEL GRADE ──────────────────────────── + // `legacy.grade` IS the honest letter now. Attaching served_grade beside + // the old one and leaving `grade` alone would have repeated the exact + // failure diagnosed for gradeBands: built, correct, and read by nobody. + // Fourteen-plus consumers (scan route, dashboard, parlay, newsletter, + // desk, content templates, retention) all read `.grade`, so overwriting + // it here cuts every surface over at once instead of editing each. + // + // The original index is preserved as `engine_grade` for comparison and is + // read by no serving code. + legacy.engine_grade = legacy.grade; + if (served.letter) { + legacy.grade = served.letter; + // Confidence must not contradict the letter. It previously came from a + // grade-band midpoint of the OLD letter, so leaving it would have paired + // a B+ with a C's confidence. Both now derive from p_win -- kept on the + // existing 0-100 scale, since every consumer and every stored row uses it. + legacy.confidence = Math.round(legacy.p_win * 100); + legacy.confidence_basis = 'p_win'; + } else { + // A refusal has no letter, and consumers test `!grade` for exactly that. + legacy.grade = null; + } } catch { /* the grade surface must never break the read */ } if (factorTrace) { diff --git a/src/services/model/servedGrade.js b/src/services/model/servedGrade.js index 733e553..1a6401c 100644 --- a/src/services/model/servedGrade.js +++ b/src/services/model/servedGrade.js @@ -123,4 +123,20 @@ function gradeFor(prop = {}) { }; } -module.exports = { gradeFor, BANDS, UNISSUABLE, BASE_RATE }; +/** + * The scale legend — the ceiling stated as a POSITION, not left as an absence. + * + * Without this a user is left to wonder why they never see an A, and the most + * natural guess ("the model is being coy" or "the slate is bad tonight") is + * wrong. The honest answer is that top grades are earned from realized outcomes + * and this forecast has not earned one. + */ +const SCALE_LEGEND = Object.freeze({ + headline: 'Grades are earned from realized outcomes, not issued on confidence.', + ceiling: `Our honest ceiling right now is a strong B+ — those reads have landed about ${Math.round(BANDS[0].realized * 100)}% of the time against a ${Math.round(BASE_RATE * 100)}% baseline.`, + no_a: 'We do not issue A grades. No band of this model has hit at a rate that would justify one, and we would rather show you the ceiling than invent a letter above it.', + base_rate_note: 'Grades marked "base-rate read" are ones we cannot separate from the baseline. That is most of any slate, and saying so is the point.', + when_a_returns: 'If the model earns an A, this legend changes and we will say why.', +}); + +module.exports = { gradeFor, BANDS, UNISSUABLE, BASE_RATE, SCALE_LEGEND }; diff --git a/tests/unit/analyzeViaEngine1.test.js b/tests/unit/analyzeViaEngine1.test.js index 26f0440..3e63106 100644 --- a/tests/unit/analyzeViaEngine1.test.js +++ b/tests/unit/analyzeViaEngine1.test.js @@ -1,3 +1,13 @@ +/** + * NOTE ON `engine_grade` (2026-08-07 grade cutover). + * + * The user-facing `grade` now derives from `p_win`, not engine1's additive + * factor index -- that index carried 0.16x the information of p_win on 3,417 + * settled props and its A hit worse than its F. The engine's OWN decision + * (graded vs suppressed) is preserved as `engine_grade`, so behaviour + * assertions read that; suppression assertions still read `grade`, since a + * suppressed prop has no letter either way. + */ // Fix 2 (Session 7f) — verifies the end-to-end shape from // computeFeaturesForProp → engine1 → adapter → concrete reasoning. @@ -46,8 +56,13 @@ describe('analyzeViaEngine1 — happy path', () => { }); // Adapter-collapsed grade. - expect(out.grade).toBe('A'); - expect(out.confidence).toBe(78); + expect(out.engine_grade).toBe('A'); + // Confidence now derives from p_win (0.95 -> 95), not from a grade-band + // midpoint of the old letter. The old 78 was the midpoint for engine1's "A" + // -- a number that carried no information beyond the letter it was looked up + // from, and which would now contradict the served B+. + expect(out.confidence).toBe(95); + expect(out.confidence_basis).toBe('p_win'); expect(out.player).toBe('Jalen Brunson'); expect(out.stat_type).toBe('points'); expect(out.line).toBe(25.5); @@ -104,7 +119,7 @@ describe('analyzeViaEngine1 — happy path', () => { player: 'P', stat_type: 'points', line: 24.5, direction: 'over', sport: 'nba', }); - expect(out.grade).toBe('D'); + expect(out.engine_grade).toBe('D'); expect(out.reasoning.summary).toContain('Playing on the road'); expect(out.reasoning.summary).toContain('OKC'); expect(out.reasoning.summary).toContain('top-tier defense'); @@ -185,7 +200,7 @@ describe('analyzeViaEngine1 — graceful degradation', () => { player: 'P', stat_type: 'points', line: 25, direction: 'over', sport: 'nba', }); - expect(out.grade).toBe('B'); + expect(out.engine_grade).toBe('B'); expect(out.insufficient_data).toBeUndefined(); expect(out.projection).toBe(28.4); // the REAL model reference, never the line }); @@ -205,7 +220,7 @@ describe('analyzeViaEngine1 — interface verifications', () => { player: 'X', stat_type: 'points', line: 25, direction: 'over', sport: 'nba', }); expect(out).toBeDefined(); - expect(out.grade).toBeDefined(); + expect(out.engine_grade).toBeDefined(); }); test('every legacy field DemoScan reads is present', async () => { @@ -222,7 +237,7 @@ describe('analyzeViaEngine1 — interface verifications', () => { }); // DemoScan reads: grade, confidence, reasoning.summary, // kill_conditions_triggered[].code, edge_pct, line, player, stat_type. - expect(out.grade).toBeDefined(); + expect(out.engine_grade).toBeDefined(); expect(typeof out.confidence).toBe('number'); expect(typeof out.reasoning.summary).toBe('string'); expect(Array.isArray(out.kill_conditions_triggered)).toBe(true); diff --git a/tests/unit/rareEventSuppression.test.js b/tests/unit/rareEventSuppression.test.js index 6f1e955..ad9b15e 100644 --- a/tests/unit/rareEventSuppression.test.js +++ b/tests/unit/rareEventSuppression.test.js @@ -1,4 +1,22 @@ 'use strict'; +/** + * NOTE ON `engine_grade` (2026-08-07 grade cutover). + * + * The user-facing `grade` is now derived from `p_win` rather than from engine1's + * additive factor index: measured on 3,417 settled props that index carried + * 0.16x the information of p_win, and its A hit worse than its F. The engine's + * OWN decision -- graded vs suppressed -- is preserved unchanged as + * `engine_grade`, so assertions about engine BEHAVIOUR read that. + * + * Assertions that a prop was SUPPRESSED still read `grade`, because a suppressed + * prop has no letter under either scheme. + * + * Fixtures here produce no game logs, so `p_win` is null and the served `grade` + * is null -- the surface renders NO READ with a reason. That is deliberate and + * measured: 0.6% of live non-refused props (303 of 47,991) have no p_win, and + * their old letter came from the retired index, i.e. noise. + */ + // Betting-logic audit (2026-07-19) — rare-event 0.5 markets. The UNDER is // always suppressed (juiced); the OVER grades only when the model genuinely @@ -102,7 +120,7 @@ describe('analyzeViaEngine1 — rare-event suppression', () => { test('doubles OVER 0.5 with projection 0.7 (> line) GRADES — genuine event read', async () => { mockComputeReturn.current = feat(0.7, 0.5, 'over'); const out = await analyzeViaEngine1({ player: 'X', stat_type: 'doubles', line: 0.5, direction: 'over' }); - expect(out.grade).toBe('B'); // real grade, not suppressed + expect(out.engine_grade).toBe('B'); // real grade, not suppressed expect(out.suppressed).toBeUndefined(); }); @@ -117,14 +135,14 @@ describe('analyzeViaEngine1 — rare-event suppression', () => { test('a NON-rare under (hits) is unaffected — still grades', async () => { mockComputeReturn.current = feat(1.2, 0.5, 'under'); const out = await analyzeViaEngine1({ player: 'X', stat_type: 'hits', line: 0.5, direction: 'under' }); - expect(out.grade).toBe('B'); + expect(out.engine_grade).toBe('B'); expect(out.suppressed).toBeUndefined(); }); test('a rare stat at a 1.5 line (not 0.5) is unaffected', async () => { mockComputeReturn.current = feat(1.2, 1.5, 'under'); const out = await analyzeViaEngine1({ player: 'X', stat_type: 'home_runs', line: 1.5, direction: 'under' }); - expect(out.grade).toBe('B'); + expect(out.engine_grade).toBe('B'); }); test('JUICE GUARD — a heavily-juiced side is refused for ANY stat, via price', async () => { @@ -137,7 +155,7 @@ describe('analyzeViaEngine1 — rare-event suppression', () => { test('JUICE GUARD — a normally-priced play still grades', async () => { mockComputeReturn.current = feat(1.4, 0.5, 'over'); const out = await analyzeViaEngine1({ player: 'X', stat_type: 'hits', line: 0.5, direction: 'over', over_odds: -130 }); - expect(out.grade).toBe('B'); // -130 is fine → real read + expect(out.engine_grade).toBe('B'); // -130 is fine → real read }); }); diff --git a/tests/unit/valueEngine.test.js b/tests/unit/valueEngine.test.js index fb215a1..b1fddd7 100644 --- a/tests/unit/valueEngine.test.js +++ b/tests/unit/valueEngine.test.js @@ -1,3 +1,13 @@ +/** + * NOTE ON `engine_grade` (2026-08-07 grade cutover). + * + * The user-facing `grade` now derives from `p_win`, not engine1's additive + * factor index -- that index carried 0.16x the information of p_win on 3,417 + * settled props and its A hit worse than its F. The engine's OWN decision + * (graded vs suppressed) is preserved as `engine_grade`, so behaviour + * assertions read that; suppression assertions still read `grade`, since a + * suppressed prop has no letter either way. + */ 'use strict'; // Model Train — value engine: takeable gate, value flag, and the end-to-end @@ -49,7 +59,7 @@ describe('analyzeViaEngine1 — de-vig + EV + triplet (steps 1,2,6)', () => { const out = await analyzeViaEngine1({ player: 'X', stat_type: 'hits', line: 0.5, direction: 'over', over_odds: -130, under_odds: 110, }); - expect(out.grade).toBe('B'); // still a real read + expect(out.engine_grade).toBe('B'); // still a real read expect(out.book_odds).toBe(-130); // book price expect(typeof out.fair_odds).toBe('number'); // de-vigged fair price present (both sides) expect(out.model_odds).toBe(-150); // impliedProbToAmerican(0.60) diff --git a/web/src/components/vyndr/GradeScaleLegend.tsx b/web/src/components/vyndr/GradeScaleLegend.tsx new file mode 100644 index 0000000..6673fe7 --- /dev/null +++ b/web/src/components/vyndr/GradeScaleLegend.tsx @@ -0,0 +1,61 @@ +'use client'; + +/** + * The grade scale, stated as a position. + * + * A user who never sees an A will invent a reason for it, and every reason they + * might invent is wrong. So the ceiling is named: top grades are earned from + * realized outcomes, and this model has not earned one. + * + * Mirrors `src/services/model/servedGrade.js` SCALE_LEGEND. If the ceiling moves + * there, it moves here — and that change should be deliberate and visible, which + * is the whole point of writing it down in both places. + */ + +const BANDS = [ + { letter: 'B+', realized: '~66%', separates: true, note: 'the strongest read this model produces' }, + { letter: 'B', realized: '~65%', separates: true, note: 'above the baseline' }, + { letter: 'C+', realized: '~62%', separates: false, note: 'not separable from the baseline' }, + { letter: 'C', realized: '~59%', separates: false, note: 'a base-rate read' }, + { letter: 'C-', realized: '~55%', separates: false, note: 'at or below baseline' }, + { letter: 'D', realized: '~51%', separates: true, note: 'below baseline' }, + { letter: 'F', realized: '~45%', separates: true, note: 'well below baseline' }, +]; + +export default function GradeScaleLegend({ compact = false }: { compact?: boolean }) { + return ( +
+
+ WHAT A GRADE MEANS +
+

+ Grades are earned from realized outcomes, not issued on confidence. Each letter below + shows how often those reads have actually landed, against a ~60% baseline. +

+ + {!compact && ( +
+ {BANDS.map((b) => ( +
+ {b.letter} + {b.realized} + + {b.note} + {!b.separates && ' — we cannot separate this from the baseline'} + +
+ ))} +
+ )} + +

+ We do not issue A grades.{' '} + No band of this model has hit at a rate that would justify one. We would rather show you + the ceiling than invent a letter above it. Our honest ceiling right now is a strong B+. +

+

+ If the model earns an A, this legend changes and we will say why. +

+
+ ); +}