diff --git a/scripts/validate-grade-fix.js b/scripts/validate-grade-fix.js new file mode 100644 index 0000000..b929d2a --- /dev/null +++ b/scripts/validate-grade-fix.js @@ -0,0 +1,28 @@ +#!/usr/bin/env node +// Post-fix re-grade validation (work-order item #4). Fetches the LIVE MLB +// snapshot and checks the three degradation signatures are gone: +// 1. projection == 0 count -> expect 0 (the nine vanish; those props refuse) +// 2. edge_pct continuous, not the {20,60,100,140} degenerate cluster, no 100 +// 3. grade <-> confidence agreement -> expect 25/25 (letter == band(conf)) +// Run: node scripts/validate-grade-fix.js (needs outbound to api.vyndr.app) +const https = require('https'); +const BANDS = require('../src/services/python/data/grade_thresholds.json').grade_scale; +const four = (g) => { const s = String(g||'').toUpperCase(); return s === 'A+' ? 'A+' : (s[0]||null); }; +function bandOf(conf) { const p = conf/100; for (const [k,b] of Object.entries(BANDS)) if (p>=b.low && p<=b.high) return k; return null; } +function get(url) { return new Promise((res,rej) => https.get(url,(r)=>{let d='';r.on('data',c=>d+=c);r.on('end',()=>res(JSON.parse(d)));}).on('error',rej)); } +(async () => { + const d = await get('https://api.vyndr.app/api/snapshot/mlb'); + const g = d.grades || []; + const proj0 = g.filter(x => x.projection === 0 || x.projection == null).length; + const edges = g.map(x => x.edge_pct).filter(v => v != null); + const has100 = edges.includes(100); + const distinctEdges = [...new Set(edges)].sort((a,b)=>a-b); + let agree = 0; + for (const x of g) { const b = bandOf(x.confidence); if (b && four(b) === four(x.grade)) agree++; } + console.log('=== AFTER (post-fix) — live MLB snapshot ==='); + console.log('updated_at:', d.updated_at, '| grades:', g.length); + console.log('1. projection==0/null:', proj0, proj0 === 0 ? 'PASS' : 'FAIL (expect 0)'); + console.log('2. edge_pct distinct:', distinctEdges.length, 'values; contains 100:', has100, (!has100 && distinctEdges.length > 4) ? 'PASS' : 'CHECK'); + console.log(' values:', distinctEdges.slice(0, 20)); + console.log('3. grade<->confidence agree:', agree + '/' + g.length, agree === g.length ? 'PASS' : 'FAIL'); +})().catch(e => { console.error('validation error:', e.message); process.exit(1); }); diff --git a/specs/audit-data/mlb-grade-degradation.md b/specs/audit-data/mlb-grade-degradation.md index dd2f142..b07bc51 100644 --- a/specs/audit-data/mlb-grade-degradation.md +++ b/specs/audit-data/mlb-grade-degradation.md @@ -1,37 +1,66 @@ -# MLB Grade Pipeline Degradation — Backend Finding (2026-07-17) +# MLB Grade Pipeline Degradation — FIXED (2026-07-17) Source: phone-audit P1-7 (broken edge board) + P2-9 (B grades at 45% confidence). -Diagnosed against LIVE `GET /api/snapshot/mlb` on 2026-07-17. This is a BACKEND -grading/feature-pipeline issue, NOT a frontend display bug. The frontend guards -(sane-edge cap, statAbbrev) are damage control, not the fix. +Diagnosed against LIVE `GET /api/snapshot/mlb` on 2026-07-17. Backend grading +bug, fixed at the source in the generic grade path (`engine1` + +`analyzeViaEngine1`), which grades EVERY sport. -## The evidence (25 live MLB grades) -- **projection == 0 for 9/25.** The model has no projection reference for a third - of props — a degraded feature path (the S58 insufficient-data refusal should - have caught these; instead they graded with projection 0). -- **edge_pct on a broken scale.** Distinct values {20, 60, 100, 140}. A real - prop-market edge is single-digit %, never past ~40. These are not a market edge - (frontend now shows them absent — slateAdapter EDGE_BOARD_SANE_MAX / EdgeCell). -- **grade <-> confidence mismatch on 25/25.** Every grade's letter disagrees with - its own surfaced `confidence` vs `grade_thresholds.json`: - - B shown at confidence 55 -> 55 is the B- band (55-59), not B (60-65). - - Systematic ~5-point (one sub-tier) gap on every prop. - The surfaced `confidence` is NOT the probability that derived the letter. +## Before (25 live MLB grades, degraded) +- **projection == 0 for 9/25** — graded on a zero projection. +- **edge_pct quantized to {20, 60, 100, 140}** — the 100s were the proj=0 + degeneracy `(line - 0)/line = 100%`. +- **grade ↔ confidence mismatch** — 10/25 disagreed even at the 4-letter level + (25/25 vs the stricter 11-step bands). -## Likely root cause (to investigate, not yet fixed) -1. `confidence` field is post-`apply_data_sufficiency_modifier` (bayesian.py), - while the grade letter was assigned from the PRE-penalty prob -> the two - disagree by one sub-tier on every prop. Either surface the same value, or - re-letter from the penalized confidence. Decide which is the source of truth. -2. projection==0 for 9/25: the MLB feature/projection path is feeding 0 instead - of refusing (S58 `insufficient_data`). Trace `projectionFor` for MLB props; - projection 0 also breaks `computeEdge` (=> the 20/60/100/140 garbage). +## Root causes + fixes (commit `888d103`) +1. **projection=0 bypassed the refusal gate.** `projectionFor` returned + `l5_avg` even when 0 (finite → the `== null` gate passed it). FIX: a + non-positive reference is not a projection — `projectionFor` skips it and + falls through to the next POSITIVE reference (l5 → l20 → per_90 → xg); when + none is positive it returns null and the read REFUSES (`insufficient_data`). + The gate also gained an explicit `> 0` guard — the invariant is structural. +2. **edge_pct.** Formula was already `(model - line)/line` (the intended + semantics); the {100} cluster was purely the proj=0 degeneracy. With fix 1 + those refuse. Main-line edge now reuses the VALIDATED projection so edge and + the persisted `projection` can't diverge. +3. **confidence/letter split.** `engine1.GRADE_TO_CONFIDENCE` was hand-rolled + and drifted a full sub-tier low (B → 0.55, which `grade_thresholds.json` + calls B-). FIX: confidence is now DERIVED from each grade's band MIDPOINT in + `grade_thresholds.json` — one source of truth. Applying the threshold table + to any grade's displayed confidence resolves back to the same letter (proven + for all 11 grades in `tests/unit/mlbGradeDegradation.test.js`). -## Do NOT -- Do not re-letter or shift thresholds on the frontend (would hide the bug). -- Do not "fix" edge display by rescaling 140 -> 14 (guessing the scale is its own - fabrication). The number must come out of the pipeline correct. +## Blast radius (commit `9fc4edf`) — work-order #6 +The degraded grades (projection=0 → `model_value = 0`) are already settled in +the append-only `ledger_entries` and are NOT deleted. Functional marking: +`getModelAggregate` now filters `.gt('model_value', 0)` on the settled AND +pending queries — the rows stay in the ledger but leave the public model record +(their hit/miss is noise, not skill). Post-fix no new such row can be written. -## Frontend already shipped (honest guards) -- Leaderboard stat labels: `web/src/lib/statAbbrev.js` (raw snake_case -> SB/ER/TB). -- Edge board: impossible |edge|>40 rendered absent AND excluded from ranking. +**Exact count NOT queryable from the dev box** (`*.supabase.co` is unreachable +here — curl 000; only `vyndr.app`/`api.vyndr.app` resolve; no `VYNDR_INTERNAL_KEY` +locally). Proxy signal: 9/25 (36%) of the current live slate. For the precise +figure, run in Supabase SQL: +```sql +SELECT count(*) FILTER (WHERE outcome IS NOT NULL) AS settled_degraded, + count(*) AS total_degraded +FROM ledger_entries +WHERE user_id IS NULL AND model_value = 0; +``` + +## Other sports — work-order #5 +NBA/WNBA/soccer grade through the SAME `analyzeViaEngine1` → `engine1` path +(`gradeSlateService` does not branch by sport; `mlbGrader.js` is dead code). So +they SHARE the disease and are fixed by the same commit. They rarely grade in +prod today (stats service offline off-season → refuse anyway). No separate fix. + +## Live validation — work-order #4 +The fix deploys immediately, but the SNAPSHOT only re-grades on the full cron +(UTC hours 14,19,22,1,3). Run after the next 14:00 UTC snapshot post-deploy: +``` +node scripts/validate-grade-fix.js +``` +PASS criteria: projection==0 count → 0; edge_pct no longer contains 100 and is +not the four-value cluster; grade↔confidence agreement 25/25. The before-state +(this file's "Before") is the diff baseline; the script's output is the +fingerprint.