From 89a2977f57d5f099d9347217556a51234bb0f870 Mon Sep 17 00:00:00 2001 From: Kev Date: Fri, 17 Jul 2026 16:08:18 -0400 Subject: [PATCH] =?UTF-8?q?Item=207=20=E2=80=94=20public=20accuracy=20read?= =?UTF-8?q?s=20the=20CLEAN=20ledger;=20BEAT=20CLOSE=20hidden=20until=20C4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kev's call: the 30D accuracy surfaces must read TRUTH, not a cache that can't be filtered. My earlier degraded-row exclusion only touched getModelAggregate (Postgres); the public buckets/badge still read outcomeService (Redis outcome log), which counts degraded projection-0 outcomes and has no field to filter on. - /api/accuracy (AccuracyBadge) + /api/ledger/accuracy (buckets/ModelRecord) now source from the clean Postgres ledger aggregate via new ledgerService.getAccuracyView + accuracyBucketsFromAgg (model_value > 0 excludes degraded rows). Same response shapes → no frontend change. Redis outcome log is now read by nothing public; it can age out or be rebuilt. - BEAT CLOSE is a MEASURED-WRONG ZERO: captureClosing re-records the locked line as the "closing" line, so clv is flat on the whole sample and beat_close reads 0% (comparing a number to itself). Full write-up: specs/audit-data/ clv-capture-broken.md (the fix belongs to C4). Until then, beat_close_pct + clv_distribution are SUPPRESSED at the source (getModelAggregate, gated by clvCaptureReliable() / CLV_CAPTURE_RELIABLE=1). Every public surface already renders BEAT CLOSE only when non-null, so they all hide it now — no wrong zero anywhere. HIT RATE (real) is unaffected. Suite 271/3261 green, web build exit 0. Co-Authored-By: Claude Opus 4.8 (1M context) --- specs/audit-data/clv-capture-broken.md | 41 ++++++++++++ src/routes/accuracy.js | 18 ++++-- src/routes/ledger.js | 13 +++- src/services/ledgerService.js | 86 ++++++++++++++++++++++++- tests/integration/accuracy.test.js | 73 ++++++++++++++------- tests/unit/ledgerAggregateScope.test.js | 2 + tests/unit/ledgerService.test.js | 24 +++++++ 7 files changed, 222 insertions(+), 35 deletions(-) create mode 100644 specs/audit-data/clv-capture-broken.md diff --git a/specs/audit-data/clv-capture-broken.md b/specs/audit-data/clv-capture-broken.md new file mode 100644 index 0000000..d7e0562 --- /dev/null +++ b/specs/audit-data/clv-capture-broken.md @@ -0,0 +1,41 @@ +# CLV Capture Broken — C4 Finding (2026-07-17) + +Source: Truth-Everywhere Part 2, item 7. Escalated from the "0% BEAT CLOSE" +observation on the public accuracy surfaces. + +## The bug +Every settled ledger row has `closing_line == locked_line` (and +`closing_odds == locked_odds`), so the computed `clv` is 0/flat on the ENTIRE +sample. `beat_close_pct` therefore reads a fabricated-looking **0%** — it is +comparing a number to itself, not measuring closing-line value. + +Evidence (live `/api/ledger/model`, 2026-07-17): sampled rows show +`line == closing_line`, `clv: null`/0, `clv_result: null`/flat across the board. + +## Root cause (to fix in C4) +`ledgerService.captureClosing` overwrites today's unsettled rows' +`closing_line`/`closing_odds` on every snapshot with the CURRENT feed values. +The intent (S58) was "the last write before game start is the close." But in +practice the captured value equals the locked line every time — either the +lines genuinely don't move in the captured window, or captureClosing is reading +the same feed field the lock came from and writing it back unchanged. Net: the +"closing" column is a copy of the lock, so CLV is structurally always 0. + +## The fix (C4 — not done here) +Capture the REAL last line before game start into a SEPARATE field, never +defaulted to the lock: +- Record `closing_line` only from a distinct closing snapshot (the last feed + read before first pitch/tip), and only when it actually differs from the + lock — otherwise leave it null (absent, not a copy). +- CLV = signed(locked - closing) by side, computed only when a real, distinct + closing line exists. No close captured ⇒ CLV null for that row (honest), not 0. +- Verify against a few known line moves before trusting the aggregate. + +## Interim (shipped this pass, item 7) +`beat_close_pct` and `clv_distribution` are SUPPRESSED at the source +(`getModelAggregate`, gated by `clvCaptureReliable()` / +`CLV_CAPTURE_RELIABLE=1`). Every public surface (ModelRecord, ledger MODEL tab, +public profiles, OG images) already renders BEAT CLOSE only when non-null, so +they all hide it now — no measured-wrong zero on any public surface. HIT RATE +(which is real) is unaffected. Flip `CLV_CAPTURE_RELIABLE=1` once C4 lands and +real closes are verified. diff --git a/src/routes/accuracy.js b/src/routes/accuracy.js index e740457..8c5dfda 100644 --- a/src/routes/accuracy.js +++ b/src/routes/accuracy.js @@ -3,27 +3,31 @@ /** * GET /api/accuracy (Session 55) — the system's track record. * - * Public, cache-only read of the rolling accuracy record written by - * outcomeService (settled snapshot grades vs real results). Powers the - * dashboard "A-rated: 68% hit rate" pill and the grade-card accuracy line. - * NEVER triggers settlement (that's the internal cron) → no API credits spent. + * Public, cache-only read of the model's track record. Powers the AccuracyBadge + * "A-RATED · X% HIT · 30D" pill and the grade-card accuracy line. + * + * Truth-Everywhere Part 2 (item 7) — sourced from the CLEAN Postgres ledger + * aggregate (getAccuracyView: model_value > 0 excludes the degraded + * projection-0 rows), NOT the Redis outcome log (which still counts them and + * can't be filtered). Redis is a cache; when a cache can't be filtered, read + * from truth. NEVER triggers settlement → no API credits spent. */ const express = require('express'); const { createRateLimit } = require('../middleware/rateLimit'); -const outcomeService = require('../services/outcomeService'); +const ledgerService = require('../services/ledgerService'); const router = express.Router(); router.use(createRateLimit({ windowMs: 60_000, max: 60 })); router.get('/', async (req, res) => { try { - const acc = await outcomeService.getAccuracy(); + const acc = await ledgerService.getAccuracyView({}); res.set('Cache-Control', 'public, max-age=300'); return res.json(acc); } catch (err) { console.error('[accuracy]', err.message); - return res.status(200).json({ overall: null, sports: {}, min_sample: outcomeService.MIN_SAMPLE, updated_at: null }); + return res.status(200).json({ overall: null, sports: {}, min_sample: 20, updated_at: null }); } }); diff --git a/src/routes/ledger.js b/src/routes/ledger.js index 71528b7..b78a579 100644 --- a/src/routes/ledger.js +++ b/src/routes/ledger.js @@ -41,11 +41,18 @@ function applyFilters(query, req) { } router.get('/accuracy', async (req, res) => { + // Truth-Everywhere Part 2 (item 7) — read the CLEAN Postgres ledger aggregate + // (model_value > 0 excludes degraded rows), NOT the Redis outcome log (which + // still counts degraded projection-0 outcomes and can't be filtered). try { - const acc = await outcomeService.getAccuracy(); - const buckets = outcomeService.accuracyBuckets(acc.overall); + const agg = await ledgerService.getModelAggregate({}); + const buckets = ledgerService.accuracyBucketsFromAgg(agg); + const overall = { + hits: agg.hits, misses: agg.misses, pushes: agg.pushes, + total: agg.hits + agg.misses + agg.pushes, pct: agg.hit_pct ?? null, + }; res.set('Cache-Control', 'public, max-age=300'); - return res.json({ buckets, overall: acc.overall && acc.overall.overall, updated_at: acc.updated_at }); + return res.json({ buckets, overall, updated_at: null }); } catch (err) { console.error('[ledger/accuracy]', err.message); return res.status(200).json({ buckets: [] }); diff --git a/src/services/ledgerService.js b/src/services/ledgerService.js index b5f5aa9..8c5581a 100644 --- a/src/services/ledgerService.js +++ b/src/services/ledgerService.js @@ -37,6 +37,12 @@ const AGG_WINDOW_DAYS = 30; const AGG_FETCH_LIMIT = 5000; /** Below this many settled rows, callers must not render a percentage. */ const MIN_AGG_SAMPLE = 20; +// Truth-Everywhere Part 2 (item 7) — CLV capture is broken (closing_line == +// locked_line; see the C4 finding). Until C4 records a real closing line, +// beat_close/CLV are suppressed everywhere. Read at call time (not module load) +// so C4 can flip it via CLV_CAPTURE_RELIABLE=1 without a redeploy, and tests can +// exercise the CLV math directly. +function clvCaptureReliable() { return process.env.CLV_CAPTURE_RELIABLE === '1'; } /** * S6 (A1 board) — CLV distribution buckets (the MODEL tab strip). Signed CLV: @@ -515,14 +521,22 @@ async function getModelAggregate(opts = {}) { if (agg.settled >= MIN_AGG_SAMPLE && decided > 0) { agg.hit_pct = Math.round((agg.hits / decided) * 100); } - if (agg.settled >= MIN_AGG_SAMPLE && agg.clv_sample > 0) { + // Truth-Everywhere Part 2 (item 7) — CLV is currently MEASURED WRONG: + // captureClosing re-records the LOCKED line as the "closing" line + // (closing_line == locked_line across the whole sample), so every row's CLV + // computes to 0/flat and beat_close reads a fabricated-looking 0%. That's + // comparing a number to itself. Until C4 (real closing-line capture) lands, + // CLV_CAPTURE_RELIABLE stays false and beat_close_pct / clv_distribution are + // suppressed at the SOURCE — every public surface hides BEAT CLOSE rather + // than showing a measured-wrong zero. Flip this to true when C4 ships. + if (clvCaptureReliable() && agg.settled >= MIN_AGG_SAMPLE && agg.clv_sample > 0) { agg.beat_close_pct = Math.round((agg.clv_beat / agg.clv_sample) * 100); } // S6 (A1 board) — clv_distribution rides the SAME n≥20 gate (this is the // single home of the gate — consumers never re-derive it). Null below the // sample floor or with zero settled clv values; the UI renders nothing. agg.clv_distribution = null; - if (agg.settled >= MIN_AGG_SAMPLE && agg.clv_sample > 0) { + if (clvCaptureReliable() && agg.settled >= MIN_AGG_SAMPLE && agg.clv_sample > 0) { const dist = CLV_BUCKETS.map((b) => ({ ...b, count: 0 })); let counted = 0; for (const r of settledRows || []) { @@ -534,6 +548,72 @@ async function getModelAggregate(opts = {}) { return agg; } +// Truth-Everywhere Part 2 (item 7) — the public 30D accuracy VIEW, built from +// the CLEAN ledger aggregate (model_value > 0), NOT the Redis outcome log +// (which still counts degraded projection-0 rows and can't be filtered). Same +// shape the AccuracyBadge / buckets consumed from outcomeService, so no +// frontend change. Redis is a cache; when a cache can't be filtered, read truth. +const ACCURACY_VIEW_SPORTS = ['mlb', 'wnba', 'nba', 'soccer']; +function _aggToRecord(agg, sport) { + const byGrade = {}; + for (const [tier, b] of Object.entries(agg.by_tier || {})) { + byGrade[tier] = { + hits: b.hits, misses: b.misses, pushes: b.pushes, + total: b.hits + b.misses + b.pushes, pct: b.hit_pct ?? null, + }; + } + return { + sport, + updated_at: null, + window_days: agg.window_days, + sample: agg.settled, + min_sample: agg.min_sample, + overall: { + hits: agg.hits, misses: agg.misses, pushes: agg.pushes, + total: agg.hits + agg.misses + agg.pushes, pct: agg.hit_pct ?? null, + }, + byGrade, + }; +} +async function getAccuracyView(opts = {}) { + const base = { sb: opts.sb, nowMs: opts.nowMs }; + const overallAgg = await getModelAggregate(base); + const sports = {}; + for (const s of ACCURACY_VIEW_SPORTS) { + const a = await getModelAggregate({ ...base, sport: s }); + if (a.settled > 0) sports[s] = _aggToRecord(a, s); + } + return { + overall: _aggToRecord(overallAgg, 'overall'), + sports, + min_sample: overallAgg.min_sample, + updated_at: null, + }; +} + +// Grade-tier buckets for the ledger accuracy strip, from the clean aggregate. +function accuracyBucketsFromAgg(agg) { + // First-letter buckets (A+ folds into A for the public strip, matching the + // old outcomeService.accuracyBuckets contract), n≥20 gate per bucket. + const order = ['A', 'B', 'C', 'D', 'F']; + const rolled = {}; + for (const [tier, b] of Object.entries(agg.by_tier || {})) { + const k = tier === 'A+' ? 'A' : tier[0]; + rolled[k] = rolled[k] || { hits: 0, misses: 0, total: 0 }; + rolled[k].hits += b.hits; + rolled[k].misses += b.misses; + rolled[k].total += b.hits + b.misses + b.pushes; + } + return order + .filter((k) => rolled[k] && rolled[k].total > 0) + .map((k) => { + const r = rolled[k]; + const decided = r.hits + r.misses; + const pct = r.total >= MIN_AGG_SAMPLE && decided > 0 ? Math.round((r.hits / decided) * 100) : null; + return { grade: k, hits: r.hits, total: r.total, pct }; + }); +} + module.exports = { recordPipelineGrades, captureClosing, @@ -542,6 +622,8 @@ module.exports = { applyRevision, countRowsForDate, getModelAggregate, + getAccuracyView, + accuracyBucketsFromAgg, MIN_AGG_SAMPLE, __internals: { rowsFromSnapshot, computeClv, clvResultOf, indexProps, gameIdFor, diff --git a/tests/integration/accuracy.test.js b/tests/integration/accuracy.test.js index 9d4de4c..d3587a7 100644 --- a/tests/integration/accuracy.test.js +++ b/tests/integration/accuracy.test.js @@ -1,7 +1,8 @@ 'use strict'; -// Session 55 — the self-learning loop's public read endpoints. Redis is mocked -// so these run offline; the store is seeded per-test via cacheGet. +// Truth-Everywhere Part 2 (item 7) — the public accuracy endpoints now read the +// CLEAN Postgres ledger aggregate (getModelAggregate / getAccuracyView), NOT the +// Redis outcome log. We partial-mock ledgerService so these run offline. const request = require('supertest'); @@ -14,12 +15,37 @@ jest.mock('../../src/utils/redis', () => ({ isDegraded: () => false, })); +jest.mock('../../src/services/ledgerService', () => { + const actual = jest.requireActual('../../src/services/ledgerService'); + return { + ...actual, + getAccuracyView: jest.fn(), + getModelAggregate: jest.fn(), + accuracyBucketsFromAgg: actual.accuracyBucketsFromAgg, // keep the real bucketer + }; +}); + +const ledgerService = require('../../src/services/ledgerService'); const app = require('../../src/app'); -beforeEach(() => { mockStore = {}; }); +const EMPTY_AGG = { + window_days: 30, min_sample: 20, settled: 0, hits: 0, misses: 0, pushes: 0, + hit_pct: null, beat_close_pct: null, clv_distribution: null, by_tier: {}, pending: 0, +}; + +beforeEach(() => { + mockStore = {}; + ledgerService.getAccuracyView.mockReset(); + ledgerService.getModelAggregate.mockReset(); +}); describe('GET /api/accuracy', () => { - test('cold cache → valid empty-safe shape', async () => { + test('cold (no data) → valid empty-safe shape from the ledger view', async () => { + ledgerService.getAccuracyView.mockResolvedValue({ + overall: { sport: 'overall', window_days: 30, sample: 0, min_sample: 20, + overall: { hits: 0, misses: 0, pushes: 0, total: 0, pct: null }, byGrade: {} }, + sports: {}, min_sample: 20, updated_at: null, + }); const res = await request(app).get('/api/accuracy'); expect(res.status).toBe(200); expect(res.body).toHaveProperty('overall'); @@ -27,13 +53,14 @@ describe('GET /api/accuracy', () => { expect(res.body.min_sample).toBeGreaterThan(0); }); - test('returns the persisted record when present', async () => { - mockStore['accuracy:overall'] = { - sport: 'overall', updated_at: '2026-07-10T00:00:00Z', window_days: 30, sample: 20, - overall: { hits: 14, misses: 6, pushes: 0, total: 20, pct: 70 }, - byGrade: { 'A': { hits: 8, misses: 2, pushes: 0, total: 10, pct: 80 } }, - }; - mockStore['accuracy:mlb'] = mockStore['accuracy:overall']; + test('returns the clean ledger record when present', async () => { + ledgerService.getAccuracyView.mockResolvedValue({ + overall: { sport: 'overall', window_days: 30, sample: 20, min_sample: 20, + overall: { hits: 14, misses: 6, pushes: 0, total: 20, pct: 70 }, + byGrade: { A: { hits: 8, misses: 2, pushes: 0, total: 10, pct: 80 } } }, + sports: { mlb: { sport: 'mlb', overall: { hits: 14, misses: 6, pushes: 0, total: 20, pct: 70 }, byGrade: {} } }, + min_sample: 20, updated_at: null, + }); const res = await request(app).get('/api/accuracy'); expect(res.body.overall.overall.pct).toBe(70); expect(res.body.sports.mlb).toBeTruthy(); @@ -41,25 +68,25 @@ describe('GET /api/accuracy', () => { }); describe('GET /api/ledger/accuracy', () => { - test('returns grade-tier buckets from the accuracy record', async () => { - mockStore['accuracy:overall'] = { - sport: 'overall', updated_at: '2026-07-10T00:00:00Z', window_days: 30, sample: 15, - overall: { hits: 10, misses: 5, pushes: 0, total: 15, pct: 67 }, - byGrade: { - 'A+': { hits: 3, misses: 0, pushes: 0, total: 3, pct: 100 }, - 'A': { hits: 5, misses: 2, pushes: 0, total: 7, pct: 71 }, - 'B': { hits: 2, misses: 3, pushes: 0, total: 5, pct: 40 }, + test('returns grade-tier buckets from the clean ledger aggregate', async () => { + ledgerService.getModelAggregate.mockResolvedValue({ + ...EMPTY_AGG, settled: 30, hits: 20, misses: 10, hit_pct: 67, + by_tier: { + 'A+': { settled: 3, hits: 3, misses: 0, pushes: 0, hit_pct: null }, + 'A': { settled: 22, hits: 15, misses: 7, pushes: 0, hit_pct: 68 }, + 'B': { settled: 5, hits: 2, misses: 3, pushes: 0, hit_pct: null }, }, - }; + }); const res = await request(app).get('/api/ledger/accuracy'); expect(res.status).toBe(200); expect(Array.isArray(res.body.buckets)).toBe(true); const grades = res.body.buckets.map((b) => b.grade); - expect(grades).toContain('A+'); - expect(grades).toContain('A'); + expect(grades).toContain('A'); // A+ folds into A in the public strip + expect(res.body.overall.pct).toBe(67); }); - test('cold cache → empty buckets, never 500', async () => { + test('empty aggregate → empty buckets, never 500', async () => { + ledgerService.getModelAggregate.mockResolvedValue({ ...EMPTY_AGG }); const res = await request(app).get('/api/ledger/accuracy'); expect(res.status).toBe(200); expect(res.body.buckets).toEqual([]); diff --git a/tests/unit/ledgerAggregateScope.test.js b/tests/unit/ledgerAggregateScope.test.js index 4bd0cfb..284e1fa 100644 --- a/tests/unit/ledgerAggregateScope.test.js +++ b/tests/unit/ledgerAggregateScope.test.js @@ -65,6 +65,7 @@ describe('getModelAggregate scoping', () => { }); test('user scope renders percentages at n≥20 like the public record', async () => { + process.env.CLV_CAPTURE_RELIABLE = '1'; // exercise the CLV math (suppressed by default, item 7) const rows = [ ...Array.from({ length: 14 }, () => ({ outcome: 'hit', clv_result: 'beat', grade: 'A' })), ...Array.from({ length: 7 }, () => ({ outcome: 'miss', clv_result: 'faded', grade: 'B' })), @@ -73,5 +74,6 @@ describe('getModelAggregate scoping', () => { expect(agg.settled).toBe(21); expect(agg.hit_pct).toBe(Math.round((14 / 21) * 100)); expect(agg.beat_close_pct).toBe(Math.round((14 / 21) * 100)); + delete process.env.CLV_CAPTURE_RELIABLE; }); }); diff --git a/tests/unit/ledgerService.test.js b/tests/unit/ledgerService.test.js index 8570e8f..669efeb 100644 --- a/tests/unit/ledgerService.test.js +++ b/tests/unit/ledgerService.test.js @@ -206,6 +206,27 @@ describe('captureClosing — real feed values only', () => { }); describe('getModelAggregate — never a % under min sample', () => { + // beat_close/CLV are suppressed by default (item 7 — CLV capture broken until + // C4). These tests exercise the CLV MATH, so enable the reliable flag; a + // separate test below locks the default-suppressed behavior. + beforeAll(() => { process.env.CLV_CAPTURE_RELIABLE = '1'; }); + afterAll(() => { delete process.env.CLV_CAPTURE_RELIABLE; }); + + test('beat_close is SUPPRESSED by default until C4 (CLV capture broken)', async () => { + delete process.env.CLV_CAPTURE_RELIABLE; // default state + const sb = fakeSb(); + sb._state.selectResults = [[ + ...Array.from({ length: 13 }, () => ({ outcome: 'hit', clv_result: 'beat' })), + ...Array.from({ length: 7 }, () => ({ outcome: 'miss', clv_result: 'faded' })), + ]]; + sb._state.countResult = 0; + const agg = await ledger.getModelAggregate({ sb }); + expect(agg.hit_pct).toBe(65); // hit rate still renders (it's real) + expect(agg.beat_close_pct).toBeNull(); // BEAT CLOSE hidden — measured-wrong + expect(agg.clv_distribution).toBeNull(); + process.env.CLV_CAPTURE_RELIABLE = '1'; // restore for the rest of the block + }); + test('below 20 settles → hit_pct/beat_close_pct null, counts real', async () => { const sb = fakeSb(); sb._state.selectResults = [ @@ -296,6 +317,9 @@ describe('indexProps — prefers a book row with both sides priced', () => { // centralized HERE (getModelAggregate) — consumers never re-derive it. describe('getModelAggregate — clv_distribution (n>=20 gate lives in the service)', () => { const { clvBucketIndex, CLV_BUCKETS } = ledger.__internals; + // CLV suppressed by default (item 7); enable to test the distribution math. + beforeAll(() => { process.env.CLV_CAPTURE_RELIABLE = '1'; }); + afterAll(() => { delete process.env.CLV_CAPTURE_RELIABLE; }); test('below 20 settles → clv_distribution is null (never a small-sample chart)', async () => { const sb = fakeSb();