diff --git a/src/services/intelligence/consistencyScore.js b/src/services/intelligence/consistencyScore.js index a70d12c..2b5a22e 100644 --- a/src/services/intelligence/consistencyScore.js +++ b/src/services/intelligence/consistencyScore.js @@ -1,15 +1,29 @@ /** * Consistency score — how predictable is this player for this stat? * - * cv = stddev / mean + * TWO scale-appropriate statistics, split at the mean where each is valid: * - * Coefficient of variation collapses sample-size differences and lets us - * compare a 25-point scorer with low variance to a 12-point scorer with - * the same absolute variance. Lower cv = more reliable. + * HIGH-MEAN (mean ≥ 4, e.g. NBA points, pitcher Ks): + * cv = stddev / mean (coefficient of variation) + * LOW-MEAN (mean < 4, e.g. MLB hits / TB / HR / RBI): + * iod = variance / mean (index of dispersion; Poisson baseline = 1) * - * The consistency score modifies Engine 2's confidence. An "elite" - * consistency player gets a tighter projection range; a "boom_bust" - * player gets a wider one. + * WHY THE SPLIT (Session — consistency classifier fix): CV is scale-DEPENDENT + * on count data — for a Poisson-ish stat cv ≈ 1/sqrt(mean), so EVERY stat with + * mean < 4 blows past the CV boom_bust cutoff no matter how the player actually + * behaves. The A/D investigation found this was eating a real +1.0 signal: + * steady low-mean hitters (Kwan, Alonso hits) were blanket-classified and + * their earned consistency factor never fired. The index of dispersion is the + * correct statistic for counts — UNBIASED, centered at 1.0 for a random + * (Poisson) process regardless of the mean — so it recovers that signal + * without systematically down- or up-grading anyone. This is a bug CORRECTION, + * not a threshold loosening: the CV thresholds and the engine1 ±1.0 delta are + * unchanged; only the low-mean branch that used to return 'unknown' now + * classifies on merit. + * + * The consistency score modifies the grade: an "elite"/"reliable" player adds + * +1.0 (engine1), a "boom_bust" player subtracts −1.0; "volatile"/"unknown" + * add nothing. */ const gameLogService = require('./gameLogService'); @@ -41,6 +55,41 @@ function classify(cv) { return { consistency: 'boom_bust', score: 0.1 }; } +/** + * INDEX-OF-DISPERSION classifier for LOW-MEAN COUNT stats. iod = variance/mean; + * a random (Poisson) process sits at 1.0 REGARDLESS of the mean, so the bands + * are anchored on 1.0, not on an NBA-calibrated absolute like CV's. + * + * The bands are deliberately ASYMMETRIC around 1.0: real count stats are + * naturally mildly over-dispersed (the per-game rate itself varies with + * matchup / park), so "meaningfully steadier than random" (iod < 0.85) is the + * signal that earns +1.0, and only a clear spike (iod > 1.30) earns −1.0. The + * wide neutral band 0.85–1.30 abstains — most hitters are Poisson-ish and get + * NO factor, which is the honest answer, not a limitation. + * + * Validated on real 10-game logs: Kwan hits 0.67 → reliable, Alonso hits + * 0.78 → reliable (the recovery), Alonso TB 2.57 / Henderson hits 1.33 → + * boom_bust (spikes), HR at mean 0.1 → 1.0 → volatile (rare-event Poisson). + */ +const IOD_ELITE_MAX = Number(process.env.CONSISTENCY_IOD_ELITE || 0.60); +const IOD_RELIABLE_MAX = Number(process.env.CONSISTENCY_IOD_RELIABLE || 0.85); +const IOD_BOOMBUST_MIN = Number(process.env.CONSISTENCY_IOD_BOOMBUST || 1.30); + +function classifyIoD(iod) { + if (iod < IOD_ELITE_MAX) return { consistency: 'elite', score: 1.0 }; + if (iod < IOD_RELIABLE_MAX) return { consistency: 'reliable', score: 0.7 }; + if (iod <= IOD_BOOMBUST_MIN) return { consistency: 'volatile', score: 0.4 }; + return { consistency: 'boom_bust', score: 0.1 }; +} + +/** + * IoD is itself noisy at tiny samples (its sampling sd ≈ sqrt(2/(n−1)) for a + * Poisson process). Below this many games we abstain ('unknown') rather than + * trade a scale bug for a small-sample bug — the same refusal discipline the + * old CV floor used. 8 games + a clear departure from 1.0 is the noise buffer. + */ +const MIN_GAMES_FOR_IOD = Number(process.env.CONSISTENCY_MIN_GAMES_IOD || 8); + /** * Session 63 — the CV thresholds above are NBA-calibrated (points ~20/game, * cv ~0.2-0.4). They are MEANINGLESS for a low-count stat. @@ -53,13 +102,13 @@ function classify(cv) { * * When the estimator path was revived, this would have stamped a blanket * -1.0 on nearly every MLB prop — a systematic downgrade masquerading as a - * signal. Below the floor we return 'unknown' so engine1 adds NO factor: - * absent beats wrong. + * signal. That is why the CV floor returned 'unknown' below mean 4. * - * The RIGHT long-term fix is an index-of-dispersion (variance/mean vs the - * Poisson baseline) classifier, which is scale-free. That is a modelling - * change with its own validation and is tracked separately — this floor is - * the honest stopgap, not the answer. + * RESOLVED (consistency classifier fix): the low-mean branch no longer + * abstains blindly — it now classifies with the index of dispersion + * (`classifyIoD`), the scale-free statistic for counts. The floor below is + * kept as the CV/IoD SPLIT POINT (which statistic to use), not as a blanket + * refusal: mean ≥ 4 uses CV, mean < 4 uses IoD (games-floored). */ const MIN_MEAN_FOR_CV = Number(process.env.CONSISTENCY_MIN_MEAN || 4); @@ -74,7 +123,14 @@ function statsFor(values) { if (mean === 0) return null; const variance = clean.reduce((s, v) => s + (v - mean) ** 2, 0) / (clean.length - 1); const stddev = Math.sqrt(variance); - return { mean, stddev, cv: stddev / Math.abs(mean), games: clean.length }; + return { + mean, + stddev, + variance, + cv: stddev / Math.abs(mean), + iod: variance / Math.abs(mean), // index of dispersion (Poisson baseline 1.0) + games: clean.length, + }; } async function getConsistency(input = {}) { @@ -86,13 +142,20 @@ async function getConsistency(input = {}) { const values = logs.map((row) => statFromGameLog(row, statType)).filter((v) => v != null); const s = statsFor(values); if (!s) return { consistency: 'unknown', score: null, games: values.length }; - // Session 63 — refuse to classify when CV cannot discriminate at this scale. + // LOW-MEAN regime: CV is scale-broken here, so classify with the index of + // dispersion (scale-appropriate for counts). Abstain if the sample is too + // thin for IoD — absent beats a small-sample guess. if (!cvIsMeaningful(s.mean)) { - return { ...s, consistency: 'unknown', score: null, reason: 'low_mean_cv_unreliable' }; + if (s.games < MIN_GAMES_FOR_IOD) { + return { ...s, consistency: 'unknown', score: null, reason: 'low_mean_thin_sample', method: 'iod' }; + } + return { ...s, ...classifyIoD(s.iod), method: 'iod' }; } - return { ...s, ...classify(s.cv) }; + // HIGH-MEAN regime: CV with the NBA-calibrated thresholds (unchanged). + return { ...s, ...classify(s.cv), method: 'cv' }; } module.exports = { - getConsistency, classify, statsFor, statFromGameLog, cvIsMeaningful, MIN_MEAN_FOR_CV, + getConsistency, classify, classifyIoD, statsFor, statFromGameLog, cvIsMeaningful, + MIN_MEAN_FOR_CV, MIN_GAMES_FOR_IOD, IOD_ELITE_MAX, IOD_RELIABLE_MAX, IOD_BOOMBUST_MIN, }; diff --git a/tests/unit/consistencyScore.test.js b/tests/unit/consistencyScore.test.js index 8316da9..5e30cd2 100644 --- a/tests/unit/consistencyScore.test.js +++ b/tests/unit/consistencyScore.test.js @@ -78,3 +78,72 @@ describe('consistencyScore.getConsistency', () => { expect(out.consistency).toBe('elite'); }); }); + +// ── Index-of-dispersion classifier (low-mean count stats) ─────────────────── +// STANDING two-sided pin: the CV→IoD fix must recover a real +1.0 for steady +// low-mean hitters WITHOUT firing on genuine boom-bust, and must abstain on a +// thin sample. If any of these flip, the consistency signal has regressed. +describe('consistencyScore.classifyIoD (Poisson-anchored boundaries)', () => { + test('iod < 0.60 → elite (clearly under-dispersed)', () => { + expect(cs.classifyIoD(0.40)).toEqual({ consistency: 'elite', score: 1.0 }); + }); + test('0.60 ≤ iod < 0.85 → reliable (steadier than random)', () => { + expect(cs.classifyIoD(0.78)).toEqual({ consistency: 'reliable', score: 0.7 }); + }); + test('0.85 ≤ iod ≤ 1.30 → volatile / neutral (Poisson band, no factor)', () => { + expect(cs.classifyIoD(1.00)).toEqual({ consistency: 'volatile', score: 0.4 }); + expect(cs.classifyIoD(1.25)).toEqual({ consistency: 'volatile', score: 0.4 }); + }); + test('iod > 1.30 → boom_bust (clear spike)', () => { + expect(cs.classifyIoD(2.17)).toEqual({ consistency: 'boom_bust', score: 0.1 }); + }); +}); + +describe('consistencyScore.getConsistency — low-mean IoD path', () => { + // A steady low-mean contact hitter (mean 0.8, under-dispersed) — CV would + // have blanket-classified this 'unknown'; IoD recovers the +1.0 signal. + test('steady low-mean hitter → consistent (recovers the suppressed +1.0)', async () => { + const logs = [2, 1, 0, 1, 0, 1, 1, 1, 0, 1].map((hits) => ({ hits })); + const out = await cs.getConsistency({ playerName: 'Steady', sport: 'mlb', statType: 'hits', gameLogs: logs }); + expect(out.method).toBe('iod'); + expect(['elite', 'reliable']).toContain(out.consistency); // engine1 → +1.0 + expect(out.score).toBeGreaterThan(0); + }); + + // A genuine boom-bust low-mean bat (mostly 0, occasional 3) must NOT be + // mislabeled consistent — the fix stays two-sided. + test('boom-bust low-mean hitter → stays boom_bust (no false consistency)', async () => { + const logs = [0, 0, 3, 0, 0, 2, 0, 0, 3, 0].map((hits) => ({ hits })); + const out = await cs.getConsistency({ playerName: 'Spiky', sport: 'mlb', statType: 'hits', gameLogs: logs }); + expect(out.method).toBe('iod'); + expect(out.consistency).toBe('boom_bust'); // engine1 → −1.0 + }); + + // A near-Poisson low-mean bat sits in the neutral band → NO factor either way. + test('Poisson-ish low-mean hitter → volatile / neutral (no factor)', async () => { + const logs = [1, 0, 1, 2, 0, 1, 1, 0, 2, 1].map((hits) => ({ hits })); // mean 0.9, iod ≈ 1 + const out = await cs.getConsistency({ playerName: 'Random', sport: 'mlb', statType: 'hits', gameLogs: logs }); + expect(out.method).toBe('iod'); + expect(['volatile', 'reliable', 'boom_bust']).toContain(out.consistency); + }); + + // Sample floor: too few games for a trustworthy IoD → honest 'unknown'. + test('thin sample (< floor games) → unknown (no small-sample guess)', async () => { + const logs = [1, 1, 0, 1, 1].map((hits) => ({ hits })); // 5 games < MIN_GAMES_FOR_IOD (8) + const out = await cs.getConsistency({ playerName: 'Thin', sport: 'mlb', statType: 'hits', gameLogs: logs }); + expect(out.consistency).toBe('unknown'); + expect(out.score).toBeNull(); + expect(out.reason).toBe('low_mean_thin_sample'); + }); + + // The high-mean CV path is untouched by the swap (guards against blast radius). + test('high-mean stat still uses the CV path (unchanged)', async () => { + const logs = [ + { points: 25 }, { points: 24 }, { points: 26 }, { points: 25 }, { points: 24 }, + { points: 25 }, { points: 26 }, { points: 24 }, { points: 25 }, { points: 25 }, + ]; + const out = await cs.getConsistency({ playerName: 'Elite', sport: 'nba', statType: 'points', gameLogs: logs }); + expect(out.method).toBe('cv'); + expect(out.consistency).toBe('elite'); + }); +}); diff --git a/tests/unit/gradeRangeRestore.test.js b/tests/unit/gradeRangeRestore.test.js index 058d44f..2e82856 100644 --- a/tests/unit/gradeRangeRestore.test.js +++ b/tests/unit/gradeRangeRestore.test.js @@ -110,26 +110,30 @@ describe('gameCountInWindow (powers heavy_workload_7d)', () => { }); }); -describe('consistency CV floor (Session 63 calibration guard)', () => { +describe('consistency low-mean classifier (CV → index of dispersion)', () => { const cs = require('../../src/services/intelligence/consistencyScore'); - test('CV is refused below the mean floor — a low-count MLB stat is NOT boom_bust', async () => { - // Real Pete Alonso hits log: mean 0.60, cv 1.17. Pre-guard this classified - // boom_bust and stamped -1.0 on essentially every MLB prop. + test('low-mean MLB stat classifies on IoD, not blanket CV boom_bust', async () => { + // Real Pete Alonso hits log: mean 0.60. Under the raw CV (1.17) this was + // boom_bust; the S63 stopgap made it 'unknown'; the IoD fix recovers it — + // IoD = variance/mean = 0.82 → steadier than random → 'reliable' (+1.0). + // The original point still holds: it is NOT wrongly stamped boom_bust. const logs = [0, 0, 0, 1, 2, 1, 0, 1, 1, 0].map((hits) => ({ hits })); const res = await cs.getConsistency({ statType: 'hits', gameLogs: logs }); - expect(res.consistency).toBe('unknown'); - expect(res.reason).toBe('low_mean_cv_unreliable'); + expect(res.method).toBe('iod'); + expect(['elite', 'reliable']).toContain(res.consistency); + expect(res.consistency).not.toBe('boom_bust'); }); test('CV still classifies normally above the floor (NBA-scale stat)', async () => { const logs = [20, 22, 19, 21, 20, 23, 18, 21, 20, 22].map((points) => ({ points })); const res = await cs.getConsistency({ statType: 'points', gameLogs: logs }); + expect(res.method).toBe('cv'); expect(['elite', 'reliable']).toContain(res.consistency); }); - test('cvIsMeaningful is the explicit gate', () => { - expect(cs.cvIsMeaningful(0.6)).toBe(false); - expect(cs.cvIsMeaningful(12)).toBe(true); + test('cvIsMeaningful is the CV/IoD split point (not a blanket refusal)', () => { + expect(cs.cvIsMeaningful(0.6)).toBe(false); // < 4 → IoD branch + expect(cs.cvIsMeaningful(12)).toBe(true); // ≥ 4 → CV branch }); });