Per-archetype grade bands: built, gated, and the rescale blocked twice
The premise does not hold. proven-status.js run fresh: PROVEN_SET is EMPTY, no archetype x stat reaches the gate. pitcher_contact_profile has a CI upper bound of exactly 0.0000 and platoon_severity is held on 4.5%-contaminated splits, so the proven set is one factor, pooled, not three archetype-conditioned ones. The specific pattern the order names -- defense strong for GHOST/BRUSH, null for BOMBER -- is the one I measured running the OTHER WAY yesterday, both noise-dominated. But the second blocker is new and matters more, because it would stop the rescale even if the factors had proved: the grade does not separate within any archetype. Every archetype collapses to ONE band at the corrected bar, because bands merge when their intervals overlap and publishing two letters we cannot tell apart is a distinction we have not measured. Uncorrected, so the ranking is visible rather than hidden by the bar, this INVERTS the order's design. The order gives contact types the factor-rich treatment and power types honest base-rate, reasoning that single-game hits are variance for a power profile. Measured: BOMBER n=466 corr(p_win,outcome) +0.207 quintiles 0.75 0.62 0.60 0.48 0.48 GHOST n=192 corr(p_win,outcome) -0.007 quintiles 0.47 0.63 0.74 0.58 0.45 BOMBER is the one archetype the model ranks, and it splits into a real A 0.660 / B 0.481 at 95%. GHOST is flat, and non-monotone -- its most confident reads hit 47% while its middle reads hit 74%. Shipping as specified would have given the factor-rich treatment to the archetype the model reads worst and left base-rate on the one it reads best. That is mechanically sensible in hindsight: a power hitter's hit tracks whether he can damage the arm, a contact hitter's depends on balls finding holes. BOMBER's split does not survive the cumulative correction at 106 tests. Exposing it by loosening the correction is the curve-to-make-A's the order forbids, so it stays one band. BUILT: gradeBands.js -- lift against the archetype's OWN base rate (the same 62% is lift for a 45% profile and a deficit for a 68% one), indistinguishable neighbours merged, thin bands PROVISIONAL not dropped, Wilson intervals widened by the cumulative correction. The two-bar rule is structural: proven-alone, calibrated-alone and neither all return base_rate with the reason stated, so with nothing proven no factor-informed band can be produced at all. reasoning() is built and tested but NOT wired to the card -- there is no per-archetype band being served, so attaching the copy now would ship product language for a rescale that does not exist. NOT BUILT: the specified power-type reason "the matchup edge is in total_bases". total_bases is recorded INCONCLUSIVE (+0.0038, CI [-0.068,+0.075]). Wiring it would assert an edge measured as indistinguishable from zero -- the exact fabricated-reason failure this module exists to prevent. BOMBER x hits is 29 rows short of the gate and is the archetype the model actually reads. That is the first slot to test, not GHOST. Counter and frozen clusters byte-identical. No letter was moved. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W1sivYNqY2TS5ftykmHBU9
This commit is contained in:
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* build-grade-bands — publish what each letter actually means, per archetype.
|
||||
*
|
||||
* Runs the real settled ledger through gradeBands. Because no factor has passed
|
||||
* the gate for any archetype, every band comes back a BASE-RATE read — which is
|
||||
* the honest answer today, and the output states it rather than leaving a reader
|
||||
* to infer it.
|
||||
*
|
||||
* SUPABASE_URL=... node scripts/build-grade-bands.js
|
||||
*/
|
||||
|
||||
require('dotenv').config();
|
||||
const { createClient } = require('@supabase/supabase-js');
|
||||
const gb = require('../src/services/model/gradeBands');
|
||||
const tl = require('../src/services/model/testLedger');
|
||||
const cal = require('../src/services/model/calibration');
|
||||
const { knownNumber } = require('../src/utils/known');
|
||||
|
||||
const SB_URL = process.env.SUPABASE_URL;
|
||||
const SB_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SERVICE_KEY;
|
||||
const STAT = process.env.BAND_STAT || 'hits';
|
||||
const PAGE = 1000;
|
||||
|
||||
/**
|
||||
* PROVEN, PER ARCHETYPE. Empty, and that is the measured state — see
|
||||
* specs/per-archetype-re-audit.md. Nothing may be added here that has not
|
||||
* cleared the gate FOR THAT ARCHETYPE; pooled proof does not qualify a slot.
|
||||
*/
|
||||
const PROVEN_BY_ARCHETYPE = Object.freeze({});
|
||||
|
||||
async function page(sb, table, select, apply) {
|
||||
const out = [];
|
||||
for (let from = 0; ; from += PAGE) {
|
||||
const { data, error } = await apply(sb.from(table).select(select)).range(from, from + PAGE - 1);
|
||||
if (error) throw error;
|
||||
if (!data || data.length === 0) break;
|
||||
out.push(...data);
|
||||
if (data.length < PAGE) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const sb = createClient(SB_URL, SB_KEY, { auth: { persistSession: false } });
|
||||
|
||||
const snaps = await page(sb, 'model_snapshots', 'player_key, game_date, archetype, stat',
|
||||
(q) => q.eq('sport', 'mlb').eq('stat', STAT).not('archetype', 'is', null));
|
||||
const archOf = new Map();
|
||||
for (const s of snaps) archOf.set(`${s.player_key}|${s.game_date}`, s.archetype);
|
||||
|
||||
const led = await page(sb, 'ledger_entries', 'player_key, game_date, outcome, p_win, quarantine_reason',
|
||||
(q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', STAT)
|
||||
.in('outcome', ['hit', 'miss']).not('p_win', 'is', null));
|
||||
const clean = led.filter((r) => !(r.quarantine_reason || '').startsWith('nontakeable_book'));
|
||||
|
||||
const byArch = new Map();
|
||||
for (const r of clean) {
|
||||
const a = String(archOf.get(`${r.player_key}|${r.game_date}`) || 'UNLABELLED').toUpperCase();
|
||||
if (!byArch.has(a)) byArch.set(a, []);
|
||||
byArch.get(a).push({ p: knownNumber(r.p_win), won: r.outcome === 'hit' ? 1 : 0 });
|
||||
}
|
||||
|
||||
const mc = await tl.recordAndCount(tl.supabaseStore(sb),
|
||||
[...byArch.keys()].map((a) => ({
|
||||
sport: 'mlb', stat: STAT, archetype: a === 'UNLABELLED' ? null : a,
|
||||
interaction: 'grade_band_lift', target: 'outcome',
|
||||
})));
|
||||
|
||||
// Calibration is measured, not assumed. Today it is certified for hits only in
|
||||
// a middle band (specs — held-out error 0.477->0.506, 0.587->0.580), which is
|
||||
// NOT the same as an archetype's probabilities being calibrated.
|
||||
const certified = typeof cal.certifyBands === 'function';
|
||||
|
||||
const out = [];
|
||||
for (const [arch, rows] of [...byArch.entries()].sort((a, b) => b[1].length - a[1].length)) {
|
||||
out.push(gb.buildBands(rows, {
|
||||
archetype: arch,
|
||||
cumulativeTests: mc.cumulative_tests,
|
||||
proven: Boolean(PROVEN_BY_ARCHETYPE[arch]),
|
||||
calibrated: false, // no archetype's distribution is certified calibrated
|
||||
}));
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({
|
||||
stat: STAT,
|
||||
settled_rows: clean.length,
|
||||
archetypes: byArch.size,
|
||||
cumulative_tests: mc.cumulative_tests,
|
||||
calibration_helper_present: certified,
|
||||
proven_by_archetype: PROVEN_BY_ARCHETYPE,
|
||||
note: 'every band is a BASE-RATE read — no factor has passed the gate for any archetype',
|
||||
bands: out,
|
||||
}, null, 2));
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch((e) => { console.error(e); process.exit(1); });
|
||||
@@ -0,0 +1,126 @@
|
||||
# Per-archetype grade bands — built, gated, and not yet shippable
|
||||
|
||||
**The rescale does not ship, for two independent reasons — and the second one is
|
||||
new and more interesting than the first.**
|
||||
|
||||
---
|
||||
|
||||
## 1. The premise, checked against the ledger
|
||||
|
||||
`scripts/proven-status.js` (run fresh, not recalled):
|
||||
|
||||
```
|
||||
PROVEN_SET: EMPTY — no stat has beaten the counter out-of-sample
|
||||
archetype_x_stat_at_or_above_gate: []
|
||||
closest: BOMBER x hits n=471 (short 29) · GHOST x hits n=196 (short 304)
|
||||
```
|
||||
|
||||
| the order states | measured |
|
||||
|---|---|
|
||||
| three proven causally-correct factors | **one.** `pitcher_contact_profile` CI upper bound is exactly `0.0000` (not proven); `platoon_severity` passes but is held on 4.5%-median-contaminated season-to-date splits |
|
||||
| several now archetype-conditioned | **zero.** No archetype slot reaches n≥500 |
|
||||
| "defense strong for GHOST/BRUSH, honestly-null for BOMBER" | measured the **opposite direction** — BOMBER −0.0036, GHOST −0.0024, both noise-dominated (specs/per-archetype-re-audit.md) |
|
||||
|
||||
Pooled proof does not qualify a slot. `PROVEN_BY_ARCHETYPE` is `{}` and the band
|
||||
builder reads it directly.
|
||||
|
||||
---
|
||||
|
||||
## 2. The new finding: the grade does not separate within archetype
|
||||
|
||||
This is the reason that matters, because it would block the rescale even if the
|
||||
factors had proved.
|
||||
|
||||
Published bands, full clean settled hits history (n=1,312, 106 cumulative tests):
|
||||
**every archetype collapses to ONE band.** Bands merge when their corrected
|
||||
intervals overlap, because publishing two letters we cannot tell apart is a
|
||||
distinction we have not measured.
|
||||
|
||||
Uncorrected, so the underlying ranking is visible rather than hidden by the bar:
|
||||
|
||||
| archetype | n | base rate | quintiles (hi→lo) | corr(p_win, outcome) | bands @95% |
|
||||
|---|---|---|---|---|---|
|
||||
| **BOMBER** | 466 | 0.588 | 0.75 · 0.62 · 0.60 · 0.48 · 0.48 | **+0.207** | 2 — A 0.660 (n=279) vs B 0.481 (n=187), A shows lift |
|
||||
| UNLABELLED | 490 | 0.539 | 0.66 · 0.53 · 0.58 · 0.51 · 0.41 | +0.155 | 2, no lift |
|
||||
| **GHOST** | 192 | 0.578 | 0.47 · 0.63 · 0.74 · 0.58 · 0.45 | **−0.007** | 1 |
|
||||
| BRUSH | 80 | 0.537 | 0.81 · 0.63 · 0.44 · 0.50 · 0.31 | +0.350 | 1 |
|
||||
|
||||
### This inverts the order's design
|
||||
|
||||
The order assigns **contact types the factor-rich treatment** and **power types
|
||||
the honest base-rate treatment**, on the reasoning that single-game hits are
|
||||
variance for a power profile.
|
||||
|
||||
Measured, it is the other way round. **BOMBER is the one archetype where the
|
||||
model ranks** (+0.207, monotone across quintiles, and it separates into a genuine
|
||||
A/B at 95%). **GHOST is where it is flat** — corr −0.007, and its quintiles are
|
||||
non-monotone: the model's most confident GHOST reads hit 47% while its middle
|
||||
reads hit 74%.
|
||||
|
||||
That ordering is mechanically sensible in hindsight. A power hitter's chance of a
|
||||
hit tracks whether he can damage the arm he is facing, which the counter's
|
||||
frequency question partly captures. A contact hitter's hits depend on batted
|
||||
balls finding holes, which is much closer to luck — the same reason the pooled
|
||||
hits negative closed (S83) and the reason `defense_by_direction` is the factor
|
||||
that survives.
|
||||
|
||||
**Had the rescale shipped as specified, it would have given the factor-rich
|
||||
treatment to the archetype the model reads worst, and left base-rate on the one
|
||||
it reads best.**
|
||||
|
||||
### BOMBER still cannot publish two letters
|
||||
|
||||
BOMBER's A/B split is real at 95% but does not survive the cumulative correction
|
||||
at 106 tests — the intervals merge. So even the best archetype gets one honest
|
||||
band today. Lowering the correction to expose it would be exactly the "curve to
|
||||
make more A's" the order forbids.
|
||||
|
||||
---
|
||||
|
||||
## 3. What was built
|
||||
|
||||
`src/services/model/gradeBands.js` — per-archetype bands from realized outcomes:
|
||||
|
||||
- **Lift, not raw rate.** A band is credited only when its interval clears *that
|
||||
archetype's own* base rate. Locked by test: the same 62% realized rate is lift
|
||||
for a 45%-base profile and a deficit for a 68%-base one.
|
||||
- **The two-bar rule is structural.** `factor_informed` requires `proven` AND
|
||||
`calibrated` for that archetype. Tests assert that proven-alone, calibrated-
|
||||
alone, and neither all return `basis: 'base_rate'` with the reason stated —
|
||||
so with nothing proven, which is today, no factor-informed band can be produced
|
||||
at all. Same shape as `featureRegistry.liveFeatures()`: the honest state is the
|
||||
default and the richer claim has to be earned past a gate.
|
||||
- **Indistinguishable neighbours merge** rather than becoming different letters.
|
||||
- **Thin bands are PROVISIONAL, not dropped** — "still counting" and "nothing
|
||||
here" are different claims.
|
||||
- **Wilson intervals**, widened by the cumulative correction. Wilson because
|
||||
these bands are small and rates sit near the edges, where a normal
|
||||
approximation runs past 0 and 1 and implies impossible rates.
|
||||
|
||||
`reasoning()` is built and tested: a base-rate band says *"base-rate read … no
|
||||
matchup factor is proven for this profile yet"*, and a factor-informed band with
|
||||
no named proven factors returns **nothing** rather than inventing a why.
|
||||
|
||||
It is **not wired to the card**, deliberately. There is no per-archetype band
|
||||
being served, so attaching per-archetype copy now would ship product language for
|
||||
a rescale that does not exist.
|
||||
|
||||
### One piece of the order I did not build
|
||||
|
||||
The specified power-type reasoning — *"the matchup edge is in total_bases"* — is
|
||||
**not supported by any measurement**. `proven-status.js` records total_bases as
|
||||
INCONCLUSIVE (delta +0.0038, CI [−0.068, +0.075]). Wiring that sentence would
|
||||
assert an edge we have measured as indistinguishable from zero, which is the
|
||||
fabricated-reason failure the rest of this module exists to prevent.
|
||||
|
||||
---
|
||||
|
||||
## 4. What would unblock the rescale
|
||||
|
||||
1. **BOMBER × hits is 29 rows short** of the gate — days away, and it is the
|
||||
archetype the model actually reads. That is the first slot to test, not GHOST.
|
||||
2. **Point-in-time platoon splits** would convert two held passes into real ones.
|
||||
3. **Separation must survive the correction**, not just 95%. More sample tightens
|
||||
the intervals; nothing else legitimately does.
|
||||
|
||||
Counter and frozen clusters untouched. No letter was moved.
|
||||
@@ -0,0 +1,235 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* gradeBands — WHAT DOES A LETTER MEAN, PER ARCHETYPE, AND WHO SAYS SO?
|
||||
*
|
||||
* A grade is the product. So a letter has to be backed by a realized rate the
|
||||
* ledger can stand behind — not by a threshold someone chose. This builds bands
|
||||
* from settled outcomes, per archetype, and refuses to publish any band the
|
||||
* evidence cannot support.
|
||||
*
|
||||
* ── LIFT, NOT RAW RATE ────────────────────────────────────────────────────
|
||||
* An A must beat the archetype's OWN naive base rate. A power hitter and a
|
||||
* contact hitter have different base rates for a hit, so the same 62% realized
|
||||
* rate is a strong read for one and slightly below water for the other. Bands
|
||||
* are therefore drawn where the realized rate SEPARATES from that archetype's
|
||||
* base rate, and a band whose interval still contains the base rate has shown no
|
||||
* lift — whatever its raw number looks like.
|
||||
*
|
||||
* ── THE TWO-BAR RULE IS STRUCTURAL, NOT A HABIT ───────────────────────────
|
||||
* A band may be described as FACTOR-INFORMED only when that archetype's factors
|
||||
* are both PROVEN and CALIBRATED. There is no argument that overrides it: the
|
||||
* caller passes evidence, and without it every band comes back `basis:
|
||||
* 'base_rate'` and says so. A test asserts that with nothing proven — which is
|
||||
* the state today — no factor-informed band can be produced at all.
|
||||
*
|
||||
* This is the same shape as featureRegistry.liveFeatures(): the honest state is
|
||||
* the DEFAULT, and the richer claim has to be earned past a gate, so it cannot
|
||||
* be reached by forgetting.
|
||||
*
|
||||
* ── WHY A BAND CAN BE REFUSED ─────────────────────────────────────────────
|
||||
* Thin bands are labelled PROVISIONAL rather than dropped, because "we are still
|
||||
* counting" and "there is nothing here" are different claims. But a band whose
|
||||
* corrected interval spans the base rate does NOT get a lift letter — it is
|
||||
* reported as the base-rate read it is.
|
||||
*/
|
||||
|
||||
const { knownNumber } = require('../../utils/known');
|
||||
|
||||
/** Below this a band is published but flagged — counting, not concluded. */
|
||||
const PROVISIONAL_N = 30;
|
||||
/** Below this a band is not published at all; there is nothing to stand behind. */
|
||||
const MIN_BAND_N = 12;
|
||||
/** Letters, richest first. Assigned by measured lift, never by curve. */
|
||||
const LETTERS = ['A', 'B', 'C', 'D', 'F'];
|
||||
|
||||
/**
|
||||
* Wilson score interval, widened by the cumulative correction.
|
||||
*
|
||||
* Wilson rather than normal-approximation because these bands are small and
|
||||
* rates sit near the edges, where the normal interval runs past 0 and 1 and
|
||||
* quietly implies impossible rates.
|
||||
*/
|
||||
function wilson(hits, n, cumulativeTests = 1) {
|
||||
const k = knownNumber(hits); const N = knownNumber(n);
|
||||
if (k === null || N === null || N <= 0) return null;
|
||||
const tests = Math.max(1, Math.round(knownNumber(cumulativeTests) ?? 1));
|
||||
const z = zFor(1 - (0.05 / tests) / 2);
|
||||
const p = k / N;
|
||||
const d = 1 + (z * z) / N;
|
||||
const centre = (p + (z * z) / (2 * N)) / d;
|
||||
const half = (z * Math.sqrt((p * (1 - p)) / N + (z * z) / (4 * N * N))) / d;
|
||||
return [round4(Math.max(0, centre - half)), round4(Math.min(1, centre + half))];
|
||||
}
|
||||
|
||||
/** Inverse normal CDF (Acklam) — good to ~1e-9, enough for an interval bound. */
|
||||
function zFor(p) {
|
||||
if (p <= 0 || p >= 1) return 0;
|
||||
const a = [-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02,
|
||||
1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00];
|
||||
const b = [-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02,
|
||||
6.680131188771972e+01, -1.328068155288572e+01];
|
||||
const c = [-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00,
|
||||
-2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00];
|
||||
const d = [7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00,
|
||||
3.754408661907416e+00];
|
||||
const pl = 0.02425;
|
||||
let q; let r;
|
||||
if (p < pl) {
|
||||
q = Math.sqrt(-2 * Math.log(p));
|
||||
return (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5])
|
||||
/ ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
|
||||
}
|
||||
if (p > 1 - pl) {
|
||||
q = Math.sqrt(-2 * Math.log(1 - p));
|
||||
return -(((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5])
|
||||
/ ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
|
||||
}
|
||||
q = p - 0.5; r = q * q;
|
||||
return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q
|
||||
/ (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build bands for ONE archetype.
|
||||
*
|
||||
* @param {Array} rows [{ p, won }] — p is the model's probability, won 0/1
|
||||
* @param {object} opts
|
||||
* opts.archetype label
|
||||
* opts.cumulativeTests Bonferroni denominator (applied to every interval)
|
||||
* opts.proven this archetype's factors passed the gate
|
||||
* opts.calibrated this archetype's probabilities are certified calibrated
|
||||
* opts.targetBands how many bands to attempt (default 5)
|
||||
*/
|
||||
function buildBands(rows, opts = {}) {
|
||||
const usable = (rows || []).filter((r) =>
|
||||
knownNumber(r && r.p) !== null && knownNumber(r && r.won) !== null);
|
||||
const n = usable.length;
|
||||
const tests = opts.cumulativeTests ?? 1;
|
||||
|
||||
// THE TWO-BAR RULE. Both, or the bands are a base-rate read and say so.
|
||||
const factorInformed = Boolean(opts.proven) && Boolean(opts.calibrated);
|
||||
|
||||
if (n < MIN_BAND_N) {
|
||||
return {
|
||||
archetype: opts.archetype || null,
|
||||
basis: factorInformed ? 'factor_informed' : 'base_rate',
|
||||
n,
|
||||
base_rate: null,
|
||||
bands: [],
|
||||
refused: 'insufficient settled outcomes to stand behind any band',
|
||||
};
|
||||
}
|
||||
|
||||
const wins = usable.reduce((s, r) => s + (r.won > 0 ? 1 : 0), 0);
|
||||
const baseRate = wins / n;
|
||||
|
||||
// Candidate cuts by quantile of the model's own probability, then merged so
|
||||
// every published band is distinguishable from its neighbour.
|
||||
const sorted = [...usable].sort((a, b) => b.p - a.p);
|
||||
const target = Math.max(2, Math.min(LETTERS.length, opts.targetBands || 5));
|
||||
const per = Math.max(MIN_BAND_N, Math.floor(n / target));
|
||||
|
||||
let raw = [];
|
||||
for (let i = 0; i < sorted.length; i += per) {
|
||||
const slice = sorted.slice(i, i + per);
|
||||
// A trailing remainder too small to stand alone joins the previous band
|
||||
// rather than being published as its own thin claim.
|
||||
if (slice.length < MIN_BAND_N && raw.length) raw[raw.length - 1].push(...slice);
|
||||
else raw.push(slice);
|
||||
}
|
||||
|
||||
// Merge neighbours whose corrected intervals overlap — if we cannot tell two
|
||||
// bands apart, publishing them as different letters is a distinction we have
|
||||
// not measured.
|
||||
let merged = true;
|
||||
while (merged && raw.length > 1) {
|
||||
merged = false;
|
||||
for (let i = 0; i < raw.length - 1; i += 1) {
|
||||
const a = stats(raw[i], tests); const b = stats(raw[i + 1], tests);
|
||||
if (a.ci && b.ci && a.ci[0] <= b.ci[1] && b.ci[0] <= a.ci[1]) {
|
||||
raw.splice(i, 2, raw[i].concat(raw[i + 1]));
|
||||
merged = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const bands = raw.map((slice, i) => {
|
||||
const s = stats(slice, tests);
|
||||
// LIFT: does this band's interval clear the archetype's own base rate?
|
||||
const clearsAbove = s.ci !== null && s.ci[0] > baseRate;
|
||||
const clearsBelow = s.ci !== null && s.ci[1] < baseRate;
|
||||
return {
|
||||
letter: LETTERS[Math.min(i, LETTERS.length - 1)],
|
||||
n: slice.length,
|
||||
p_range: [round4(Math.min(...slice.map((r) => r.p))), round4(Math.max(...slice.map((r) => r.p)))],
|
||||
realized_rate: s.rate,
|
||||
ci: s.ci,
|
||||
lift_vs_archetype_base: round4(s.rate - baseRate),
|
||||
// The claim the letter is allowed to make.
|
||||
shows_lift: clearsAbove,
|
||||
shows_deficit: clearsBelow,
|
||||
separation: clearsAbove ? 'above_base_rate' : (clearsBelow ? 'below_base_rate' : 'indistinguishable_from_base_rate'),
|
||||
provisional: slice.length < PROVISIONAL_N,
|
||||
basis: factorInformed ? 'factor_informed' : 'base_rate',
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
archetype: opts.archetype || null,
|
||||
basis: factorInformed ? 'factor_informed' : 'base_rate',
|
||||
// Stated explicitly so a consumer never has to infer why it is a base-rate read.
|
||||
two_bar: {
|
||||
proven: Boolean(opts.proven),
|
||||
calibrated: Boolean(opts.calibrated),
|
||||
factor_informed_allowed: factorInformed,
|
||||
...(factorInformed ? {} : {
|
||||
reason: !opts.proven
|
||||
? 'no factor has passed the gate for this archetype'
|
||||
: 'factors proved but this archetype\'s probabilities are not certified calibrated',
|
||||
}),
|
||||
},
|
||||
n,
|
||||
base_rate: round4(baseRate),
|
||||
bands,
|
||||
bonferroni_tests: Math.max(1, Math.round(knownNumber(tests) ?? 1)),
|
||||
bands_showing_lift: bands.filter((b) => b.shows_lift).length,
|
||||
};
|
||||
}
|
||||
|
||||
function stats(slice, tests) {
|
||||
const n = slice.length;
|
||||
const w = slice.reduce((s, r) => s + (r.won > 0 ? 1 : 0), 0);
|
||||
return { rate: round4(n ? w / n : null), ci: wilson(w, n, tests) };
|
||||
}
|
||||
|
||||
/**
|
||||
* The sentence attached to a grade. TRUE or absent — never a fluent fallback.
|
||||
*
|
||||
* A factor-informed reason names the factors that proved FOR THIS ARCHETYPE. A
|
||||
* base-rate reason says plainly that it is a base-rate read, because a user who
|
||||
* is told "favourable matchup" when nothing about the matchup was read has been
|
||||
* given a fabricated reason, and that is worse than being given none.
|
||||
*/
|
||||
function reasoning(band, opts = {}) {
|
||||
if (!band) return null;
|
||||
const arch = opts.archetype ? String(opts.archetype).toUpperCase() : null;
|
||||
if (band.basis === 'factor_informed') {
|
||||
const proved = (opts.provenFactors || []).filter(Boolean);
|
||||
if (!proved.length) return null; // cannot name what did not prove
|
||||
return `${arch ? `${arch}: ` : ''}${proved.join(' + ')} — proved for this profile`;
|
||||
}
|
||||
const sep = band.separation === 'above_base_rate'
|
||||
? 'above this profile\'s base rate'
|
||||
: (band.separation === 'below_base_rate' ? 'below this profile\'s base rate' : 'at this profile\'s base rate');
|
||||
return `${arch ? `${arch}: ` : ''}base-rate read, ${sep}`
|
||||
+ ' — no matchup factor is proven for this profile yet';
|
||||
}
|
||||
|
||||
const round4 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 10000) / 10000);
|
||||
|
||||
module.exports = {
|
||||
buildBands, reasoning, wilson,
|
||||
PROVISIONAL_N, MIN_BAND_N, LETTERS,
|
||||
};
|
||||
@@ -0,0 +1,153 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* What a letter is allowed to mean.
|
||||
*
|
||||
* The failure these guard against is the one the whole programme keeps circling:
|
||||
* a grade that LOOKS like it read tonight's matchup while reading nothing. Here
|
||||
* that would be a band labelled factor-informed on an archetype where no factor
|
||||
* ever proved — which is the state of every archetype today.
|
||||
*/
|
||||
|
||||
const gb = require('../../src/services/model/gradeBands');
|
||||
|
||||
/** n rows whose outcome rate genuinely tracks p, interleaved (never front-loaded). */
|
||||
function rows(specs) {
|
||||
const out = [];
|
||||
for (const [p, n, rate] of specs) {
|
||||
for (let i = 0; i < n; i += 1) {
|
||||
const won = Math.floor((i + 1) * rate) > Math.floor(i * rate) ? 1 : 0;
|
||||
out.push({ p, won });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
describe('the two-bar rule is structural', () => {
|
||||
const data = rows([[0.75, 120, 0.75], [0.6, 120, 0.6], [0.45, 120, 0.45]]);
|
||||
|
||||
it('with NOTHING proven — the state today — no band can be factor-informed', () => {
|
||||
const out = gb.buildBands(data, { archetype: 'GHOST' });
|
||||
expect(out.basis).toBe('base_rate');
|
||||
expect(out.two_bar.factor_informed_allowed).toBe(false);
|
||||
expect(out.two_bar.reason).toMatch(/no factor has passed the gate/);
|
||||
expect(out.bands.every((b) => b.basis === 'base_rate')).toBe(true);
|
||||
});
|
||||
|
||||
it('PROVEN but not calibrated is still a base-rate read, and names why', () => {
|
||||
// Both bars, or neither claim. A proven factor whose numbers are not
|
||||
// certified honest cannot carry a letter that asserts a rate.
|
||||
const out = gb.buildBands(data, { archetype: 'GHOST', proven: true });
|
||||
expect(out.basis).toBe('base_rate');
|
||||
expect(out.two_bar.reason).toMatch(/not certified calibrated/);
|
||||
});
|
||||
|
||||
it('CALIBRATED but not proven is still a base-rate read', () => {
|
||||
const out = gb.buildBands(data, { archetype: 'GHOST', calibrated: true });
|
||||
expect(out.basis).toBe('base_rate');
|
||||
expect(out.two_bar.reason).toMatch(/no factor has passed the gate/);
|
||||
});
|
||||
|
||||
it('only BOTH unlocks factor-informed', () => {
|
||||
const out = gb.buildBands(data, { archetype: 'GHOST', proven: true, calibrated: true });
|
||||
expect(out.basis).toBe('factor_informed');
|
||||
expect(out.bands.every((b) => b.basis === 'factor_informed')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('lift is measured against the archetype\'s OWN base rate', () => {
|
||||
it('a band is only credited when its interval clears the base rate', () => {
|
||||
const out = gb.buildBands(rows([[0.8, 200, 0.80], [0.5, 200, 0.50], [0.3, 200, 0.30]]), {
|
||||
archetype: 'GHOST', targetBands: 3,
|
||||
});
|
||||
const top = out.bands[0];
|
||||
expect(top.shows_lift).toBe(true);
|
||||
expect(top.separation).toBe('above_base_rate');
|
||||
expect(top.lift_vs_archetype_base).toBeGreaterThan(0);
|
||||
const bottom = out.bands[out.bands.length - 1];
|
||||
expect(bottom.shows_deficit).toBe(true);
|
||||
});
|
||||
|
||||
it('the same realized rate is lift for one archetype and not for another', () => {
|
||||
// 62% is a real read for a profile that hits 45%, and slightly under water
|
||||
// for one that hits 68%. A raw-rate band would call both the same letter.
|
||||
const lowBase = gb.buildBands(rows([[0.62, 300, 0.62], [0.4, 300, 0.40]]), { archetype: 'LOW', targetBands: 2 });
|
||||
const highBase = gb.buildBands(rows([[0.62, 300, 0.62], [0.75, 300, 0.75]]), { archetype: 'HIGH', targetBands: 2 });
|
||||
expect(lowBase.bands[0].shows_lift).toBe(true);
|
||||
expect(highBase.bands.find((b) => b.p_range[0] === 0.62).shows_deficit).toBe(true);
|
||||
});
|
||||
|
||||
it('a band indistinguishable from the base rate claims NO lift', () => {
|
||||
const flat = gb.buildBands(rows([[0.7, 150, 0.55], [0.5, 150, 0.55], [0.3, 150, 0.55]]), { archetype: 'FLAT' });
|
||||
for (const b of flat.bands) {
|
||||
expect(b.shows_lift).toBe(false);
|
||||
expect(b.separation).toBe('indistinguishable_from_base_rate');
|
||||
}
|
||||
expect(flat.bands_showing_lift).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('bands the ledger cannot stand behind', () => {
|
||||
it('indistinguishable neighbours are MERGED, not published as different letters', () => {
|
||||
// If we cannot tell two bands apart, calling one A and one B is a
|
||||
// distinction we have not measured.
|
||||
const out = gb.buildBands(rows([[0.7, 200, 0.60], [0.65, 200, 0.60], [0.6, 200, 0.60]]), {
|
||||
archetype: 'SAME', targetBands: 5,
|
||||
});
|
||||
expect(out.bands.length).toBe(1);
|
||||
});
|
||||
|
||||
it('thin bands are PROVISIONAL, not silently dropped', () => {
|
||||
// "Still counting" and "nothing here" are different claims.
|
||||
const out = gb.buildBands(rows([[0.8, 20, 0.8], [0.3, 20, 0.3]]), { archetype: 'THIN', targetBands: 2 });
|
||||
expect(out.bands.some((b) => b.provisional)).toBe(true);
|
||||
});
|
||||
|
||||
it('too few outcomes → no bands at all, with the refusal stated', () => {
|
||||
const out = gb.buildBands(rows([[0.6, 5, 0.6]]), { archetype: 'TINY' });
|
||||
expect(out.bands).toEqual([]);
|
||||
expect(out.refused).toMatch(/insufficient settled outcomes/);
|
||||
});
|
||||
|
||||
it('the cumulative correction WIDENS every interval', () => {
|
||||
const data = rows([[0.75, 150, 0.75], [0.45, 150, 0.45]]);
|
||||
const one = gb.buildBands(data, { archetype: 'X', cumulativeTests: 1, targetBands: 2 });
|
||||
const many = gb.buildBands(data, { archetype: 'X', cumulativeTests: 99, targetBands: 2 });
|
||||
const width = (o) => o.bands[0].ci[1] - o.bands[0].ci[0];
|
||||
expect(width(many)).toBeGreaterThan(width(one));
|
||||
});
|
||||
|
||||
it('the interval never runs past 0 or 1', () => {
|
||||
const out = gb.buildBands(rows([[0.99, 60, 1.0], [0.01, 60, 0.0]]), { archetype: 'EDGE', targetBands: 2 });
|
||||
for (const b of out.bands) {
|
||||
expect(b.ci[0]).toBeGreaterThanOrEqual(0);
|
||||
expect(b.ci[1]).toBeLessThanOrEqual(1);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('the reason attached to a grade is true or absent', () => {
|
||||
it('a base-rate band SAYS it is a base-rate read', () => {
|
||||
const out = gb.buildBands(rows([[0.75, 120, 0.75], [0.45, 120, 0.45]]), { archetype: 'BOMBER', targetBands: 2 });
|
||||
const text = gb.reasoning(out.bands[0], { archetype: 'BOMBER' });
|
||||
expect(text).toMatch(/base-rate read/);
|
||||
expect(text).toMatch(/no matchup factor is proven/);
|
||||
// The thing it must never do is imply a matchup was read.
|
||||
expect(text).not.toMatch(/matchup edge|favourable matchup/i);
|
||||
});
|
||||
|
||||
it('a factor-informed band with NO named proven factors returns nothing', () => {
|
||||
// Better to say nothing than to invent the why.
|
||||
const out = gb.buildBands(rows([[0.75, 120, 0.75], [0.45, 120, 0.45]]),
|
||||
{ archetype: 'GHOST', proven: true, calibrated: true, targetBands: 2 });
|
||||
expect(gb.reasoning(out.bands[0], { archetype: 'GHOST', provenFactors: [] })).toBeNull();
|
||||
});
|
||||
|
||||
it('a factor-informed band names the factors that actually proved', () => {
|
||||
const out = gb.buildBands(rows([[0.75, 120, 0.75], [0.45, 120, 0.45]]),
|
||||
{ archetype: 'GHOST', proven: true, calibrated: true, targetBands: 2 });
|
||||
const text = gb.reasoning(out.bands[0], { archetype: 'GHOST', provenFactors: ['defense_by_direction'] });
|
||||
expect(text).toMatch(/GHOST: defense_by_direction/);
|
||||
expect(text).toMatch(/proved for this profile/);
|
||||
});
|
||||
});
|
||||
+1
-1
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user