Per-archetype re-audit: no slot reaches 500, and the replication unit

decided everything

The premise does not hold. prove-hit-factors.js has no date filter
anywhere in it and pages the full table -- there was never a window to
widen. Full clean history is 1,266 rows, not 2,715. platoon was not
"proved" last session, it was explicitly held on 4.5%-median-contaminated
season-to-date splits, and pitcher_contact_profile was demoted. The
proven set going in was one factor, not three.

STEP 1: no archetype slot reaches n>=500 on full history. Best is BOMBER
at 408, and BOMBER is the most common archetype on the board. GHOST 173,
BRUSH 64, DRIVER 43, CATALYST 16. These are confirmed genuinely short,
not artifacts.

STEP 2 is where the real finding is. park_hits initially PROVED at 619
rows across 45 games -- but those games only ever visited 14 distinct
park values. A park effect is replicated across parks, and unmodelled
park heterogeneity is confounded with the thing being estimated. Each
factor is now clustered on the coarser of the game and the entity its
treatment rides on.

That flipped two verdicts and confirms Kev's causal-correctness thesis
from a new direction: defense_by_direction has 442 hitter-team units of
replication where crude team defense has 26. The correct atom is not just
more accurate, it is the only one measurable at all. park_hits (14) and
defense (26) can never be validated however long the ledger runs -- the
same ceiling as park dimensions, reached independently.

Also fixed a bar I got wrong last session: I transplanted the 500-row
floor onto clusters, which refused a factor with 1,059 rows over 85 games
while answering neither question. Two floors now -- rows>=500 for a stable
estimate, clusters>=40 for a trustworthy interval. Not a lowered bar:
park_hits and defense are still refused.

PROVEN: defense_by_direction only, pooled, [-0.0054,-0.0012] at 99 tests.
It stays POOLED-ONLY -- no per-archetype reasoning wired, nothing
grandfathered. The card must not say "GHOST: defence matchup strong"
because we have not earned that sentence. The predicted fingerprint did
not appear either: BOMBER -0.0036 vs GHOST -0.0024, the opposite
direction, both noise-dominated. Recorded so it is not claimed later.

RESCALE: NOT READY. One proven factor worth -0.0031 Brier. Rescaling on
that is relabelling.

Counter and frozen clusters byte-identical.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01W1sivYNqY2TS5ftykmHBU9

scripts/prove-hit-factors.js

@@ -54,12 +54,14 @@ const FACTORS = [
   {
     key: 'defense_by_direction',
     needs: ['spray_multiplier'],
+    entity: (r) => `${r.player_key}|${r.opp}`,
     mechanism: 'CAUSALLY-CORRECT DEFENCE. Where the hitter puts the ball (pull/straight/oppo x ground/air) crossed with the OAA of the fielders actually standing in those zones, joined by handedness. Team-average failed the gate because it averages in five fielders who will never touch his ball.',
     apply: (r) => r.spray_multiplier,
   },
   {
     key: 'defense',
     needs: ['team_defense'],
+    entity: (r) => r.opp,
     mechanism: 'A ball in play becomes a hit or an out partly by who is standing behind the pitcher. Should matter most where contact stays in the park.',
     // More outs converted above average -> fewer hits.
     apply: (r) => 1 - Math.max(-0.12, Math.min(0.12, r.team_defense / 250)),
@@ -67,12 +69,14 @@ const FACTORS = [
   {
     key: 'pitcher_contact_profile',
     needs: ['pitcher_hard_hit_allowed'],
+    entity: (r) => r.starter_id,
     mechanism: 'A contact-allowing arm concedes better contact than a bat-misser; hit probability should follow the quality of contact he permits.',
     apply: (r) => 1 + Math.max(-0.15, Math.min(0.15, (r.pitcher_hard_hit_allowed - 0.389) * 1.2)),
   },
   {
     key: 'park_hits',
     needs: ['park_factor'],
+    entity: (r) => r.park_factor,
     mechanism: 'Some parks turn outs into hits without producing runs — big outfields, high walls, deep gaps.',
     apply: (r) => r.park_factor,
     caveat: 'STAT_BASE maps hits -> run_base, so this is a RUN factor standing in for a HITS factor. A park that converts outs to hits without scoring is invisible to it.',
@@ -80,12 +84,14 @@ const FACTORS = [
   {
     key: 'platoon_severity',
     needs: ['platoon_severity_mult'],
+    entity: (r) => r.player_key,
     mechanism: "CAUSALLY-CORRECT PLATOON. The advantage is worth only what THIS hitter's measured split is worth, shrunk toward league by the smaller side's PA and refused outright below a floor. Flat handedness applies the same boost to a 63-point split and to none.",
     apply: (r) => r.platoon_severity_mult,
   },
   {
     key: 'platoon',
     needs: ['platoon_edge'],
+    entity: (r) => r.player_key,
     mechanism: 'Handedness advantage — a hitter facing the opposite hand sees the ball better and hits it harder.',
     apply: (r) => (r.platoon_edge > 0 ? 1.06 : 0.96),
   },
@@ -128,7 +134,7 @@ async function main() {
   for (const s of snaps) archOf.set(`${s.player_key}|${s.game_date}`, s.archetype);
 
   const led = await page(sb, 'ledger_entries',
-    'id, player_key, player_name, line, side, outcome, game_date, p_win, quarantine_reason, env_park_base',
+    'id, game_id, player_key, player_name, line, side, outcome, game_date, p_win, quarantine_reason, env_park_base',
     (q) => q.eq('sport', 'mlb').is('user_id', null).eq('stat', 'hits')
       .in('outcome', ['hit', 'miss']).not('p_win', 'is', null));
   const clean = led.filter((r) => !(r.quarantine_reason || '').startsWith('nontakeable_book'));
@@ -184,6 +190,15 @@ async function main() {
     loss.kept += 1;
     rows.push({
       id: r.id,
+      // Errors are correlated WITHIN a game — shared starter, park, weather and
+      // the game's own randomness — so the interval must be clustered on it.
+      // Three of these factors (pitcher profile, team defence, park) are also
+      // CONSTANT across every hitter facing that starter, which makes row
+      // resampling straightforwardly wrong for them.
+      cluster: r.game_id,
+      opp: faced,
+      starter_id: starterId != null ? Number(starterId) : null,
+      player_key: r.player_key,
       archetype: archOf.get(`${r.player_key}|${r.game_date}`) || null,
       won: r.outcome === 'hit' ? 1 : 0,
       baseline,
@@ -219,15 +234,43 @@ async function main() {
   const mc = await tl.recordAndCount(store, FACTORS.flatMap((f) =>
     ARCHS.map((a) => ({ sport: 'mlb', stat: 'hits', archetype: a === 'ALL' ? null : a, interaction: `factor:${f.key}`, target: 'outcome' }))));
 
+  // STEP 1 — FULL-HISTORY SAMPLE AUDIT PER SLOT, before any gating.
+  const audit = [];
+  for (const f of FACTORS) {
+    for (const arch of ARCHS) {
+      const slot = arch === 'ALL' ? rows : rows.filter((r) => String(r.archetype || '').toUpperCase() === arch);
+      const usable = slot.filter((r) => f.needs.every((k) => knownNumber(r[k]) !== null));
+      audit.push({
+        factor: f.key,
+        archetype: arch,
+        rows: usable.length,
+        games: new Set(usable.map((r) => r.cluster).filter(Boolean)).size,
+        players: new Set(usable.map((r) => r.player_key)).size,
+      });
+    }
+  }
+
   const results = [];
   for (const arch of ARCHS) {
     const slot = arch === 'ALL' ? rows : rows.filter((r) => String(r.archetype || '').toUpperCase() === arch);
     for (const f of FACTORS) {
       const usable = slot.filter((r) => f.needs.every((k) => knownNumber(r[k]) !== null));
+      // A park effect is replicated across PARKS, not across games: 619 rows in
+      // 45 games still only ever saw ~23 ballparks, and unmodelled park
+      // heterogeneity is confounded with the very thing being estimated. So the
+      // cluster is the COARSER of the game and the entity the treatment rides on.
+      const ents = f.entity ? new Set(usable.map((r) => String(f.entity(r)))) : null;
+      const games = new Set(usable.map((r) => String(r.cluster)));
+      const useEntity = ents && ents.size < games.size;
       const paired = usable.map((r) => {
         const mult = f.apply(r);
         const cond = mult === null ? null : Math.min(0.99, Math.max(0.01, r.baseline * mult));
-        return { baseline: r.baseline, conditioned: cond, won: r.won };
+        return {
+          baseline: r.baseline,
+          conditioned: cond,
+          won: r.won,
+          cluster: useEntity ? `e:${f.entity(r)}` : r.cluster,
+        };
       });
       const v = fg.adjudicate(paired, {
         factor: f.key, archetype: arch, stat: 'hits',
@@ -235,6 +278,10 @@ async function main() {
       });
       results.push({
         archetype: arch, factor: f.key, n: v.movement.n,
+        clusters: v.improvement ? v.improvement.effective_n : null,
+        cluster_unit: useEntity ? 'treatment_entity' : 'game',
+        distinct_games: games.size,
+        distinct_entities: ents ? ents.size : null,
         mean_abs_shift: v.movement.mean_abs_shift,
         brier_delta: v.improvement ? v.improvement.brier_delta : null,
         ci: v.improvement ? v.improvement.ci : null,
@@ -249,6 +296,7 @@ async function main() {
   console.log(JSON.stringify({
     baseline: "each row scored against the player's OWN leave-one-out base rate — the honest 'he's due' null",
     total_rows: rows.length,
+    slot_audit: audit,
     clean_settled_rows_available: clean.length,
     row_loss: loss,
     cumulative_bonferroni: mc,

specs/per-archetype-re-audit.md

@@ -0,0 +1,134 @@
+# Per-archetype re-audit — and the replication unit that decided it
+
+**The premise this order opened with does not hold, and the query it blames was
+never windowed.** `prove-hit-factors.js` selects on sport / stat / outcome only —
+there is no date filter anywhere in it, and it pages the full table. Nothing was
+being clipped.
+
+| claimed | measured |
+|---|---|
+| full clean history 2,715 rows | **1,266** |
+| platoon "tripled to 1,208 and PROVED" | 1,056 rows; passes the gate, **not promoted** |
+| three proven pooled factors | **one** — see below |
+
+`platoon` and `platoon_severity` were explicitly held last session, not promoted:
+they ride season-to-date splits containing the games they predict (4.5% median
+contamination, 12.4% p90) and passed with an upper bound of −0.0001. That still
+stands. `pitcher_contact_profile` was **demoted** last session. So the proven set
+going in was one factor, not three.
+
+---
+
+## STEP 1 — Full-history sample audit per slot
+
+Run against full clean settled history, deduped, non-quarantined:
+
+| factor | ALL | BOMBER | GHOST | BRUSH | DRIVER | CATALYST |
+|---|---|---|---|---|---|---|
+| pitcher_contact_profile | 1,059 | 408 | 173 | 64 | 43 | 16 |
+| platoon | 1,056 | 408 | 173 | 64 | 43 | 16 |
+| defense | 912 | 357 | 150 | 57 | 37 | 12 |
+| defense_by_direction | 782 | 319 | 117 | 49 | 32 | 12 |
+| platoon_severity | 700 | 343 | 121 | 24 | 38 | 16 |
+| park_hits | 619 | 239 | 96 | 36 | 22 | 9 |
+
+**No archetype slot reaches n≥500 on full history.** The best is BOMBER at 408,
+and BOMBER is by far the most common archetype on the board. These are
+**CONFIRMED genuinely short — not windowed-query artifacts.**
+
+---
+
+## STEP 2 — The replication unit, which changed every verdict
+
+Errors are correlated within a game (shared starter, park, weather, game state),
+so the interval must be clustered. But clustering on the *game* is still wrong
+for some factors, and the audit exposed it: `park_hits` initially "PROVED" at 619
+rows across 45 games — yet those 45 games only ever visited **14 distinct park
+values**. A park effect is replicated across parks. Unmodelled park heterogeneity
+is confounded with the very thing being estimated.
+
+So each factor is now clustered on the **coarser of the game and the entity its
+treatment rides on**:
+
+| factor | rows | games | treatment entities | clustered on | k | Brier Δ | CI (corrected, 99 tests) | verdict |
+|---|---|---|---|---|---|---|---|---|
+| `defense_by_direction` | 782 | 84 | **442** hitter×team | game | 84 | −0.0031 | [−0.0054, −0.0012] | **PROVES** |
+| `platoon_severity` | 700 | 85 | 145 hitters | game | 85 | −0.0038 | [−0.0070, −0.0007] | PROVES\* |
+| `platoon` | 1,056 | 85 | 228 hitters | game | 85 | −0.0033 | [−0.0061, −0.0006] | PROVES\* |
+| `pitcher_contact_profile` | 1,059 | 85 | 130 pitchers | game | 85 | −0.0034 | [−0.0067, **0.0000**] | NOT_PROVEN |
+| `defense` | 912 | 84 | **26** teams | entity | 26 | −0.0038 | [−0.0075, −0.0006] | PENDING — k<40 |
+| `park_hits` | 619 | 45 | **14** park values | entity | 14 | −0.0037 | [−0.0093, **+0.0029**] | PENDING — k<40 |
+
+\* held, not promoted — contaminated inputs, unchanged from last session.
+
+### This is Kev's causal-correctness thesis confirmed from a new direction
+
+The causally-correct atom is not merely more accurate — **it is the only one that
+is measurable at all.** `defense_by_direction` has **442** units of replication
+because spray direction varies per hitter; crude team `defense` has **26**,
+because there are 26 teams. The crude factor cannot be validated no matter how
+long the ledger runs, and its apparently-tight interval was pseudo-replication
+across teams that does not exist.
+
+`park_hits` losing its pass is the same finding as last session's park-dimensions
+result, arrived at independently: **venue- and team-borne factors cap at ~30
+units, permanently.**
+
+### Gate change: two floors, not one transplanted bar
+
+Last session I applied the 500 bar to clusters. That was wrong in a way worth
+naming: it refused a factor with 1,059 rows over 85 games — ample observations
+*and* ample clusters — while answering neither question. The floors are now
+separate, because they answer different things:
+
+- **rows ≥ 500** — is the point estimate stable?
+- **clusters ≥ 40** — can the interval around it be believed?
+
+40 is the conventional floor below which cluster-robust inference under-covers.
+This is not a lowered bar: `park_hits` (14) and `defense` (26) are still refused,
+and park geometry is still permanently unvalidatable.
+
+---
+
+## STEP 3 — What gets wired
+
+**Nothing new.** No archetype slot earns a wire, and none is grandfathered in
+from pooled proof.
+
+- `defense_by_direction` — **PROVES pooled, stays POOLED-ONLY.** Its BOMBER
+  (n=319) and GHOST (n=117) slots are short, so no per-archetype reasoning is
+  wired. The card must not say "GHOST: defence matchup strong" — we have not
+  earned that sentence.
+- `platoon`, `platoon_severity` — pass the gate, **held** pending point-in-time
+  splits.
+- `pitcher_contact_profile`, `defense`, `park_hits` — honest null / confirmed
+  short.
+
+### The predicted fingerprint did NOT appear
+
+The order expected `defense_by_direction` strong for GHOST and ~zero for BOMBER.
+Measured point estimates run the other way — BOMBER −0.0036, GHOST −0.0024 — and
+at n=319/117 both are noise-dominated. **Recorded so it is not claimed later.**
+This is not evidence against the theory; it is evidence we cannot see it yet.
+
+---
+
+## STEP 4 — Rescale readiness: NOT READY
+
+One proven factor, worth −0.0031 Brier, clustered-honest. Two more held behind a
+contaminated input. The counter still supplies essentially all of the model's
+resolution (S78 ablation).
+
+Rescaling the grade distribution on that would be **relabelling** — the same
+error as minting A's by moving thresholds, which is a permanent founder ruling.
+The distribution is not factor-rich enough.
+
+**What would change the answer**, in order of cost:
+
+1. **Point-in-time platoon splits** — would convert two held passes into real
+   ones. Cheapest, no waiting; needs per-game split reconstruction.
+2. **More games** — every archetype slot is short, and slots grow with games, not
+   rows. BOMBER needs ~92 more rows to reach 500.
+3. **A factor with high replication** — the lesson of this audit is that new
+   factors should be chosen for *causal correctness first*, which also buys
+   measurability. Anything venue- or team-borne is dead on arrival.

src/services/model/factorGate.js

@@ -40,7 +40,24 @@ const { knownNumber } = require('../../utils/known');
 
 /** Minimum mean |Δp| for a factor to count as having moved anything at all. */
 const MIN_MOVEMENT = 0.01;
+/** Observations needed for the effect ESTIMATE to be stable. */
 const MIN_N = 500;
+/**
+ * Clusters needed for the cluster-robust INTERVAL to be trustworthy.
+ *
+ * These two floors answer different questions and must not be collapsed. Rows
+ * govern whether the point estimate is stable; clusters govern whether the
+ * interval around it means anything. Transplanting the 500-row bar onto clusters
+ * refuses a factor measured over 1,059 rows and 85 games — which has ample
+ * observations AND ample clusters — while telling us nothing about either.
+ *
+ * 40 is the conventional floor below which cluster-robust inference is known to
+ * under-cover regardless of how many rows sit inside the clusters. It is a
+ * statement about when the bootstrap can be believed, not a bar tuned to let
+ * anything through: a venue-constant factor still caps at 30 ballparks and is
+ * still refused, permanently.
+ */
+const MIN_CLUSTERS = 40;
 
 const brier = (ps, ys) => (ps.length
   ? ps.reduce((s, p, i) => s + (p - ys[i]) ** 2, 0) / ps.length : null);
@@ -171,19 +188,20 @@ function adjudicate(rows, opts = {}) {
 
   const base = { factor: opts.factor || null, archetype: opts.archetype || null, stat: opts.stat || 'hits', movement: mv, improvement: imp };
 
-  // Sample is judged in the unit the FACTOR varies over, not the unit the rows
-  // happen to arrive in. A game-level factor with 928 rows across 53 games has
-  // 53 readings, and calling that 928 is how a gate passes something on sample
-  // it never had.
-  const effN = imp && imp.effective_n != null ? imp.effective_n : mv.n;
-  if (effN < minN) {
-    const unit = imp && imp.cluster_unit === 'cluster' ? 'independent clusters' : 'rows';
+  // TWO FLOORS, because they answer different questions. Rows decide whether the
+  // point estimate is stable; clusters decide whether the interval around it can
+  // be believed. A factor needs both.
+  if (mv.n < minN) {
+    return { ...base, verdict: 'CANDIDATE_PENDING_SAMPLE', reason: `n ${mv.n} < ${minN}`, rows_needed: minN - mv.n };
+  }
+  const minClusters = opts.minClusters ?? MIN_CLUSTERS;
+  if (imp && imp.cluster_unit === 'cluster' && imp.effective_n < minClusters) {
     return {
       ...base,
       verdict: 'CANDIDATE_PENDING_SAMPLE',
-      reason: `${effN} ${unit} < ${minN}`
-        + (effN !== mv.n ? ` (${mv.n} rows, but the factor varies over ${effN} clusters — the rows are not independent readings)` : ''),
-      rows_needed: minN - effN,
+      reason: `${mv.n} rows but only ${imp.effective_n} independent clusters < ${minClusters}`
+        + ' — the rows are not independent readings and the interval cannot be trusted at this cluster count',
+      clusters_needed: minClusters - imp.effective_n,
     };
   }
   if (mv.mean_abs_shift === null || mv.mean_abs_shift < minMove) {
@@ -221,4 +239,5 @@ function adjudicate(rows, opts = {}) {
 
 const round4 = (v) => (v == null || !Number.isFinite(v) ? null : Math.round(v * 10000) / 10000);
 
-module.exports = { movement, improvement, adjudicate, MIN_MOVEMENT, MIN_N };
+module.exports = {
+  MIN_CLUSTERS, movement, improvement, adjudicate, MIN_MOVEMENT, MIN_N };

tests/unit/factorGate.test.js

@@ -179,7 +179,8 @@ describe('pseudo-replication — sample counted in the unit the factor varies ov
 
   it('judges sample by CLUSTERS, so 900 rows over 50 games is 50 readings', () => {
     const rows = build(50, 18);
-    const v = fg.adjudicate(rows, { factor: 'park', minN: 500 });
+    // 900 rows clears the row floor; 50 clusters is what actually decides it.
+    const v = fg.adjudicate(rows, { factor: 'park', minN: 500, minClusters: 100 });
     expect(rows.length).toBeGreaterThan(500);          // looks like plenty
     expect(v.verdict).toBe('CANDIDATE_PENDING_SAMPLE'); // and is not
     expect(v.improvement.effective_n).toBe(50);
@@ -191,7 +192,7 @@ describe('pseudo-replication — sample counted in the unit the factor varies ov
     // This is the whole hazard: resampling rows would have manufactured a
     // confidence the evidence never supported.
     const rows = build(40, 20, 7);
-    const clustered = fg.adjudicate(rows, { factor: 'park', minN: 10 });
+    const clustered = fg.adjudicate(rows, { factor: 'park', minN: 10, minClusters: 5 });
     const flat = fg.adjudicate(rows.map(({ cluster, ...r }) => r), { factor: 'park', minN: 10 });
     const width = (v) => v.improvement.ci[1] - v.improvement.ci[0];
     expect(width(clustered)).toBeGreaterThan(width(flat));
@@ -199,8 +200,50 @@ describe('pseudo-replication — sample counted in the unit the factor varies ov
 
   it('rows with no cluster keep the original row-resampling behaviour', () => {
     const rows = build(40, 20, 3).map(({ cluster, ...r }) => r);
-    const v = fg.adjudicate(rows, { factor: 'x', minN: 10 });
+    const v = fg.adjudicate(rows, { factor: 'x', minN: 10, minClusters: 5 });
     expect(v.improvement.cluster_unit).toBe('row');
     expect(v.improvement.effective_n).toBe(v.improvement.n);
   });
 });
+
+describe('the two floors answer different questions', () => {
+  const rows = [];
+  let s2 = 11;
+  const rnd = () => (s2 = (s2 * 1103515245 + 12345) % 2147483648) / 2147483648;
+  for (let g = 0; g < 60; g += 1) {
+    for (let i = 0; i < 20; i += 1) {
+      const base = 0.35 + rnd() * 0.3;
+      rows.push({ cluster: `g${g}`, baseline: base, conditioned: base - 0.02, won: rnd() < base ? 1 : 0 });
+    }
+  }
+
+  it('ample rows with too FEW clusters is refused on the cluster floor', () => {
+    const v = fg.adjudicate(rows, { minN: 500, minClusters: 200 });
+    expect(v.verdict).toBe('CANDIDATE_PENDING_SAMPLE');
+    expect(v.reason).toMatch(/independent clusters/);
+    expect(v.clusters_needed).toBe(140);
+  });
+
+  it('too few ROWS is refused on the row floor even with many clusters', () => {
+    const thin = rows.filter((_, i) => i % 20 === 0);   // 60 rows, 60 clusters
+    const v = fg.adjudicate(thin, { minN: 500, minClusters: 40 });
+    expect(v.verdict).toBe('CANDIDATE_PENDING_SAMPLE');
+    expect(v.reason).toMatch(/^n 60 < 500/);
+  });
+
+  it('a venue-constant factor stays refused however many rows accrue', () => {
+    // 30 ballparks is the whole universe; rows can grow forever and the
+    // interval never becomes trustworthy.
+    const venues = [];
+    for (let v = 0; v < 30; v += 1) {
+      for (let i = 0; i < 200; i += 1) {
+        const base = 0.4 + rnd() * 0.2;
+        venues.push({ cluster: `v${v}`, baseline: base, conditioned: base - 0.03, won: rnd() < base ? 1 : 0 });
+      }
+    }
+    const v = fg.adjudicate(venues, { minN: 500 });
+    expect(venues.length).toBe(6000);
+    expect(v.verdict).toBe('CANDIDATE_PENDING_SAMPLE');
+    expect(v.improvement.effective_n).toBe(30);
+  });
+});