Part 1 diagnostic: read-only refusal categoriser (25-cap + 72% refusal)

READ-ONLY. Runs the REAL grade path over a REAL slate and categorises
every refusal; writes nothing. Reproduces gradeSlateService.dedupeProps
exactly (MODEL_BOOKS, first-row-wins) and calls analyzeViaEngine1 the same
way, so it measures what the pipeline does rather than a re-implementation.

Adds a FIFTH bucket the order did not anticipate, and it is likely to
change how the 72% is read: (e) POLICY-SUPPRESSION. The 2026-07-19
betting-logic audit deliberately refuses rare-event 0.5 markets (doubles/
triples/HR/SB) on the juiced under, plus any over-juiced price -- and it
sets the SAME insufficient_data flag as a genuine data gap. Counting those
as a data problem would send us hunting for data that is not missing, and
"fixing" them would re-introduce bets we removed on purpose.

Separates (b) FETCHABLE-GAP from (d) GENUINE-ABSENCE by asking the stats
layer directly whether the player has ANY game log, rather than assuming:
no log -> genuine absence, keep refusing; a log that exists while the grade
path found no projection -> a wiring gap with something to fix.

Also measures per-grade latency (mean/median/p90/max, serial and at
concurrency) so Part 2 can decide the cap on cost rather than on taste.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QJs13VsyiSKYQP6rj3NNmc

src/routes/internal.js

@@ -601,4 +601,28 @@ router.get('/ranking-delta', async (req, res) => {
   }
 });
 
+/**
+ * GET /api/internal/diagnose-refusals (25-cap + 72% refusal, Part 1)
+ *
+ * READ-ONLY diagnosis: runs the real grade path over a real slate, categorises
+ * every refusal, and measures per-grade cost so we know what raising the cap
+ * would actually cost. Writes nothing.
+ *
+ *   ?sport=mlb&sample=60&concurrency=5
+ */
+router.get('/diagnose-refusals', async (req, res) => {
+  try {
+    const { diagnose } = require('../services/refusalDiagnostics');
+    const out = await diagnose({
+      sport: req.query.sport || 'mlb',
+      sample: parseInt(req.query.sample, 10) || undefined,
+      concurrency: parseInt(req.query.concurrency, 10) || undefined,
+    });
+    res.set('Cache-Control', 'no-store');
+    return res.json({ ok: true, ...out });
+  } catch (err) {
+    return res.status(500).json({ ok: false, error: err && err.message });
+  }
+});
+
 module.exports = router;

src/services/refusalDiagnostics.js

@@ -0,0 +1,221 @@
+'use strict';
+
+/**
+ * refusalDiagnostics — Order: THE 25-CAP + 72% REFUSAL, Part 1. READ-ONLY.
+ *
+ * Runs the REAL grade path over a REAL slate and categorises every refusal.
+ * Writes nothing — no cache, no ledger, no snapshot. It reproduces
+ * `gradeSlateService.dedupeProps` exactly (MODEL_BOOKS, first-row-wins) and
+ * calls `analyzeViaEngine1` the same way, so what it measures is what the
+ * pipeline actually does rather than a re-implementation of it.
+ *
+ * BUCKETS — the order's four, plus one it did not anticipate:
+ *
+ *   (a) FALSE-THRESHOLD    data exists, a grading threshold rejected it
+ *   (b) FETCHABLE-GAP      data exists somewhere, not wired to the grade path
+ *   (c) ARCHETYPE-GAP      the prop cannot be classified
+ *   (d) GENUINE-ABSENCE    no history exists → CORRECT refusal, keep refusing
+ *   (e) POLICY-SUPPRESSION *** NOT A DATA GAP ***
+ *
+ * (e) is the finding that changes how this order should be read. The
+ * 2026-07-19 betting-logic audit deliberately refuses rare-event 0.5 markets
+ * (doubles / triples / HR / SB) on the juiced under, and any over-juiced price
+ * — and it sets the SAME `insufficient_data: true` flag as a genuine data gap.
+ * Counting those as a data problem would send us hunting for data that is not
+ * missing, and "fixing" them would re-introduce bets we removed on purpose.
+ *
+ * (b) vs (d) is separated by asking the stats layer directly whether the player
+ * has ANY game log: no log at all → genuine absence; a log that exists while
+ * the grade path still found no projection → a wiring gap, not an absence.
+ */
+
+const DEFAULT_SAMPLE = 60;
+const DEFAULT_CONCURRENCY = 5;
+
+/** Bounded-concurrency map (mirrors gradeSlateService's own helper). */
+async function mapLimit(items, limit, fn) {
+  const out = new Array(items.length);
+  let cursor = 0;
+  const workers = Array.from({ length: Math.max(1, limit) }, async () => {
+    for (;;) {
+      const idx = cursor;
+      if (idx >= items.length) return;
+      cursor += 1;
+      out[idx] = await fn(items[idx], idx);
+    }
+  });
+  await Promise.all(workers);
+  return out;
+}
+
+/** Reproduce gradeSlateService.dedupeProps — MODEL books, first row wins. */
+function uniqueGradeable(props, isModelBook) {
+  const seen = new Set();
+  const out = [];
+  for (const p of props || []) {
+    if (!p || !p.player || !p.stat_type || p.line == null) continue;
+    if (!isModelBook(p.book)) continue;
+    const k = `${p.player}::${p.stat_type}::${p.line}`;
+    if (seen.has(k)) continue;
+    seen.add(k);
+    out.push(p);
+  }
+  return out;
+}
+
+/**
+ * Does this player have ANY usable stat history? This is what separates a
+ * FETCHABLE-GAP (b) from a GENUINE-ABSENCE (d) — and the distinction decides
+ * whether there is anything to fix at all.
+ */
+async function probeHistory(player, sport, statType, deps) {
+  try {
+    const getStatRows = deps.getStatRows
+      || require('./intelligence/featureCache').getStatRows;
+    const rows = await getStatRows(player, sport, statType);
+    if (!Array.isArray(rows)) return { rows: 0, withStat: 0 };
+    const withStat = rows.filter((r) => r && r[statType] != null).length;
+    return { rows: rows.length, withStat };
+  } catch {
+    return { rows: 0, withStat: 0, probe_failed: true };
+  }
+}
+
+/**
+ * The Part-1 report. All deps injectable so tests never touch the network.
+ */
+async function diagnose(opts = {}) {
+  const sport = String(opts.sport || 'mlb').toLowerCase();
+  const sample = Math.max(1, Math.min(300, opts.sample || DEFAULT_SAMPLE));
+  const concurrency = Math.max(1, Math.min(10, opts.concurrency || DEFAULT_CONCURRENCY));
+
+  const getOdds = opts.getOdds || require('./oddsService').getOdds;
+  const analyze = opts.analyze || require('./intelligence/analyzeViaEngine1').analyzeViaEngine1;
+  const isModelBook = opts.isModelBook || require('../config/bookRoles').isModelBook;
+
+  const startedAt = Date.now();
+  const odds = await getOdds(sport);
+  const allRows = (odds && odds.props) || [];
+  const unique = uniqueGradeable(allRows, isModelBook);
+  const batch = unique.slice(0, sample);
+
+  const latencies = [];
+  const results = await mapLimit(batch, concurrency, async (p) => {
+    const t = Date.now();
+    let res = null;
+    let threw = null;
+    try {
+      res = await analyze({
+        player: p.player, stat_type: p.stat_type, line: p.line, sport,
+        direction: 'over', book: p.book,
+        over_odds: p.over_odds ?? null, under_odds: p.under_odds ?? null,
+        home_team: p.home_team, away_team: p.away_team, game_time: p.game_time,
+      });
+    } catch (err) { threw = (err && err.message) || String(err); }
+    latencies.push(Date.now() - t);
+    return { p, res, threw };
+  });
+
+  const buckets = {};
+  const suppressedReasons = {};
+  const refusalSummaries = {};
+  const byStat = {};
+  let graded = 0;
+  const noProjection = [];
+
+  const bump = (o, k) => { o[k] = (o[k] || 0) + 1; };
+
+  for (const { p, res, threw } of results) {
+    const statKey = String(p.stat_type || '?');
+    byStat[statKey] = byStat[statKey] || { graded: 0, refused: 0, suppressed: 0 };
+    if (threw) { bump(buckets, 'x_THREW'); byStat[statKey].refused += 1; continue; }
+    if (res && res.grade && !res.insufficient_data) {
+      graded += 1; byStat[statKey].graded += 1; continue;
+    }
+    if (res && res.suppressed) {
+      bump(buckets, 'e_POLICY_SUPPRESSION');
+      bump(suppressedReasons, res.suppressed_reason || 'unknown');
+      byStat[statKey].suppressed += 1;
+      continue;
+    }
+    // Everything else claims "no projection". Whether that is (b) or (d) is
+    // decided by probing the stats layer, not by assuming.
+    bump(buckets, 'no_projection_PENDING_SPLIT');
+    byStat[statKey].refused += 1;
+    const s = (res && res.reasoning && res.reasoning.summary) || '(no summary)';
+    bump(refusalSummaries, s.slice(0, 160));
+    noProjection.push(p);
+  }
+
+  // (b) vs (d): probe history for the no-projection refusals.
+  const probes = await mapLimit(noProjection.slice(0, 40), concurrency,
+    (p) => probeHistory(p.player, sport, p.stat_type, opts));
+  let fetchableGap = 0;
+  let genuineAbsence = 0;
+  let probeFailed = 0;
+  const fetchableExamples = [];
+  probes.forEach((h, i) => {
+    if (!h) return;
+    if (h.probe_failed) { probeFailed += 1; return; }
+    if (h.withStat > 0) {
+      fetchableGap += 1;
+      if (fetchableExamples.length < 8) {
+        fetchableExamples.push({
+          player: noProjection[i].player, stat: noProjection[i].stat_type,
+          log_rows: h.rows, rows_with_stat: h.withStat,
+        });
+      }
+    } else genuineAbsence += 1;
+  });
+
+  const n = batch.length || 1;
+  const sorted = [...latencies].sort((a, b) => a - b);
+  const sum = latencies.reduce((a, b) => a + b, 0);
+  const pct = (v) => Math.round((1000 * v) / n) / 10;
+
+  return {
+    read_only: true,
+    sport,
+    generated_at: new Date().toISOString(),
+    slate: {
+      rows_in_feed: allRows.length,
+      unique_gradeable_props: unique.length,
+      sampled: batch.length,
+      // The number the order is really about: the cap vs what exists.
+      current_cap: 25,
+      capped_out: Math.max(0, unique.length - 25),
+    },
+    outcome: {
+      graded, graded_pct: pct(graded),
+      ...Object.fromEntries(Object.entries(buckets).map(([k, v]) => [k, v])),
+      buckets_pct: Object.fromEntries(Object.entries(buckets).map(([k, v]) => [k, pct(v)])),
+    },
+    // (e) — deliberate, correct, NOT a data gap.
+    policy_suppression_reasons: suppressedReasons,
+    // (b) vs (d) — the only split that says whether there is anything to fix.
+    no_projection_split: {
+      probed: probes.length,
+      b_fetchable_gap: fetchableGap,
+      d_genuine_absence: genuineAbsence,
+      probe_failed: probeFailed,
+      fetchable_examples: fetchableExamples,
+    },
+    refusal_summaries: refusalSummaries,
+    by_stat: byStat,
+    cost: {
+      n: latencies.length,
+      mean_ms: Math.round(sum / (latencies.length || 1)),
+      median_ms: sorted[Math.floor(sorted.length / 2)] ?? null,
+      p90_ms: sorted[Math.floor(0.9 * sorted.length)] ?? null,
+      max_ms: sorted[sorted.length - 1] ?? null,
+      serial_total_s: Math.round(sum / 100) / 10,
+      est_wall_s_at_concurrency: Math.round(sum / (concurrency * 100)) / 10,
+      run_wall_s: Math.round((Date.now() - startedAt) / 100) / 10,
+    },
+  };
+}
+
+module.exports = {
+  diagnose,
+  __internals: { uniqueGradeable, probeHistory, mapLimit, DEFAULT_SAMPLE, DEFAULT_CONCURRENCY },
+};