Build the gate, run it, and find we were proving things on the wrong stat
PREMISE CORRECTION FIRST. statModel.js and correlateValidator.js do not exist in this repository. The validation spec's only prior form is src/services/python/blueprints/unconventional.py -- a Flask blueprint in the Python service that is offline in production, scoring NBA factors against a warehouse that was never populated -- and tests/unit/supplementSystems.test.js requires only fs and path while defining its own validateFactor inline at line 368. Those tests assert a re-implementation of the thresholds, not an implementation, which is exactly why they passed for months while nothing was connected. The diagnosis behind the order is right -- every challenger was measured without a gate -- but the cause is that there was no gate on the Node side to import. So it is built, to the exact spec. correlateValidator: n>=500, |r|>=0.15, p<0.05, Bonferroni across the sweep. The p-value is exact rather than approximated (t-transform through a regularized incomplete beta) and is verified in the suite against known values, because scipy is not available here. Pairs with an unknown side are dropped, never zero-filled -- a zero-fill inside a correlation does not add noise, it invents a point at the origin. THE RUN, hits, n=570, Bonferroni-8: every skill feature fails, and not narrowly. The strongest marginal correlation against the counter's residual is 0.062 against a 0.15 bar. That is an effect-size failure at a sample that would have found a real effect comfortably -- a clean, well-powered negative. The head-to-head agrees: value engine 0.0499 against the counter's 0.166, delta -0.116 with CI [-0.189, -0.043]. Not promoted. THE RUN, total bases, n=295: cannot be tested, and that is the finding. hard_hit_pct shows a marginal r of 0.153 -- above the threshold -- and exit velo 0.124, refused solely because n is 205 short of 500. It is the most encouraging number this work has produced, and it is what the physics predicts: contact quality governs extra bases, not whether a grounder finds a hole. We have been testing skill inputs on the one stat where they should not matter much. Two things the run forced. Feature verdicts are now PER STAT, because marking these DEAD sport-wide on hits evidence would have killed, for total bases, the features that look most alive there -- per-sport doctrine one level deeper. And the gate now reports r and p even when underpowered, because "not enough data yet" and "nothing here" demand opposite decisions and a bare refusal was hiding the best signal on the board. Next: build the compound TB projection (skillProjection still refuses total bases by design, since a deterministic bases-per-hit made P(TB>=2) identical to P(hits>=1)), accrue to n>=500, re-run this gate. Leave hits alone. 4,200 tests green (334 suites); web build exit 0; counter byte-identical. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W1sivYNqY2TS5ftykmHBU9
This commit is contained in:
@@ -0,0 +1,208 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* correlateValidator — THE GATE A FEATURE MUST PASS TO EARN ITS PLACE.
|
||||
*
|
||||
* Kev's spec, exactly: n ≥ 500 · |Pearson r| ≥ 0.15 · p < 0.05 · Bonferroni-
|
||||
* corrected across the number of factors being tested at once.
|
||||
*
|
||||
* PROVENANCE, because it matters. This is a JS implementation of the thresholds
|
||||
* declared in `src/services/python/blueprints/unconventional.py`
|
||||
* (`VALIDATION_REQUIREMENTS`), which is the spec's only prior existence — a
|
||||
* Flask blueprint in the Python service that is offline in production, scoring
|
||||
* NBA "unconventional factors" against a data warehouse that was never
|
||||
* populated. `tests/unit/supplementSystems.test.js` asserts the same thresholds
|
||||
* but INLINES its own `validateFactor` (line 368) rather than importing one, so
|
||||
* those tests passed for months without any implementation existing in the Node
|
||||
* path at all. That is the whole reason every challenger to date was measured
|
||||
* without a gate: there was no gate on this side of the codebase to call.
|
||||
*
|
||||
* WHY BONFERRONI IS NOT OPTIONAL HERE. The S78 residual scan ran 14 features ×
|
||||
* 5 stats = 70 tests at α=.05 and produced six "significant" results — almost
|
||||
* exactly the 3–4 the null hypothesis predicts by chance. Without a correction,
|
||||
* a wide feature sweep manufactures discoveries. The correction is what makes
|
||||
* "PROVEN" mean something.
|
||||
*
|
||||
* THE P-VALUE IS REAL, NOT APPROXIMATED. scipy's `pearsonr` is not available in
|
||||
* Node, so the two-sided p is computed from the exact t transform
|
||||
*
|
||||
* t = r · sqrt((n−2) / (1−r²)), p = I_{v/(v+t²)}(v/2, 1/2), v = n−2
|
||||
*
|
||||
* using a regularized incomplete beta (Lentz continued fraction). Verified in
|
||||
* the unit suite against known values rather than trusted.
|
||||
*
|
||||
* HONESTY: pairs where either side is unknown are DROPPED, never zero-filled —
|
||||
* `Number(null) === 0` inside a correlation does not merely add noise, it
|
||||
* invents a data point at the origin and drags r toward whatever the means are.
|
||||
*/
|
||||
|
||||
const { knownNumber } = require('../../utils/known');
|
||||
|
||||
const VALIDATION_REQUIREMENTS = Object.freeze({
|
||||
min_historical_instances: 500,
|
||||
min_pearson_r: 0.15,
|
||||
max_p_value: 0.05, // BEFORE Bonferroni
|
||||
bonferroni_correction: true,
|
||||
});
|
||||
|
||||
/** log-gamma (Lanczos) — for the incomplete beta. */
|
||||
function logGamma(x) {
|
||||
const c = [76.18009172947146, -86.50532032941677, 24.01409824083091,
|
||||
-1.231739572450155, 0.1208650973866179e-2, -0.5395239384953e-5];
|
||||
let y = x;
|
||||
let tmp = x + 5.5;
|
||||
tmp -= (x + 0.5) * Math.log(tmp);
|
||||
let ser = 1.000000000190015;
|
||||
for (let j = 0; j < 6; j += 1) { y += 1; ser += c[j] / y; }
|
||||
return -tmp + Math.log((2.5066282746310005 * ser) / x);
|
||||
}
|
||||
|
||||
/** Continued fraction for the incomplete beta (modified Lentz). */
|
||||
function betaCf(a, b, x) {
|
||||
const MAXIT = 200;
|
||||
const EPS = 3e-14;
|
||||
const FPMIN = 1e-300;
|
||||
const qab = a + b;
|
||||
const qap = a + 1;
|
||||
const qam = a - 1;
|
||||
let c = 1;
|
||||
let d = 1 - (qab * x) / qap;
|
||||
if (Math.abs(d) < FPMIN) d = FPMIN;
|
||||
d = 1 / d;
|
||||
let h = d;
|
||||
for (let m = 1; m <= MAXIT; m += 1) {
|
||||
const m2 = 2 * m;
|
||||
let aa = (m * (b - m) * x) / ((qam + m2) * (a + m2));
|
||||
d = 1 + aa * d; if (Math.abs(d) < FPMIN) d = FPMIN;
|
||||
c = 1 + aa / c; if (Math.abs(c) < FPMIN) c = FPMIN;
|
||||
d = 1 / d;
|
||||
h *= d * c;
|
||||
aa = (-(a + m) * (qab + m) * x) / ((a + m2) * (qap + m2));
|
||||
d = 1 + aa * d; if (Math.abs(d) < FPMIN) d = FPMIN;
|
||||
c = 1 + aa / c; if (Math.abs(c) < FPMIN) c = FPMIN;
|
||||
d = 1 / d;
|
||||
const del = d * c;
|
||||
h *= del;
|
||||
if (Math.abs(del - 1) < EPS) break;
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
/** Regularized incomplete beta I_x(a,b). */
|
||||
function incompleteBeta(a, b, x) {
|
||||
if (x <= 0) return 0;
|
||||
if (x >= 1) return 1;
|
||||
const bt = Math.exp(logGamma(a + b) - logGamma(a) - logGamma(b)
|
||||
+ a * Math.log(x) + b * Math.log(1 - x));
|
||||
if (x < (a + 1) / (a + b + 2)) return (bt * betaCf(a, b, x)) / a;
|
||||
return 1 - (bt * betaCf(b, a, 1 - x)) / b;
|
||||
}
|
||||
|
||||
/**
|
||||
* Two-sided p-value for a Pearson r on n paired observations.
|
||||
* Returns null when it is not defined (n < 3), never a comforting 1.0.
|
||||
*/
|
||||
function pearsonPValue(r, n) {
|
||||
const rr = knownNumber(r);
|
||||
const nn = knownNumber(n);
|
||||
if (rr === null || nn === null || nn < 3) return null;
|
||||
const v = nn - 2;
|
||||
const r2 = Math.min(1, rr * rr);
|
||||
if (r2 >= 1) return 0; // perfect correlation
|
||||
const t2 = (r2 * v) / (1 - r2);
|
||||
return incompleteBeta(v / 2, 0.5, v / (v + t2));
|
||||
}
|
||||
|
||||
/**
|
||||
* Pearson r over PAIRED observations, dropping any pair with an unknown side.
|
||||
* Returns { r, n } so the caller always knows how many pairs actually counted —
|
||||
* a correlation quoted without its surviving n hides exactly this kind of loss.
|
||||
*/
|
||||
function pearson(xs, ys) {
|
||||
const X = []; const Y = [];
|
||||
const len = Math.min((xs || []).length, (ys || []).length);
|
||||
for (let i = 0; i < len; i += 1) {
|
||||
const a = knownNumber(xs[i]);
|
||||
const b = knownNumber(ys[i]);
|
||||
if (a === null || b === null) continue; // DROP the pair, never zero-fill
|
||||
X.push(a); Y.push(b);
|
||||
}
|
||||
const n = X.length;
|
||||
if (n < 3) return { r: null, n };
|
||||
const mx = X.reduce((s, v) => s + v, 0) / n;
|
||||
const my = Y.reduce((s, v) => s + v, 0) / n;
|
||||
let sxy = 0; let sxx = 0; let syy = 0;
|
||||
for (let i = 0; i < n; i += 1) {
|
||||
const dx = X[i] - mx; const dy = Y[i] - my;
|
||||
sxy += dx * dy; sxx += dx * dx; syy += dy * dy;
|
||||
}
|
||||
if (sxx <= 0 || syy <= 0) return { r: null, n }; // a constant has no correlation
|
||||
return { r: sxy / Math.sqrt(sxx * syy), n };
|
||||
}
|
||||
|
||||
/**
|
||||
* THE GATE.
|
||||
*
|
||||
* @param {number[]} factorValues the feature, per observation
|
||||
* @param {number[]} outcomeValues what actually happened, per observation
|
||||
* @param {number} numActiveTests how many factors are being tested together
|
||||
* (the Bonferroni denominator). 1 means no correction.
|
||||
* @returns {object} a verdict that always states WHY, so a failure is readable
|
||||
* without re-running anything.
|
||||
*/
|
||||
function validateFactor(factorValues, outcomeValues, numActiveTests = 1, opts = {}) {
|
||||
const req = { ...VALIDATION_REQUIREMENTS, ...(opts.requirements || {}) };
|
||||
const { r, n } = pearson(factorValues, outcomeValues);
|
||||
|
||||
if (n < req.min_historical_instances) {
|
||||
// REPORT r AND p ANYWAY. `validated` stays false — the bar is the bar — but
|
||||
// hiding the numbers turns "not enough data yet" into "nothing here", and
|
||||
// those need different decisions: one waits, the other stops. The caller can
|
||||
// see the trend without being allowed to act on it.
|
||||
const pUnder = r === null ? null : pearsonPValue(r, n);
|
||||
return {
|
||||
validated: false,
|
||||
reason: 'insufficient_data',
|
||||
detail: `${n} < ${req.min_historical_instances} required instances`,
|
||||
sample_size: n,
|
||||
rows_needed: req.min_historical_instances - n,
|
||||
pearson_r: r === null ? null : round6(r),
|
||||
p_value: pUnder === null ? null : round8(pUnder),
|
||||
underpowered: true,
|
||||
};
|
||||
}
|
||||
if (r === null) {
|
||||
return { validated: false, reason: 'no_variation', sample_size: n, pearson_r: null };
|
||||
}
|
||||
|
||||
const p = pearsonPValue(r, n);
|
||||
const tests = Math.max(1, Math.round(knownNumber(numActiveTests) ?? 1));
|
||||
const correctedAlpha = req.bonferroni_correction ? req.max_p_value / tests : req.max_p_value;
|
||||
|
||||
const strongEnough = Math.abs(r) >= req.min_pearson_r;
|
||||
const significant = p !== null && p < correctedAlpha;
|
||||
|
||||
let reason = null;
|
||||
if (!strongEnough && !significant) reason = 'weak_correlation_and_not_significant';
|
||||
else if (!strongEnough) reason = 'weak_correlation';
|
||||
else if (!significant) reason = 'not_significant_after_bonferroni';
|
||||
|
||||
return {
|
||||
validated: strongEnough && significant,
|
||||
reason,
|
||||
pearson_r: round6(r),
|
||||
p_value: p === null ? null : round8(p),
|
||||
corrected_alpha: round6(correctedAlpha),
|
||||
bonferroni_tests: tests,
|
||||
sample_size: n,
|
||||
thresholds: { min_n: req.min_historical_instances, min_abs_r: req.min_pearson_r, base_alpha: req.max_p_value },
|
||||
};
|
||||
}
|
||||
|
||||
const round6 = (v) => Math.round(v * 1e6) / 1e6;
|
||||
const round8 = (v) => Math.round(v * 1e8) / 1e8;
|
||||
|
||||
module.exports = {
|
||||
VALIDATION_REQUIREMENTS,
|
||||
validateFactor, pearson, pearsonPValue, incompleteBeta,
|
||||
};
|
||||
@@ -130,6 +130,53 @@ for (const f of FEATURES) byKey.set(`${f.sport}|${f.key}`, { ...f, evidence: f.e
|
||||
|
||||
const idOf = (sport, key) => `${String(sport || '').toLowerCase()}|${key}`;
|
||||
|
||||
/**
|
||||
* PER-STAT VERDICTS — added 2026-08-03, because the first real gate run demanded
|
||||
* it. Every skill feature FAILED the gate for `hits` at n=570 (max |r| 0.062),
|
||||
* while the same features show the strongest correlations yet seen for
|
||||
* `total_bases` (hard-hit marginal r = 0.153). Marking them DEAD sport-wide
|
||||
* would have killed, on hits evidence, the exact features that look most alive
|
||||
* on total bases — which is the per-sport doctrine one level deeper: a feature
|
||||
* earns or loses its place PER STAT, because the physics differ. Exit velocity
|
||||
* governs extra bases; it barely governs whether a single finds a hole.
|
||||
*
|
||||
* A stat-level verdict OVERRIDES the sport-level status for that stat only.
|
||||
*/
|
||||
const statVerdicts = new Map(); // `${sport}|${stat}|${key}` -> {status, evidence}
|
||||
const statIdOf = (sport, stat, key) => `${String(sport || '').toLowerCase()}|${String(stat || '').toLowerCase()}|${key}`;
|
||||
|
||||
/** Record a measured verdict for ONE feature on ONE stat. */
|
||||
function recordStatVerdict(sport, stat, key, status, evidence) {
|
||||
if (!byKey.has(idOf(sport, key))) return { ok: false, reason: 'unknown_feature' };
|
||||
if (!Object.values(STATUS).includes(status)) return { ok: false, reason: 'bad_status' };
|
||||
statVerdicts.set(statIdOf(sport, stat, key), { status, evidence: evidence || null });
|
||||
return { ok: true, status };
|
||||
}
|
||||
|
||||
/** Status for a feature on a specific stat — falls back to the sport-level one. */
|
||||
function statusForStat(sport, stat, key) {
|
||||
const v = statVerdicts.get(statIdOf(sport, stat, key));
|
||||
if (v) return v.status;
|
||||
return statusOf(sport, key);
|
||||
}
|
||||
|
||||
/** The live gate for ONE stat: PROVEN at the stat level, or proven sport-wide and not stat-DEAD. */
|
||||
function liveFeaturesForStat(sport, stat) {
|
||||
return new Set(allFeatures(sport)
|
||||
.filter((f) => statusForStat(sport, stat, f.key) === STATUS.PROVEN)
|
||||
.map((f) => f.key));
|
||||
}
|
||||
|
||||
/** Candidates still worth measuring for ONE stat — a stat-DEAD feature is not. */
|
||||
function candidateFeaturesForStat(sport, stat) {
|
||||
return new Set(allFeatures(sport)
|
||||
.filter((f) => {
|
||||
const st = statusForStat(sport, stat, f.key);
|
||||
return st === STATUS.CANDIDATE || st === STATUS.PROVEN;
|
||||
})
|
||||
.map((f) => f.key));
|
||||
}
|
||||
|
||||
/** Every registered feature for a sport (any status). */
|
||||
function allFeatures(sport) {
|
||||
const sp = String(sport || '').toLowerCase();
|
||||
@@ -223,6 +270,7 @@ function summary(sport) {
|
||||
|
||||
/** Test-only: restore the declared statuses so suites cannot leak into each other. */
|
||||
function __reset() {
|
||||
statVerdicts.clear();
|
||||
byKey.clear();
|
||||
for (const f of FEATURES) byKey.set(`${f.sport}|${f.key}`, { ...f, evidence: f.evidence || null, history: [] });
|
||||
}
|
||||
@@ -230,5 +278,6 @@ function __reset() {
|
||||
module.exports = {
|
||||
STATUS, MIN_PROMOTION_N,
|
||||
allFeatures, liveFeatures, candidateFeatures, statusOf, isLive,
|
||||
recordStatVerdict, statusForStat, liveFeaturesForStat, candidateFeaturesForStat,
|
||||
isSufficient, promote, demote, summary, __reset,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user