From 94f7c3c3ef134874cbb6dcceb5e9b0b0ca8baa91 Mon Sep 17 00:00:00 2001 From: Kev Date: Thu, 3 Sep 2026 18:33:31 -0400 Subject: [PATCH] The probability never leaked across stats; the attribution did MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-fix cohort df4ec562 closed the primary question: eleven non-hits stats, 1,910 rows, ZERO certified and ZERO numeric served probabilities. The cross-stat repair holds. But every one of those 1,910 rows still recorded `artifact_id: mlb-hits-isotonic@2026-09-03` beside state UNSUPPORTED. A `doubles` row named the hits artifact. Nothing was calibrated by it, so no number leaked — but a later query for "rows this artifact produced" would have returned 2,186 instead of 133, and that is the shape of footgun this programme keeps finding. The contract check runs BEFORE any artifact is relevant: with no certified contract for the sport/stat, no artifact applies, and naming one asserts a relationship that does not exist. UNSUPPORTED now carries null artifact, artifact_id, estimator_type, estimator_version, certification_version and procedure_version. Attribution is KEPT where the artifact is genuinely the thing that declined — UNCERTIFIED (out of support) and VERSION_MISMATCH both still name it. A test holds both directions so this does not over-correct into erasing real provenance. One existing test called resolve() without naming a stat and relied on the service substituting one. That substitution was the original defect, so the test now names its stat, as production does. Artifact unchanged. Live OFF. Suite 405/405, 5,662 passed. Teeth 35/35 + 10/10 + 23/23. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CQJeAG8vcDoL5zkiaJyVb8 --- scripts/teeth-artifact-governance.js | 8 ++++ src/services/model/probabilityContract.js | 18 ++++++++- tests/unit/artifactGovernance.test.js | 45 +++++++++++++++++++++++ tests/unit/probabilityContract.test.js | 5 ++- 4 files changed, 73 insertions(+), 3 deletions(-) diff --git a/scripts/teeth-artifact-governance.js b/scripts/teeth-artifact-governance.js index 5a539a3..2aa6ae6 100644 --- a/scripts/teeth-artifact-governance.js +++ b/scripts/teeth-artifact-governance.js @@ -292,6 +292,14 @@ inject(34, 'an artifact for another stat is accepted', ` || false)) {`, GOV); +inject(35, 'an UNSUPPORTED row is attributed to the hits artifact', + 'src/services/model/probabilityContract.js', + ` estimator_type: null, estimator_version: null, certification_version: null, + contract_model_version: null, artifact: null, artifact_id: null, + procedure_version: null,`, + ``, + GOV); + const landed = results.filter((r) => r.landed).length; console.log(JSON.stringify({ teeth_landed: `${landed}/${results.length}`, results }, null, 2)); process.exit(landed === results.length ? 0 : 1); diff --git a/src/services/model/probabilityContract.js b/src/services/model/probabilityContract.js index 9f71790..d14c004 100644 --- a/src/services/model/probabilityContract.js +++ b/src/services/model/probabilityContract.js @@ -173,7 +173,23 @@ function resolve(read = {}, deps = {}) { }; if (!contract) { - return { ...base, probability_state: STATE.UNSUPPORTED, reason: 'no certified contract for this sport/stat' }; + // NO ARTIFACT ATTRIBUTION ON AN UNSUPPORTED ROW. + // + // The contract check happens BEFORE any artifact is relevant: there is no + // certified contract for this sport/stat, so no artifact applies and naming + // one asserts a relationship that does not exist. Measured on cohort + // df4ec562: every one of 1,910 non-hits rows recorded + // `artifact_id: mlb-hits-isotonic@...` beside state UNSUPPORTED, so a query + // for "rows this artifact produced" would have returned 2,186 instead of + // 133. The probability never leaked; the attribution did. + return { + ...base, + estimator_type: null, estimator_version: null, certification_version: null, + contract_model_version: null, artifact: null, artifact_id: null, + procedure_version: null, + probability_state: STATE.UNSUPPORTED, + reason: 'no certified contract for this sport/stat', + }; } if (read.model_version !== contract.model_version) { // A calibration artifact is only meaningful against the forecast it was diff --git a/tests/unit/artifactGovernance.test.js b/tests/unit/artifactGovernance.test.js index 02b370b..6e775fc 100644 --- a/tests/unit/artifactGovernance.test.js +++ b/tests/unit/artifactGovernance.test.js @@ -274,3 +274,48 @@ describe('ONE ARTIFACT, ONE STAT — the cross-stat leak found by cohort 0353c55 expect(r.served_probability).toBeNull(); }); }); + +describe('an UNSUPPORTED row attributes no artifact', () => { + const retention = require('../../src/services/retentionService'); + + it('names no artifact for a stat that has no contract', () => { + const r = pc.resolve({ sport: 'mlb', stat: 'doubles', model_version: ERA, p_win: 0.65 }, + { estimate: est, artifact: good }); + expect(r.probability_state).toBe(pc.STATE.UNSUPPORTED); + expect(r.served_probability).toBeNull(); + // the attribution, not just the number + expect(r.artifact).toBeNull(); + expect(r.artifact_id).toBeNull(); + expect(r.estimator_type).toBeNull(); + expect(r.procedure_version).toBeNull(); + expect(JSON.stringify(r)).not.toContain(good.artifact_id); + }); + + it('but a hits row in the same batch still carries the full identity', async () => { + const contract = await svc.build(null, { sport: 'mlb', stat: 'hits' }); + const c = retention.createCollector({ + snapshotId: 's2', sport: 'mlb', modelVersion: ERA, codeSha: 't', + gameDate: '2026-09-03', gameIdFor: () => 'g', + }); + for (const stat of ['hits', 'doubles']) { + c.onGraded({ player: `P ${stat}`, stat_type: stat, line: 0.5, sport: 'mlb', over_odds: -110, under_odds: -110 }, + [{ direction: 'over', grade: 'C+', p_win: 0.65, confidence: 65 }]); + } + const merged = retention.mergeProbabilityContract(c.rows, contract); + const hits = merged.find((r) => r.stat === 'hits').probability_contract; + const doubles = merged.find((r) => r.stat === 'doubles').probability_contract; + expect(hits.artifact.artifact_id).toBe(good.artifact_id); + expect(hits.served_probability).not.toBeNull(); + expect(doubles.artifact).toBeNull(); + expect(doubles.served_probability).toBeNull(); + expect(doubles.probability_state).toBe(pc.STATE.UNSUPPORTED); + }); + + it('an artifact IS still named where it is the relevant one', () => { + // uncertified (out of support) and version-mismatch rows keep attribution: + // there the artifact was consulted and is the thing that declined. + expect(pc.resolve(read(0.91), { estimate: est, artifact: good }).artifact).not.toBeNull(); + expect(pc.resolve(read(0.65, { model_version: 'other' }), { estimate: est, artifact: good }).artifact) + .not.toBeNull(); + }); +}); diff --git a/tests/unit/probabilityContract.test.js b/tests/unit/probabilityContract.test.js index b5cec9f..72b136f 100644 --- a/tests/unit/probabilityContract.test.js +++ b/tests/unit/probabilityContract.test.js @@ -217,8 +217,9 @@ describe('probabilityContractService — loads, never fits', () => { const b1 = await svc.build(null, { sport: 'mlb', stat: 'hits' }); const b2 = await svc.build(null, { sport: 'mlb', stat: 'hits' }); for (const p of [0.50, 0.55, 0.601, 0.72, 0.799]) { - const a = b1.resolve({ model_version: ERA, p_win: p }); - const c = b2.resolve({ model_version: ERA, p_win: p }); + // the read NAMES its stat — the resolver no longer substitutes one + const a = b1.resolve({ sport: 'mlb', stat: 'hits', model_version: ERA, p_win: p }); + const c = b2.resolve({ sport: 'mlb', stat: 'hits', model_version: ERA, p_win: p }); expect(a.served_probability).toBe(c.served_probability); expect(a.artifact.knot_digest).toBe(c.artifact.knot_digest); }