From f9c02550607e5d38f13430d0fb637b79472b623e Mon Sep 17 00:00:00 2001 From: Kev Date: Sun, 30 Aug 2026 23:17:33 -0400 Subject: [PATCH] The observer must run on the index that exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The coverage audit returned AUDIT_UNAVAILABLE on the real 03:00Z cohort with `canceling statement due to statement timeout`, three attempts running. `model_snapshots` carries no index on `snapshot_id` and none on `captured_at`. The only usable one is `(game_date, sport)`. So "the latest published row, ordered by captured_at" scans and sorts the whole 377k-row table, and dropping the `game_date` filter from the cohort walk removed the last index the read had. The audit degraded honestly rather than reporting a false zero — which is the designed behaviour and is why this was diagnosable — but an observer that cannot run is not an observer. Both reads are now bounded on `game_date`: a lookback window for the head, and a +/-2 day span around it for the cohort. The result declares `date_window` and `game_dates_audited`, so the audit states its own scope instead of implying a completeness it did not check. No index was added; this uses what is there. Suite 397/5,499/0 · 15/15 teeth. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CQJeAG8vcDoL5zkiaJyVb8 --- src/services/lineageCoverage.js | 82 +++++++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 20 deletions(-) diff --git a/src/services/lineageCoverage.js b/src/services/lineageCoverage.js index c3920a7..f1ada9b 100644 --- a/src/services/lineageCoverage.js +++ b/src/services/lineageCoverage.js @@ -183,58 +183,100 @@ const AUDIT_COLUMNS = [ * published and recorded nothing surfaces as MISSING_COVERAGE rather than as * silence. */ +/** ISO date offset by whole days, for the query-plan window below. */ +function shiftDate(isoDate, days) { + const d = new Date(`${String(isoDate).slice(0, 10)}T00:00:00Z`); + d.setUTCDate(d.getUTCDate() + days); + return d.toISOString().slice(0, 10); +} + +/** + * Audit the most recent cohort that could have produced lineage, chosen from + * durable retained state. This is the "writer never ran" detector: the cohort + * is selected without reference to whether any lineage exists, so a slate that + * published and recorded nothing surfaces as MISSING_COVERAGE rather than as + * silence. + * + * EVERY READ IS BOUNDED BY `game_date`, AND THAT IS NOT A STYLE CHOICE. + * `model_snapshots` carries no index on `snapshot_id` or `captured_at` — the + * only usable one is `(game_date, sport)`. An unbounded "latest published row" + * sort therefore scans the whole table and dies on the statement timeout, which + * is exactly what this audit did in production at ~377k rows. The window is + * declared in the result (`date_window`, `game_dates_audited`) so the audit + * states its own scope instead of implying completeness it did not check. + */ async function auditLatestCohort(sport, deps = {}) { const sp = String(sport || '').toLowerCase(); + const unavailable = (reason) => Object.freeze({ + audit_available: false, reason, sport: sp, health: HEALTH.AUDIT_UNAVAILABLE, + }); try { const getClient = deps.getClient || require('../utils/supabase').getSupabaseServiceClient; const supabase = getClient(); - if (!supabase) return Object.freeze({ audit_available: false, reason: 'no_supabase_env', sport: sp, health: HEALTH.AUDIT_UNAVAILABLE }); + if (!supabase) return unavailable('no_supabase_env'); + + const nowIso = deps.now ? deps.now() : new Date().toISOString(); + const lookbackDays = deps.lookbackDays ?? 4; + const from = shiftDate(nowIso, -lookbackDays); - // The head identifies the COHORT (snapshot_id), not one date within it. const { data: head, error: headErr } = await supabase .from('model_snapshots') .select('snapshot_id, game_date, captured_at, code_sha') .eq('sport', sp).eq('published', true) + .gte('game_date', from) .order('captured_at', { ascending: false }).limit(1); - if (headErr) return Object.freeze({ audit_available: false, reason: headErr.message, sport: sp, health: HEALTH.AUDIT_UNAVAILABLE }); + if (headErr) return unavailable(headErr.message); if (!head || head.length === 0) { - return Object.freeze({ audit_available: true, sport: sp, expected_keys: 0, covered_keys: 0, health: HEALTH.NO_ELIGIBLE_CLAIMS, reason: 'no published rows' }); + return Object.freeze({ + audit_available: true, sport: sp, expected_keys: 0, covered_keys: 0, + health: HEALTH.NO_ELIGIBLE_CLAIMS, reason: 'no published rows in window', + date_window: [from, null], + }); } const h = head[0]; + // A COHORT IS A snapshot_id, NOT A DATE SLICE OF ONE. Measured on the + // 2026-08-31 03:00Z run: 1,970 rows across TWO game_dates (1,572 on 08-31, + // 398 on 08-30 — the late slot spanning midnight ET). Auditing only the head + // row's date measured 646 of 801 eligible keys and still said HEALTHY, so a + // whole date slice with no lineage would have been invisible. The span is + // walked date by date, which also keeps every read on the index. + const lo = shiftDate(h.game_date, -2); + const hi = shiftDate(h.game_date, 2); const { paginate } = require('../utils/safePaginate'); - // A COHORT IS A snapshot_id, NOT A DATE SLICE OF ONE. - // - // Measured on the 2026-08-31 03:00Z run: the cohort held 1,970 rows across - // TWO game_dates (1,572 on 08-31 and 398 on 08-30, the late slot spanning - // midnight ET). Filtering to the head row's date measured 646 of 801 - // eligible keys and still reported HEALTHY — so a whole date slice with - // zero lineage would have been invisible. The audit covers the cohort. const rows = await paginate(() => supabase .from('model_snapshots') .select(AUDIT_COLUMNS) + .eq('sport', sp) .eq('snapshot_id', h.snapshot_id) - .eq('sport', sp), { key: 'id', label: 'lineageCoverage.auditLatestCohort' }); + .gte('game_date', lo) + .lte('game_date', hi), { key: 'id', label: 'lineageCoverage.auditLatestCohort' }); + const dates = [...new Set(rows.map((r) => r.game_date))].sort(); const audit = auditRows(rows, { - snapshot_id: h.snapshot_id, sport: sp, - game_date: [...new Set(rows.map((r) => r.game_date))].sort().join(','), + snapshot_id: h.snapshot_id, sport: sp, game_date: dates.join(','), code_sha: h.code_sha, captured_at: h.captured_at, retention_terminal: deps.retentionTerminal ?? null, }); - const ageMs = deps.now ? (new Date(deps.now()).getTime() - new Date(h.captured_at).getTime()) : null; + const ageMs = new Date(nowIso).getTime() - new Date(h.captured_at).getTime(); const staleAfter = deps.staleAfterMs ?? (6 * 60 * 60 * 1000); - if (ageMs !== null && ageMs > staleAfter && audit.health === HEALTH.HEALTHY) { - return Object.freeze({ ...audit, health: HEALTH.STALE, age_ms: ageMs }); + const out = { + ...audit, + age_ms: Number.isFinite(ageMs) ? ageMs : null, + date_window: [lo, hi], + game_dates_audited: dates, + }; + if (Number.isFinite(ageMs) && ageMs > staleAfter && audit.health === HEALTH.HEALTHY) { + return Object.freeze({ ...out, health: HEALTH.STALE }); } - return Object.freeze({ ...audit, age_ms: ageMs }); + return Object.freeze(out); } catch (e) { - return Object.freeze({ audit_available: false, reason: e && e.message ? e.message : String(e), sport: sp, health: HEALTH.AUDIT_UNAVAILABLE }); + return unavailable(e && e.message ? e.message : String(e)); } } module.exports = { HEALTH, VALID_ACTION_FIELDS, AUDIT_COLUMNS, - isValidAction, expectedKeys, coveredKeys, graphDefects, classify, + isValidAction, expectedKeys, coveredKeys, graphDefects, classify, shiftDate, auditRows, auditLatestCohort, };