The observer must run on the index that exists
The coverage audit returned AUDIT_UNAVAILABLE on the real 03:00Z cohort with `canceling statement due to statement timeout`, three attempts running. `model_snapshots` carries no index on `snapshot_id` and none on `captured_at`. The only usable one is `(game_date, sport)`. So "the latest published row, ordered by captured_at" scans and sorts the whole 377k-row table, and dropping the `game_date` filter from the cohort walk removed the last index the read had. The audit degraded honestly rather than reporting a false zero — which is the designed behaviour and is why this was diagnosable — but an observer that cannot run is not an observer. Both reads are now bounded on `game_date`: a lookback window for the head, and a +/-2 day span around it for the cohort. The result declares `date_window` and `game_dates_audited`, so the audit states its own scope instead of implying a completeness it did not check. No index was added; this uses what is there. Suite 397/5,499/0 · 15/15 teeth. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CQJeAG8vcDoL5zkiaJyVb8
This commit is contained in:
@@ -183,58 +183,100 @@ const AUDIT_COLUMNS = [
|
|||||||
* published and recorded nothing surfaces as MISSING_COVERAGE rather than as
|
* published and recorded nothing surfaces as MISSING_COVERAGE rather than as
|
||||||
* silence.
|
* silence.
|
||||||
*/
|
*/
|
||||||
|
/** ISO date offset by whole days, for the query-plan window below. */
|
||||||
|
function shiftDate(isoDate, days) {
|
||||||
|
const d = new Date(`${String(isoDate).slice(0, 10)}T00:00:00Z`);
|
||||||
|
d.setUTCDate(d.getUTCDate() + days);
|
||||||
|
return d.toISOString().slice(0, 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Audit the most recent cohort that could have produced lineage, chosen from
|
||||||
|
* durable retained state. This is the "writer never ran" detector: the cohort
|
||||||
|
* is selected without reference to whether any lineage exists, so a slate that
|
||||||
|
* published and recorded nothing surfaces as MISSING_COVERAGE rather than as
|
||||||
|
* silence.
|
||||||
|
*
|
||||||
|
* EVERY READ IS BOUNDED BY `game_date`, AND THAT IS NOT A STYLE CHOICE.
|
||||||
|
* `model_snapshots` carries no index on `snapshot_id` or `captured_at` — the
|
||||||
|
* only usable one is `(game_date, sport)`. An unbounded "latest published row"
|
||||||
|
* sort therefore scans the whole table and dies on the statement timeout, which
|
||||||
|
* is exactly what this audit did in production at ~377k rows. The window is
|
||||||
|
* declared in the result (`date_window`, `game_dates_audited`) so the audit
|
||||||
|
* states its own scope instead of implying completeness it did not check.
|
||||||
|
*/
|
||||||
async function auditLatestCohort(sport, deps = {}) {
|
async function auditLatestCohort(sport, deps = {}) {
|
||||||
const sp = String(sport || '').toLowerCase();
|
const sp = String(sport || '').toLowerCase();
|
||||||
|
const unavailable = (reason) => Object.freeze({
|
||||||
|
audit_available: false, reason, sport: sp, health: HEALTH.AUDIT_UNAVAILABLE,
|
||||||
|
});
|
||||||
try {
|
try {
|
||||||
const getClient = deps.getClient || require('../utils/supabase').getSupabaseServiceClient;
|
const getClient = deps.getClient || require('../utils/supabase').getSupabaseServiceClient;
|
||||||
const supabase = getClient();
|
const supabase = getClient();
|
||||||
if (!supabase) return Object.freeze({ audit_available: false, reason: 'no_supabase_env', sport: sp, health: HEALTH.AUDIT_UNAVAILABLE });
|
if (!supabase) return unavailable('no_supabase_env');
|
||||||
|
|
||||||
|
const nowIso = deps.now ? deps.now() : new Date().toISOString();
|
||||||
|
const lookbackDays = deps.lookbackDays ?? 4;
|
||||||
|
const from = shiftDate(nowIso, -lookbackDays);
|
||||||
|
|
||||||
// The head identifies the COHORT (snapshot_id), not one date within it.
|
|
||||||
const { data: head, error: headErr } = await supabase
|
const { data: head, error: headErr } = await supabase
|
||||||
.from('model_snapshots')
|
.from('model_snapshots')
|
||||||
.select('snapshot_id, game_date, captured_at, code_sha')
|
.select('snapshot_id, game_date, captured_at, code_sha')
|
||||||
.eq('sport', sp).eq('published', true)
|
.eq('sport', sp).eq('published', true)
|
||||||
|
.gte('game_date', from)
|
||||||
.order('captured_at', { ascending: false }).limit(1);
|
.order('captured_at', { ascending: false }).limit(1);
|
||||||
if (headErr) return Object.freeze({ audit_available: false, reason: headErr.message, sport: sp, health: HEALTH.AUDIT_UNAVAILABLE });
|
if (headErr) return unavailable(headErr.message);
|
||||||
if (!head || head.length === 0) {
|
if (!head || head.length === 0) {
|
||||||
return Object.freeze({ audit_available: true, sport: sp, expected_keys: 0, covered_keys: 0, health: HEALTH.NO_ELIGIBLE_CLAIMS, reason: 'no published rows' });
|
return Object.freeze({
|
||||||
|
audit_available: true, sport: sp, expected_keys: 0, covered_keys: 0,
|
||||||
|
health: HEALTH.NO_ELIGIBLE_CLAIMS, reason: 'no published rows in window',
|
||||||
|
date_window: [from, null],
|
||||||
|
});
|
||||||
}
|
}
|
||||||
const h = head[0];
|
const h = head[0];
|
||||||
|
|
||||||
|
// A COHORT IS A snapshot_id, NOT A DATE SLICE OF ONE. Measured on the
|
||||||
|
// 2026-08-31 03:00Z run: 1,970 rows across TWO game_dates (1,572 on 08-31,
|
||||||
|
// 398 on 08-30 — the late slot spanning midnight ET). Auditing only the head
|
||||||
|
// row's date measured 646 of 801 eligible keys and still said HEALTHY, so a
|
||||||
|
// whole date slice with no lineage would have been invisible. The span is
|
||||||
|
// walked date by date, which also keeps every read on the index.
|
||||||
|
const lo = shiftDate(h.game_date, -2);
|
||||||
|
const hi = shiftDate(h.game_date, 2);
|
||||||
const { paginate } = require('../utils/safePaginate');
|
const { paginate } = require('../utils/safePaginate');
|
||||||
// A COHORT IS A snapshot_id, NOT A DATE SLICE OF ONE.
|
|
||||||
//
|
|
||||||
// Measured on the 2026-08-31 03:00Z run: the cohort held 1,970 rows across
|
|
||||||
// TWO game_dates (1,572 on 08-31 and 398 on 08-30, the late slot spanning
|
|
||||||
// midnight ET). Filtering to the head row's date measured 646 of 801
|
|
||||||
// eligible keys and still reported HEALTHY — so a whole date slice with
|
|
||||||
// zero lineage would have been invisible. The audit covers the cohort.
|
|
||||||
const rows = await paginate(() => supabase
|
const rows = await paginate(() => supabase
|
||||||
.from('model_snapshots')
|
.from('model_snapshots')
|
||||||
.select(AUDIT_COLUMNS)
|
.select(AUDIT_COLUMNS)
|
||||||
|
.eq('sport', sp)
|
||||||
.eq('snapshot_id', h.snapshot_id)
|
.eq('snapshot_id', h.snapshot_id)
|
||||||
.eq('sport', sp), { key: 'id', label: 'lineageCoverage.auditLatestCohort' });
|
.gte('game_date', lo)
|
||||||
|
.lte('game_date', hi), { key: 'id', label: 'lineageCoverage.auditLatestCohort' });
|
||||||
|
|
||||||
|
const dates = [...new Set(rows.map((r) => r.game_date))].sort();
|
||||||
const audit = auditRows(rows, {
|
const audit = auditRows(rows, {
|
||||||
snapshot_id: h.snapshot_id, sport: sp,
|
snapshot_id: h.snapshot_id, sport: sp, game_date: dates.join(','),
|
||||||
game_date: [...new Set(rows.map((r) => r.game_date))].sort().join(','),
|
|
||||||
code_sha: h.code_sha, captured_at: h.captured_at,
|
code_sha: h.code_sha, captured_at: h.captured_at,
|
||||||
retention_terminal: deps.retentionTerminal ?? null,
|
retention_terminal: deps.retentionTerminal ?? null,
|
||||||
});
|
});
|
||||||
const ageMs = deps.now ? (new Date(deps.now()).getTime() - new Date(h.captured_at).getTime()) : null;
|
const ageMs = new Date(nowIso).getTime() - new Date(h.captured_at).getTime();
|
||||||
const staleAfter = deps.staleAfterMs ?? (6 * 60 * 60 * 1000);
|
const staleAfter = deps.staleAfterMs ?? (6 * 60 * 60 * 1000);
|
||||||
if (ageMs !== null && ageMs > staleAfter && audit.health === HEALTH.HEALTHY) {
|
const out = {
|
||||||
return Object.freeze({ ...audit, health: HEALTH.STALE, age_ms: ageMs });
|
...audit,
|
||||||
|
age_ms: Number.isFinite(ageMs) ? ageMs : null,
|
||||||
|
date_window: [lo, hi],
|
||||||
|
game_dates_audited: dates,
|
||||||
|
};
|
||||||
|
if (Number.isFinite(ageMs) && ageMs > staleAfter && audit.health === HEALTH.HEALTHY) {
|
||||||
|
return Object.freeze({ ...out, health: HEALTH.STALE });
|
||||||
}
|
}
|
||||||
return Object.freeze({ ...audit, age_ms: ageMs });
|
return Object.freeze(out);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return Object.freeze({ audit_available: false, reason: e && e.message ? e.message : String(e), sport: sp, health: HEALTH.AUDIT_UNAVAILABLE });
|
return unavailable(e && e.message ? e.message : String(e));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
HEALTH, VALID_ACTION_FIELDS, AUDIT_COLUMNS,
|
HEALTH, VALID_ACTION_FIELDS, AUDIT_COLUMNS,
|
||||||
isValidAction, expectedKeys, coveredKeys, graphDefects, classify,
|
isValidAction, expectedKeys, coveredKeys, graphDefects, classify, shiftDate,
|
||||||
auditRows, auditLatestCohort,
|
auditRows, auditLatestCohort,
|
||||||
};
|
};
|
||||||
|
|||||||
Reference in New Issue
Block a user