Runtime observability: report the build and canary state the system acts on

The rollout stalled at RUNTIME_UNVERIFIED because two facts were answerable only as a side effect of a scheduled snapshot writing a row: which build is running, and whether MLB lineage is effectively enabled. Every state transition therefore waited on cron rather than on asking the service. - src/services/lineageCanaryConfig.js — THE canary resolver. Parsed once at module load (unchanged semantics), normalised sorted/deduped/trimmed, frozen. snapshotService's write gate now delegates to it, and the status probe reads the SAME state. A route that parsed the environment itself would be a second version of the truth, free to drift from the gate it claims to report. - GET /api/internal/snapshot/status gains runtime.code_sha (the production codeSha resolver — never git, never gitea/main; null when unavailable), runtime.started_at (computed ONCE at module load, so it marks a boundary rather than reading as now; deliberately not called deployed_at), and lineage_canary {enabled, sports, configuration_source}. - No raw environment value is returned; sports is the normalised set and configuration_source says only ENVIRONMENT vs DEFAULT. Router-wide requireInternalAuth is unchanged: 200 with key, 401 without. - Effective lineage config is fixed for the process lifetime, so runtime.started_at is a defensible lower bound for how long that state held. Strictly observational — the handler still only reads Redis. Model and decision code byte-identical to 8c6aef1: analyzeViaEngine1, probabilityEstimator, gradeRanking, eventIdentity, gradeSlateService, retentionService, ledgerService, mlbStatsAdapter. Suite 381/5,081/0 from the release worktree; web tsc exit 0. Lineage stays OFF. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CQJeAG8vcDoL5zkiaJyVb8
2026-08-27 17:16:05 -04:00
parent 8c6aef1e12
commit ceaa896f77
4 changed files with 338 additions and 3 deletions
@@ -0,0 +1,210 @@
+'use strict';
+
+/**
+ * RUNTIME OBSERVABILITY — the status probe must report the reality the system
+ * acts on, not its own version of it.
+ *
+ * The rollout stalled at RUNTIME_UNVERIFIED because "which build is running?"
+ * and "is lineage effectively on?" were answerable only as a side effect of a
+ * scheduled snapshot writing a row. These tests lock the two properties that
+ * make the answer trustworthy: ONE build-identity resolver and ONE canary
+ * parser, shared with production.
+ */
+
+const path = require('path');
+const fs = require('fs');
+
+const ROOT = path.resolve(__dirname, '..', '..');
+const ROUTE_RAW = fs.readFileSync(path.join(ROOT, 'src/routes/internal.js'), 'utf8');
+
+/**
+ * Comments are stripped before any forbidden-string scan.
+ *
+ * The header of this route EXPLAINS that it must never use gitea/main and must
+ * never be named `deployed_at` — and a raw scan flagged that prose as the
+ * violation. A guard that cannot tell code from the comment describing it will
+ * eventually be silenced by deleting the explanation, which is the worst
+ * possible fix. Strip, then scan.
+ */
+const stripComments = (src) => src
+  .replace(/\/\*[\s\S]*?\*\//g, '')
+  .replace(/(^|[^:])\/\/.*$/gm, '$1');
+const ROUTE_SRC = stripComments(ROUTE_RAW);
+
+const loadConfig = (val) => {
+  const prev = process.env.LINEAGE_CANARY_SPORTS;
+  if (val === undefined) delete process.env.LINEAGE_CANARY_SPORTS;
+  else process.env.LINEAGE_CANARY_SPORTS = val;
+  jest.resetModules();
+  // eslint-disable-next-line global-require
+  const cfg = require('../../src/services/lineageCanaryConfig');
+  if (prev === undefined) delete process.env.LINEAGE_CANARY_SPORTS;
+  else process.env.LINEAGE_CANARY_SPORTS = prev;
+  return cfg;
+};
+
+describe('RUNTIME SHA — one build identity', () => {
+  test('the probe uses the production codeSha resolver, not git', () => {
+    expect(ROUTE_SRC).toMatch(/const \{ codeSha \} = require\('\.\.\/services\/retentionService'\)/);
+    expect(ROUTE_SRC).toMatch(/code_sha: codeSha\(\)/);
+    // Never repository state.
+    expect(ROUTE_SRC).not.toMatch(/rev-parse|child_process|execSync|gitea|refs\/heads/);
+  });
+
+  test('a known runtime SHA is returned exactly', () => {
+    const prev = process.env.SOURCE_COMMIT;
+    process.env.SOURCE_COMMIT = 'abc123def456';
+    jest.resetModules();
+    // eslint-disable-next-line global-require
+    const { codeSha } = require('../../src/services/retentionService');
+    expect(codeSha()).toBe('abc123def456');
+    if (prev === undefined) delete process.env.SOURCE_COMMIT; else process.env.SOURCE_COMMIT = prev;
+  });
+
+  test('an unavailable runtime SHA is null, never a substitute', () => {
+    const saved = {};
+    for (const k of ['SOURCE_COMMIT', 'GIT_SHA', 'COOLIFY_GIT_COMMIT_SHA']) {
+      saved[k] = process.env[k]; delete process.env[k];
+    }
+    jest.resetModules();
+    // eslint-disable-next-line global-require
+    const { codeSha } = require('../../src/services/retentionService');
+    expect(codeSha()).toBeNull();
+    for (const [k, v] of Object.entries(saved)) if (v !== undefined) process.env[k] = v;
+  });
+});
+
+describe('RUNTIME START — one process lifetime', () => {
+  test('started_at is computed ONCE at module load, not per request', () => {
+    // Recomputing per call would make it read as "now" and destroy its only
+    // use: marking a boundary.
+    expect(ROUTE_SRC).toMatch(/const PROCESS_STARTED_AT = new Date\(Date\.now\(\) - Math\.round\(process\.uptime\(\) \* 1000\)\)\.toISOString\(\)/);
+    expect(ROUTE_SRC).toMatch(/started_at: PROCESS_STARTED_AT/);
+    // Exactly one assignment — no shadowing recompute.
+    expect((ROUTE_SRC.match(/PROCESS_STARTED_AT\s*=/g) || []).length).toBe(1);
+  });
+
+  test('it is NOT named deployed_at — a restart moves it without a deploy', () => {
+    expect(ROUTE_SRC).not.toMatch(/deployed_at/);
+  });
+
+  test('the derived instant is in the past and plausible for this process', () => {
+    const started = Date.now() - Math.round(process.uptime() * 1000);
+    expect(started).toBeLessThanOrEqual(Date.now());
+    expect(Number.isFinite(started)).toBe(true);
+  });
+});
+
+describe('LINEAGE CONFIG — one parser', () => {
+  test('the write gate and the probe consume the SAME module', () => {
+    const snap = fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8');
+    expect(snap).toMatch(/require\('\.\/lineageCanaryConfig'\)/);
+    expect(ROUTE_SRC).toMatch(/require\('\.\.\/services\/lineageCanaryConfig'\)/);
+    // Neither may re-parse the environment itself.
+    expect(ROUTE_SRC).not.toMatch(/process\.env\.LINEAGE_CANARY_SPORTS/);
+    expect(snap).not.toMatch(/process\.env\.LINEAGE_CANARY_SPORTS/);
+  });
+
+  test('the gate delegates to the shared resolver', () => {
+    jest.resetModules();
+    // eslint-disable-next-line global-require
+    const snap = require('../../src/services/snapshotService');
+    // eslint-disable-next-line global-require
+    const cfg = require('../../src/services/lineageCanaryConfig');
+    for (const sp of ['mlb', 'wnba', 'nba', 'soccer']) {
+      expect(snap.lineageCanaryEnabled(sp)).toBe(cfg.isEnabled(sp));
+    }
+    expect(snap.LINEAGE_CANARY_SPORTS).toBe(cfg.SPORTS);
+  });
+
+  test('unset reports disabled + [] + DEFAULT', () => {
+    const c = loadConfig(undefined);
+    expect(c.state()).toEqual({ enabled: false, sports: [], configuration_source: 'DEFAULT' });
+  });
+
+  test('mlb reports enabled + ["mlb"] + ENVIRONMENT', () => {
+    const c = loadConfig('mlb');
+    expect(c.state()).toEqual({ enabled: true, sports: ['mlb'], configuration_source: 'ENVIRONMENT' });
+  });
+
+  test('an EXPLICIT empty is ENVIRONMENT, not DEFAULT', () => {
+    // An operator deliberately blanking the value reads differently from never
+    // having set it, even though both are dark.
+    const c = loadConfig('');
+    expect(c.state()).toEqual({ enabled: false, sports: [], configuration_source: 'ENVIRONMENT' });
+  });
+
+  test('multiple sports normalise deterministically — sorted, deduped, trimmed, lowercased', () => {
+    expect(loadConfig('MLB, mlb ,wnba').state().sports).toEqual(['mlb', 'wnba']);
+    expect(loadConfig('wnba,mlb').state().sports).toEqual(['mlb', 'wnba']);
+    expect(loadConfig('  MLB  ').state().sports).toEqual(['mlb']);
+    expect(loadConfig(',,mlb,,').state().sports).toEqual(['mlb']);
+  });
+
+  test('an unsupported value behaves exactly as the gate behaves', () => {
+    const c = loadConfig('cricket');
+    expect(c.state().sports).toEqual(['cricket']);
+    expect(c.isEnabled('cricket')).toBe(true);   // the flag is scope, not validation
+    expect(c.isEnabled('mlb')).toBe(false);
+  });
+
+  test('the source-code default cannot hide an environment override', () => {
+    // The whole point: if Coolify sets mlb, the probe must say mlb.
+    expect(loadConfig('mlb').state().enabled).toBe(true);
+    expect(loadConfig(undefined).state().enabled).toBe(false);
+  });
+});
+
+describe('SECURITY — no raw config, no weakened access', () => {
+  test('the raw environment value is never returned', () => {
+    const c = loadConfig('MLB, wnba ');
+    const payload = JSON.stringify(c.state());
+    expect(payload).not.toContain('MLB, wnba');
+    expect(payload).not.toContain('  ');
+  });
+
+  test('the response exposes no secret-like config', () => {
+    const forbidden = [
+      'VYNDR_INTERNAL_KEY', 'SUPABASE_SERVICE', 'STRIPE', 'REDIS_URL',
+      'PROPLINE_API_KEY', 'ODDS_API_KEY', 'SUPABASE_DB_PASSWORD',
+    ];
+    const handler = ROUTE_SRC.slice(ROUTE_SRC.indexOf("router.get('/snapshot/status'"));
+    const body = handler.slice(0, handler.indexOf('router.'));
+    for (const f of forbidden) expect(body).not.toContain(f);
+  });
+
+  test('router-wide internal auth is unchanged', () => {
+    expect(ROUTE_SRC).toMatch(/router\.use\(requireInternalAuth\(\{ loopbackOnly: false \}\)\)/);
+    // The status route must not opt itself out.
+    expect(ROUTE_SRC).not.toMatch(/\/snapshot\/status'[^)]*skipAuth/);
+  });
+});
+
+describe('READ ONLY — the probe observes and nothing else', () => {
+  test('the handler performs no writes of any kind', () => {
+    const start = ROUTE_SRC.indexOf("router.get('/snapshot/status'");
+    const body = ROUTE_SRC.slice(start, ROUTE_SRC.indexOf('\nrouter.', start + 10));
+    for (const verb of ['cacheSet', '.insert(', '.upsert(', '.update(', '.delete(', 'runSnapshot', 'commitPublication', 'attachLineage']) {
+      expect(body).not.toContain(verb);
+    }
+    // Reading Redis is the only side-effect-free access it needs.
+    expect(body).toContain('cacheGet');
+  });
+
+  test('it cannot enable lineage — it only reads the frozen state', () => {
+    const c = loadConfig(undefined);
+    const before = JSON.stringify(c.state());
+    c.state(); c.state();
+    expect(JSON.stringify(c.state())).toBe(before);
+    // No setter exists.
+    expect(Object.keys(c).filter((k) => /^(set|enable|disable|update)/i.test(k))).toHaveLength(0);
+  });
+
+  test('the reported state is a copy — a caller cannot mutate the gate', () => {
+    const c = loadConfig('mlb');
+    const s1 = c.state();
+    s1.sports.push('wnba');
+    expect(c.state().sports).toEqual(['mlb']);
+    expect(c.isEnabled('wnba')).toBe(false);
+  });
+});