Runtime observability: report the build and canary state the system acts on

The rollout stalled at RUNTIME_UNVERIFIED because two facts were answerable only
as a side effect of a scheduled snapshot writing a row: which build is running,
and whether MLB lineage is effectively enabled. Every state transition therefore
waited on cron rather than on asking the service.

- src/services/lineageCanaryConfig.js — THE canary resolver. Parsed once at
  module load (unchanged semantics), normalised sorted/deduped/trimmed, frozen.
  snapshotService's write gate now delegates to it, and the status probe reads
  the SAME state. A route that parsed the environment itself would be a second
  version of the truth, free to drift from the gate it claims to report.
- GET /api/internal/snapshot/status gains runtime.code_sha (the production
  codeSha resolver — never git, never gitea/main; null when unavailable),
  runtime.started_at (computed ONCE at module load, so it marks a boundary
  rather than reading as now; deliberately not called deployed_at), and
  lineage_canary {enabled, sports, configuration_source}.
- No raw environment value is returned; sports is the normalised set and
  configuration_source says only ENVIRONMENT vs DEFAULT. Router-wide
  requireInternalAuth is unchanged: 200 with key, 401 without.
- Effective lineage config is fixed for the process lifetime, so
  runtime.started_at is a defensible lower bound for how long that state held.

Strictly observational — the handler still only reads Redis.

Model and decision code byte-identical to 8c6aef1: analyzeViaEngine1,
probabilityEstimator, gradeRanking, eventIdentity, gradeSlateService,
retentionService, ledgerService, mlbStatsAdapter.

Suite 381/5,081/0 from the release worktree; web tsc exit 0. Lineage stays OFF.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CQJeAG8vcDoL5zkiaJyVb8

src/routes/internal.js

@@ -16,6 +16,13 @@
 
 const express = require('express');
 const { requireInternalAuth } = require('../middleware/internalAuth');
+
+/**
+ * The start of THIS Node process, computed ONCE so every call reports the same
+ * instant for one process lifetime. Deriving it per request from `Date.now()`
+ * would make it read as "now" and destroy its only use — marking a boundary.
+ */
+const PROCESS_STARTED_AT = new Date(Date.now() - Math.round(process.uptime() * 1000)).toISOString();
 const tank01Prefetch = require('../../scripts/tank01-prefetch');
 const quotaTracker = require('../services/quotaTracker');
 
@@ -133,6 +140,28 @@ router.post('/snapshot/all', async (req, res) => {
  * whether the in-process cron is armed, the freshest snapshot per sport, which
  * pipeline Redis keys exist, and the ticker item count. Read-only; safe to poll.
  * GET vs the POST /snapshot/:sport below — no route collision.
+ *
+ * -- RUNTIME IDENTITY (added for the MLB rollout) -------------------------
+ * The rollout stalled because nothing could answer "which build is running?"
+ * or "is lineage effectively on?" except as a side effect of a scheduled
+ * snapshot writing a row — so every state transition waited on cron.
+ *
+ * Two read-only fields close that:
+ *
+ *   runtime.code_sha        the SAME resolver production provenance uses
+ *                           (`retentionService.codeSha`). NEVER git HEAD, never
+ *                           gitea/main, never deployment intent. Unavailable
+ *                           returns null rather than a guess.
+ *   runtime.started_at      the start of THIS process. Deliberately not named
+ *                           `deployed_at` — a restart without a deploy moves it.
+ *   lineage_canary          the SAME frozen state the write gate consults
+ *                           (`lineageCanaryConfig`), never a second parse.
+ *
+ * No raw environment value is returned: `lineage_canary.sports` is the
+ * normalised set, and `configuration_source` says only whether the value came
+ * from the environment or the default.
+ *
+ * Still strictly observational — the handler only reads Redis.
  */
 router.get('/snapshot/status', async (req, res) => {
   const { cacheGet } = require('../utils/redis');
@@ -159,9 +188,22 @@ router.get('/snapshot/status', async (req, res) => {
     }
     const ticker = await cacheGet('ticker:items');
     redis_keys['ticker:items'] = !!ticker;
+    // Same helpers the pipeline itself uses — one build identity, one canary parser.
+    const { codeSha } = require('../services/retentionService');
+    const lineageCanary = require('../services/lineageCanaryConfig');
     // Session 56 — surface the missed-cron signal in the health probe.
     const mlbTs = last_snapshot.mlb && last_snapshot.mlb.updated_at;
     return res.json({
+      runtime: {
+        code_sha: codeSha(),
+        // Computed once at module load from process.uptime(), so repeated calls
+        // report one stable value for one process lifetime.
+        started_at: PROCESS_STARTED_AT,
+      },
+      // The effective lineage state is resolved once at module load and cannot
+      // change without a new process, so `runtime.started_at` is a defensible
+      // lower bound for how long this state has held.
+      lineage_canary: lineageCanary.state(),
       cron_armed: process.env.SNAPSHOT_CRON === '1',
       cron_hours_utc: HOURS_UTC,
       last_snapshot,

src/services/lineageCanaryConfig.js

@@ -0,0 +1,79 @@
+'use strict';
+
+/**
+ * LINEAGE CANARY CONFIG — the ONE resolver.
+ *
+ * -- WHY THIS MODULE EXISTS ----------------------------------------------
+ * The rollout stalled at RUNTIME_UNVERIFIED because nothing could answer
+ * "is MLB lineage effectively enabled?" without waiting for a scheduled
+ * snapshot to write a row. Exposing the answer requires a status route to read
+ * the config — and a status route that parses the environment ITSELF would be a
+ * second version of the truth, free to drift from the gate it claims to report.
+ *
+ * So the parse lives here, once. `snapshotService` consumes it for the actual
+ * lineage write gate, and the internal status probe consumes the SAME frozen
+ * state. Observability reports the reality the system acts on, or it is not
+ * observability.
+ *
+ * -- EFFECTIVE-TIME SEMANTICS --------------------------------------------
+ * The value is resolved ONCE, at module load, exactly as the previous inline
+ * constant was. It is therefore FIXED FOR THE LIFETIME OF THE PROCESS:
+ *
+ *   read from process.env            yes
+ *   parsed once at module startup    yes
+ *   re-read per lineage decision     no
+ *   can change without a restart     no
+ *
+ * That is what makes the process start time a defensible lower bound for
+ * "this runtime has had this effective lineage state since at least then".
+ * Changing the variable in Coolify restarts the container, which yields a new
+ * process and therefore a new boundary.
+ */
+
+/** Where the effective value came from. Never the value itself. */
+const CONFIGURATION_SOURCE = Object.freeze({
+  ENVIRONMENT: 'ENVIRONMENT',
+  DEFAULT: 'DEFAULT',
+});
+
+const RAW = process.env.LINEAGE_CANARY_SPORTS;
+
+/**
+ * Normalised, deterministic, frozen. Sorted so two processes with the same
+ * effective configuration report byte-identical state regardless of the order
+ * it was written in.
+ */
+const SPORTS = Object.freeze(
+  String(RAW || '')
+    .split(',')
+    .map((x) => x.trim().toLowerCase())
+    .filter(Boolean)
+    .filter((x, i, a) => a.indexOf(x) === i)
+    .sort(),
+);
+
+/**
+ * DEFAULT means the variable was never set — the dark default. ENVIRONMENT
+ * means something set it, INCLUDING setting it to empty, because an explicit
+ * empty is an operator decision and reads differently from an absent one.
+ */
+const SOURCE = RAW === undefined ? CONFIGURATION_SOURCE.DEFAULT : CONFIGURATION_SOURCE.ENVIRONMENT;
+
+/** THE gate. Lineage writes for a sport only if this says so. */
+function isEnabled(sport) {
+  return SPORTS.includes(String(sport || '').toLowerCase());
+}
+
+/**
+ * The reportable state. Contains no raw environment value by construction —
+ * `RAW` is never returned, only the normalised set derived from it.
+ */
+function state() {
+  return {
+    enabled: SPORTS.length > 0,
+    sports: [...SPORTS],
+    configuration_source: SOURCE,
+  };
+}
+
+module.exports = { SPORTS, isEnabled, state, CONFIGURATION_SOURCE };

src/services/snapshotService.js

@@ -311,11 +311,15 @@ const CALIBRATION_DEPLOYED = Object.freeze([]);
 // The flag governs the observer only. It does NOT control canonical event
 // identity, the impossible-binding refusal, event-aware dedupe, or
 // publication-commit ordering — those are integrity repairs and ship active.
-const LINEAGE_CANARY_SPORTS = String(process.env.LINEAGE_CANARY_SPORTS || '')
+// ONE resolver, shared with the internal status probe. A route that parsed the
-  .split(',').map((x) => x.trim().toLowerCase()).filter(Boolean);
+// environment itself would be a second version of the truth, free to drift from
+// the gate it claims to report.
+const lineageCanaryConfig = require('./lineageCanaryConfig');
+
+const LINEAGE_CANARY_SPORTS = lineageCanaryConfig.SPORTS;
 
 function lineageCanaryEnabled(sport) {
-  return LINEAGE_CANARY_SPORTS.includes(String(sport || '').toLowerCase());
+  return lineageCanaryConfig.isEnabled(sport);
 }
 
 /**

tests/unit/runtimeObservability.test.js

@@ -0,0 +1,210 @@
+'use strict';
+
+/**
+ * RUNTIME OBSERVABILITY — the status probe must report the reality the system
+ * acts on, not its own version of it.
+ *
+ * The rollout stalled at RUNTIME_UNVERIFIED because "which build is running?"
+ * and "is lineage effectively on?" were answerable only as a side effect of a
+ * scheduled snapshot writing a row. These tests lock the two properties that
+ * make the answer trustworthy: ONE build-identity resolver and ONE canary
+ * parser, shared with production.
+ */
+
+const path = require('path');
+const fs = require('fs');
+
+const ROOT = path.resolve(__dirname, '..', '..');
+const ROUTE_RAW = fs.readFileSync(path.join(ROOT, 'src/routes/internal.js'), 'utf8');
+
+/**
+ * Comments are stripped before any forbidden-string scan.
+ *
+ * The header of this route EXPLAINS that it must never use gitea/main and must
+ * never be named `deployed_at` — and a raw scan flagged that prose as the
+ * violation. A guard that cannot tell code from the comment describing it will
+ * eventually be silenced by deleting the explanation, which is the worst
+ * possible fix. Strip, then scan.
+ */
+const stripComments = (src) => src
+  .replace(/\/\*[\s\S]*?\*\//g, '')
+  .replace(/(^|[^:])\/\/.*$/gm, '$1');
+const ROUTE_SRC = stripComments(ROUTE_RAW);
+
+const loadConfig = (val) => {
+  const prev = process.env.LINEAGE_CANARY_SPORTS;
+  if (val === undefined) delete process.env.LINEAGE_CANARY_SPORTS;
+  else process.env.LINEAGE_CANARY_SPORTS = val;
+  jest.resetModules();
+  // eslint-disable-next-line global-require
+  const cfg = require('../../src/services/lineageCanaryConfig');
+  if (prev === undefined) delete process.env.LINEAGE_CANARY_SPORTS;
+  else process.env.LINEAGE_CANARY_SPORTS = prev;
+  return cfg;
+};
+
+describe('RUNTIME SHA — one build identity', () => {
+  test('the probe uses the production codeSha resolver, not git', () => {
+    expect(ROUTE_SRC).toMatch(/const \{ codeSha \} = require\('\.\.\/services\/retentionService'\)/);
+    expect(ROUTE_SRC).toMatch(/code_sha: codeSha\(\)/);
+    // Never repository state.
+    expect(ROUTE_SRC).not.toMatch(/rev-parse|child_process|execSync|gitea|refs\/heads/);
+  });
+
+  test('a known runtime SHA is returned exactly', () => {
+    const prev = process.env.SOURCE_COMMIT;
+    process.env.SOURCE_COMMIT = 'abc123def456';
+    jest.resetModules();
+    // eslint-disable-next-line global-require
+    const { codeSha } = require('../../src/services/retentionService');
+    expect(codeSha()).toBe('abc123def456');
+    if (prev === undefined) delete process.env.SOURCE_COMMIT; else process.env.SOURCE_COMMIT = prev;
+  });
+
+  test('an unavailable runtime SHA is null, never a substitute', () => {
+    const saved = {};
+    for (const k of ['SOURCE_COMMIT', 'GIT_SHA', 'COOLIFY_GIT_COMMIT_SHA']) {
+      saved[k] = process.env[k]; delete process.env[k];
+    }
+    jest.resetModules();
+    // eslint-disable-next-line global-require
+    const { codeSha } = require('../../src/services/retentionService');
+    expect(codeSha()).toBeNull();
+    for (const [k, v] of Object.entries(saved)) if (v !== undefined) process.env[k] = v;
+  });
+});
+
+describe('RUNTIME START — one process lifetime', () => {
+  test('started_at is computed ONCE at module load, not per request', () => {
+    // Recomputing per call would make it read as "now" and destroy its only
+    // use: marking a boundary.
+    expect(ROUTE_SRC).toMatch(/const PROCESS_STARTED_AT = new Date\(Date\.now\(\) - Math\.round\(process\.uptime\(\) \* 1000\)\)\.toISOString\(\)/);
+    expect(ROUTE_SRC).toMatch(/started_at: PROCESS_STARTED_AT/);
+    // Exactly one assignment — no shadowing recompute.
+    expect((ROUTE_SRC.match(/PROCESS_STARTED_AT\s*=/g) || []).length).toBe(1);
+  });
+
+  test('it is NOT named deployed_at — a restart moves it without a deploy', () => {
+    expect(ROUTE_SRC).not.toMatch(/deployed_at/);
+  });
+
+  test('the derived instant is in the past and plausible for this process', () => {
+    const started = Date.now() - Math.round(process.uptime() * 1000);
+    expect(started).toBeLessThanOrEqual(Date.now());
+    expect(Number.isFinite(started)).toBe(true);
+  });
+});
+
+describe('LINEAGE CONFIG — one parser', () => {
+  test('the write gate and the probe consume the SAME module', () => {
+    const snap = fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8');
+    expect(snap).toMatch(/require\('\.\/lineageCanaryConfig'\)/);
+    expect(ROUTE_SRC).toMatch(/require\('\.\.\/services\/lineageCanaryConfig'\)/);
+    // Neither may re-parse the environment itself.
+    expect(ROUTE_SRC).not.toMatch(/process\.env\.LINEAGE_CANARY_SPORTS/);
+    expect(snap).not.toMatch(/process\.env\.LINEAGE_CANARY_SPORTS/);
+  });
+
+  test('the gate delegates to the shared resolver', () => {
+    jest.resetModules();
+    // eslint-disable-next-line global-require
+    const snap = require('../../src/services/snapshotService');
+    // eslint-disable-next-line global-require
+    const cfg = require('../../src/services/lineageCanaryConfig');
+    for (const sp of ['mlb', 'wnba', 'nba', 'soccer']) {
+      expect(snap.lineageCanaryEnabled(sp)).toBe(cfg.isEnabled(sp));
+    }
+    expect(snap.LINEAGE_CANARY_SPORTS).toBe(cfg.SPORTS);
+  });
+
+  test('unset reports disabled + [] + DEFAULT', () => {
+    const c = loadConfig(undefined);
+    expect(c.state()).toEqual({ enabled: false, sports: [], configuration_source: 'DEFAULT' });
+  });
+
+  test('mlb reports enabled + ["mlb"] + ENVIRONMENT', () => {
+    const c = loadConfig('mlb');
+    expect(c.state()).toEqual({ enabled: true, sports: ['mlb'], configuration_source: 'ENVIRONMENT' });
+  });
+
+  test('an EXPLICIT empty is ENVIRONMENT, not DEFAULT', () => {
+    // An operator deliberately blanking the value reads differently from never
+    // having set it, even though both are dark.
+    const c = loadConfig('');
+    expect(c.state()).toEqual({ enabled: false, sports: [], configuration_source: 'ENVIRONMENT' });
+  });
+
+  test('multiple sports normalise deterministically — sorted, deduped, trimmed, lowercased', () => {
+    expect(loadConfig('MLB, mlb ,wnba').state().sports).toEqual(['mlb', 'wnba']);
+    expect(loadConfig('wnba,mlb').state().sports).toEqual(['mlb', 'wnba']);
+    expect(loadConfig('  MLB  ').state().sports).toEqual(['mlb']);
+    expect(loadConfig(',,mlb,,').state().sports).toEqual(['mlb']);
+  });
+
+  test('an unsupported value behaves exactly as the gate behaves', () => {
+    const c = loadConfig('cricket');
+    expect(c.state().sports).toEqual(['cricket']);
+    expect(c.isEnabled('cricket')).toBe(true);   // the flag is scope, not validation
+    expect(c.isEnabled('mlb')).toBe(false);
+  });
+
+  test('the source-code default cannot hide an environment override', () => {
+    // The whole point: if Coolify sets mlb, the probe must say mlb.
+    expect(loadConfig('mlb').state().enabled).toBe(true);
+    expect(loadConfig(undefined).state().enabled).toBe(false);
+  });
+});
+
+describe('SECURITY — no raw config, no weakened access', () => {
+  test('the raw environment value is never returned', () => {
+    const c = loadConfig('MLB, wnba ');
+    const payload = JSON.stringify(c.state());
+    expect(payload).not.toContain('MLB, wnba');
+    expect(payload).not.toContain('  ');
+  });
+
+  test('the response exposes no secret-like config', () => {
+    const forbidden = [
+      'VYNDR_INTERNAL_KEY', 'SUPABASE_SERVICE', 'STRIPE', 'REDIS_URL',
+      'PROPLINE_API_KEY', 'ODDS_API_KEY', 'SUPABASE_DB_PASSWORD',
+    ];
+    const handler = ROUTE_SRC.slice(ROUTE_SRC.indexOf("router.get('/snapshot/status'"));
+    const body = handler.slice(0, handler.indexOf('router.'));
+    for (const f of forbidden) expect(body).not.toContain(f);
+  });
+
+  test('router-wide internal auth is unchanged', () => {
+    expect(ROUTE_SRC).toMatch(/router\.use\(requireInternalAuth\(\{ loopbackOnly: false \}\)\)/);
+    // The status route must not opt itself out.
+    expect(ROUTE_SRC).not.toMatch(/\/snapshot\/status'[^)]*skipAuth/);
+  });
+});
+
+describe('READ ONLY — the probe observes and nothing else', () => {
+  test('the handler performs no writes of any kind', () => {
+    const start = ROUTE_SRC.indexOf("router.get('/snapshot/status'");
+    const body = ROUTE_SRC.slice(start, ROUTE_SRC.indexOf('\nrouter.', start + 10));
+    for (const verb of ['cacheSet', '.insert(', '.upsert(', '.update(', '.delete(', 'runSnapshot', 'commitPublication', 'attachLineage']) {
+      expect(body).not.toContain(verb);
+    }
+    // Reading Redis is the only side-effect-free access it needs.
+    expect(body).toContain('cacheGet');
+  });
+
+  test('it cannot enable lineage — it only reads the frozen state', () => {
+    const c = loadConfig(undefined);
+    const before = JSON.stringify(c.state());
+    c.state(); c.state();
+    expect(JSON.stringify(c.state())).toBe(before);
+    // No setter exists.
+    expect(Object.keys(c).filter((k) => /^(set|enable|disable|update)/i.test(k))).toHaveLength(0);
+  });
+
+  test('the reported state is a copy — a caller cannot mutate the gate', () => {
+    const c = loadConfig('mlb');
+    const s1 = c.state();
+    s1.sports.push('wnba');
+    expect(c.state().sports).toEqual(['mlb']);
+    expect(c.isEnabled('wnba')).toBe(false);
+  });
+});