Files
vyndr/tests/unit/runtimeObservability.test.js
builtbykev cc5bdf5797 A canary is temporary: bound activation with a fail-closed lease
`LINEAGE_CANARY_SPORTS=mlb` was parsed once at module load and frozen, so an
enabled canary stayed writable for the whole process lifetime. On 2026-08-29 it
was left set and the 22:00Z, 01:00Z and 03:00Z scheduled snapshots each wrote
lineage overnight with nobody watching. That history happened to be correct.
Lineage is append-only evidence, so a defective canary would have written
irreversible WRONG evidence exactly as quietly. Correct history was luck, not a
safety property.

NEW CONTRACT:  LINEAGE_CANARY_SPORTS=mlb@2026-08-30T18:00:00Z

Absolute UTC instant only -- no duration, no local timezone. A relative "4h"
would silently restart on every redeploy, which is the defect being removed.

LEGACY `mlb` NO LONGER ACTIVATES ANYTHING. It is INVALID_MISSING_EXPIRY. Leaving
it working would have left the defect in place behind a nicer-looking
alternative, so this is the load-bearing half of the repair.

EXPIRY IS EVALUATED AT THE WRITE GATE, not at startup. `isEnabled(sport, now)`
re-reads the clock on every call and `lineageCanaryEnabled` threads it through,
so a lease turns itself off with no operator, no restart, no Redis and no
network. A design where an expired canary keeps writing until someone restarts
is the same failure in a different hat.

MAX_LEASE is FOUR HOURS, and the bound is proven rather than chosen. MLB ticks
are [14,19,22,1,3] UTC; exhaustively over a minute grid across UTC date
boundaries, the shortest span enclosing THREE consecutive ticks is
22:00 -> 01:00 -> 03:00 = five hours. Four hours encloses at most two, with an
hour of margin. (An earlier note claimed six hours admitted two. It admits
three; that claim was false and the test now pins the arithmetic.) The bound is
a function of the scheduler, so a test reads the hours from sportCadence and
fails if a cadence change invalidates the proof.

Everything fails closed: absent, empty, bare sport, comma list, two leases,
duplicate @, non-leasable sport, missing timezone, numeric offset, malformed,
over-long, already-expired, expiry-equals-now, and an unreadable clock.
Configured-but-EXPIRED stays distinguishable from never-configured so automatic
containment is auditable.

No Redis, no Supabase, no counter, no network in the lease path -- an
unreachable dependency must never decide whether lineage may write.

Lineage algorithms, cache-date, participant and retention identity untouched.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CQJeAG8vcDoL5zkiaJyVb8
2026-08-30 14:58:11 -04:00

250 lines
11 KiB
JavaScript

'use strict';
/**
* RUNTIME OBSERVABILITY — the status probe must report the reality the system
* acts on, not its own version of it.
*
* The rollout stalled at RUNTIME_UNVERIFIED because "which build is running?"
* and "is lineage effectively on?" were answerable only as a side effect of a
* scheduled snapshot writing a row. These tests lock the two properties that
* make the answer trustworthy: ONE build-identity resolver and ONE canary
* parser, shared with production.
*/
const path = require('path');
const fs = require('fs');
const ROOT = path.resolve(__dirname, '..', '..');
const ROUTE_RAW = fs.readFileSync(path.join(ROOT, 'src/routes/internal.js'), 'utf8');
/**
* Comments are stripped before any forbidden-string scan.
*
* The header of this route EXPLAINS that it must never use gitea/main and must
* never be named `deployed_at` — and a raw scan flagged that prose as the
* violation. A guard that cannot tell code from the comment describing it will
* eventually be silenced by deleting the explanation, which is the worst
* possible fix. Strip, then scan.
*/
const stripComments = (src) => src
.replace(/\/\*[\s\S]*?\*\//g, '')
.replace(/(^|[^:])\/\/.*$/gm, '$1');
const ROUTE_SRC = stripComments(ROUTE_RAW);
const loadConfig = (val) => {
const prev = process.env.LINEAGE_CANARY_SPORTS;
if (val === undefined) delete process.env.LINEAGE_CANARY_SPORTS;
else process.env.LINEAGE_CANARY_SPORTS = val;
jest.resetModules();
// eslint-disable-next-line global-require
const cfg = require('../../src/services/lineageCanaryConfig');
if (prev === undefined) delete process.env.LINEAGE_CANARY_SPORTS;
else process.env.LINEAGE_CANARY_SPORTS = prev;
return cfg;
};
describe('RUNTIME SHA — one build identity', () => {
test('the probe uses the production codeSha resolver, not git', () => {
// The destructure now also pulls `lastRetention` (terminal retention
// status), so match the resolver rather than the exact import list.
expect(ROUTE_SRC).toMatch(/const \{ codeSha[^}]*\} = require\('\.\.\/services\/retentionService'\)/);
expect(ROUTE_SRC).toMatch(/code_sha: codeSha\(\)/);
// Never repository state.
expect(ROUTE_SRC).not.toMatch(/rev-parse|child_process|execSync|gitea|refs\/heads/);
});
test('a known runtime SHA is returned exactly', () => {
const prev = process.env.SOURCE_COMMIT;
process.env.SOURCE_COMMIT = 'abc123def456';
jest.resetModules();
// eslint-disable-next-line global-require
const { codeSha } = require('../../src/services/retentionService');
expect(codeSha()).toBe('abc123def456');
if (prev === undefined) delete process.env.SOURCE_COMMIT; else process.env.SOURCE_COMMIT = prev;
});
test('an unavailable runtime SHA is null, never a substitute', () => {
const saved = {};
for (const k of ['SOURCE_COMMIT', 'GIT_SHA', 'COOLIFY_GIT_COMMIT_SHA']) {
saved[k] = process.env[k]; delete process.env[k];
}
jest.resetModules();
// eslint-disable-next-line global-require
const { codeSha } = require('../../src/services/retentionService');
expect(codeSha()).toBeNull();
for (const [k, v] of Object.entries(saved)) if (v !== undefined) process.env[k] = v;
});
});
describe('RUNTIME START — one process lifetime', () => {
test('started_at is computed ONCE at module load, not per request', () => {
// Recomputing per call would make it read as "now" and destroy its only
// use: marking a boundary.
expect(ROUTE_SRC).toMatch(/const PROCESS_STARTED_AT = new Date\(Date\.now\(\) - Math\.round\(process\.uptime\(\) \* 1000\)\)\.toISOString\(\)/);
expect(ROUTE_SRC).toMatch(/started_at: PROCESS_STARTED_AT/);
// Exactly one assignment — no shadowing recompute.
expect((ROUTE_SRC.match(/PROCESS_STARTED_AT\s*=/g) || []).length).toBe(1);
});
test('it is NOT named deployed_at — a restart moves it without a deploy', () => {
expect(ROUTE_SRC).not.toMatch(/deployed_at/);
});
test('the derived instant is in the past and plausible for this process', () => {
const started = Date.now() - Math.round(process.uptime() * 1000);
expect(started).toBeLessThanOrEqual(Date.now());
expect(Number.isFinite(started)).toBe(true);
});
});
describe('LINEAGE CONFIG — one parser', () => {
test('the write gate and the probe consume the SAME module', () => {
const snap = fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8');
expect(snap).toMatch(/require\('\.\/lineageCanaryConfig'\)/);
expect(ROUTE_SRC).toMatch(/require\('\.\.\/services\/lineageCanaryConfig'\)/);
// Neither may re-parse the environment itself.
expect(ROUTE_SRC).not.toMatch(/process\.env\.LINEAGE_CANARY_SPORTS/);
expect(snap).not.toMatch(/process\.env\.LINEAGE_CANARY_SPORTS/);
});
test('the gate delegates to the shared resolver', () => {
jest.resetModules();
// eslint-disable-next-line global-require
const snap = require('../../src/services/snapshotService');
// eslint-disable-next-line global-require
const cfg = require('../../src/services/lineageCanaryConfig');
const now = new Date('2026-08-30T12:00:00Z');
for (const sp of ['mlb', 'wnba', 'nba', 'soccer']) {
expect(snap.lineageCanaryEnabled(sp, now)).toBe(cfg.isEnabled(sp, now));
}
expect(snap.LINEAGE_CANARY_SPORTS).toBe(cfg.SPORTS);
});
test('unset reports OFF + [] + DEFAULT', () => {
const c = loadConfig(undefined);
const st = c.state(new Date('2026-08-30T12:00:00Z'));
expect(st.enabled).toBe(false);
expect(st.sports).toEqual([]);
expect(st.configuration_source).toBe('DEFAULT');
expect(st.lease_state).toBe('OFF');
expect(st.configured).toBe(false);
});
test('LEGACY plain `mlb` no longer activates anything', () => {
// This is the safety repair, expressed as a test. The bare sport form used
// to mean "write forever until somebody remembers"; it is now refused.
const c = loadConfig('mlb');
const st = c.state(new Date('2026-08-30T12:00:00Z'));
expect(st.enabled).toBe(false);
expect(st.effective_enabled).toBe(false);
expect(st.sports).toEqual([]);
expect(st.lease_state).toBe('INVALID');
expect(st.invalid_reason).toBe('INVALID_MISSING_EXPIRY');
expect(c.isEnabled('mlb', new Date('2026-08-30T12:00:00Z'))).toBe(false);
});
test('a bounded lease reports ACTIVE + ["mlb"] + ENVIRONMENT', () => {
const c = loadConfig('mlb@2026-08-30T13:00:00Z');
const st = c.state(new Date('2026-08-30T12:00:00Z'));
expect(st.enabled).toBe(true);
expect(st.sports).toEqual(['mlb']);
expect(st.configuration_source).toBe('ENVIRONMENT');
expect(st.lease_state).toBe('ACTIVE');
expect(st.configured_expires_at).toBe('2026-08-30T13:00:00.000Z');
});
test('an EXPLICIT empty is ENVIRONMENT, not DEFAULT', () => {
// An operator deliberately blanking the value reads differently from never
// having set it, even though both are dark.
const c = loadConfig('');
const st = c.state(new Date('2026-08-30T12:00:00Z'));
expect(st.enabled).toBe(false);
expect(st.sports).toEqual([]);
expect(st.configuration_source).toBe('ENVIRONMENT');
expect(st.lease_state).toBe('OFF');
});
test('a comma list can no longer widen the canary', () => {
for (const v of ['MLB, mlb ,wnba', 'wnba,mlb', 'mlb@2026-08-30T13:00:00Z,nba@2026-08-30T13:00:00Z']) {
const st = loadConfig(v).state(new Date('2026-08-30T12:00:00Z'));
expect(st.effective_enabled).toBe(false);
expect(st.sports).toEqual([]);
expect(st.lease_state).toBe('INVALID');
}
});
test('a sport outside the leasable set cannot be leased', () => {
const c = loadConfig('cricket@2026-08-30T13:00:00Z');
const st = c.state(new Date('2026-08-30T12:00:00Z'));
expect(st.lease_state).toBe('INVALID');
expect(st.invalid_reason).toBe('INVALID_SPORT_NOT_LEASABLE');
expect(c.isEnabled('cricket', new Date('2026-08-30T12:00:00Z'))).toBe(false);
expect(c.isEnabled('mlb', new Date('2026-08-30T12:00:00Z'))).toBe(false);
});
test('the source-code default cannot hide an environment override', () => {
const now = new Date('2026-08-30T12:00:00Z');
expect(loadConfig('mlb@2026-08-30T13:00:00Z').state(now).enabled).toBe(true);
expect(loadConfig(undefined).state(now).enabled).toBe(false);
});
});
describe('SECURITY — no raw config, no weakened access', () => {
test('the raw environment value is never returned', () => {
const c = loadConfig('MLB, wnba ');
const payload = JSON.stringify(c.state(new Date('2026-08-30T12:00:00Z')));
expect(payload).not.toContain('MLB, wnba');
expect(payload).not.toContain(' ');
});
test('the response exposes no secret-like config', () => {
const forbidden = [
'VYNDR_INTERNAL_KEY', 'SUPABASE_SERVICE', 'STRIPE', 'REDIS_URL',
'PROPLINE_API_KEY', 'ODDS_API_KEY', 'SUPABASE_DB_PASSWORD',
];
const handler = ROUTE_SRC.slice(ROUTE_SRC.indexOf("router.get('/snapshot/status'"));
const body = handler.slice(0, handler.indexOf('router.'));
for (const f of forbidden) expect(body).not.toContain(f);
});
test('router-wide internal auth is unchanged', () => {
expect(ROUTE_SRC).toMatch(/router\.use\(requireInternalAuth\(\{ loopbackOnly: false \}\)\)/);
// The status route must not opt itself out.
expect(ROUTE_SRC).not.toMatch(/\/snapshot\/status'[^)]*skipAuth/);
});
});
describe('READ ONLY — the probe observes and nothing else', () => {
test('the handler performs no writes of any kind', () => {
const start = ROUTE_SRC.indexOf("router.get('/snapshot/status'");
const body = ROUTE_SRC.slice(start, ROUTE_SRC.indexOf('\nrouter.', start + 10));
for (const verb of ['cacheSet', '.insert(', '.upsert(', '.update(', '.delete(', 'runSnapshot', 'commitPublication', 'attachLineage']) {
expect(body).not.toContain(verb);
}
// Reading Redis is the only side-effect-free access it needs.
expect(body).toContain('cacheGet');
});
test('it cannot enable lineage — it only reads the frozen state', () => {
const c = loadConfig(undefined);
const fixed = new Date('2026-08-30T12:00:00Z');
const before = JSON.stringify(c.state(fixed));
c.state(fixed); c.state(fixed);
expect(JSON.stringify(c.state(fixed))).toBe(before);
// No setter exists.
expect(Object.keys(c).filter((k) => /^(set|enable|disable|update)/i.test(k))).toHaveLength(0);
});
test('the reported state is a copy — a caller cannot mutate the gate', () => {
const c = loadConfig('mlb@2026-08-30T13:00:00Z');
const now = new Date('2026-08-30T12:00:00Z');
const s1 = c.state(now);
s1.sports.push('wnba');
s1.effective_active_sports.push('nba');
expect(c.state(now).sports).toEqual(['mlb']);
expect(c.state(now).effective_active_sports).toEqual(['mlb']);
expect(c.isEnabled('wnba', now)).toBe(false);
expect(c.isEnabled('nba', now)).toBe(false);
});
});