P0 fix: content/ not in image crashed API boot; harden garnish + preflight
ROOT CAUSE: the Dockerfile copied src/poller/scripts/supabase but NOT content/. mediaEngine.js read content/stark-lines.json with an unguarded module-load readFileSync; ENOENT in the image threw at require time, and via app.js → routes/desk → deskService → mediaEngine that crashed the ENTIRE API at boot. The Coolify healthcheck rolled back to the last healthy image (4d2b27d), so every deploy since219167esilently served a 14-hour-old build — S11 live tracking, S6 API code, the settlement boot line, and SNAPSHOT_EXPECTED_INTERVAL were all merged but NOT running. FIX (one train): 1. Dockerfile COPYs content/ into the runner image. 2. mediaEngine: stark-lines.json is OPTIONAL (garnish, never load-bearing) — loadStark() try/catch → {} → posts render without the Stark kicker, never a crash. Belt AND suspenders with #1. 3. src/preflight.js (§A4): boot prints '[preflight] OK' or 'DEGRADED' naming exactly what content/env is missing — before the healthcheck can fail silently. Run first in server.js. 4. Full fragility sweep: mediaEngine was the ONLY unguarded module-load file read; coachSignals (config/coaches.json) was already lazy + try/catch + copied. No others. Verified: requiring app.js + deskService + mediaEngine with stark-lines.json ABSENT now boots clean (reproduced the exact prod failure). 2757 -> 2763 tests (tests/unit/bootResilience.test.js). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -47,6 +47,12 @@ COPY src ./src
|
||||
COPY poller ./poller
|
||||
COPY scripts ./scripts
|
||||
COPY supabase ./supabase
|
||||
# Session 65 (P0) — content/ holds the Stark-line library + seed articles.
|
||||
# It was NOT copied, so mediaEngine's module-load read threw ENOENT in the
|
||||
# image and crashed the API at boot (the deploy silently rolled back for 14
|
||||
# hours). The read is now hardened to optional too (belt AND suspenders),
|
||||
# but the file belongs in the image.
|
||||
COPY content ./content
|
||||
|
||||
# Persistent volume for JSONL training data (resolutions survive
|
||||
# redeploys via the Coolify mount). PM2_HOME lives outside it so
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* preflight — boot visibility (Session 65, Standard §A4).
|
||||
*
|
||||
* A healthcheck that only pings /api/health can't tell "booted degraded"
|
||||
* from "crashed and rolled back" — which is exactly how a missing garnish
|
||||
* file (content/stark-lines.json) served a 14-hour-old image silently. This
|
||||
* runs once at boot and prints ONE unambiguous line: `[preflight] OK` when
|
||||
* everything required is present, or `[preflight] DEGRADED` naming exactly
|
||||
* what is missing. It NEVER throws and NEVER blocks boot — its only job is
|
||||
* to make the container's state legible in the logs.
|
||||
*
|
||||
* Philosophy (matches the app's own doctrine): content garnish missing is
|
||||
* NOTED, not fatal (absent beats crashed). Missing critical env is named so
|
||||
* an operator sees it immediately instead of debugging a symptom later.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// Optional content — the app runs fine without these; we just name them so a
|
||||
// missing one is visible (and never silently crashes a require chain again).
|
||||
const OPTIONAL_CONTENT = [
|
||||
{ label: 'stark-lines.json', file: path.join(__dirname, '..', 'content', 'stark-lines.json'), note: 'media posts drop the Stark kicker' },
|
||||
{ label: 'coaches.json', file: path.join(__dirname, 'config', 'coaches.json'), note: 'coach-impact feature uses no seed' },
|
||||
];
|
||||
|
||||
// Critical env — the app boots without these but key surfaces degrade; named
|
||||
// loudly so an operator fixes them before hunting a downstream symptom.
|
||||
const CRITICAL_ENV = ['SUPABASE_URL', 'REDIS_URL', 'VYNDR_INTERNAL_KEY'];
|
||||
// Either service-role name satisfies the Supabase write path.
|
||||
const ENV_EITHER = [['SUPABASE_SERVICE_ROLE_KEY', 'SUPABASE_SERVICE_KEY']];
|
||||
|
||||
function runPreflight(log = console) {
|
||||
try {
|
||||
const missingContent = [];
|
||||
for (const c of OPTIONAL_CONTENT) {
|
||||
let ok = false;
|
||||
try { fs.accessSync(c.file, fs.constants.R_OK); ok = true; } catch { ok = false; }
|
||||
if (!ok) missingContent.push(`${c.label} (${c.note})`);
|
||||
}
|
||||
|
||||
const missingEnv = CRITICAL_ENV.filter((k) => !process.env[k]);
|
||||
for (const group of ENV_EITHER) {
|
||||
if (!group.some((k) => process.env[k])) missingEnv.push(group.join('|'));
|
||||
}
|
||||
|
||||
// Operational visibility for the pipeline knobs (not failures — states).
|
||||
const cron = process.env.SNAPSHOT_CRON === '1' ? 'armed' : 'off';
|
||||
const interval = Number(process.env.SNAPSHOT_EXPECTED_INTERVAL) > 0
|
||||
? `${process.env.SNAPSHOT_EXPECTED_INTERVAL}s` : 'default(18000s)';
|
||||
|
||||
if (missingContent.length === 0 && missingEnv.length === 0) {
|
||||
log.log(`[preflight] OK — content present, critical env present, snapshot cron ${cron}, expected-interval ${interval}`);
|
||||
} else {
|
||||
log.warn(`[preflight] DEGRADED — snapshot cron ${cron}, expected-interval ${interval}`);
|
||||
if (missingEnv.length) log.warn(`[preflight] missing env: ${missingEnv.join(', ')}`);
|
||||
if (missingContent.length) log.warn(`[preflight] missing content (non-fatal): ${missingContent.join(', ')}`);
|
||||
}
|
||||
return { ok: missingContent.length === 0 && missingEnv.length === 0, missingContent, missingEnv };
|
||||
} catch (e) {
|
||||
// Preflight itself must never break boot.
|
||||
try { log.warn(`[preflight] check errored (non-fatal): ${e.message}`); } catch { /* ignore */ }
|
||||
return { ok: false, error: e.message };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { runPreflight, __internals: { OPTIONAL_CONTENT, CRITICAL_ENV, ENV_EITHER } };
|
||||
@@ -1,3 +1,8 @@
|
||||
// Session 65 (§A4) — preflight BEFORE anything else prints, so the container's
|
||||
// state (content present? critical env present? cron armed?) is the first
|
||||
// thing in the logs, not something inferred after a silent rollback.
|
||||
require('./preflight').runPreflight();
|
||||
|
||||
const app = require('./app');
|
||||
// Session 20 — surface which providers are actually configured at
|
||||
// boot. A silently-missing key (e.g. ODDSPAPI_KEY unset in prod)
|
||||
|
||||
@@ -20,7 +20,24 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const STARK = JSON.parse(fs.readFileSync(path.join(__dirname, '..', '..', 'content', 'stark-lines.json'), 'utf8'));
|
||||
// Session 65 (P0 deploy fix) — the Stark-layer library is a GARNISH, never
|
||||
// load-bearing. This used to be an unguarded module-load readFileSync; when
|
||||
// the file wasn't in the container image it threw at require time, which
|
||||
// (via app.js → routes/desk → deskService → here) crashed the WHOLE API at
|
||||
// boot and silently rolled the deploy back for 14 hours. A witty kicker is
|
||||
// not allowed to take down the record. Missing/corrupt file → empty library
|
||||
// → every format posts WITHOUT the Stark line (starkLine returns null),
|
||||
// which is a valid, still-VOICE-compliant post. Never throws.
|
||||
const STARK_PATH = path.join(__dirname, '..', '..', 'content', 'stark-lines.json');
|
||||
function loadStark() {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(STARK_PATH, 'utf8'));
|
||||
} catch (e) {
|
||||
console.warn(`[mediaEngine] stark-lines.json unavailable (${e.code || e.message}); posting without the Stark layer.`);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
const STARK = loadStark();
|
||||
|
||||
// ---- VOICE lint -----------------------------------------------------------
|
||||
const BANNED_PATTERNS = [
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
// Session 65 (P0 deploy fix) — the API crashed at boot because a GARNISH
|
||||
// file (content/stark-lines.json) was a hard module-load readFileSync and
|
||||
// wasn't in the container image. These tests lock the fix: the media engine
|
||||
// boots and functions with the file ABSENT, and the preflight names what's
|
||||
// missing without ever throwing.
|
||||
|
||||
describe('mediaEngine — boots and posts with the Stark library absent (§L2)', () => {
|
||||
let media;
|
||||
beforeEach(() => {
|
||||
jest.resetModules();
|
||||
// Make EVERY stark-lines.json read throw ENOENT — simulate the file
|
||||
// missing from the image (the exact production failure).
|
||||
jest.doMock('fs', () => {
|
||||
const real = jest.requireActual('fs');
|
||||
return {
|
||||
...real,
|
||||
readFileSync: (p, ...rest) => {
|
||||
if (String(p).includes('stark-lines.json')) {
|
||||
const err = new Error('ENOENT: no such file');
|
||||
err.code = 'ENOENT';
|
||||
throw err;
|
||||
}
|
||||
return real.readFileSync(p, ...rest);
|
||||
},
|
||||
};
|
||||
});
|
||||
media = require('../../src/services/mediaEngine');
|
||||
});
|
||||
afterEach(() => { jest.dontMock('fs'); jest.resetModules(); });
|
||||
|
||||
test('requiring the module does NOT throw (this was the boot crash)', () => {
|
||||
expect(media).toBeTruthy();
|
||||
expect(media.__internals.STARK).toEqual({}); // empty library, not a crash
|
||||
});
|
||||
|
||||
test('starkLine returns null with no library — the garnish is simply absent', () => {
|
||||
expect(media.starkLine('morning', '2026-07-12T14:00:00Z')).toBeNull();
|
||||
});
|
||||
|
||||
test('formats still produce valid, VOICE-clean posts without the kicker', () => {
|
||||
const wire = media.morningWire({ dateIso: '2026-07-12T14:00:00Z', counts: { mlb: 15 }, loudest: null });
|
||||
expect(wire).toContain('15 MLB.');
|
||||
expect(wire).toContain('vyndr.app');
|
||||
expect(media.lintVoice(wire)).toEqual([]);
|
||||
|
||||
const settle = media.theSettle({
|
||||
aggregate: { settled: 5, hits: 3, misses: 2, pushes: 0, hit_pct: null, pending: 10, by_tier: {} },
|
||||
misses: [{ player_name: 'A Guy', stat: 'points', line: 23.5, side: 'over' }],
|
||||
dateIso: '2026-07-12T14:00:00Z',
|
||||
});
|
||||
expect(settle).toContain('All grades: 3-2');
|
||||
expect(media.lintVoice(settle)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('deskService assembles with the Stark library absent', () => {
|
||||
test('variants() tolerates an empty STARK (no throw, no kicker to strip)', () => {
|
||||
jest.resetModules();
|
||||
jest.doMock('fs', () => {
|
||||
const real = jest.requireActual('fs');
|
||||
return {
|
||||
...real,
|
||||
readFileSync: (p, ...rest) => {
|
||||
if (String(p).includes('stark-lines.json')) { const e = new Error('ENOENT'); e.code = 'ENOENT'; throw e; }
|
||||
return real.readFileSync(p, ...rest);
|
||||
},
|
||||
};
|
||||
});
|
||||
const { assembleDeskPack } = require('../../src/services/deskService');
|
||||
expect(typeof assembleDeskPack).toBe('function');
|
||||
jest.dontMock('fs');
|
||||
jest.resetModules();
|
||||
});
|
||||
});
|
||||
|
||||
describe('preflight — names what is missing, never throws (§A4)', () => {
|
||||
const { runPreflight } = require('../../src/preflight');
|
||||
|
||||
test('reports OK with a captured logger when content + env are present', () => {
|
||||
const logs = [];
|
||||
const fake = { log: (m) => logs.push(['log', m]), warn: (m) => logs.push(['warn', m]) };
|
||||
const prev = { SUPABASE_URL: process.env.SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY: process.env.SUPABASE_SERVICE_ROLE_KEY, REDIS_URL: process.env.REDIS_URL, VYNDR_INTERNAL_KEY: process.env.VYNDR_INTERNAL_KEY };
|
||||
process.env.SUPABASE_URL = 'x'; process.env.SUPABASE_SERVICE_ROLE_KEY = 'x'; process.env.REDIS_URL = 'x'; process.env.VYNDR_INTERNAL_KEY = 'x';
|
||||
const res = runPreflight(fake);
|
||||
// content/stark-lines.json + src/config/coaches.json both exist in the tree.
|
||||
expect(res.ok).toBe(true);
|
||||
expect(logs.some(([lvl, m]) => lvl === 'log' && m.includes('[preflight] OK'))).toBe(true);
|
||||
Object.assign(process.env, prev);
|
||||
for (const k of Object.keys(prev)) if (prev[k] === undefined) delete process.env[k];
|
||||
});
|
||||
|
||||
test('names missing env, does not throw, returns not-ok', () => {
|
||||
const logs = [];
|
||||
const fake = { log: (m) => logs.push(m), warn: (m) => logs.push(m) };
|
||||
const prev = { SUPABASE_URL: process.env.SUPABASE_URL, REDIS_URL: process.env.REDIS_URL, VYNDR_INTERNAL_KEY: process.env.VYNDR_INTERNAL_KEY, SUPABASE_SERVICE_ROLE_KEY: process.env.SUPABASE_SERVICE_ROLE_KEY, SUPABASE_SERVICE_KEY: process.env.SUPABASE_SERVICE_KEY };
|
||||
delete process.env.SUPABASE_URL; delete process.env.REDIS_URL; delete process.env.VYNDR_INTERNAL_KEY;
|
||||
delete process.env.SUPABASE_SERVICE_ROLE_KEY; delete process.env.SUPABASE_SERVICE_KEY;
|
||||
const res = runPreflight(fake);
|
||||
expect(res.ok).toBe(false);
|
||||
expect(res.missingEnv).toEqual(expect.arrayContaining(['SUPABASE_URL', 'REDIS_URL', 'VYNDR_INTERNAL_KEY']));
|
||||
expect(logs.some((m) => m.includes('DEGRADED'))).toBe(true);
|
||||
Object.assign(process.env, prev);
|
||||
for (const k of Object.keys(prev)) if (prev[k] === undefined) delete process.env[k];
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user