ef7f17610f
BACKUP_DIR is now a persistent volume (/app/backups), so the dump already survives redeploys — the container-ephemeral risk that made this urgent is closed. Storage Box SSH auth is not sorted yet, so the off-box push is explicitly DEFERRED rather than failing: - gated on BACKUP_OFFBOX=1 (plus BACKUP_REMOTE and BACKUP_SSH_KEY); until then the script logs "off-box push DEFERRED" and exits clean. - if an enabled push DOES fail, it is a LOW-priority "deferred" notice, not a failure — the durable on-box dump succeeded, and calling that an incident would train us to ignore backup alerts. Adds the read-back check, because a backup nobody has read is a hope: countRowsInDump() runs `pg_restore --data-only --table=X -f -` and counts the rows between `FROM stdin;` and the terminating `\.`, proving the archive CONTAINS the data rather than merely parsing. Needs no Postgres server, so it runs inside the API container. Validated against a real pg_dump from a scratch Postgres: counted exactly 604 rows. GET /api/internal/backup/verify exposes it (newest dump in BACKUP_DIR, size, table, rows_in_dump). Unit tests inject spawn/fs so CI needs neither docker nor pg_restore. Suite 278/3310 green, build exit 0. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SmNjJAwEnqHPtXbvSZR8kA
159 lines
5.5 KiB
JavaScript
159 lines
5.5 KiB
JavaScript
/**
|
|
* Session 64 — nightly backup scheduler.
|
|
*
|
|
* The S62 backup script was mechanism-verified but never installed, so the
|
|
* database went unbacked every night. These lock the arming semantics that make
|
|
* "deploy == installed" true, and the honesty rules around durability.
|
|
*/
|
|
|
|
const sched = require('../../src/backupScheduler');
|
|
|
|
describe('arming (opt-OUT, because opt-in is what failed)', () => {
|
|
test('armed whenever SUPABASE_DB_URL is present', () => {
|
|
expect(sched.shouldArm({ SUPABASE_DB_URL: 'postgres://x' }).armed).toBe(true);
|
|
});
|
|
|
|
test('BACKUP_CRON=0 is the kill switch', () => {
|
|
const r = sched.shouldArm({ SUPABASE_DB_URL: 'postgres://x', BACKUP_CRON: '0' });
|
|
expect(r.armed).toBe(false);
|
|
expect(r.reason).toMatch(/kill switch/);
|
|
});
|
|
|
|
test('no SUPABASE_DB_URL → disarmed with a reason (nothing to dump)', () => {
|
|
const r = sched.shouldArm({});
|
|
expect(r.armed).toBe(false);
|
|
expect(r.reason).toMatch(/SUPABASE_DB_URL/);
|
|
});
|
|
|
|
test('does NOT require an opt-in env var to arm', () => {
|
|
// The whole S62 failure: BACKUP_CRON=1 was never set by anyone.
|
|
expect(sched.shouldArm({ SUPABASE_DB_URL: 'postgres://x' }).armed).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe('durability honesty', () => {
|
|
test('warns when neither a volume nor an off-box target is configured', () => {
|
|
expect(sched.durabilityWarning({})).toMatch(/ephemeral/);
|
|
});
|
|
|
|
test('no warning once BACKUP_REMOTE is set', () => {
|
|
expect(sched.durabilityWarning({ BACKUP_REMOTE: 'u1@box:vyndr/' })).toBeNull();
|
|
});
|
|
|
|
test('no warning once BACKUP_DIR is set (persistent volume)', () => {
|
|
expect(sched.durabilityWarning({ BACKUP_DIR: '/var/backups/vyndr' })).toBeNull();
|
|
});
|
|
});
|
|
|
|
describe('scheduling', () => {
|
|
const env = { SUPABASE_DB_URL: 'postgres://x', BACKUP_REMOTE: 'u1@box:v/' };
|
|
|
|
test('runs at the configured minute, and only once per day', async () => {
|
|
const runs = [];
|
|
const at = (h, m, day = 19) => new Date(Date.UTC(2026, 6, day, h, m));
|
|
let clock = at(3, 10);
|
|
const s = sched.startBackupScheduler({
|
|
env,
|
|
now: () => clock,
|
|
notify: async () => {},
|
|
runBackup: async () => { runs.push(clock.toISOString()); return { ok: true, code: 0, ms: 1 }; },
|
|
});
|
|
await s.tick();
|
|
await s.tick(); // same minute again — must not re-run
|
|
expect(runs).toHaveLength(1);
|
|
|
|
clock = at(3, 10, 20); // next day
|
|
await s.tick();
|
|
expect(runs).toHaveLength(2);
|
|
});
|
|
|
|
test('does nothing outside the scheduled minute', async () => {
|
|
const runs = [];
|
|
const s = sched.startBackupScheduler({
|
|
env,
|
|
now: () => new Date(Date.UTC(2026, 6, 19, 14, 0)),
|
|
notify: async () => {},
|
|
runBackup: async () => { runs.push(1); return { ok: true, code: 0, ms: 1 }; },
|
|
});
|
|
await s.tick();
|
|
expect(runs).toHaveLength(0);
|
|
});
|
|
|
|
test('a failed backup pages high priority — silence is the danger', async () => {
|
|
const alerts = [];
|
|
const s = sched.startBackupScheduler({
|
|
env,
|
|
now: () => new Date(Date.UTC(2026, 6, 19, 3, 10)),
|
|
notify: async (msg, opts) => { alerts.push({ msg, opts }); },
|
|
runBackup: async () => ({ ok: false, code: 2, ms: 5 }),
|
|
});
|
|
await s.tick();
|
|
expect(alerts).toHaveLength(1);
|
|
expect(alerts[0].opts.priority).toBe('high');
|
|
expect(alerts[0].msg).toMatch(/FAILED/);
|
|
});
|
|
|
|
test('disarmed scheduler returns null and schedules nothing', () => {
|
|
expect(sched.startBackupScheduler({ env: {}, notify: async () => {} })).toBeNull();
|
|
});
|
|
});
|
|
|
|
describe('countRowsInDump — reading the backup back', () => {
|
|
const { EventEmitter } = require('events');
|
|
const { Readable } = require('stream');
|
|
|
|
function fakeSpawn(stdoutText, code = 0) {
|
|
return () => {
|
|
const child = new EventEmitter();
|
|
child.stdout = Readable.from([stdoutText]);
|
|
child.stderr = Readable.from([]);
|
|
child.stdout.on('end', () => setImmediate(() => child.emit('close', code)));
|
|
return child;
|
|
};
|
|
}
|
|
|
|
const COPY_BLOCK = [
|
|
'--',
|
|
'COPY public.ledger_entries (id, player_name) FROM stdin;',
|
|
'1\tAlonso',
|
|
'2\tHenderson',
|
|
'3\tReese',
|
|
'\\.',
|
|
'',
|
|
].join('\n');
|
|
|
|
test('counts only the rows inside the COPY block', async () => {
|
|
const r = await sched.countRowsInDump('/x.dump', 'ledger_entries', { spawn: fakeSpawn(COPY_BLOCK) });
|
|
expect(r.ok).toBe(true);
|
|
expect(r.rows).toBe(3);
|
|
});
|
|
|
|
test('an empty dump reports 0 rows, not a false success count', async () => {
|
|
const empty = 'COPY public.ledger_entries (id) FROM stdin;\n\\.\n';
|
|
const r = await sched.countRowsInDump('/x.dump', 'ledger_entries', { spawn: fakeSpawn(empty) });
|
|
expect(r.rows).toBe(0);
|
|
});
|
|
|
|
test('a failing pg_restore is reported, never counted as ok', async () => {
|
|
const r = await sched.countRowsInDump('/x.dump', 'ledger_entries', { spawn: fakeSpawn('', 1) });
|
|
expect(r.ok).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe('latestDump', () => {
|
|
test('picks the newest dump and ignores unrelated files', () => {
|
|
const fs = {
|
|
readdirSync: () => ['vyndr-old.dump', 'notes.txt', 'vyndr-new.dump'],
|
|
statSync: (p) => ({ size: p.includes('new') ? 999 : 111, mtimeMs: p.includes('new') ? 200 : 100 }),
|
|
};
|
|
const d = sched.latestDump('/app/backups', { fs });
|
|
expect(d.file).toBe('vyndr-new.dump');
|
|
expect(d.size).toBe(999);
|
|
});
|
|
|
|
test('missing directory degrades to null, never throws', () => {
|
|
const fs = { readdirSync: () => { throw new Error('ENOENT'); } };
|
|
expect(sched.latestDump('/nope', { fs })).toBeNull();
|
|
});
|
|
});
|