Backup: durable on-box volume, off-box DEFERRED, and a real read-back check

BACKUP_DIR is now a persistent volume (/app/backups), so the dump already
survives redeploys — the container-ephemeral risk that made this urgent is
closed. Storage Box SSH auth is not sorted yet, so the off-box push is
explicitly DEFERRED rather than failing:

- gated on BACKUP_OFFBOX=1 (plus BACKUP_REMOTE and BACKUP_SSH_KEY); until
  then the script logs "off-box push DEFERRED" and exits clean.
- if an enabled push DOES fail, it is a LOW-priority "deferred" notice, not
  a failure — the durable on-box dump succeeded, and calling that an
  incident would train us to ignore backup alerts.

Adds the read-back check, because a backup nobody has read is a hope:
countRowsInDump() runs `pg_restore --data-only --table=X -f -` and counts
the rows between `FROM stdin;` and the terminating `\.`, proving the
archive CONTAINS the data rather than merely parsing. Needs no Postgres
server, so it runs inside the API container. Validated against a real
pg_dump from a scratch Postgres: counted exactly 604 rows.

GET /api/internal/backup/verify exposes it (newest dump in BACKUP_DIR,
size, table, rows_in_dump). Unit tests inject spawn/fs so CI needs neither
docker nor pg_restore.

Suite 278/3310 green, build exit 0.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SmNjJAwEnqHPtXbvSZR8kA

src/backupScheduler.js

@@ -110,6 +110,71 @@ function startBackupScheduler(opts = {}) {
   return { interval, tick };
 }
 
+/**
+ * Count rows for a table INSIDE a dump, without needing a Postgres server.
+ * `pg_restore --data-only --table=X` emits a COPY block; the rows are the lines
+ * between `FROM stdin;` and the terminating `\.`. This proves the dump actually
+ * CONTAINS the data (not merely that it parses), which is the thing a backup has
+ * to guarantee. A real server restore is still the gold standard — this is the
+ * strongest check available from inside the API container.
+ */
+function countRowsInDump(dumpPath, table = 'ledger_entries', deps = {}) {
+  const spawn = deps.spawn || require('child_process').spawn;
+  return new Promise((resolve) => {
+    let child;
+    try {
+      child = spawn('pg_restore', ['--data-only', `--table=${table}`, '-f', '-', dumpPath], {
+        stdio: ['ignore', 'pipe', 'pipe'],
+      });
+    } catch (e) {
+      return resolve({ ok: false, rows: null, error: e.message });
+    }
+    let out = '';
+    let err = '';
+    let inCopy = false;
+    let rows = 0;
+    let leftover = '';
+    child.stdout.on('data', (b) => {
+      const text = leftover + b.toString();
+      const lines = text.split('\n');
+      leftover = lines.pop() ?? '';
+      for (const line of lines) {
+        if (!inCopy) {
+          if (/FROM stdin;\s*$/.test(line)) inCopy = true;
+        } else if (line === '\\.') {
+          inCopy = false;
+        } else {
+          rows += 1;
+        }
+      }
+      if (out.length < 4000) out += text.slice(0, 4000);
+    });
+    child.stderr.on('data', (b) => { err = (err + b.toString()).slice(-2000); });
+    child.on('error', (e) => resolve({ ok: false, rows: null, error: e.message }));
+    child.on('close', (code) => resolve({
+      ok: code === 0, rows, code, error: code === 0 ? null : (err || `exit ${code}`),
+    }));
+  });
+}
+
+/** Newest *.dump in BACKUP_DIR, with its size. */
+function latestDump(dir = process.env.BACKUP_DIR || '/var/backups/vyndr', deps = {}) {
+  const fs = deps.fs || require('fs');
+  try {
+    const files = fs.readdirSync(dir)
+      .filter((f) => f.startsWith('vyndr-') && f.endsWith('.dump'))
+      .map((f) => {
+        const full = path.join(dir, f);
+        return { file: f, path: full, size: fs.statSync(full).size, mtime: fs.statSync(full).mtimeMs };
+      })
+      .sort((a, b) => b.mtime - a.mtime);
+    return files[0] || null;
+  } catch (e) {
+    return null;
+  }
+}
+
 module.exports = {
   startBackupScheduler, runBackup, shouldArm, durabilityWarning, SCRIPT,
+  countRowsInDump, latestDump,
 };

src/routes/internal.js

@@ -271,6 +271,38 @@ router.post('/backup/run', async (req, res) => {
   }
 });
 
+/**
+ * GET /api/internal/backup/verify (Session 64) — prove the newest dump on the
+ * persistent volume actually CONTAINS the data, by counting `ledger_entries`
+ * rows out of the archive with pg_restore. A backup nobody has read back is a
+ * hope, not a backup.
+ */
+router.get('/backup/verify', async (req, res) => {
+  const { latestDump, countRowsInDump } = require('../backupScheduler');
+  try {
+    const dir = process.env.BACKUP_DIR || '/var/backups/vyndr';
+    const dump = latestDump(dir);
+    if (!dump) {
+      return res.json({ ok: false, backup_dir: dir, error: 'no dump found in BACKUP_DIR' });
+    }
+    const table = String(req.query.table || 'ledger_entries');
+    const counted = await countRowsInDump(dump.path, table);
+    return res.json({
+      ok: counted.ok,
+      backup_dir: dir,
+      dump: dump.file,
+      dump_bytes: dump.size,
+      table,
+      rows_in_dump: counted.rows,
+      error: counted.error || null,
+    });
+  } catch (err) {
+    const message = err && err.message ? err.message : String(err);
+    console.error('[internal/backup/verify] failed:', message);
+    return res.status(500).json({ ok: false, error: message });
+  }
+});
+
 /**
  * POST /api/internal/ledger/settle (Session 58, Phase 1) — settle the
  * persistent ledger (outcome + actual_value + CLV) across every sport.