From c4c9b9760461da304cda06b47722cc2da914216a Mon Sep 17 00:00:00 2001 From: Kev Date: Mon, 20 Jul 2026 01:10:23 -0400 Subject: [PATCH] Off-box backup: pin the host key, guarantee the remote dir, page on failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PHASE 1 — HOST KEY STATICALLY PINNED. ssh-keyscan -p 23 returned an ED25519 key whose fingerprint EQUALS the out-of-band value SHA256:XqONwb1S0zuj5A1CDxpOSuD2hnAArV1A3wKY7Z3sdgM, so it is safe to pin. scripts/storagebox_known_hosts now carries that verified line and ships to the container (Dockerfile already COPYs scripts/). backup-db.sh uses StrictHostKeyChecking=yes + UserKnownHostsFile= instead of accept-new, which was trust-on-first-use and would have accepted an impostor on the very first run. A missing pin file REFUSES the push rather than silently falling back. Never weakened to accept-new/=no//dev/null — a test asserts that on executable lines. PHASE 1b — REMOTE DIR GUARANTEED. The box has only .ssh/, and rsyncing a file into a missing parent either fails or silently writes the dump AS the directory name — one file, overwritten nightly, reading as "backups exist" while retaining exactly one. Uses rsync --mkpath when available, else an explicit remote mkdir -p ahead of the push. PHASE 2b — FAILED OFF-BOX PUSH IS NOW LOUD. Off-box is required, so the failed-push path pages at "urgent" (was "low"/deferred) and the script emits a machine-readable OFFBOX_OK=1/0/deferred that POST /api/internal/backup/run surfaces as a distinct offbox_ok field. Exit code deliberately still reflects ON-BOX durability — a good on-box dump must not raise a false total-failure alarm. Surfacing the truth, not manufacturing a failure. No key material is echoed anywhere; only the PUBLIC host key is committed. Suite 280/3338 green, build exit 0. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01SmNjJAwEnqHPtXbvSZR8kA --- scripts/backup-db.sh | 42 ++++++++++++--- scripts/storagebox_known_hosts | 8 +++ src/routes/internal.js | 8 +++ tests/unit/backupOffbox.test.js | 95 +++++++++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 6 deletions(-) create mode 100644 scripts/storagebox_known_hosts create mode 100644 tests/unit/backupOffbox.test.js diff --git a/scripts/backup-db.sh b/scripts/backup-db.sh index 961a587..a137bc0 100644 --- a/scripts/backup-db.sh +++ b/scripts/backup-db.sh @@ -72,7 +72,15 @@ find "${BACKUP_DIR}" -name 'vyndr-*.dump' -type f -mtime "+${KEEP_DAYS}" -delete SSH_KEY_FILE="" cleanup_key() { [ -n "${SSH_KEY_FILE}" ] && rm -f "${SSH_KEY_FILE}" || true; } trap cleanup_key EXIT -RSYNC_SSH="ssh -p ${BACKUP_SSH_PORT:-23} -o StrictHostKeyChecking=accept-new -o BatchMode=yes" +# HOST KEY IS STATICALLY PINNED (Session 64). This used to be +# StrictHostKeyChecking=accept-new — trust-on-first-use, which accepts whatever +# host key it meets first and would happily trust an impostor on the first run. +# scripts/storagebox_known_hosts carries the ED25519 line verified out-of-band +# (SHA256:XqONwb1S0zuj5A1CDxpOSuD2hnAArV1A3wKY7Z3sdgM). A mismatch is now a HARD +# FAIL — which is the point. Never weaken this to accept-new/=no//dev/null. +KNOWN_HOSTS="${BACKUP_KNOWN_HOSTS:-$(dirname "$0")/storagebox_known_hosts}" +[ -f "${KNOWN_HOSTS}" ] || fail "pinned known_hosts missing at ${KNOWN_HOSTS} — refusing to push without host-key verification" +RSYNC_SSH="ssh -p ${BACKUP_SSH_PORT:-23} -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${KNOWN_HOSTS} -o BatchMode=yes" if [ -n "${BACKUP_SSH_KEY:-}" ]; then SSH_KEY_FILE="$(mktemp)" chmod 600 "${SSH_KEY_FILE}" @@ -106,16 +114,38 @@ fi # Set BACKUP_OFFBOX=1 (with BACKUP_SSH_KEY) to re-enable. Until then we log # and page at LOW priority, and we never call a deferred push a failure. if [ "${BACKUP_OFFBOX:-0}" = "1" ] && [ -n "${BACKUP_REMOTE:-}" ] && [ -n "${BACKUP_SSH_KEY:-}" ]; then - if rsync -az --timeout=120 -e "${RSYNC_SSH}" "${DUMP}" "${BACKUP_REMOTE}"; then - echo "off-box push OK -> ${BACKUP_REMOTE%%:*}" + # Phase 1b — GUARANTEE THE REMOTE DIRECTORY EXISTS. The box starts with only + # .ssh/, and rsync of a file into a missing parent either fails or silently + # writes the dump AS the directory name (one file, overwritten nightly, which + # would read as "backups exist" while retaining exactly one). Prefer rsync's + # own --mkpath; fall back to an explicit ssh mkdir -p for older rsync. + REMOTE_HOST="${BACKUP_REMOTE%%:*}" + REMOTE_PATH="${BACKUP_REMOTE#*:}" + MKPATH_FLAG="" + if rsync --help 2>&1 | grep -q -- '--mkpath'; then + MKPATH_FLAG="--mkpath" + else + ${RSYNC_SSH} "${REMOTE_HOST}" "mkdir -p '${REMOTE_PATH}'" \ + || echo "warn: remote mkdir -p failed; relying on an existing directory" + fi + + # OFF-BOX IS REQUIRED NOW (Session 64, Phase 2b). A failed push must never + # again read as success: it PAGES, and the run reports offbox_ok:false. + # Exit code deliberately still reflects ON-BOX durability — a good on-box dump + # must not raise a false total-failure alarm. Surface the truth; don't + # manufacture a failure. + if rsync -az --timeout=120 ${MKPATH_FLAG} -e "${RSYNC_SSH}" "${DUMP}" "${BACKUP_REMOTE}"; then + echo "off-box push OK -> ${BACKUP_REMOTE}" + echo "OFFBOX_OK=1" notify "VYNDR backup OK (+off-box)" "default" "Nightly dump ${STAMP} (${SIZE} bytes) pushed off-box." else - # NOT a failure: the durable on-box dump succeeded. - echo "off-box push FAILED (deferred; on-box dump is durable)" - notify "VYNDR off-box push deferred" "low" "Dump ${STAMP} (${SIZE} bytes) is durable on the persistent volume; the off-box rsync failed and is deferred." + echo "off-box push FAILED (on-box dump is durable, but OFF-BOX IS REQUIRED)" + echo "OFFBOX_OK=0" + notify "VYNDR OFF-BOX PUSH FAILED" "urgent" "Dump ${STAMP} (${SIZE} bytes) is on the persistent volume but did NOT reach the Storage Box. The database has no off-box copy tonight." fi else echo "off-box push DEFERRED (BACKUP_OFFBOX!=1 or remote/key unset) — on-box dump is durable at ${DUMP}" + echo "OFFBOX_OK=deferred" fi echo "backup ok: ${DUMP} (${SIZE} bytes)" diff --git a/scripts/storagebox_known_hosts b/scripts/storagebox_known_hosts new file mode 100644 index 0000000..4ff8a69 --- /dev/null +++ b/scripts/storagebox_known_hosts @@ -0,0 +1,8 @@ +# VYNDR — PINNED Storage Box host key (Session 64). +# Verified 2026-07-20: SHA256:XqONwb1S0zuj5A1CDxpOSuD2hnAArV1A3wKY7Z3sdgM +# The backup used StrictHostKeyChecking=accept-new, which is trust-on-first-use: +# it accepts whatever host key it meets first. This file makes the trust STATIC. +# Never replace with accept-new / =no / UserKnownHostsFile=/dev/null. +# Rotating the box regenerates this line — re-verify the fingerprint out-of-band +# before changing it. A mismatch at run time is a HARD FAIL, by design. +[u635423.your-storagebox.de]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs diff --git a/src/routes/internal.js b/src/routes/internal.js index 8c13501..e5d72ae 100644 --- a/src/routes/internal.js +++ b/src/routes/internal.js @@ -255,8 +255,16 @@ router.post('/backup/run', async (req, res) => { try { const started = Date.now(); const result = await runBackup(); + // Session 64 Phase 2b — off-box is REQUIRED now, so its outcome is reported + // SEPARATELY from the exit code. The script deliberately still exits 0 on a + // failed push (a durable on-box dump must not raise a false total-failure + // alarm) — so without this field a failed off-box push reads as success. + const tail = result.tail || ''; + const offboxOk = /OFFBOX_OK=1/.test(tail) ? true + : (/OFFBOX_OK=0/.test(tail) ? false : null); // null = deferred/not attempted return res.json({ ok: result.ok, + offbox_ok: offboxOk, exit_code: result.code, duration_ms: Date.now() - started, durability_warning: durabilityWarning() || null, diff --git a/tests/unit/backupOffbox.test.js b/tests/unit/backupOffbox.test.js new file mode 100644 index 0000000..2384328 --- /dev/null +++ b/tests/unit/backupOffbox.test.js @@ -0,0 +1,95 @@ +/** + * Session 64 — off-box backup hardening. + * + * Locks three properties that are easy to silently undo later: + * 1. the Storage Box host key is STATICALLY PINNED (no trust-on-first-use) + * 2. the remote directory is guaranteed before a push + * 3. a failed REQUIRED off-box push pages and reports offbox_ok:false + */ + +const fs = require('fs'); +const path = require('path'); +const { execFileSync } = require('child_process'); + +const ROOT = path.join(__dirname, '..', '..'); +const SCRIPT = fs.readFileSync(path.join(ROOT, 'scripts', 'backup-db.sh'), 'utf8'); +const KNOWN_HOSTS_PATH = path.join(ROOT, 'scripts', 'storagebox_known_hosts'); + +const EXPECTED_FP = 'SHA256:XqONwb1S0zuj5A1CDxpOSuD2hnAArV1A3wKY7Z3sdgM'; + +describe('host key is statically pinned', () => { + test('the pinned known_hosts file ships in the repo', () => { + expect(fs.existsSync(KNOWN_HOSTS_PATH)).toBe(true); + }); + + test('it contains the VERIFIED Storage Box ED25519 fingerprint', () => { + // ssh-keygen is the authority — parse the file the way ssh will. + const out = execFileSync('ssh-keygen', ['-lf', KNOWN_HOSTS_PATH], { encoding: 'utf8' }); + expect(out).toContain(EXPECTED_FP); + expect(out).toContain('[u635423.your-storagebox.de]:23'); + }); + + test('the script uses StrictHostKeyChecking=yes with the pinned file', () => { + expect(SCRIPT).toMatch(/StrictHostKeyChecking=yes/); + expect(SCRIPT).toMatch(/UserKnownHostsFile=\$\{KNOWN_HOSTS\}/); + }); + + test('trust-on-first-use and disabled checking are GONE and stay gone', () => { + // Assert on EXECUTABLE lines only — the comments deliberately name + // accept-new to explain what was removed and why it must not come back. + const code = SCRIPT.split('\n') + .filter((l) => !l.trim().startsWith('#')) + .join('\n'); + expect(code).not.toMatch(/StrictHostKeyChecking=accept-new/); + expect(code).not.toMatch(/StrictHostKeyChecking=no\b/); + expect(code).not.toMatch(/UserKnownHostsFile=\/dev\/null/); + }); + + test('a missing pin file refuses the push rather than falling back', () => { + expect(SCRIPT).toMatch(/refusing to push without host-key verification/); + }); +}); + +describe('remote directory is guaranteed', () => { + test('uses --mkpath when available, else an explicit remote mkdir -p', () => { + expect(SCRIPT).toMatch(/--mkpath/); + expect(SCRIPT).toMatch(/mkdir -p/); + }); +}); + +describe('off-box failure is loud (Phase 2b)', () => { + test('a failed REQUIRED push pages at urgent, not low', () => { + const failBlock = SCRIPT.slice(SCRIPT.indexOf('off-box push FAILED')); + expect(failBlock).toMatch(/notify "VYNDR OFF-BOX PUSH FAILED" "urgent"/); + expect(SCRIPT).not.toMatch(/notify "VYNDR off-box push deferred" "low"/); + }); + + test('the script emits a machine-readable off-box result', () => { + expect(SCRIPT).toMatch(/OFFBOX_OK=1/); + expect(SCRIPT).toMatch(/OFFBOX_OK=0/); + }); + + test('exit code still reflects ON-BOX durability (no false total-failure)', () => { + // The push lives in an if/else; neither branch exits non-zero. + const pushSection = SCRIPT.slice(SCRIPT.indexOf('OFF-BOX COPY')); + expect(pushSection).not.toMatch(/exit 1/); + }); + + test('the API surfaces offbox_ok distinctly from ok', () => { + const route = fs.readFileSync(path.join(ROOT, 'src', 'routes', 'internal.js'), 'utf8'); + expect(route).toMatch(/offbox_ok/); + expect(route).toMatch(/OFFBOX_OK=1/); + }); +}); + +describe('offbox_ok parsing', () => { + // Mirrors the route's parse so the three-state logic is locked. + const parse = (tail) => (/OFFBOX_OK=1/.test(tail) ? true : (/OFFBOX_OK=0/.test(tail) ? false : null)); + + test('success → true', () => expect(parse('...\nOFFBOX_OK=1\n')).toBe(true)); + test('failure → false (never null, never true)', () => expect(parse('...\nOFFBOX_OK=0\n')).toBe(false)); + test('deferred/absent → null, distinct from false', () => { + expect(parse('OFFBOX_OK=deferred')).toBeNull(); + expect(parse('')).toBeNull(); + }); +});