builtbykev
ef7f17610f
BACKUP_DIR is now a persistent volume (/app/backups), so the dump already survives redeploys — the container-ephemeral risk that made this urgent is closed. Storage Box SSH auth is not sorted yet, so the off-box push is explicitly DEFERRED rather than failing: - gated on BACKUP_OFFBOX=1 (plus BACKUP_REMOTE and BACKUP_SSH_KEY); until then the script logs "off-box push DEFERRED" and exits clean. - if an enabled push DOES fail, it is a LOW-priority "deferred" notice, not a failure — the durable on-box dump succeeded, and calling that an incident would train us to ignore backup alerts. Adds the read-back check, because a backup nobody has read is a hope: countRowsInDump() runs `pg_restore --data-only --table=X -f -` and counts the rows between `FROM stdin;` and the terminating `\.`, proving the archive CONTAINS the data rather than merely parsing. Needs no Postgres server, so it runs inside the API container. Validated against a real pg_dump from a scratch Postgres: counted exactly 604 rows. GET /api/internal/backup/verify exposes it (newest dump in BACKUP_DIR, size, table, rows_in_dump). Unit tests inject spawn/fs so CI needs neither docker nor pg_restore. Suite 278/3310 green, build exit 0. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SmNjJAwEnqHPtXbvSZR8kA
109 lines
5.6 KiB
Bash
109 lines
5.6 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# VYNDR nightly database backup (security follow-up item 2).
|
|
#
|
|
# Supabase free tier has ZERO backups (no scheduled, no PITR) — the ledger and
|
|
# everything else have no safety net. This dumps the WHOLE database nightly via
|
|
# the direct connection string, keeps 14 days locally, pushes a weekly copy
|
|
# off-box, and pages ntfy on ANY failure. Runs on the Hetzner box via cron.
|
|
#
|
|
# REQUIRED env (set on the box / in the container that runs the cron):
|
|
# SUPABASE_DB_URL the Supabase DIRECT connection string (session mode, the
|
|
# db.<ref>.supabase.co:5432 URL — NOT the :6543 pooler;
|
|
# pg_dump needs a real session). Kev pastes this in Coolify.
|
|
# OPTIONAL env:
|
|
# BACKUP_DIR local dump dir (default /var/backups/vyndr)
|
|
# BACKUP_KEEP_DAYS local retention (default 14)
|
|
# BACKUP_REMOTE off-box rsync target for the weekly copy, e.g.
|
|
# u123456@u123456.your-storagebox.de:vyndr-backups/
|
|
# (empty = skip the off-box push; a WARN is paged)
|
|
# NTFY_URL (default https://ntfy.sh)
|
|
# NTFY_TOPIC (default vyndr-backups-kev2026)
|
|
#
|
|
set -Eeuo pipefail
|
|
|
|
BACKUP_DIR="${BACKUP_DIR:-/var/backups/vyndr}"
|
|
KEEP_DAYS="${BACKUP_KEEP_DAYS:-14}"
|
|
NTFY_URL="${NTFY_URL:-https://ntfy.sh}"
|
|
NTFY_TOPIC="${NTFY_TOPIC:-vyndr-backups-kev2026}"
|
|
STAMP="$(date -u +%Y%m%d-%H%M%S)"
|
|
DUMP="${BACKUP_DIR}/vyndr-${STAMP}.dump"
|
|
MIN_BYTES="${BACKUP_MIN_BYTES:-50000}" # a real dump of this DB is far bigger; guards an empty/failed dump
|
|
|
|
notify() { # notify <title> <priority> <message>
|
|
curl -fsS --max-time 15 \
|
|
-H "Title: ${1}" -H "Priority: ${2}" -H "Tags: floppy_disk" \
|
|
-d "${3}" "${NTFY_URL}/${NTFY_TOPIC}" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
fail() { notify "VYNDR backup FAILED" "urgent" "${1}"; echo "ERROR: ${1}" >&2; exit 1; }
|
|
trap 'fail "backup script errored near line ${LINENO}"' ERR
|
|
|
|
[ -n "${SUPABASE_DB_URL:-}" ] || fail "SUPABASE_DB_URL is not set — cannot back up"
|
|
command -v pg_dump >/dev/null 2>&1 || fail "pg_dump not installed (apt-get install postgresql-client)"
|
|
mkdir -p "${BACKUP_DIR}"
|
|
|
|
# 1. Dump the whole DB in custom format (-Fc: compressed, restorable with pg_restore).
|
|
pg_dump "${SUPABASE_DB_URL}" -Fc --no-owner --no-privileges -f "${DUMP}" \
|
|
|| fail "pg_dump failed"
|
|
|
|
# 2. Sanity: a real dump is not tiny. An empty/near-empty file is a silent failure.
|
|
SIZE="$(stat -c%s "${DUMP}" 2>/dev/null || echo 0)"
|
|
[ "${SIZE}" -ge "${MIN_BYTES}" ] || fail "dump is only ${SIZE} bytes (< ${MIN_BYTES}) — treating as a failed backup"
|
|
|
|
# 2b. Integrity fingerprint: a valid custom-format archive lists its objects via
|
|
# pg_restore --list (no target DB needed). Confirm it parses AND contains the
|
|
# ledger — proves it's a real, restorable archive, not just a file of bytes.
|
|
TOC="$(pg_restore --list "${DUMP}" 2>/dev/null)" || fail "pg_restore --list failed — dump is not a valid archive"
|
|
OBJECTS="$(printf '%s\n' "${TOC}" | grep -c ';' || true)"
|
|
printf '%s\n' "${TOC}" | grep -qi 'TABLE DATA public ledger_entries' \
|
|
|| fail "dump archive does not contain ledger_entries — refusing to trust it"
|
|
echo "backup validated: ${DUMP} (${SIZE} bytes, ${OBJECTS} archive objects, ledger_entries present)"
|
|
|
|
# 3. Rotate: drop local dumps older than KEEP_DAYS.
|
|
find "${BACKUP_DIR}" -name 'vyndr-*.dump' -type f -mtime "+${KEEP_DAYS}" -delete || true
|
|
|
|
# 3b. SSH key for the off-box push (Session 64).
|
|
# The container filesystem is EPHEMERAL — a keypair generated inside it dies
|
|
# on the next redeploy and the off-box push would silently start failing. So
|
|
# the PRIVATE key is injected as an env var (Coolify secret) and written to a
|
|
# 0600 temp file per run. Hetzner Storage Box speaks full OpenSSH on PORT 23
|
|
# (port 22 is SFTP-only, mod_sftp) — verified live; rsync must target 23.
|
|
SSH_KEY_FILE=""
|
|
cleanup_key() { [ -n "${SSH_KEY_FILE}" ] && rm -f "${SSH_KEY_FILE}" || true; }
|
|
trap cleanup_key EXIT
|
|
RSYNC_SSH="ssh -p ${BACKUP_SSH_PORT:-23} -o StrictHostKeyChecking=accept-new -o BatchMode=yes"
|
|
if [ -n "${BACKUP_SSH_KEY:-}" ]; then
|
|
SSH_KEY_FILE="$(mktemp)"
|
|
chmod 600 "${SSH_KEY_FILE}"
|
|
# Accept the key with literal \n escapes (how env vars usually carry it).
|
|
printf '%b\n' "${BACKUP_SSH_KEY}" | sed -e 's/[[:space:]]*$//' > "${SSH_KEY_FILE}"
|
|
RSYNC_SSH="${RSYNC_SSH} -i ${SSH_KEY_FILE}"
|
|
fi
|
|
|
|
# 4. OFF-BOX COPY — DEFERRED (Session 64).
|
|
# The dump now lands on a PERSISTENT VOLUME (BACKUP_DIR=/app/backups), so it
|
|
# already survives redeploys — the container-ephemeral risk is closed. Storage
|
|
# Box SSH auth is not working yet, so the off-box push is explicitly DEFERRED:
|
|
# it must never fail the backup. A durable on-box dump is a real backup; a
|
|
# failing rsync on top of it is a follow-up, not an incident.
|
|
#
|
|
# Set BACKUP_OFFBOX=1 (with BACKUP_SSH_KEY) to re-enable. Until then we log
|
|
# and page at LOW priority, and we never call a deferred push a failure.
|
|
if [ "${BACKUP_OFFBOX:-0}" = "1" ] && [ -n "${BACKUP_REMOTE:-}" ] && [ -n "${BACKUP_SSH_KEY:-}" ]; then
|
|
if rsync -az --timeout=120 -e "${RSYNC_SSH}" "${DUMP}" "${BACKUP_REMOTE}"; then
|
|
echo "off-box push OK -> ${BACKUP_REMOTE%%:*}"
|
|
notify "VYNDR backup OK (+off-box)" "default" "Nightly dump ${STAMP} (${SIZE} bytes) pushed off-box."
|
|
else
|
|
# NOT a failure: the durable on-box dump succeeded.
|
|
echo "off-box push FAILED (deferred; on-box dump is durable)"
|
|
notify "VYNDR off-box push deferred" "low" "Dump ${STAMP} (${SIZE} bytes) is durable on the persistent volume; the off-box rsync failed and is deferred."
|
|
fi
|
|
else
|
|
echo "off-box push DEFERRED (BACKUP_OFFBOX!=1 or remote/key unset) — on-box dump is durable at ${DUMP}"
|
|
fi
|
|
|
|
echo "backup ok: ${DUMP} (${SIZE} bytes)"
|
|
|
|
exit 0
|