diff --git a/Dockerfile b/Dockerfile index 1608f76..a69beac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,8 +26,11 @@ RUN npm ci --omit=dev --no-audit --no-fund FROM node:20-alpine AS runner WORKDIR /app -# curl is used by the /api/health smoke check (Coolify HEALTHCHECK). -RUN apk add --no-cache curl tini +# curl — /api/health smoke check (Coolify HEALTHCHECK). +# postgresql-client (pg_dump/pg_restore) + rsync + bash — the nightly DB backup +# (scripts/backup-db.sh) runs INSIDE this container, where SUPABASE_DB_URL and +# the Supabase network are available. See docs/BACKUP-RUNBOOK.md. +RUN apk add --no-cache curl tini bash postgresql-client rsync # PM2 is installed globally so the entrypoint can call `pm2 start` to # boot all three pollers (NBA / WNBA / MLB) alongside the Express API. diff --git a/docs/BACKUP-RUNBOOK.md b/docs/BACKUP-RUNBOOK.md index 83b97ac..41b4dc0 100644 --- a/docs/BACKUP-RUNBOOK.md +++ b/docs/BACKUP-RUNBOOK.md @@ -4,29 +4,35 @@ Supabase free tier has **zero** backups (no scheduled, no PITR). `scripts/backup is the safety net: a nightly full-database `pg_dump`, 14 days kept locally, a weekly copy pushed off-box, ntfy alert on any failure. -## What Kev needs to set (one env var) +## Where it runs -**`SUPABASE_DB_URL`** — the Supabase **direct** connection string (session mode). -Supabase → Project → Settings → Database → **Connection string → URI**, the -`db..supabase.co:5432` one (NOT the `:6543` transaction pooler — `pg_dump` -needs a real session). Paste it in Coolify as an env var; never commit it. +`SUPABASE_DB_URL` is set in Coolify **on the VYNDR API service** — so the backup +runs **inside that container**, which already has the env, the Supabase network, +and now `pg_dump`/`pg_restore`/`rsync` (added to the Dockerfile). The Hetzner box +can't reach `db..supabase.co` directly and doesn't hold the connection +string; the container is the right place. -Optional: `BACKUP_REMOTE` (off-box rsync target for the weekly copy — see below). +**`SUPABASE_DB_URL`** must be the **direct** connection string (session mode): +Supabase → Settings → Database → **Connection string → URI**, the +`db..supabase.co:5432` one (NOT the `:6543` pooler — `pg_dump` needs a real +session). Already set. Optional: `BACKUP_DIR` (mount a Coolify **persistent +volume** here so dumps survive redeploys — e.g. `/var/backups/vyndr`), +`BACKUP_REMOTE` (off-box rsync target, below). -## Install the cron (on the Hetzner box) +## Install the cron (host → docker exec into the API container) + +Find the API container name (`docker ps | grep vyndr`), then a host cron: ```bash -# 1. Ensure the client is present -sudo apt-get install -y postgresql-client rsync - -# 2. Nightly at 03:10 UTC. Pass the env the script needs (or source an env file). +# Nightly at 03:10 UTC. BACKUP_REMOTE can also be set in Coolify instead. sudo crontab -e -# add: -10 3 * * * SUPABASE_DB_URL='postgresql://...' BACKUP_REMOTE='u123456@u123456.your-storagebox.de:vyndr-backups/' /path/to/vyndr/scripts/backup-db.sh >> /var/log/vyndr-backup.log 2>&1 +# add (replace ): +10 3 * * * docker exec -e BACKUP_REMOTE='u123456@u123456.your-storagebox.de:vyndr-backups/' sh /app/scripts/backup-db.sh >> /var/log/vyndr-backup.log 2>&1 ``` -(If the cron runs inside the Coolify container instead, the env vars are already -present — just schedule `scripts/backup-db.sh`.) +`docker exec` inherits the container's env (`SUPABASE_DB_URL`) + network + the +newly-installed `pg_dump`. (If you'd rather, Coolify's **Scheduled Tasks** can run +`sh /app/scripts/backup-db.sh` on the API service on the same cadence.) ## Off-box target (simplest reliable pick) @@ -38,14 +44,24 @@ rather keep it off Hetzner entirely — swap the `rsync` line for `rclone copy`. ## Restore / FINGERPRINT (proves it's a real backup, not just a file) -A dump only counts once a restore of it succeeds. Load one into a scratch DB: +**Built-in (runs on every backup):** the script validates each dump with +`pg_restore --list` — a dump that isn't a valid archive, or that doesn't contain +`ledger_entries`, is treated as a FAILURE and paged. So every successful run has +already proven the archive parses and holds the ledger. + +**Full restore proof (run once to fingerprint):** from the host, restore the +newest dump into a throwaway postgres and count the ledger: ```bash -# spin a throwaway local postgres, restore the newest dump, count a known table +# 1. Produce a dump on demand (writes into the container's BACKUP_DIR) +docker exec sh /app/scripts/backup-db.sh +# 2. Copy the newest dump out of the container +newest=$(docker exec sh -lc 'ls -t /var/backups/vyndr/vyndr-*.dump | head -1') +docker cp ":${newest}" /tmp/vyndr-latest.dump +# 3. Restore into a scratch postgres + count a known table docker run -d --name vyndr-restore-test -e POSTGRES_PASSWORD=x -p 55432:5432 postgres:15 -sleep 5 -newest=$(ls -t /var/backups/vyndr/vyndr-*.dump | head -1) -pg_restore --no-owner --no-privileges -d "postgresql://postgres:x@localhost:55432/postgres" "$newest" +sleep 6 +pg_restore --no-owner --no-privileges -d "postgresql://postgres:x@localhost:55432/postgres" /tmp/vyndr-latest.dump psql "postgresql://postgres:x@localhost:55432/postgres" -c "select count(*) from public.ledger_entries;" docker rm -f vyndr-restore-test ``` diff --git a/scripts/backup-db.sh b/scripts/backup-db.sh index 983dca5..6ea51a6 100644 --- a/scripts/backup-db.sh +++ b/scripts/backup-db.sh @@ -51,6 +51,15 @@ pg_dump "${SUPABASE_DB_URL}" -Fc --no-owner --no-privileges -f "${DUMP}" \ SIZE="$(stat -c%s "${DUMP}" 2>/dev/null || echo 0)" [ "${SIZE}" -ge "${MIN_BYTES}" ] || fail "dump is only ${SIZE} bytes (< ${MIN_BYTES}) — treating as a failed backup" +# 2b. Integrity fingerprint: a valid custom-format archive lists its objects via +# pg_restore --list (no target DB needed). Confirm it parses AND contains the +# ledger — proves it's a real, restorable archive, not just a file of bytes. +TOC="$(pg_restore --list "${DUMP}" 2>/dev/null)" || fail "pg_restore --list failed — dump is not a valid archive" +OBJECTS="$(printf '%s\n' "${TOC}" | grep -c ';' || true)" +printf '%s\n' "${TOC}" | grep -qi 'TABLE DATA public ledger_entries' \ + || fail "dump archive does not contain ledger_entries — refusing to trust it" +echo "backup validated: ${DUMP} (${SIZE} bytes, ${OBJECTS} archive objects, ledger_entries present)" + # 3. Rotate: drop local dumps older than KEEP_DAYS. find "${BACKUP_DIR}" -name 'vyndr-*.dump' -type f -mtime "+${KEEP_DAYS}" -delete || true