Phase 1: ship the backup cron as CODE + a manual regrade trigger

FOUNDATION-FIRST re-order, phase 1 (tooling + safety).

BACKUP (highest-severity open item) — INSTALLED, not re-proven.
src/backupScheduler.js runs scripts/backup-db.sh nightly from inside the
API container, armed at boot in server.js. The container already has
SUPABASE_DB_URL, pg_dump and the Supabase route, so deploy == installed:
no host crontab, no Coolify click. Arming is deliberately opt-OUT (armed
whenever SUPABASE_DB_URL exists; BACKUP_CRON=0 kills it) because the S62
design was opt-in and nobody ever opted in — the DB went unbacked every
night for weeks. A failed run pages high-priority ntfy; silence is the
danger with backups.

Durability is the one part still needing a human: the container FS is
ephemeral, so a dump dies on redeploy unless BACKUP_REMOTE (off-box
rsync) or BACKUP_DIR (persistent volume) is set. The scheduler detects
that and pages a WARNING at boot rather than letting an undurable backup
read as "backed up". Runbook rewritten to lead with the code path.

MANUAL REGRADE TRIGGER — scripts/run-snapshot.js, runnable via
docker exec with no VYNDR_INTERNAL_KEY and no new HTTP surface. Runs the
SAME snapshotService.runSnapshot the cron runs (including the team-stats
refresh that powers opp_rank_stat), supports `all` and `--settle`, and
prints the grade/confidence distribution plus p_win/ev_pct presence —
which is the thing you actually want when verifying a grading change.

ACCESS BLOCKER, logged honestly in specs/model-train.md: there is no
VYNDR_INTERNAL_KEY in the local .env and SSH to the box times out from
WSL2, so I can neither curl the internal endpoints (which already exist
from S45) nor docker exec. The trigger is built and correct but only Kev
can run it until a key or SSH access exists. This is the highest-leverage
unblock for phases 2 and 3, which both need on-demand regrade+settle to
verify anything.

Also logged the standing cautions: CLV ledger stays private until
backtest-proven; "self-improving model" is unsupported marketing until
the loop closes; the engine is MLB/WNBA-calibrated and NFL/NBA/soccer
need their own calibration before the hub grades them (scaling gate).

Suite 277/3300 green.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SmNjJAwEnqHPtXbvSZR8kA

src/backupScheduler.js

@@ -0,0 +1,115 @@
+'use strict';
+
+/**
+ * NIGHTLY DATABASE BACKUP — in-process scheduler (Session 64, Phase 1).
+ *
+ * Supabase free tier has ZERO backups. `scripts/backup-db.sh` was written and
+ * mechanism-verified in Session 62, but it was never INSTALLED: it depended on
+ * someone adding a host cron or a Coolify Scheduled Task by hand, and that never
+ * happened. The ledger has been unbacked every night since.
+ *
+ * So the cron ships as CODE, in the container that already holds
+ * SUPABASE_DB_URL, pg_dump and the Supabase network route. Deploy = installed.
+ * No host shell, no dashboard click, nothing to forget.
+ *
+ * ARMING (deliberately opt-OUT, not opt-in — the whole failure mode was an
+ * opt-in that nobody opted into):
+ *   - armed whenever SUPABASE_DB_URL is present
+ *   - BACKUP_CRON=0 is the explicit kill switch
+ *   - BACKUP_HOUR_UTC (default 3) / BACKUP_MINUTE_UTC (default 10)
+ *
+ * DURABILITY CAVEAT (reported, not hidden): the container filesystem is
+ * ephemeral. Dumps survive a redeploy ONLY if BACKUP_DIR points at a persistent
+ * volume, or BACKUP_REMOTE pushes them off-box. We detect and page when neither
+ * is set, because an undurable backup that reads as "backed up" is worse than a
+ * loud gap.
+ */
+
+const path = require('path');
+
+const SCRIPT = path.join(__dirname, '..', 'scripts', 'backup-db.sh');
+
+function hourUtc() { return Number(process.env.BACKUP_HOUR_UTC || 3); }
+function minuteUtc() { return Number(process.env.BACKUP_MINUTE_UTC || 10); }
+
+/** Is durable retention configured? Neither → the dump dies with the container. */
+function durabilityWarning(env = process.env) {
+  const hasRemote = !!(env.BACKUP_REMOTE && String(env.BACKUP_REMOTE).trim());
+  const hasVolume = !!(env.BACKUP_DIR && String(env.BACKUP_DIR).trim());
+  if (hasRemote || hasVolume) return null;
+  return 'backups are written inside an ephemeral container — set BACKUP_REMOTE (off-box rsync) or point BACKUP_DIR at a persistent volume, or every dump is lost on redeploy';
+}
+
+function shouldArm(env = process.env) {
+  if (env.BACKUP_CRON === '0') return { armed: false, reason: 'BACKUP_CRON=0 (kill switch)' };
+  if (!env.SUPABASE_DB_URL) return { armed: false, reason: 'SUPABASE_DB_URL unset (nothing to dump)' };
+  return { armed: true, reason: null };
+}
+
+/** One backup run. Resolves { ok, code, ms } — never throws. */
+function runBackup(deps = {}) {
+  const spawn = deps.spawn || require('child_process').spawn;
+  const started = Date.now();
+  return new Promise((resolve) => {
+    let child;
+    try {
+      child = spawn('sh', [SCRIPT], { env: process.env, stdio: ['ignore', 'pipe', 'pipe'] });
+    } catch (e) {
+      return resolve({ ok: false, code: null, ms: Date.now() - started, error: e.message });
+    }
+    let tail = '';
+    const grab = (b) => { tail = (tail + b.toString()).slice(-2000); };
+    if (child.stdout) child.stdout.on('data', grab);
+    if (child.stderr) child.stderr.on('data', grab);
+    child.on('error', (e) => resolve({ ok: false, code: null, ms: Date.now() - started, error: e.message, tail }));
+    child.on('close', (code) => resolve({ ok: code === 0, code, ms: Date.now() - started, tail }));
+  });
+}
+
+function startBackupScheduler(opts = {}) {
+  const notify = opts.notify || require('./utils/opsNotify').notify;
+  const now = opts.now || (() => new Date());
+  const doBackup = opts.runBackup || runBackup;
+
+  const { armed, reason } = shouldArm(opts.env || process.env);
+  if (!armed) {
+    console.log(`[backupScheduler] disarmed — ${reason}`);
+    return null;
+  }
+
+  const warn = durabilityWarning(opts.env || process.env);
+  console.log(`[backupScheduler] armed — nightly ${String(hourUtc()).padStart(2, '0')}:${String(minuteUtc()).padStart(2, '0')} UTC via ${SCRIPT}${warn ? ` — WARNING: ${warn}` : ''}`);
+  if (warn) {
+    void notify(`⚠️ VYNDR backup: ${warn}`, { title: 'VYNDR backup', priority: 'default', tags: ['warning'] });
+  }
+
+  let lastRunDay = null;
+
+  async function tick() {
+    const d = now();
+    if (d.getUTCHours() !== hourUtc() || d.getUTCMinutes() !== minuteUtc()) return;
+    const dayKey = d.toISOString().slice(0, 10);
+    if (lastRunDay === dayKey) return;      // once per day, even if the tick repeats
+    lastRunDay = dayKey;
+
+    console.log(`[backupScheduler] starting nightly backup (${dayKey})`);
+    const res = await doBackup(opts);
+    if (res.ok) {
+      console.log(`[backupScheduler] backup OK in ${Math.round(res.ms / 1000)}s`);
+    } else {
+      console.error(`[backupScheduler] backup FAILED (code ${res.code}): ${res.error || ''}`);
+      await notify(
+        `❌ VYNDR nightly backup FAILED (exit ${res.code}). The database is unprotected tonight.`,
+        { title: 'VYNDR backup', priority: 'high', tags: ['x'] },
+      );
+    }
+  }
+
+  const interval = setInterval(() => { void tick(); }, 60_000);
+  if (interval.unref) interval.unref();
+  return { interval, tick };
+}
+
+module.exports = {
+  startBackupScheduler, runBackup, shouldArm, durabilityWarning, SCRIPT,
+};

src/server.js

@@ -14,6 +14,7 @@ const { getConfiguredProviders, listProviderIds } = require('./config/providers'
 const { scheduleStartupPrefetch } = require('./startupPrefetch');
 // Session 45 — in-process snapshot cron (gated on SNAPSHOT_CRON=1).
 const { startSnapshotScheduler } = require('./snapshotScheduler');
+const { startBackupScheduler } = require('./backupScheduler');
 
 // Default 3001 — Next.js owns 3000 locally and in production. The poller,
 // internal cron, and BASE_URL conventions all assume 3001 for the Express
@@ -35,4 +36,10 @@ app.listen(PORT, () => {
 
   // Session 45 — arm the snapshot cron (no-op unless SNAPSHOT_CRON=1).
   startSnapshotScheduler();
+
+  // Session 64 — arm the NIGHTLY BACKUP. Opt-OUT (armed whenever
+  // SUPABASE_DB_URL exists; BACKUP_CRON=0 kills it). The host cron was written
+  // in S62 and never installed, so the database went unbacked every night;
+  // shipping it as code means deploy == installed.
+  startBackupScheduler();
 });