From b742230d94831719396a79ca5765d56578d020d6 Mon Sep 17 00:00:00 2001 From: Kev Date: Sun, 19 Jul 2026 19:28:15 -0400 Subject: [PATCH] Phase 1: ship the backup cron as CODE + a manual regrade trigger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FOUNDATION-FIRST re-order, phase 1 (tooling + safety). BACKUP (highest-severity open item) — INSTALLED, not re-proven. src/backupScheduler.js runs scripts/backup-db.sh nightly from inside the API container, armed at boot in server.js. The container already has SUPABASE_DB_URL, pg_dump and the Supabase route, so deploy == installed: no host crontab, no Coolify click. Arming is deliberately opt-OUT (armed whenever SUPABASE_DB_URL exists; BACKUP_CRON=0 kills it) because the S62 design was opt-in and nobody ever opted in — the DB went unbacked every night for weeks. A failed run pages high-priority ntfy; silence is the danger with backups. Durability is the one part still needing a human: the container FS is ephemeral, so a dump dies on redeploy unless BACKUP_REMOTE (off-box rsync) or BACKUP_DIR (persistent volume) is set. The scheduler detects that and pages a WARNING at boot rather than letting an undurable backup read as "backed up". Runbook rewritten to lead with the code path. MANUAL REGRADE TRIGGER — scripts/run-snapshot.js, runnable via docker exec with no VYNDR_INTERNAL_KEY and no new HTTP surface. Runs the SAME snapshotService.runSnapshot the cron runs (including the team-stats refresh that powers opp_rank_stat), supports `all` and `--settle`, and prints the grade/confidence distribution plus p_win/ev_pct presence — which is the thing you actually want when verifying a grading change. ACCESS BLOCKER, logged honestly in specs/model-train.md: there is no VYNDR_INTERNAL_KEY in the local .env and SSH to the box times out from WSL2, so I can neither curl the internal endpoints (which already exist from S45) nor docker exec. The trigger is built and correct but only Kev can run it until a key or SSH access exists. This is the highest-leverage unblock for phases 2 and 3, which both need on-demand regrade+settle to verify anything. Also logged the standing cautions: CLV ledger stays private until backtest-proven; "self-improving model" is unsupported marketing until the loop closes; the engine is MLB/WNBA-calibrated and NFL/NBA/soccer need their own calibration before the hub grades them (scaling gate). Suite 277/3300 green. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01SmNjJAwEnqHPtXbvSZR8kA --- docs/BACKUP-RUNBOOK.md | 33 +++++++++ scripts/run-snapshot.js | 89 ++++++++++++++++++++++ specs/model-train.md | 44 +++++++++++ src/backupScheduler.js | 115 +++++++++++++++++++++++++++++ src/server.js | 7 ++ tests/unit/backupScheduler.test.js | 99 +++++++++++++++++++++++++ 6 files changed, 387 insertions(+) create mode 100644 scripts/run-snapshot.js create mode 100644 src/backupScheduler.js create mode 100644 tests/unit/backupScheduler.test.js diff --git a/docs/BACKUP-RUNBOOK.md b/docs/BACKUP-RUNBOOK.md index 41b4dc0..3c46591 100644 --- a/docs/BACKUP-RUNBOOK.md +++ b/docs/BACKUP-RUNBOOK.md @@ -19,6 +19,39 @@ session). Already set. Optional: `BACKUP_DIR` (mount a Coolify **persistent volume** here so dumps survive redeploys — e.g. `/var/backups/vyndr`), `BACKUP_REMOTE` (off-box rsync target, below). +## ✅ THE CRON NOW SHIPS AS CODE (Session 64) — no install step + +**Read this before following the manual instructions below; they are now the +FALLBACK, not the primary path.** + +`src/backupScheduler.js` runs the nightly backup **inside the API container**, +armed from `server.js` at boot. The container already holds `SUPABASE_DB_URL`, +`pg_dump` and the Supabase network route, so **deploy == installed**. Nothing to +add to crontab, nothing to click in Coolify. + +- **Arming is opt-OUT:** armed whenever `SUPABASE_DB_URL` is set. The S62 design + was opt-in (a host cron someone had to add) and nobody ever added it — the + database went unbacked every night for weeks. That failure mode is now + impossible. +- **Kill switch:** `BACKUP_CRON=0`. +- **Schedule:** `BACKUP_HOUR_UTC` (default 3) / `BACKUP_MINUTE_UTC` (default 10). +- **Failure pages high-priority ntfy.** Silence is the danger with backups. +- Boot log line: `[backupScheduler] armed — nightly 03:10 UTC ...` + +### ⚠️ DURABILITY — the one thing still requiring a human + +The container filesystem is **ephemeral**: a dump written inside it is LOST on the +next redeploy. The scheduler detects this and pages a warning at boot when +neither is configured. Set ONE of: + +1. **`BACKUP_REMOTE`** — off-box rsync target (Hetzner Storage Box, ~€3/mo). Best. +2. **`BACKUP_DIR`** pointed at a **Coolify persistent volume** (e.g. `/var/backups/vyndr`). + +Until one is set, backups run but do not survive a deploy. An undurable backup +that reads as "backed up" is worse than a loud gap — hence the boot-time page. + +--- + ## Install the cron (host → docker exec into the API container) Find the API container name (`docker ps | grep vyndr`), then a host cron: diff --git a/scripts/run-snapshot.js b/scripts/run-snapshot.js new file mode 100644 index 0000000..16c6015 --- /dev/null +++ b/scripts/run-snapshot.js @@ -0,0 +1,89 @@ +#!/usr/bin/env node +/** + * MANUAL REGRADE TRIGGER (Session 64, Phase 1). + * + * Fires the snapshot pipeline on demand so a fix can be verified in minutes + * instead of waiting for the 14/19/22/01/03 UTC cron. Runs INSIDE the API + * container, so it needs no VYNDR_INTERNAL_KEY and no open HTTP surface: + * + * docker exec node scripts/run-snapshot.js mlb + * docker exec node scripts/run-snapshot.js all + * docker exec node scripts/run-snapshot.js mlb --settle + * + * It runs the SAME `snapshotService.runSnapshot` the cron runs — including the + * team-stats refresh that populates `opp_rank_stat` — so what you verify is what + * production does, not a parallel code path. + * + * `--settle` additionally runs the outcome + ledger settle pass first, matching + * the scheduler's real order (settle yesterday, then grade today). + * + * Prints a grade-distribution summary at the end, which is the thing you + * actually want when verifying a grading change. + */ + +const args = process.argv.slice(2); +const target = (args[0] || 'all').toLowerCase(); +const doSettle = args.includes('--settle'); + +function tally(list, key) { + return list.reduce((m, g) => { const k = g?.[key] ?? 'null'; m[k] = (m[k] || 0) + 1; return m; }, {}); +} + +(async () => { + const snapshotService = require('../src/services/snapshotService'); + + if (doSettle) { + console.log('--- settle pass (outcomes + ledger) ---'); + try { + const outcomeService = require('../src/services/outcomeService'); + for (const sp of ['mlb', 'wnba', 'nba']) { + try { + const r = await outcomeService.settleSnapshot(sp); + console.log(` ${sp}: ${JSON.stringify(r)}`); + } catch (e) { console.warn(` ${sp}: settle failed — ${e.message}`); } + } + const ledger = require('../src/services/ledgerService'); + if (typeof ledger.settleLedger === 'function') { + for (const sp of ['mlb', 'wnba']) { + try { console.log(` ledger ${sp}: ${JSON.stringify(await ledger.settleLedger(sp))}`); } + catch (e) { console.warn(` ledger ${sp}: ${e.message}`); } + } + } + } catch (e) { console.warn('settle pass failed:', e.message); } + } + + const sports = target === 'all' ? ['mlb', 'wnba', 'nba', 'soccer'] : [target]; + console.log(`--- snapshot: ${sports.join(', ')} ---`); + + const results = []; + for (const sp of sports) { + const t = Date.now(); + try { + const r = await snapshotService.runSnapshot(sp); + console.log(` ${sp}: status=${r.status} grades=${r.gradeCount}${r.reason ? ` reason=${r.reason}` : ''} (${Math.round((Date.now() - t) / 1000)}s)`); + results.push({ sp, r }); + } catch (e) { + console.error(` ${sp}: THREW — ${e.message}`); + } + } + + // Distribution — the point of running this by hand. + console.log('\n--- GRADE DISTRIBUTION (from the freshly written cache) ---'); + const { cacheGet } = require('../src/utils/redis'); + for (const { sp } of results) { + try { + const snap = await cacheGet(`snapshot:${sp}:latest`); + const grades = (snap && Array.isArray(snap.grades)) ? snap.grades : []; + if (!grades.length) { console.log(` ${sp}: (no grades)`); continue; } + const withEv = grades.filter((g) => Number.isFinite(Number(g.ev_pct))).length; + const withP = grades.filter((g) => Number.isFinite(Number(g.p_win))).length; + const withOpp = grades.filter((g) => g.opp_rank_stat != null).length; + console.log(` ${sp}: n=${grades.length} ${JSON.stringify(tally(grades, 'grade'))}`); + console.log(` confidence: ${JSON.stringify(tally(grades, 'confidence'))}`); + console.log(` p_win present: ${withP}/${grades.length} · ev_pct present: ${withEv}/${grades.length} · opp_rank on grade: ${withOpp}`); + } catch (e) { console.warn(` ${sp}: could not read cache — ${e.message}`); } + } + + await new Promise((r) => process.stdout.write('', r)); + process.exit(0); +})().catch((e) => { console.error('run-snapshot failed:', e); process.exit(1); }); diff --git a/specs/model-train.md b/specs/model-train.md index 6ee663e..568fff2 100644 --- a/specs/model-train.md +++ b/specs/model-train.md @@ -153,6 +153,50 @@ timestamp) is unchanged. `toHero` now also passes through --- +## 2B. FOUNDATION-FIRST RE-ORDER (Kev, 2026-07-19) — supersedes the arc order + +New features are paused until the foundation is real. Phases run in order: +1 (tooling+safety) → 2 (the instrument) → 3 (corruption fixes) → 4 (depth). + +| # | Item | Phase | Status | +|---|---|---|---| +| 1 | Manual regrade trigger | 1 | ✅ `scripts/run-snapshot.js` shipped — **but see ACCESS BLOCKER** | +| 2 | Backup cron INSTALLED | 1 | ✅ shipped as CODE (`src/backupScheduler.js`) — deploy == installed | +| 3 | Backtest harness | 2 | open — REPORT-FIRST on replayable history | +| 4 | Settlement-correctness audit | 2 | open — REPORT | +| 5 | Sample-size discipline | 2 | open — report the floor | +| 6 | edge_pct → ev_pct migration | 3 | open — REPORT-FIRST (ledger bleed) | +| 7 | Grade-lock + directional CLV + C4 | 3 | open — REPORT-FIRST (does a snapshot overwrite a prior grade?) | +| 8 | Model uncertainty | 4 | open — REPORT-FIRST (design proposal) | +| 9 | Calibration by odds band | 4 | open — runs ON the harness | + +### 🔴 ACCESS BLOCKER (Session 64) — I cannot reach the box + +Verified this session: **no `VYNDR_INTERNAL_KEY` in the local `.env`** (it holds +only Supabase + `ODDS_API_KEY`), and **SSH to `git.builtbykev.com` / +`api.vyndr.app` / `vyndr.app` times out from WSL2.** So I can neither call the +internal endpoints that already exist (S45 shipped +`POST /api/internal/snapshot/:sport|/all`) nor `docker exec` anything. + +Consequences, stated plainly: +- The manual trigger is BUILT and correct, but **only Kev can run it** until one + of these exists: `VYNDR_INTERNAL_KEY` shared with the agent env, or box SSH. +- Verifying a grading fix still depends on the cron or on Kev running one command. +- **This is the single highest-leverage unblock for every future phase** — Phase 2 + and 3 both need on-demand regrade + settle runs to verify anything. + +### Standing cautions (Kev, logged 2026-07-19) + +- **CLV ledger stays PRIVATE** until the model is backtest-proven. Publishing CLV + before then broadcasts our weaknesses to sharps. (Also currently broken — C4.) +- **"Self-improving model" is UNSUPPORTED marketing** until the loop actually + closes: backtest → calibration → weight correction. No such loop exists today + (no harness, no calibration gate, `weightAdjuster` is not in the grade path). + Do not claim it. +- **The engine is MLB/WNBA-calibrated. NFL/NBA/soccer are NOT.** "Unified engine" + is currently "MLB engine, others guessing." Each sport needs its own calibration + before the offseason hub grades it — this is a **scaling gate**, not a nice-to-have. + ## 2A. ARC 2+ BOARD — STATUS (Kev's arc list, 2026-07-19) Full arc definitions live in the Session-63 order. Status only here; update as each ships. diff --git a/src/backupScheduler.js b/src/backupScheduler.js new file mode 100644 index 0000000..9740c7c --- /dev/null +++ b/src/backupScheduler.js @@ -0,0 +1,115 @@ +'use strict'; + +/** + * NIGHTLY DATABASE BACKUP — in-process scheduler (Session 64, Phase 1). + * + * Supabase free tier has ZERO backups. `scripts/backup-db.sh` was written and + * mechanism-verified in Session 62, but it was never INSTALLED: it depended on + * someone adding a host cron or a Coolify Scheduled Task by hand, and that never + * happened. The ledger has been unbacked every night since. + * + * So the cron ships as CODE, in the container that already holds + * SUPABASE_DB_URL, pg_dump and the Supabase network route. Deploy = installed. + * No host shell, no dashboard click, nothing to forget. + * + * ARMING (deliberately opt-OUT, not opt-in — the whole failure mode was an + * opt-in that nobody opted into): + * - armed whenever SUPABASE_DB_URL is present + * - BACKUP_CRON=0 is the explicit kill switch + * - BACKUP_HOUR_UTC (default 3) / BACKUP_MINUTE_UTC (default 10) + * + * DURABILITY CAVEAT (reported, not hidden): the container filesystem is + * ephemeral. Dumps survive a redeploy ONLY if BACKUP_DIR points at a persistent + * volume, or BACKUP_REMOTE pushes them off-box. We detect and page when neither + * is set, because an undurable backup that reads as "backed up" is worse than a + * loud gap. + */ + +const path = require('path'); + +const SCRIPT = path.join(__dirname, '..', 'scripts', 'backup-db.sh'); + +function hourUtc() { return Number(process.env.BACKUP_HOUR_UTC || 3); } +function minuteUtc() { return Number(process.env.BACKUP_MINUTE_UTC || 10); } + +/** Is durable retention configured? Neither → the dump dies with the container. */ +function durabilityWarning(env = process.env) { + const hasRemote = !!(env.BACKUP_REMOTE && String(env.BACKUP_REMOTE).trim()); + const hasVolume = !!(env.BACKUP_DIR && String(env.BACKUP_DIR).trim()); + if (hasRemote || hasVolume) return null; + return 'backups are written inside an ephemeral container — set BACKUP_REMOTE (off-box rsync) or point BACKUP_DIR at a persistent volume, or every dump is lost on redeploy'; +} + +function shouldArm(env = process.env) { + if (env.BACKUP_CRON === '0') return { armed: false, reason: 'BACKUP_CRON=0 (kill switch)' }; + if (!env.SUPABASE_DB_URL) return { armed: false, reason: 'SUPABASE_DB_URL unset (nothing to dump)' }; + return { armed: true, reason: null }; +} + +/** One backup run. Resolves { ok, code, ms } — never throws. */ +function runBackup(deps = {}) { + const spawn = deps.spawn || require('child_process').spawn; + const started = Date.now(); + return new Promise((resolve) => { + let child; + try { + child = spawn('sh', [SCRIPT], { env: process.env, stdio: ['ignore', 'pipe', 'pipe'] }); + } catch (e) { + return resolve({ ok: false, code: null, ms: Date.now() - started, error: e.message }); + } + let tail = ''; + const grab = (b) => { tail = (tail + b.toString()).slice(-2000); }; + if (child.stdout) child.stdout.on('data', grab); + if (child.stderr) child.stderr.on('data', grab); + child.on('error', (e) => resolve({ ok: false, code: null, ms: Date.now() - started, error: e.message, tail })); + child.on('close', (code) => resolve({ ok: code === 0, code, ms: Date.now() - started, tail })); + }); +} + +function startBackupScheduler(opts = {}) { + const notify = opts.notify || require('./utils/opsNotify').notify; + const now = opts.now || (() => new Date()); + const doBackup = opts.runBackup || runBackup; + + const { armed, reason } = shouldArm(opts.env || process.env); + if (!armed) { + console.log(`[backupScheduler] disarmed — ${reason}`); + return null; + } + + const warn = durabilityWarning(opts.env || process.env); + console.log(`[backupScheduler] armed — nightly ${String(hourUtc()).padStart(2, '0')}:${String(minuteUtc()).padStart(2, '0')} UTC via ${SCRIPT}${warn ? ` — WARNING: ${warn}` : ''}`); + if (warn) { + void notify(`⚠️ VYNDR backup: ${warn}`, { title: 'VYNDR backup', priority: 'default', tags: ['warning'] }); + } + + let lastRunDay = null; + + async function tick() { + const d = now(); + if (d.getUTCHours() !== hourUtc() || d.getUTCMinutes() !== minuteUtc()) return; + const dayKey = d.toISOString().slice(0, 10); + if (lastRunDay === dayKey) return; // once per day, even if the tick repeats + lastRunDay = dayKey; + + console.log(`[backupScheduler] starting nightly backup (${dayKey})`); + const res = await doBackup(opts); + if (res.ok) { + console.log(`[backupScheduler] backup OK in ${Math.round(res.ms / 1000)}s`); + } else { + console.error(`[backupScheduler] backup FAILED (code ${res.code}): ${res.error || ''}`); + await notify( + `❌ VYNDR nightly backup FAILED (exit ${res.code}). The database is unprotected tonight.`, + { title: 'VYNDR backup', priority: 'high', tags: ['x'] }, + ); + } + } + + const interval = setInterval(() => { void tick(); }, 60_000); + if (interval.unref) interval.unref(); + return { interval, tick }; +} + +module.exports = { + startBackupScheduler, runBackup, shouldArm, durabilityWarning, SCRIPT, +}; diff --git a/src/server.js b/src/server.js index 4c1e551..c7b59da 100644 --- a/src/server.js +++ b/src/server.js @@ -14,6 +14,7 @@ const { getConfiguredProviders, listProviderIds } = require('./config/providers' const { scheduleStartupPrefetch } = require('./startupPrefetch'); // Session 45 — in-process snapshot cron (gated on SNAPSHOT_CRON=1). const { startSnapshotScheduler } = require('./snapshotScheduler'); +const { startBackupScheduler } = require('./backupScheduler'); // Default 3001 — Next.js owns 3000 locally and in production. The poller, // internal cron, and BASE_URL conventions all assume 3001 for the Express @@ -35,4 +36,10 @@ app.listen(PORT, () => { // Session 45 — arm the snapshot cron (no-op unless SNAPSHOT_CRON=1). startSnapshotScheduler(); + + // Session 64 — arm the NIGHTLY BACKUP. Opt-OUT (armed whenever + // SUPABASE_DB_URL exists; BACKUP_CRON=0 kills it). The host cron was written + // in S62 and never installed, so the database went unbacked every night; + // shipping it as code means deploy == installed. + startBackupScheduler(); }); diff --git a/tests/unit/backupScheduler.test.js b/tests/unit/backupScheduler.test.js new file mode 100644 index 0000000..211740d --- /dev/null +++ b/tests/unit/backupScheduler.test.js @@ -0,0 +1,99 @@ +/** + * Session 64 — nightly backup scheduler. + * + * The S62 backup script was mechanism-verified but never installed, so the + * database went unbacked every night. These lock the arming semantics that make + * "deploy == installed" true, and the honesty rules around durability. + */ + +const sched = require('../../src/backupScheduler'); + +describe('arming (opt-OUT, because opt-in is what failed)', () => { + test('armed whenever SUPABASE_DB_URL is present', () => { + expect(sched.shouldArm({ SUPABASE_DB_URL: 'postgres://x' }).armed).toBe(true); + }); + + test('BACKUP_CRON=0 is the kill switch', () => { + const r = sched.shouldArm({ SUPABASE_DB_URL: 'postgres://x', BACKUP_CRON: '0' }); + expect(r.armed).toBe(false); + expect(r.reason).toMatch(/kill switch/); + }); + + test('no SUPABASE_DB_URL → disarmed with a reason (nothing to dump)', () => { + const r = sched.shouldArm({}); + expect(r.armed).toBe(false); + expect(r.reason).toMatch(/SUPABASE_DB_URL/); + }); + + test('does NOT require an opt-in env var to arm', () => { + // The whole S62 failure: BACKUP_CRON=1 was never set by anyone. + expect(sched.shouldArm({ SUPABASE_DB_URL: 'postgres://x' }).armed).toBe(true); + }); +}); + +describe('durability honesty', () => { + test('warns when neither a volume nor an off-box target is configured', () => { + expect(sched.durabilityWarning({})).toMatch(/ephemeral/); + }); + + test('no warning once BACKUP_REMOTE is set', () => { + expect(sched.durabilityWarning({ BACKUP_REMOTE: 'u1@box:vyndr/' })).toBeNull(); + }); + + test('no warning once BACKUP_DIR is set (persistent volume)', () => { + expect(sched.durabilityWarning({ BACKUP_DIR: '/var/backups/vyndr' })).toBeNull(); + }); +}); + +describe('scheduling', () => { + const env = { SUPABASE_DB_URL: 'postgres://x', BACKUP_REMOTE: 'u1@box:v/' }; + + test('runs at the configured minute, and only once per day', async () => { + const runs = []; + const at = (h, m, day = 19) => new Date(Date.UTC(2026, 6, day, h, m)); + let clock = at(3, 10); + const s = sched.startBackupScheduler({ + env, + now: () => clock, + notify: async () => {}, + runBackup: async () => { runs.push(clock.toISOString()); return { ok: true, code: 0, ms: 1 }; }, + }); + await s.tick(); + await s.tick(); // same minute again — must not re-run + expect(runs).toHaveLength(1); + + clock = at(3, 10, 20); // next day + await s.tick(); + expect(runs).toHaveLength(2); + }); + + test('does nothing outside the scheduled minute', async () => { + const runs = []; + const s = sched.startBackupScheduler({ + env, + now: () => new Date(Date.UTC(2026, 6, 19, 14, 0)), + notify: async () => {}, + runBackup: async () => { runs.push(1); return { ok: true, code: 0, ms: 1 }; }, + }); + await s.tick(); + expect(runs).toHaveLength(0); + }); + + test('a failed backup pages high priority — silence is the danger', async () => { + const alerts = []; + const s = sched.startBackupScheduler({ + env, + now: () => new Date(Date.UTC(2026, 6, 19, 3, 10)), + notify: async (msg, opts) => { alerts.push({ msg, opts }); }, + runBackup: async () => ({ ok: false, code: 2, ms: 5 }), + }); + await s.tick(); + expect(alerts).toHaveLength(1); + expect(alerts[0].opts.priority).toBe('high'); + expect(alerts[0].msg).toMatch(/FAILED/); + }); + + test('disarmed scheduler returns null and schedules nothing', () => { + expect(sched.startBackupScheduler({ env: {}, notify: async () => {} })).toBeNull(); + }); +});