S8 (a1): ops — the product watches itself

Settlement alarm (settle-pass THROW pages high; morning zero-settle alarm keyed off the Postgres ledger settle results, once per ET date, never on an empty yesterday), per-sport 3-consecutive-slot failure pager (pure opsWatch.createFailureTracker, pages once per losing streak), odds-api >=80% quota alert (once per day, Redis-deduped), systemHealth (statfs + os mem, disk>85 / mem>90 pages), daily 9 AM ET pulse (ONE notification: ledger rows yesterday via ledgerService.countRowsForDate, settles 24h, quota, disk/mem, desk line), docs/OPS-RUNBOOK.md (Uptime Kuma monitors, Coolify deploy-failure -> ntfy, phone subscription). All copy VOICE v1.1 — deadpan, numbers, no exclamation points (tests lint for it). 2398 -> 2437 tests. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-11 14:34:45 -04:00
parent e4d2e79f95
commit 044941d406
11 changed files with 1083 additions and 4 deletions
@@ -0,0 +1,60 @@
+'use strict';
+
+/**
+ * systemHealth — box vitals (Session 8, A1 board — ops).
+ *
+ * Pure + injectable: disk usage via fs.promises.statfs('/'), memory via
+ * os.freemem/os.totalmem. Returns percentages (0-100, rounded) or null when a
+ * probe fails — the caller renders "n/a", never a fabricated number (Data
+ * Semantics Rule applies to ops copy too). Zero dependencies, zero cost.
+ */
+
+const fsp = require('fs').promises;
+const os = require('os');
+
+/** Alerting thresholds — checked by the daily pulse. */
+const THRESHOLDS = { disk_pct: 85, mem_pct: 90 };
+
+const roundPct = (frac) => Math.min(100, Math.max(0, Math.round(frac * 100)));
+
+/**
+ * { disk_pct, mem_pct } — each null when its probe fails.
+ * opts: { fsImpl (statfs), osImpl (freemem/totalmem), path }.
+ */
+async function getSystemHealth(opts = {}) {
+  const fsImpl = opts.fsImpl || fsp;
+  const osImpl = opts.osImpl || os;
+  const out = { disk_pct: null, mem_pct: null };
+  try {
+    const s = await fsImpl.statfs(opts.path || '/');
+    if (s && Number.isFinite(s.blocks) && s.blocks > 0 && Number.isFinite(s.bavail)) {
+      out.disk_pct = roundPct(1 - s.bavail / s.blocks);
+    }
+  } catch { /* disk_pct stays null */ }
+  try {
+    const total = osImpl.totalmem();
+    const free = osImpl.freemem();
+    if (Number.isFinite(total) && total > 0 && Number.isFinite(free)) {
+      out.mem_pct = roundPct(1 - free / total);
+    }
+  } catch { /* mem_pct stays null */ }
+  return out;
+}
+
+/**
+ * Deadpan issue lines for anything over threshold (strictly >). Empty array =
+ * healthy. Copy carries the numbers; no punctuation theatrics.
+ */
+function healthIssues(health, thresholds = THRESHOLDS) {
+  const issues = [];
+  if (!health) return issues;
+  if (Number.isFinite(health.disk_pct) && health.disk_pct > thresholds.disk_pct) {
+    issues.push(`disk at ${health.disk_pct}% (threshold ${thresholds.disk_pct}%)`);
+  }
+  if (Number.isFinite(health.mem_pct) && health.mem_pct > thresholds.mem_pct) {
+    issues.push(`memory at ${health.mem_pct}% (threshold ${thresholds.mem_pct}%)`);
+  }
+  return issues;
+}
+
+module.exports = { getSystemHealth, healthIssues, THRESHOLDS };