From 909ae8a713aaff132afcd67d5b6142ea35b35a48 Mon Sep 17 00:00:00 2001
From: SkateSpots <skatespots>
Date: Mon, 12 Oct 2026 12:54:08 +0700
Subject: [PATCH] chore(relay): host health check + self-heal + ntfy alerts

scripts/host/relay-health.sh (run by skatespots-health.timer every 5 min on the relay box): checks relay/ai/push/cvm/caddy + public endpoints + disk/mem, restarts the relay when its localhost NIP-11 dies, and alerts via ntfy.sh on state changes.
---
 AGENTS.md                                 |  7 +++
 scripts/host/relay-health.sh              | 72 +++++++++++++++++++++++
 scripts/systemd/skatespots-health.service |  8 +++
 scripts/systemd/skatespots-health.timer   | 10 ++++
 4 files changed, 97 insertions(+)
 create mode 100644 scripts/host/relay-health.sh
 create mode 100644 scripts/systemd/skatespots-health.service
 create mode 100644 scripts/systemd/skatespots-health.timer

diff --git a/AGENTS.md b/AGENTS.md
index 64501e0..1ae7189 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -2098,6 +2098,13 @@ The community relay moved off the hosted relay.tools instance
   `enable --now`; tracked at `scripts/systemd/skatespots-relay.service`). Restart
   with `sudo systemctl restart skatespots-relay`. The container also carries
   `restart: unless-stopped`.
+- **Monitoring (2026-10-12):** `skatespots-health.{service,timer}` on the host run
+  `scripts/host/relay-health.sh` every 5 min — checks relay/ai/push/cvm/caddy + the
+  public endpoints + disk/memory, **restarts the relay** if its localhost NIP-11
+  dies, and posts to **ntfy.sh** when the problem set changes (deduped; also a
+  "recovered" message). The topic is in `/home/ubuntu/skatespots-health.env`
+  (mode 600, not tracked) — subscribe to it in the ntfy app. Units tracked at
+  `scripts/systemd/skatespots-health.{service,timer}`.
 - **Cutover (2026-10-12):** `SKATE_RELAY` (`src/lib/types.ts`) → the new relay; the
   default pools, the DM/CVM/zap relay arrays, the `cvm-server` + `push-service`
   defaults, and the archive default all point at the new relay **with the legacy
diff --git a/scripts/host/relay-health.sh b/scripts/host/relay-health.sh
new file mode 100644
index 0000000..fa0d0ef
--- /dev/null
+++ b/scripts/host/relay-health.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+#
+# SkateSpots relay-host health check (runs ON the relay box every 5 minutes).
+#
+# - verifies the host services (relay/ai/push/cvm/caddy) and the public endpoints
+# - restarts `skatespots-relay` if its localhost NIP-11 endpoint is dead
+# - checks disk + memory headroom
+# - sends an alert to ntfy.sh only when the problem set CHANGES (no spam)
+#
+# The ntfy topic lives in /home/ubuntu/skatespots-health.env (mode 600, not in
+# the repo):  NTFY_TOPIC=skatespots-<random>
+# Subscribe to it with the ntfy app (or `curl -s ntfy.sh/<topic>/json`).
+#
+# Manual run:  sudo /home/ubuntu/skatespots-health.sh
+set -uo pipefail
+
+ENV_FILE="/home/ubuntu/skatespots-health.env"
+# shellcheck disable=SC1090
+[ -f "$ENV_FILE" ] && . "$ENV_FILE"
+
+STATE_FILE="/home/ubuntu/.skatespots-health.state"
+ISSUES=()
+
+svc() { systemctl is-active --quiet "$1" || ISSUES+=("service $1 is not active"); }
+code() { curl -s -o /dev/null -m 8 -w '%{http_code}' "$@" || echo 000; }
+
+# Relay on localhost: if dead, attempt an automatic restart.
+rc=$(code -H 'Accept: application/nostr+json' http://127.0.0.1:7777/)
+if [ "$rc" != "200" ]; then
+  ISSUES+=("relay localhost HTTP $rc - restarting skatespots-relay")
+  systemctl restart skatespots-relay.service || ISSUES+=("relay restart FAILED")
+fi
+
+svc skatespots-relay.service
+svc skatespots-ai.service
+svc skatespots-push.service
+svc skatespots-cvm.service
+svc caddy.service
+
+[ "$(code -H 'Accept: application/nostr+json' https://relay.skate-spots.org/)" != "200" ] \
+  && ISSUES+=("relay public TLS not answering 200")
+[ "$(code https://push.skate-spots.org/health)" != "200" ] \
+  && ISSUES+=("push /health not 200")
+
+disk=$(df --output=pcent / | tail -1 | tr -d ' %')
+[ "${disk:-0}" -ge 85 ] && ISSUES+=("disk ${disk}% used")
+avail=$(free -m | awk '/Mem:/{print $7}')
+[ "${avail:-0}" -lt 300 ] && ISSUES+=("low memory: ${avail} MiB available")
+
+sig=""
+[ "${#ISSUES[@]}" -gt 0 ] && sig=$(printf '%s\n' "${ISSUES[@]}" | sort | tr '\n' '|')
+prev=""
+[ -f "$STATE_FILE" ] && prev=$(cat "$STATE_FILE")
+
+if [ "$sig" != "$prev" ]; then
+  if [ -n "$sig" ] && [ -n "${NTFY_TOPIC:-}" ]; then
+    curl -s -m 8 \
+      -H "Title: SkateSpots host problem" -H "Priority: high" -H "Tags: warning" \
+      -d "$(printf '%s\n' "${ISSUES[@]}")" "https://ntfy.sh/${NTFY_TOPIC}" >/dev/null || true
+  elif [ -z "$sig" ] && [ -n "$prev" ] && [ -n "${NTFY_TOPIC:-}" ]; then
+    curl -s -m 8 \
+      -H "Title: SkateSpots host recovered" -H "Tags: white_check_mark" \
+      -d "All checks passing again." "https://ntfy.sh/${NTFY_TOPIC}" >/dev/null || true
+  fi
+  printf '%s' "$sig" > "$STATE_FILE"
+fi
+
+if [ "${#ISSUES[@]}" -gt 0 ]; then
+  printf 'PROBLEM: %s\n' "${ISSUES[@]}"
+  exit 1
+fi
+echo "ok"
diff --git a/scripts/systemd/skatespots-health.service b/scripts/systemd/skatespots-health.service
new file mode 100644
index 0000000..fa5083b
--- /dev/null
+++ b/scripts/systemd/skatespots-health.service
@@ -0,0 +1,8 @@
+[Unit]
+Description=SkateSpots relay host health check
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/home/ubuntu/skatespots-health.sh
diff --git a/scripts/systemd/skatespots-health.timer b/scripts/systemd/skatespots-health.timer
new file mode 100644
index 0000000..5d2e200
--- /dev/null
+++ b/scripts/systemd/skatespots-health.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Run the SkateSpots relay host health check every 5 minutes
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=5min
+AccuracySec=30s
+
+[Install]
+WantedBy=timers.target