909ae8a713
scripts/host/relay-health.sh (run by skatespots-health.timer every 5 min on the relay box): checks relay/ai/push/cvm/caddy + public endpoints + disk/mem, restarts the relay when its localhost NIP-11 dies, and alerts via ntfy.sh on state changes.
72 lines
2.6 KiB
Bash
72 lines
2.6 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# SkateSpots relay-host health check (runs ON the relay box every 5 minutes).
|
|
#
|
|
# - verifies the host services (relay/ai/push/cvm/caddy) and the public endpoints
|
|
# - restarts `skatespots-relay` if its localhost NIP-11 endpoint is dead
|
|
# - checks disk + memory headroom
|
|
# - sends an alert to ntfy.sh only when the problem set CHANGES (no spam)
|
|
#
|
|
# The ntfy topic lives in /home/ubuntu/skatespots-health.env (mode 600, not in
|
|
# the repo): NTFY_TOPIC=skatespots-<random>
|
|
# Subscribe to it with the ntfy app (or `curl -s ntfy.sh/<topic>/json`).
|
|
#
|
|
# Manual run: sudo /home/ubuntu/skatespots-health.sh
|
|
set -uo pipefail
|
|
|
|
ENV_FILE="/home/ubuntu/skatespots-health.env"
|
|
# shellcheck disable=SC1090
|
|
[ -f "$ENV_FILE" ] && . "$ENV_FILE"
|
|
|
|
STATE_FILE="/home/ubuntu/.skatespots-health.state"
|
|
ISSUES=()
|
|
|
|
svc() { systemctl is-active --quiet "$1" || ISSUES+=("service $1 is not active"); }
|
|
code() { curl -s -o /dev/null -m 8 -w '%{http_code}' "$@" || echo 000; }
|
|
|
|
# Relay on localhost: if dead, attempt an automatic restart.
|
|
rc=$(code -H 'Accept: application/nostr+json' http://127.0.0.1:7777/)
|
|
if [ "$rc" != "200" ]; then
|
|
ISSUES+=("relay localhost HTTP $rc - restarting skatespots-relay")
|
|
systemctl restart skatespots-relay.service || ISSUES+=("relay restart FAILED")
|
|
fi
|
|
|
|
svc skatespots-relay.service
|
|
svc skatespots-ai.service
|
|
svc skatespots-push.service
|
|
svc skatespots-cvm.service
|
|
svc caddy.service
|
|
|
|
[ "$(code -H 'Accept: application/nostr+json' https://relay.skate-spots.org/)" != "200" ] \
|
|
&& ISSUES+=("relay public TLS not answering 200")
|
|
[ "$(code https://push.skate-spots.org/health)" != "200" ] \
|
|
&& ISSUES+=("push /health not 200")
|
|
|
|
disk=$(df --output=pcent / | tail -1 | tr -d ' %')
|
|
[ "${disk:-0}" -ge 85 ] && ISSUES+=("disk ${disk}% used")
|
|
avail=$(free -m | awk '/Mem:/{print $7}')
|
|
[ "${avail:-0}" -lt 300 ] && ISSUES+=("low memory: ${avail} MiB available")
|
|
|
|
sig=""
|
|
[ "${#ISSUES[@]}" -gt 0 ] && sig=$(printf '%s\n' "${ISSUES[@]}" | sort | tr '\n' '|')
|
|
prev=""
|
|
[ -f "$STATE_FILE" ] && prev=$(cat "$STATE_FILE")
|
|
|
|
if [ "$sig" != "$prev" ]; then
|
|
if [ -n "$sig" ] && [ -n "${NTFY_TOPIC:-}" ]; then
|
|
curl -s -m 8 \
|
|
-H "Title: SkateSpots host problem" -H "Priority: high" -H "Tags: warning" \
|
|
-d "$(printf '%s\n' "${ISSUES[@]}")" "https://ntfy.sh/${NTFY_TOPIC}" >/dev/null || true
|
|
elif [ -z "$sig" ] && [ -n "$prev" ] && [ -n "${NTFY_TOPIC:-}" ]; then
|
|
curl -s -m 8 \
|
|
-H "Title: SkateSpots host recovered" -H "Tags: white_check_mark" \
|
|
-d "All checks passing again." "https://ntfy.sh/${NTFY_TOPIC}" >/dev/null || true
|
|
fi
|
|
printf '%s' "$sig" > "$STATE_FILE"
|
|
fi
|
|
|
|
if [ "${#ISSUES[@]}" -gt 0 ]; then
|
|
printf 'PROBLEM: %s\n' "${ISSUES[@]}"
|
|
exit 1
|
|
fi
|
|
echo "ok"
|