103 lines
4.3 KiB
Bash
103 lines
4.3 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# SkateSpots relay-host health check (runs ON the relay box every 5 minutes).
|
|
#
|
|
# - verifies the host services (relay/ai/push/cvm/caddy) and the public endpoints
|
|
# - restarts `skatespots-relay` if its localhost NIP-11 endpoint is dead
|
|
# - checks disk + memory headroom
|
|
# - sends an alert to ntfy.sh only when the problem set CHANGES (no spam)
|
|
#
|
|
# The ntfy topic lives in /home/ubuntu/skatespots-health.env (mode 600, not in
|
|
# the repo): NTFY_TOPIC=skatespots-<random>
|
|
# Subscribe to it with the ntfy app (or `curl -s ntfy.sh/<topic>/json`).
|
|
#
|
|
# Manual run: sudo /home/ubuntu/skatespots-health.sh
|
|
set -uo pipefail
|
|
|
|
ENV_FILE="/home/ubuntu/skatespots-health.env"
|
|
# shellcheck disable=SC1090
|
|
[ -f "$ENV_FILE" ] && . "$ENV_FILE"
|
|
|
|
STATE_FILE="/home/ubuntu/.skatespots-health.state"
|
|
ISSUES=()
|
|
|
|
svc() { systemctl is-active --quiet "$1" || ISSUES+=("service $1 is not active"); }
|
|
code() { curl -s -o /dev/null -m 8 -w '%{http_code}' "$@" || echo 000; }
|
|
|
|
# Relay on localhost: if dead, attempt an automatic restart.
|
|
rc=$(code -H 'Accept: application/nostr+json' http://127.0.0.1:7777/)
|
|
if [ "$rc" != "200" ]; then
|
|
ISSUES+=("relay localhost HTTP $rc - restarting skatespots-relay")
|
|
systemctl restart skatespots-relay.service || ISSUES+=("relay restart FAILED")
|
|
fi
|
|
|
|
svc skatespots-relay.service
|
|
svc skatespots-ai.service
|
|
svc skatespots-push.service
|
|
svc skatespots-cvm.service
|
|
svc skatespots-quota.service
|
|
svc caddy.service
|
|
|
|
[ "$(code -H 'Accept: application/nostr+json' https://relay.skate-spots.org/)" != "200" ] \
|
|
&& ISSUES+=("relay public TLS not answering 200")
|
|
[ "$(code https://push.skate-spots.org/health)" != "200" ] \
|
|
&& ISSUES+=("push /health not 200")
|
|
[ "$(code http://127.0.0.1:8790/health)" != "200" ] \
|
|
&& ISSUES+=("blossom quota proxy /health not 200")
|
|
|
|
disk=$(df --output=pcent / | tail -1 | tr -d ' %')
|
|
[ "${disk:-0}" -ge 85 ] && ISSUES+=("disk ${disk}% used")
|
|
avail=$(free -m | awk '/Mem:/{print $7}')
|
|
[ "${avail:-0}" -lt 300 ] && ISSUES+=("low memory: ${avail} MiB available")
|
|
|
|
# Cordn/Staircase RPC health. Newlay logs one warning per rejected request, so
|
|
# a burst is easy to miss in routine log noise. Surface it as an operational
|
|
# problem before users experience a stuck private chat.
|
|
cordn_rejects=$(docker logs --since 15m newlay-newlay-1 2>&1 | grep -c 'cordn rate limit' || true)
|
|
[ "${cordn_rejects:-0}" -ge 40 ] && ISSUES+=("cordn rate limits: ${cordn_rejects} rejects in 15m")
|
|
|
|
# `grep -q` + pipefail makes `docker logs` exit 141 on SIGPIPE, which used to
|
|
# make this check fire even when an announcement was present. Count instead.
|
|
cordn_announce=$(docker logs --since 20m newlay-newlay-1 2>&1 | grep -c 'cordn announce' || true)
|
|
[ "${cordn_announce:-0}" -eq 0 ] && ISSUES+=("cordn coordinator has not announced in 20m")
|
|
|
|
# Blossom media budget (usage is persisted by the quota proxy).
|
|
quota_state=/home/ubuntu/skatespots-quota/usage.json
|
|
if [ -f "$quota_state" ]; then
|
|
total_bytes=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("totalBytes",0))' "$quota_state" 2>/dev/null || echo 0)
|
|
[ "${total_bytes:-0}" -ge 19327352832 ] && ISSUES+=("blossom media >18 GiB")
|
|
fi
|
|
|
|
# TLS expiry (Caddy auto-renews; this catches a broken renewal early).
|
|
expiry=$(echo | openssl s_client -servername relay.skate-spots.org -connect relay.skate-spots.org:443 2>/dev/null \
|
|
| openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
|
if [ -n "$expiry" ]; then
|
|
days=$(( ( $(date -d "$expiry" +%s) - $(date +%s) ) / 86400 ))
|
|
[ "$days" -lt 14 ] && ISSUES+=("relay TLS expires in ${days}d")
|
|
else
|
|
ISSUES+=("could not read relay TLS certificate")
|
|
fi
|
|
|
|
sig=""
|
|
[ "${#ISSUES[@]}" -gt 0 ] && sig=$(printf '%s\n' "${ISSUES[@]}" | sort | tr '\n' '|')
|
|
prev=""
|
|
[ -f "$STATE_FILE" ] && prev=$(cat "$STATE_FILE")
|
|
|
|
if [ "$sig" != "$prev" ]; then
|
|
if [ -n "$sig" ] && [ -n "${NTFY_TOPIC:-}" ]; then
|
|
curl -s -m 8 \
|
|
-H "Title: SkateSpots host problem" -H "Priority: high" -H "Tags: warning" \
|
|
-d "$(printf '%s\n' "${ISSUES[@]}")" "https://ntfy.sh/${NTFY_TOPIC}" >/dev/null || true
|
|
elif [ -z "$sig" ] && [ -n "$prev" ] && [ -n "${NTFY_TOPIC:-}" ]; then
|
|
curl -s -m 8 \
|
|
-H "Title: SkateSpots host recovered" -H "Tags: white_check_mark" \
|
|
-d "All checks passing again." "https://ntfy.sh/${NTFY_TOPIC}" >/dev/null || true
|
|
fi
|
|
printf '%s' "$sig" > "$STATE_FILE"
|
|
fi
|
|
|
|
if [ "${#ISSUES[@]}" -gt 0 ]; then
|
|
printf 'PROBLEM: %s\n' "${ISSUES[@]}"
|
|
exit 1
|
|
fi
|
|
echo "ok"
|