146 lines
6.8 KiB
Bash
146 lines
6.8 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# SkateSpots relay-host health check (runs ON the relay box every 5 minutes).
|
|
#
|
|
# - verifies the host services (relay/ai/push/cvm/caddy) and the public endpoints
|
|
# - restarts `skatespots-relay` if its localhost NIP-11 endpoint is dead
|
|
# - checks disk + memory headroom
|
|
# - sends an alert to ntfy.sh only when the problem set CHANGES (no spam)
|
|
#
|
|
# The ntfy topic lives in /home/ubuntu/skatespots-health.env (mode 600, not in
|
|
# the repo): NTFY_TOPIC=skatespots-<random>
|
|
# Subscribe to it with the ntfy app (or `curl -s ntfy.sh/<topic>/json`).
|
|
#
|
|
# Manual run: sudo /home/ubuntu/skatespots-health.sh
|
|
set -uo pipefail
|
|
|
|
ENV_FILE="/home/ubuntu/skatespots-health.env"
|
|
# shellcheck disable=SC1090
|
|
[ -f "$ENV_FILE" ] && . "$ENV_FILE"
|
|
|
|
STATE_FILE="/home/ubuntu/.skatespots-health.state"
|
|
ISSUES=()
|
|
|
|
svc() { systemctl is-active --quiet "$1" || ISSUES+=("service $1 is not active"); }
|
|
code() { curl -s -o /dev/null -m 8 -w '%{http_code}' "$@" || echo 000; }
|
|
|
|
# Relay on localhost: if dead, attempt an automatic restart.
|
|
rc=$(code -H 'Accept: application/nostr+json' http://127.0.0.1:7777/)
|
|
if [ "$rc" != "200" ]; then
|
|
ISSUES+=("relay localhost HTTP $rc - restarting skatespots-relay")
|
|
systemctl restart skatespots-relay.service || ISSUES+=("relay restart FAILED")
|
|
fi
|
|
|
|
svc skatespots-relay.service
|
|
svc skatespots-ai.service
|
|
svc skatespots-push.service
|
|
svc skatespots-cvm.service
|
|
svc skatespots-quota.service
|
|
svc caddy.service
|
|
|
|
[ "$(code -H 'Accept: application/nostr+json' https://relay.skate-spots.org/)" != "200" ] \
|
|
&& ISSUES+=("relay public TLS not answering 200")
|
|
[ "$(code https://push.skate-spots.org/health)" != "200" ] \
|
|
&& ISSUES+=("push /health not 200")
|
|
[ "$(code http://127.0.0.1:8790/health)" != "200" ] \
|
|
&& ISSUES+=("blossom quota proxy /health not 200")
|
|
|
|
disk=$(df --output=pcent / | tail -1 | tr -d ' %')
|
|
[ "${disk:-0}" -ge 85 ] && ISSUES+=("disk ${disk}% used")
|
|
avail=$(free -m | awk '/Mem:/{print $7}')
|
|
[ "${avail:-0}" -lt 300 ] && ISSUES+=("low memory: ${avail} MiB available")
|
|
|
|
# Cordn/Staircase RPC health. Newlay logs one warning per rejected request, so
|
|
# a burst is easy to miss in routine log noise. Surface it as an operational
|
|
# problem before users experience a stuck private chat.
|
|
cordn_rejects=$(docker logs --since 15m newlay-newlay-1 2>&1 | grep -c 'cordn rate limit' || true)
|
|
[ "${cordn_rejects:-0}" -ge 40 ] && ISSUES+=("cordn rate limits: ${cordn_rejects} rejects in 15m")
|
|
|
|
# `grep -q` + pipefail makes `docker logs` exit 141 on SIGPIPE, which used to
|
|
# make this check fire even when an announcement was present. Count instead.
|
|
cordn_announce=$(docker logs --since 20m newlay-newlay-1 2>&1 | grep -c 'cordn announce' || true)
|
|
[ "${cordn_announce:-0}" -eq 0 ] && ISSUES+=("cordn coordinator has not announced in 20m")
|
|
|
|
# Blossom media budget (usage is persisted by the quota proxy).
|
|
quota_state=/home/ubuntu/skatespots-quota/usage.json
|
|
if [ -f "$quota_state" ]; then
|
|
total_bytes=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("totalBytes",0))' "$quota_state" 2>/dev/null || echo 0)
|
|
[ "${total_bytes:-0}" -ge 19327352832 ] && ISSUES+=("blossom media >18 GiB")
|
|
fi
|
|
|
|
# TLS expiry (Caddy auto-renews; this catches a broken renewal early).
|
|
expiry=$(echo | openssl s_client -servername relay.skate-spots.org -connect relay.skate-spots.org:443 2>/dev/null \
|
|
| openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
|
if [ -n "$expiry" ]; then
|
|
days=$(( ( $(date -d "$expiry" +%s) - $(date +%s) ) / 86400 ))
|
|
[ "$days" -lt 14 ] && ISSUES+=("relay TLS expires in ${days}d")
|
|
else
|
|
ISSUES+=("could not read relay TLS certificate")
|
|
fi
|
|
|
|
# Machine-readable snapshot for the signed admin dashboard. Aggregate numbers
|
|
# only: no IPs, pubkeys, paths or secrets.
|
|
local_relay=$(code -H 'Accept: application/nostr+json' http://127.0.0.1:7777/)
|
|
public_relay=$(code -H 'Accept: application/nostr+json' https://relay.skate-spots.org/)
|
|
push_status=$(code https://push.skate-spots.org/health)
|
|
quota_status=$(code http://127.0.0.1:8790/health)
|
|
version=$(curl -s -m 8 -H 'Accept: application/nostr+json' https://relay.skate-spots.org/ \
|
|
| python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("version",""))' 2>/dev/null || true)
|
|
svc_on() { systemctl is-active --quiet "$1" && echo true || echo false; }
|
|
STATUS_DISK="${disk:-0}" STATUS_AVAIL="${avail:-0}" STATUS_MEDIA="${total_bytes:-0}" \
|
|
STATUS_CORDN_REJECTS="${cordn_rejects:-0}" STATUS_CORDN_ANNOUNCE="${cordn_announce:-0}" \
|
|
STATUS_TLS_DAYS="${days:-0}" STATUS_LOCAL="${local_relay:-000}" STATUS_PUBLIC="${public_relay:-000}" \
|
|
STATUS_PUSH="${push_status:-000}" STATUS_QUOTA="${quota_status:-000}" STATUS_VERSION="$version" \
|
|
STATUS_SERVICES="$(printf 'relay=%s\nai=%s\npush=%s\ncvm=%s\nquota=%s\ncaddy=%s\n' \
|
|
"$(svc_on skatespots-relay.service)" "$(svc_on skatespots-ai.service)" \
|
|
"$(svc_on skatespots-push.service)" "$(svc_on skatespots-cvm.service)" \
|
|
"$(svc_on skatespots-quota.service)" "$(svc_on caddy.service)")" \
|
|
STATUS_ISSUES="$(printf '%s\n' "${ISSUES[@]}")" \
|
|
python3 - <<'PY'
|
|
import datetime, json, os
|
|
services = {}
|
|
for line in os.environ.get('STATUS_SERVICES','').splitlines():
|
|
if '=' in line:
|
|
k,v = line.split('=',1); services[k] = v == 'true'
|
|
issues = [x for x in os.environ.get('STATUS_ISSUES','').splitlines() if x]
|
|
payload = {
|
|
'updatedAt': datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
'relay': {'local': os.environ.get('STATUS_LOCAL'), 'public': os.environ.get('STATUS_PUBLIC')},
|
|
'version': os.environ.get('STATUS_VERSION',''),
|
|
'cordn': {'announces20m': int(os.environ.get('STATUS_CORDN_ANNOUNCE','0') or 0),
|
|
'rejects15m': int(os.environ.get('STATUS_CORDN_REJECTS','0') or 0)},
|
|
'storage': {'diskPct': int(os.environ.get('STATUS_DISK','0') or 0),
|
|
'memAvailMiB': int(os.environ.get('STATUS_AVAIL','0') or 0),
|
|
'mediaBytes': int(os.environ.get('STATUS_MEDIA','0') or 0)},
|
|
'tls': {'days': int(os.environ.get('STATUS_TLS_DAYS','0') or 0)},
|
|
'services': services,
|
|
'issues': issues,
|
|
}
|
|
path = '/home/ubuntu/skatespots-quota/status.json'
|
|
with open(path,'w') as f: json.dump(payload,f)
|
|
os.chmod(path, 0o644)
|
|
PY
|
|
|
|
sig=""
|
|
[ "${#ISSUES[@]}" -gt 0 ] && sig=$(printf '%s\n' "${ISSUES[@]}" | sort | tr '\n' '|')
|
|
prev=""
|
|
[ -f "$STATE_FILE" ] && prev=$(cat "$STATE_FILE")
|
|
|
|
if [ "$sig" != "$prev" ]; then
|
|
if [ -n "$sig" ] && [ -n "${NTFY_TOPIC:-}" ]; then
|
|
curl -s -m 8 \
|
|
-H "Title: SkateSpots host problem" -H "Priority: high" -H "Tags: warning" \
|
|
-d "$(printf '%s\n' "${ISSUES[@]}")" "https://ntfy.sh/${NTFY_TOPIC}" >/dev/null || true
|
|
elif [ -z "$sig" ] && [ -n "$prev" ] && [ -n "${NTFY_TOPIC:-}" ]; then
|
|
curl -s -m 8 \
|
|
-H "Title: SkateSpots host recovered" -H "Tags: white_check_mark" \
|
|
-d "All checks passing again." "https://ntfy.sh/${NTFY_TOPIC}" >/dev/null || true
|
|
fi
|
|
printf '%s' "$sig" > "$STATE_FILE"
|
|
fi
|
|
|
|
if [ "${#ISSUES[@]}" -gt 0 ]; then
|
|
printf 'PROBLEM: %s\n' "${ISSUES[@]}"
|
|
exit 1
|
|
fi
|
|
echo "ok"
|