#!/usr/bin/env bash
#
# SkateSpots relay-host health check (runs ON the relay box every 5 minutes).
#
# - verifies the host services (relay/ai/push/cvm/caddy) and the public endpoints
# - restarts `skatespots-relay` if its localhost NIP-11 endpoint is dead
# - checks disk + memory headroom
# - sends an alert to ntfy.sh only when the problem set CHANGES (no spam)
#
# The ntfy topic lives in /home/ubuntu/skatespots-health.env (mode 600, not in
# the repo):  NTFY_TOPIC=skatespots-<random>
# Subscribe to it with the ntfy app (or `curl -s ntfy.sh/<topic>/json`).
#
# Manual run:  sudo /home/ubuntu/skatespots-health.sh
set -uo pipefail

ENV_FILE="/home/ubuntu/skatespots-health.env"
# shellcheck disable=SC1090
[ -f "$ENV_FILE" ] && . "$ENV_FILE"

STATE_FILE="/home/ubuntu/.skatespots-health.state"
ISSUES=()

svc() { systemctl is-active --quiet "$1" || ISSUES+=("service $1 is not active"); }
code() { curl -s -o /dev/null -m 8 -w '%{http_code}' "$@" || echo 000; }

# Relay on localhost: if dead, attempt an automatic restart.
rc=$(code -H 'Accept: application/nostr+json' http://127.0.0.1:7777/)
if [ "$rc" != "200" ]; then
  ISSUES+=("relay localhost HTTP $rc - restarting skatespots-relay")
  systemctl restart skatespots-relay.service || ISSUES+=("relay restart FAILED")
fi

svc skatespots-relay.service
svc skatespots-ai.service
svc skatespots-push.service
svc skatespots-cvm.service
svc caddy.service

[ "$(code -H 'Accept: application/nostr+json' https://relay.skate-spots.org/)" != "200" ] \
  && ISSUES+=("relay public TLS not answering 200")
[ "$(code https://push.skate-spots.org/health)" != "200" ] \
  && ISSUES+=("push /health not 200")

disk=$(df --output=pcent / | tail -1 | tr -d ' %')
[ "${disk:-0}" -ge 85 ] && ISSUES+=("disk ${disk}% used")
avail=$(free -m | awk '/Mem:/{print $7}')
[ "${avail:-0}" -lt 300 ] && ISSUES+=("low memory: ${avail} MiB available")

# Cordn/Staircase RPC health. Newlay logs one warning per rejected request, so
# a burst is easy to miss in routine log noise. Surface it as an operational
# problem before users experience a stuck private chat.
cordn_rejects=$(docker logs --since 15m newlay-newlay-1 2>&1 | grep -c 'cordn rate limit' || true)
[ "${cordn_rejects:-0}" -ge 40 ] && ISSUES+=("cordn rate limits: ${cordn_rejects} rejects in 15m")

# `grep -q` + pipefail makes `docker logs` exit 141 on SIGPIPE, which used to
# make this check fire even when an announcement was present. Count instead.
cordn_announce=$(docker logs --since 20m newlay-newlay-1 2>&1 | grep -c 'cordn announce' || true)
[ "${cordn_announce:-0}" -eq 0 ] && ISSUES+=("cordn coordinator has not announced in 20m")

# TLS expiry (Caddy auto-renews; this catches a broken renewal early).
expiry=$(echo | openssl s_client -servername relay.skate-spots.org -connect relay.skate-spots.org:443 2>/dev/null \
  | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
if [ -n "$expiry" ]; then
  days=$(( ( $(date -d "$expiry" +%s) - $(date +%s) ) / 86400 ))
  [ "$days" -lt 14 ] && ISSUES+=("relay TLS expires in ${days}d")
else
  ISSUES+=("could not read relay TLS certificate")
fi

sig=""
[ "${#ISSUES[@]}" -gt 0 ] && sig=$(printf '%s\n' "${ISSUES[@]}" | sort | tr '\n' '|')
prev=""
[ -f "$STATE_FILE" ] && prev=$(cat "$STATE_FILE")

if [ "$sig" != "$prev" ]; then
  if [ -n "$sig" ] && [ -n "${NTFY_TOPIC:-}" ]; then
    curl -s -m 8 \
      -H "Title: SkateSpots host problem" -H "Priority: high" -H "Tags: warning" \
      -d "$(printf '%s\n' "${ISSUES[@]}")" "https://ntfy.sh/${NTFY_TOPIC}" >/dev/null || true
  elif [ -z "$sig" ] && [ -n "$prev" ] && [ -n "${NTFY_TOPIC:-}" ]; then
    curl -s -m 8 \
      -H "Title: SkateSpots host recovered" -H "Tags: white_check_mark" \
      -d "All checks passing again." "https://ntfy.sh/${NTFY_TOPIC}" >/dev/null || true
  fi
  printf '%s' "$sig" > "$STATE_FILE"
fi

if [ "${#ISSUES[@]}" -gt 0 ]; then
  printf 'PROBLEM: %s\n' "${ISSUES[@]}"
  exit 1
fi
echo "ok"