#!/usr/bin/env bash
#
# SkateSpots relay-host health check (runs ON the relay box every 5 minutes).
#
# - verifies the host services (relay/ai/push/cvm/caddy) and the public endpoints
# - restarts `skatespots-relay` if its localhost NIP-11 endpoint is dead
# - checks disk + memory headroom
# - sends an alert to ntfy.sh only when the problem set CHANGES (no spam)
#
# The ntfy topic lives in /home/ubuntu/skatespots-health.env (mode 600, not in
# the repo):  NTFY_TOPIC=skatespots-<random>
# Subscribe to it with the ntfy app (or `curl -s ntfy.sh/<topic>/json`).
#
# Manual run:  sudo /home/ubuntu/skatespots-health.sh
set -uo pipefail

ENV_FILE="/home/ubuntu/skatespots-health.env"
# shellcheck disable=SC1090
[ -f "$ENV_FILE" ] && . "$ENV_FILE"

STATE_FILE="/home/ubuntu/.skatespots-health.state"
ISSUES=()

svc() { systemctl is-active --quiet "$1" || ISSUES+=("service $1 is not active"); }
code() { curl -s -o /dev/null -m 8 -w '%{http_code}' "$@" || echo 000; }

# Relay on localhost: if dead, attempt an automatic restart.
rc=$(code -H 'Accept: application/nostr+json' http://127.0.0.1:7777/)
if [ "$rc" != "200" ]; then
  ISSUES+=("relay localhost HTTP $rc - restarting skatespots-relay")
  systemctl restart skatespots-relay.service || ISSUES+=("relay restart FAILED")
fi

svc skatespots-relay.service
svc skatespots-ai.service
svc skatespots-push.service
svc skatespots-cvm.service
svc skatespots-quota.service
svc caddy.service

[ "$(code -H 'Accept: application/nostr+json' https://relay.skate-spots.org/)" != "200" ] \
  && ISSUES+=("relay public TLS not answering 200")
[ "$(code https://push.skate-spots.org/health)" != "200" ] \
  && ISSUES+=("push /health not 200")
[ "$(code http://127.0.0.1:8790/health)" != "200" ] \
  && ISSUES+=("blossom quota proxy /health not 200")

disk=$(df --output=pcent / | tail -1 | tr -d ' %')
[ "${disk:-0}" -ge 85 ] && ISSUES+=("disk ${disk}% used")
avail=$(free -m | awk '/Mem:/{print $7}')
[ "${avail:-0}" -lt 300 ] && ISSUES+=("low memory: ${avail} MiB available")

# Cordn/Staircase RPC health. Newlay logs one warning per rejected request, so
# a burst is easy to miss in routine log noise. Surface it as an operational
# problem before users experience a stuck private chat.
cordn_rejects=$(docker logs --since 15m newlay-newlay-1 2>&1 | grep -c 'cordn rate limit' || true)
[ "${cordn_rejects:-0}" -ge 40 ] && ISSUES+=("cordn rate limits: ${cordn_rejects} rejects in 15m")

# `grep -q` + pipefail makes `docker logs` exit 141 on SIGPIPE, which used to
# make this check fire even when an announcement was present. Count instead.
cordn_announce=$(docker logs --since 20m newlay-newlay-1 2>&1 | grep -c 'cordn announce' || true)
[ "${cordn_announce:-0}" -eq 0 ] && ISSUES+=("cordn coordinator has not announced in 20m")

# Blossom media budget (usage is persisted by the quota proxy).
quota_state=/home/ubuntu/skatespots-quota/usage.json
if [ -f "$quota_state" ]; then
  total_bytes=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("totalBytes",0))' "$quota_state" 2>/dev/null || echo 0)
  [ "${total_bytes:-0}" -ge 19327352832 ] && ISSUES+=("blossom media >18 GiB")
fi

# TLS expiry (Caddy auto-renews; this catches a broken renewal early).
expiry=$(echo | openssl s_client -servername relay.skate-spots.org -connect relay.skate-spots.org:443 2>/dev/null \
  | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
if [ -n "$expiry" ]; then
  days=$(( ( $(date -d "$expiry" +%s) - $(date +%s) ) / 86400 ))
  [ "$days" -lt 14 ] && ISSUES+=("relay TLS expires in ${days}d")
else
  ISSUES+=("could not read relay TLS certificate")
fi

# Machine-readable snapshot for the signed admin dashboard. Aggregate numbers
# only: no IPs, pubkeys, paths or secrets.
local_relay=$(code -H 'Accept: application/nostr+json' http://127.0.0.1:7777/)
public_relay=$(code -H 'Accept: application/nostr+json' https://relay.skate-spots.org/)
push_status=$(code https://push.skate-spots.org/health)
quota_status=$(code http://127.0.0.1:8790/health)
version=$(curl -s -m 8 -H 'Accept: application/nostr+json' https://relay.skate-spots.org/ \
  | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("version",""))' 2>/dev/null || true)
svc_on() { systemctl is-active --quiet "$1" && echo true || echo false; }
STATUS_DISK="${disk:-0}" STATUS_AVAIL="${avail:-0}" STATUS_MEDIA="${total_bytes:-0}" \
STATUS_CORDN_REJECTS="${cordn_rejects:-0}" STATUS_CORDN_ANNOUNCE="${cordn_announce:-0}" \
STATUS_TLS_DAYS="${days:-0}" STATUS_LOCAL="${local_relay:-000}" STATUS_PUBLIC="${public_relay:-000}" \
STATUS_PUSH="${push_status:-000}" STATUS_QUOTA="${quota_status:-000}" STATUS_VERSION="$version" \
STATUS_SERVICES="$(printf 'relay=%s\nai=%s\npush=%s\ncvm=%s\nquota=%s\ncaddy=%s\n' \
  "$(svc_on skatespots-relay.service)" "$(svc_on skatespots-ai.service)" \
  "$(svc_on skatespots-push.service)" "$(svc_on skatespots-cvm.service)" \
  "$(svc_on skatespots-quota.service)" "$(svc_on caddy.service)")" \
STATUS_ISSUES="$(printf '%s\n' "${ISSUES[@]}")" \
python3 - <<'PY'
import datetime, json, os
services = {}
for line in os.environ.get('STATUS_SERVICES','').splitlines():
    if '=' in line:
        k,v = line.split('=',1); services[k] = v == 'true'
issues = [x for x in os.environ.get('STATUS_ISSUES','').splitlines() if x]
payload = {
    'updatedAt': datetime.datetime.now(datetime.timezone.utc).isoformat(),
    'relay': {'local': os.environ.get('STATUS_LOCAL'), 'public': os.environ.get('STATUS_PUBLIC')},
    'version': os.environ.get('STATUS_VERSION',''),
    'cordn': {'announces20m': int(os.environ.get('STATUS_CORDN_ANNOUNCE','0') or 0),
              'rejects15m': int(os.environ.get('STATUS_CORDN_REJECTS','0') or 0)},
    'storage': {'diskPct': int(os.environ.get('STATUS_DISK','0') or 0),
                'memAvailMiB': int(os.environ.get('STATUS_AVAIL','0') or 0),
                'mediaBytes': int(os.environ.get('STATUS_MEDIA','0') or 0)},
    'tls': {'days': int(os.environ.get('STATUS_TLS_DAYS','0') or 0)},
    'services': services,
    'issues': issues,
}
path = '/home/ubuntu/skatespots-quota/status.json'
with open(path,'w') as f: json.dump(payload,f)
os.chmod(path, 0o644)
PY

sig=""
[ "${#ISSUES[@]}" -gt 0 ] && sig=$(printf '%s\n' "${ISSUES[@]}" | sort | tr '\n' '|')
prev=""
[ -f "$STATE_FILE" ] && prev=$(cat "$STATE_FILE")

if [ "$sig" != "$prev" ]; then
  if [ -n "$sig" ] && [ -n "${NTFY_TOPIC:-}" ]; then
    curl -s -m 8 \
      -H "Title: SkateSpots host problem" -H "Priority: high" -H "Tags: warning" \
      -d "$(printf '%s\n' "${ISSUES[@]}")" "https://ntfy.sh/${NTFY_TOPIC}" >/dev/null || true
  elif [ -z "$sig" ] && [ -n "$prev" ] && [ -n "${NTFY_TOPIC:-}" ]; then
    curl -s -m 8 \
      -H "Title: SkateSpots host recovered" -H "Tags: white_check_mark" \
      -d "All checks passing again." "https://ntfy.sh/${NTFY_TOPIC}" >/dev/null || true
  fi
  printf '%s' "$sig" > "$STATE_FILE"
fi

if [ "${#ISSUES[@]}" -gt 0 ]; then
  printf 'PROBLEM: %s\n' "${ISSUES[@]}"
  exit 1
fi
echo "ok"