313 lines
14 KiB
Bash
Executable file
313 lines
14 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
#
|
|
# ops-health.sh — "why isn't it live?" at a glance.
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Two silent failures cost real time: the ngit/Amber remote signer stalling
|
|
# mid-push, and the deploy timer skipping while origin/main had moved. Both were
|
|
# invisible. This reports everything needed to tell them apart, and can notify.
|
|
#
|
|
# scripts/ops-health.sh # read-only report
|
|
# scripts/ops-health.sh --probe-push # ALSO attempt a bounded real push
|
|
# scripts/ops-health.sh --notify # desktop-notify on actionable problems
|
|
# scripts/ops-health.sh --quiet # print/notify only when something's wrong
|
|
#
|
|
# `--probe-push` publishes any unpushed commits (that IS the probe — signing only
|
|
# happens when there's something to sign). Without it the script never writes to
|
|
# the remote.
|
|
#
|
|
# Exit: 0 healthy · 1 something needs attention · 2 bad usage.
|
|
set -uo pipefail
|
|
|
|
REPO="/home/sovereignsk8/SkateSpots"
|
|
# Overridable so the report can be tested against a fake state dir.
|
|
STATE_DIR="${SKATESPOTS_STATE_DIR:-$HOME/.local/state/skatespots-deploy}"
|
|
SHA_FILE="$STATE_DIR/last-deployed-sha"
|
|
INTERVAL_FILE="$STATE_DIR/last-deploy-epoch"
|
|
SKIP_FILE="$STATE_DIR/last-skip"
|
|
RUN_FILE="$STATE_DIR/last-run"
|
|
NOTIFY_FILE="$STATE_DIR/last-notified"
|
|
PENDING_FILE="$STATE_DIR/pending-since"
|
|
mkdir -p "$STATE_DIR"
|
|
|
|
PROBE_PUSH=0
|
|
NOTIFY=0
|
|
QUIET=0
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--probe-push) PROBE_PUSH=1 ;;
|
|
--notify) NOTIFY=1 ;;
|
|
--quiet) QUIET=1 ;;
|
|
-h|--help) sed -n '2,20p' "$0"; exit 0 ;;
|
|
*) echo "ops-health: unknown option: $arg" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
if [ -t 1 ]; then
|
|
B=$'\e[1m'; DIM=$'\e[2m'; R=$'\e[31m'; G=$'\e[32m'; Y=$'\e[33m'; N=$'\e[0m'
|
|
else
|
|
B=; DIM=; R=; G=; Y=; N=
|
|
fi
|
|
|
|
REPORT=""
|
|
p() { REPORT+="$1"$'\n'; }
|
|
|
|
problems=() # short, actionable strings (for --notify and the summary)
|
|
signature="" # stable dedupe key for notifications
|
|
|
|
human_age() {
|
|
local s="$1"
|
|
if [ "$s" -lt 60 ]; then echo "${s}s"
|
|
elif [ "$s" -lt 3600 ]; then echo "$((s / 60))m"
|
|
elif [ "$s" -lt 86400 ]; then echo "$((s / 3600))h"
|
|
else echo "$((s / 86400))d"; fi
|
|
}
|
|
|
|
# Age of a file in seconds, or empty when it doesn't exist.
|
|
file_age() {
|
|
[ -f "$1" ] && echo $(( $(date +%s) - $(cat "$1" 2>/dev/null || echo 0) )) || echo ""
|
|
}
|
|
|
|
cd "$REPO" 2>/dev/null || { echo "ops-health: repo not found: $REPO" >&2; exit 2; }
|
|
now=$(date +%s)
|
|
|
|
# ── Repo & push state ────────────────────────────────────────────────────────
|
|
p "${B}▸ Repo & push${N}"
|
|
ahead=$(git rev-list --count origin/main..HEAD 2>/dev/null || echo 0)
|
|
behind=$(git rev-list --count HEAD..origin/main 2>/dev/null || echo 0)
|
|
head_short=$(git rev-parse --short HEAD 2>/dev/null || echo '?')
|
|
origin_short=$(git rev-parse --short origin/main 2>/dev/null || echo '?')
|
|
|
|
if [ "$ahead" = "0" ] && [ "$behind" = "0" ]; then
|
|
p " ${G}✓${N} in sync with origin/main ($head_short)"
|
|
rm -f "$PENDING_FILE"
|
|
elif [ "$ahead" -gt 0 ]; then
|
|
p " ${Y}▲${N} ${B}${ahead}${N} unpushed commit(s) (HEAD $head_short vs origin $origin_short)"
|
|
[ ! -f "$PENDING_FILE" ] && date +%s > "$PENDING_FILE"
|
|
pend_age=$(( now - $(cat "$PENDING_FILE" 2>/dev/null || echo "$now") ))
|
|
p " ${DIM}waiting $(human_age "$pend_age") — push is the missing step${N}"
|
|
if [ "$pend_age" -gt 1200 ]; then
|
|
problems+=("$ahead commit(s) unpushed for $(human_age "$pend_age") — the signer may be stuck")
|
|
signature+="pending-stuck;"
|
|
fi
|
|
else
|
|
p " ${Y}▼${N} ${behind} commit(s) behind origin/main"
|
|
fi
|
|
|
|
# Optional bounded probe: signing only happens when there IS something to push.
|
|
if [ "$PROBE_PUSH" = "1" ] && [ "$ahead" -gt 0 ]; then
|
|
p " ${DIM}probing: git push origin main (bounded 45s — this publishes the commits above)…${N}"
|
|
start=$(date +%s)
|
|
out="$(timeout 45 git push origin main 2>&1)"; rc=$?
|
|
el=$(( $(date +%s) - start ))
|
|
if [ "$rc" = "0" ]; then
|
|
p " ${G}✓${N} push OK in ${el}s — signer responsive"
|
|
elif [ "$rc" = "124" ]; then
|
|
p " ${R}✗${N} push TIMED OUT after ${el}s — the ngit/Amber signer looks unresponsive"
|
|
p " ${DIM}fix: open Amber in the foreground, or: ngit account login --interactive${N}"
|
|
problems+=("signer unresponsive (push timed out)")
|
|
signature+="signer-timeout;"
|
|
else
|
|
p " ${R}✗${N} push failed (rc=$rc):"
|
|
printf '%s\n' "$out" | tail -3 | while IFS= read -r l; do p " ${DIM}$l${N}"; done
|
|
problems+=("push failed (rc=$rc)")
|
|
signature+="signer-fail;"
|
|
fi
|
|
elif [ "$ahead" -gt 0 ]; then
|
|
p " ${DIM}run with --probe-push to actually test signing (publishes the commits above)${N}"
|
|
fi
|
|
|
|
# ── Deploy state ─────────────────────────────────────────────────────────────
|
|
p ""
|
|
p "${B}▸ Deploy${N}"
|
|
deployed="$(cat "$SHA_FILE" 2>/dev/null || echo none)"
|
|
origin="$(git rev-parse origin/main 2>/dev/null || echo none)"
|
|
last_epoch="$(cat "$INTERVAL_FILE" 2>/dev/null || echo 0)"
|
|
case "$last_epoch" in ''|*[!0-9]*) last_epoch=0 ;; esac
|
|
|
|
if [ "$deployed" = "$origin" ] && [ "$origin" != "none" ]; then
|
|
if [ "$last_epoch" -gt 0 ]; then
|
|
p " ${G}✓${N} live is current (${origin:0:9}) — last production deploy $(human_age $((now - last_epoch))) ago"
|
|
else
|
|
p " ${G}✓${N} live is current (${origin:0:9})"
|
|
fi
|
|
else
|
|
p " ${Y}●${N} ${B}PENDING${N}: origin/main ${origin:0:9} but last deployed ${deployed:0:9}"
|
|
if [ -f "$SKIP_FILE" ]; then
|
|
IFS='|' read -r sk_at sk_reason sk_sha sk_detail < "$SKIP_FILE"
|
|
sk_age=$(( now - ${sk_at:-0} ))
|
|
p " last skip: ${B}${sk_reason:-unknown}${N} ($(human_age "$sk_age") ago) ${DIM}${sk_detail}${N}"
|
|
case "${sk_reason:-}" in
|
|
throttled)
|
|
p " ${DIM}→ batching to save Netlify credits; it deploys automatically when the window elapses.${N}" ;;
|
|
dirty-tree)
|
|
if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
|
|
p " ${R}→ tracked files are modified — COMMIT them or the deploy stays skipped.${N}"
|
|
problems+=("deploy blocked: working tree is dirty"); signature+="dirty;"
|
|
else
|
|
p " ${DIM}→ the tree was dirty earlier and is clean now — the next tick will deploy.${N}"
|
|
fi ;;
|
|
smoke-failed)
|
|
p " ${R}→ the built bundle failed the smoke test (real breakage — check the journal).${N}"
|
|
problems+=("smoke test failed during deploy"); signature+="smoke;" ;;
|
|
preflight-failed)
|
|
p " ${R}→ the test gate failed — fix typecheck/lint/i18n/tests before it can deploy.${N}"
|
|
problems+=("deploy blocked: preflight failed"); signature+="preflight;" ;;
|
|
deploy-failed)
|
|
p " ${R}→ Netlify rejected the deploy — check the Netlify dashboard.${N}"
|
|
problems+=("Netlify deploy failed"); signature+="deploy-failed;" ;;
|
|
health-failed)
|
|
p " ${R}→ the production URL did not answer after the deploy.${N}"
|
|
problems+=("production health check failed"); signature+="health-failed;" ;;
|
|
diverged)
|
|
p " ${R}→ local main diverged from origin/main — a manual merge is needed.${N}"
|
|
problems+=("local main diverged from origin/main"); signature+="diverged;" ;;
|
|
fetch-failed)
|
|
p " ${Y}→ couldn't fetch origin (network/relay issue).${N}" ;;
|
|
non-site-only|dist-unchanged)
|
|
p " ${DIM}→ no deploy was needed (docs-only or byte-identical build).${N}" ;;
|
|
locked|"")
|
|
p " ${DIM}→ reason: ${sk_reason:-unknown}${N}" ;;
|
|
*)
|
|
p " ${DIM}→ reason: ${sk_reason}${N}" ;;
|
|
esac
|
|
else
|
|
r_age="$(file_age "$RUN_FILE")"
|
|
if [ -n "$r_age" ] && [ "$r_age" -lt 360 ]; then
|
|
p " ${DIM}waiting for the next deploy tick (~every 3m)…${N}"
|
|
else
|
|
p " ${Y}no skip reason recorded and the deploy script hasn't run recently — the timer may be stopped.${N}"
|
|
problems+=("deploy pending with no recorded reason"); signature+="noreason;"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
min_interval="${SKATESPOTS_DEPLOY_MIN_INTERVAL:-86400}"
|
|
case "$min_interval" in ''|*[!0-9]*) min_interval=0 ;; esac
|
|
if [ "$min_interval" -gt 0 ] && [ "$last_epoch" -gt 0 ]; then
|
|
remain=$(( min_interval - (now - last_epoch) ))
|
|
if [ "$remain" -gt 0 ]; then
|
|
p " ${DIM}throttle: next auto-deploy eligible in ~$(human_age "$remain") · force now: scripts/auto-deploy.sh --force${N}"
|
|
fi
|
|
fi
|
|
|
|
# ── Timer ────────────────────────────────────────────────────────────────────
|
|
p ""
|
|
p "${B}▸ Timer${N}"
|
|
timer_state="$(systemctl --user is-active skatespots-deploy.timer 2>/dev/null || echo unknown)"
|
|
if [ "$timer_state" = "active" ]; then
|
|
next_at="$(systemctl --user list-timers skatespots-deploy.timer --no-legend --no-pager 2>/dev/null | awk '$1 != "-" {print $1" "$2" "$3}')"
|
|
p " ${G}✓${N} skatespots-deploy.timer active${next_at:+ ${DIM}next $next_at${N}}"
|
|
else
|
|
p " ${R}✗${N} skatespots-deploy.timer: ${timer_state}"
|
|
problems+=("deploy timer is not active")
|
|
signature+="timer;"
|
|
fi
|
|
if [ -f "$RUN_FILE" ]; then
|
|
run_age=$(( now - $(cat "$RUN_FILE" 2>/dev/null || echo 0) ))
|
|
if [ "$run_age" -gt 600 ]; then
|
|
p " ${Y}⚠${N} deploy script last ran $(human_age "$run_age") ago (it polls ~every 3m)"
|
|
problems+=("deploy script hasn't run in $(human_age "$run_age")")
|
|
signature+="stale-run;"
|
|
else
|
|
p " ${DIM}last run $(human_age "$run_age") ago${N}"
|
|
fi
|
|
fi
|
|
|
|
# ── Community Cordn bot ─────────────────────────────────────────────────────
|
|
p ""
|
|
p "${B}Community bot${N}"
|
|
COMMUNITY_HOST="${SKATESPOTS_COMMUNITY_HOST:-skatespots-cvm}"
|
|
if command -v ssh >/dev/null 2>&1 && grep -qsE "^Host[[:space:]]+${COMMUNITY_HOST}([[:space:]]|$)" "$HOME/.ssh/config"; then
|
|
community_state="$(timeout 8 ssh -o BatchMode=yes -o ConnectTimeout=5 "$COMMUNITY_HOST" 'systemctl is-active skatespots-community' 2>/dev/null || true)"
|
|
if [ "$community_state" = "active" ]; then
|
|
p " ${G}✓${N} skatespots-community active"
|
|
else
|
|
p " ${R}✗${N} skatespots-community: ${community_state:-unreachable}"
|
|
problems+=("community Cordn bot is ${community_state:-unreachable}")
|
|
signature+="community-bot;"
|
|
fi
|
|
community_backup="$(systemctl --user is-active skatespots-community-backup.timer 2>/dev/null || echo unknown)"
|
|
if [ "$community_backup" = "active" ]; then
|
|
p " ${G}✓${N} community state backup timer active"
|
|
else
|
|
p " ${Y}⚠${N} community state backup timer: ${community_backup}"
|
|
fi
|
|
else
|
|
p " ${DIM}ssh alias ${COMMUNITY_HOST} not configured — skipped${N}"
|
|
fi
|
|
|
|
# ── Dependency job ──────────────────────────────────────────────────────────
|
|
p ""
|
|
p "${B}Dependency job${N}"
|
|
DEPS_STATE="$HOME/.local/state/skatespots-deps"
|
|
DEPS_LAST="$DEPS_STATE/last-run"
|
|
DEPS_FAIL="$DEPS_STATE/last-failure"
|
|
DEPS_TIMER="$(systemctl --user is-active skatespots-deps.timer 2>/dev/null || echo unknown)"
|
|
SCAN_TIMER="$(systemctl --user is-active skatespots-deps-scan.timer 2>/dev/null || echo unknown)"
|
|
if [ "$DEPS_TIMER" = "active" ]; then
|
|
p " ${G}✓${N} 16-day dependency timer active ${DIM}(daily tick)${N}"
|
|
else
|
|
p " ${R}✗${N} dependency timer: ${DEPS_TIMER}"
|
|
problems+=("dependency timer is not active")
|
|
signature+="deps-timer;"
|
|
fi
|
|
if [ "$SCAN_TIMER" = "active" ]; then
|
|
p " ${G}✓${N} weekly CVE scan timer active"
|
|
else
|
|
p " ${Y}⚠${N} weekly CVE scan timer: ${SCAN_TIMER}"
|
|
fi
|
|
if [ -f "$DEPS_LAST" ]; then
|
|
deps_last="$(cat "$DEPS_LAST" 2>/dev/null || echo 0)"
|
|
# On success last-run is "now"; on failure it is "now - (interval - 3d)" so
|
|
# the next due date is the ~3-day retry, not a full 16-day wait.
|
|
deps_due=$(( deps_last + 16 * 86400 ))
|
|
if [ "$deps_due" -gt "$now" ]; then
|
|
p " ${DIM}next dependency update in ~$(human_age $((deps_due - now)))${N}"
|
|
else
|
|
p " ${Y}●${N} dependency update due at the next daily tick"
|
|
fi
|
|
fi
|
|
latest_report="$(ls -1t "$DEPS_STATE"/report-*.txt 2>/dev/null | head -1 || true)"
|
|
if [ -n "$latest_report" ]; then
|
|
osv="$(grep -E '^Total [0-9]+ packages affected' "$latest_report" | tail -1 || true)"
|
|
p " ${DIM}latest report: $(basename "$latest_report")${osv:+ — $osv}${N}"
|
|
fi
|
|
if [ -f "$DEPS_FAIL" ]; then
|
|
p " ${R}✗${N} last dependency run failed: $(cat "$DEPS_FAIL")"
|
|
problems+=("dependency update failed (see $DEPS_FAIL)")
|
|
signature+="deps-failed;"
|
|
else
|
|
p " ${G}✓${N} no recorded dependency failure"
|
|
fi
|
|
|
|
# ── Summary + optional notification ──────────────────────────────────────────
|
|
p ""
|
|
if [ ${#problems[@]} -eq 0 ]; then
|
|
p "${G}${B}All good.${N}"
|
|
else
|
|
p "${R}${B}Needs attention:${N}"
|
|
for prob in "${problems[@]}"; do p " • $prob"; done
|
|
fi
|
|
|
|
if [ "$NOTIFY" = "1" ]; then
|
|
if [ ${#problems[@]} -gt 0 ]; then
|
|
prev="$(cat "$NOTIFY_FILE" 2>/dev/null || true)"
|
|
if [ "$signature" != "$prev" ]; then
|
|
body="$(printf '%s; ' "${problems[@]}")"
|
|
if command -v notify-send >/dev/null 2>&1; then
|
|
notify-send -a SkateSpots "SkateSpots ops" "$body" 2>/dev/null || true
|
|
fi
|
|
timeout 45 "$REPO/scripts/send-ops-alert.sh" "SkateSpots needs attention: $body" || true
|
|
printf '%s' "$signature" > "$NOTIFY_FILE"
|
|
fi
|
|
else
|
|
rm -f "$NOTIFY_FILE" # healthy → re-arm notifications
|
|
fi
|
|
fi
|
|
|
|
if [ "$QUIET" = "1" ] && [ ${#problems[@]} -eq 0 ]; then
|
|
exit 0
|
|
fi
|
|
printf '%s' "$REPORT"
|
|
|
|
[ ${#problems[@]} -eq 0 ] && exit 0 || exit 1
|