50fcb419f4
Two failures today were invisible: the ngit/Amber remote signer stalling
mid-push, and the deploy timer skipping while origin/main had moved.
- scripts/auto-deploy.sh records the reason for every skip (and the run time)
to ~/.local/state/skatespots-deploy/{last-skip,last-run}.
- scripts/ops-health.sh reads that back and prints repo/push sync, the skip
reason, the throttle window and timer health; exit 1 when action is needed.
--probe-push runs a bounded real push (the only true signer probe),
--notify desktop-notifies (deduped), --quiet is silent when healthy.
- Periodic watch: skatespots-ops-health.{service,timer} (every 15 min) plus
scripts/install-ops-health-timer.sh.
Docs: AGENTS.md Deploy section.
222 lines
9.4 KiB
Bash
Executable file
222 lines
9.4 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
#
|
|
# ops-health.sh — "why isn't it live?" at a glance.
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Two silent failures cost real time: the ngit/Amber remote signer stalling
|
|
# mid-push, and the deploy timer skipping while origin/main had moved. Both were
|
|
# invisible. This reports everything needed to tell them apart, and can notify.
|
|
#
|
|
# scripts/ops-health.sh # read-only report
|
|
# scripts/ops-health.sh --probe-push # ALSO attempt a bounded real push
|
|
# scripts/ops-health.sh --notify # desktop-notify on actionable problems
|
|
# scripts/ops-health.sh --quiet # print/notify only when something's wrong
|
|
#
|
|
# `--probe-push` publishes any unpushed commits (that IS the probe — signing only
|
|
# happens when there's something to sign). Without it the script never writes to
|
|
# the remote.
|
|
#
|
|
# Exit: 0 healthy · 1 something needs attention · 2 bad usage.
|
|
set -uo pipefail
|
|
|
|
REPO="/home/sovereignsk8/SkateSpots"
|
|
# Overridable so the report can be tested against a fake state dir.
|
|
STATE_DIR="${SKATESPOTS_STATE_DIR:-$HOME/.local/state/skatespots-deploy}"
|
|
SHA_FILE="$STATE_DIR/last-deployed-sha"
|
|
INTERVAL_FILE="$STATE_DIR/last-deploy-epoch"
|
|
SKIP_FILE="$STATE_DIR/last-skip"
|
|
RUN_FILE="$STATE_DIR/last-run"
|
|
NOTIFY_FILE="$STATE_DIR/last-notified"
|
|
PENDING_FILE="$STATE_DIR/pending-since"
|
|
mkdir -p "$STATE_DIR"
|
|
|
|
PROBE_PUSH=0
|
|
NOTIFY=0
|
|
QUIET=0
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--probe-push) PROBE_PUSH=1 ;;
|
|
--notify) NOTIFY=1 ;;
|
|
--quiet) QUIET=1 ;;
|
|
-h|--help) sed -n '2,20p' "$0"; exit 0 ;;
|
|
*) echo "ops-health: unknown option: $arg" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
if [ -t 1 ]; then
|
|
B=$'\e[1m'; DIM=$'\e[2m'; R=$'\e[31m'; G=$'\e[32m'; Y=$'\e[33m'; N=$'\e[0m'
|
|
else
|
|
B=; DIM=; R=; G=; Y=; N=
|
|
fi
|
|
|
|
REPORT=""
|
|
p() { REPORT+="$1"$'\n'; }
|
|
|
|
problems=() # short, actionable strings (for --notify and the summary)
|
|
signature="" # stable dedupe key for notifications
|
|
|
|
human_age() {
|
|
local s="$1"
|
|
if [ "$s" -lt 60 ]; then echo "${s}s"
|
|
elif [ "$s" -lt 3600 ]; then echo "$((s / 60))m"
|
|
elif [ "$s" -lt 86400 ]; then echo "$((s / 3600))h"
|
|
else echo "$((s / 86400))d"; fi
|
|
}
|
|
|
|
cd "$REPO" 2>/dev/null || { echo "ops-health: repo not found: $REPO" >&2; exit 2; }
|
|
now=$(date +%s)
|
|
|
|
# ── Repo & push state ────────────────────────────────────────────────────────
|
|
p "${B}▸ Repo & push${N}"
|
|
ahead=$(git rev-list --count origin/main..HEAD 2>/dev/null || echo 0)
|
|
behind=$(git rev-list --count HEAD..origin/main 2>/dev/null || echo 0)
|
|
head_short=$(git rev-parse --short HEAD 2>/dev/null || echo '?')
|
|
origin_short=$(git rev-parse --short origin/main 2>/dev/null || echo '?')
|
|
|
|
if [ "$ahead" = "0" ] && [ "$behind" = "0" ]; then
|
|
p " ${G}✓${N} in sync with origin/main ($head_short)"
|
|
rm -f "$PENDING_FILE"
|
|
elif [ "$ahead" -gt 0 ]; then
|
|
p " ${Y}▲${N} ${B}${ahead}${N} unpushed commit(s) (HEAD $head_short vs origin $origin_short)"
|
|
[ ! -f "$PENDING_FILE" ] && date +%s > "$PENDING_FILE"
|
|
pend_age=$(( now - $(cat "$PENDING_FILE" 2>/dev/null || echo "$now") ))
|
|
p " ${DIM}waiting $(human_age "$pend_age") — push is the missing step${N}"
|
|
if [ "$pend_age" -gt 1200 ]; then
|
|
problems+=("$ahead commit(s) unpushed for $(human_age "$pend_age") — the signer may be stuck")
|
|
signature+="pending-stuck;"
|
|
fi
|
|
else
|
|
p " ${Y}▼${N} ${behind} commit(s) behind origin/main"
|
|
fi
|
|
|
|
# Optional bounded probe: signing only happens when there IS something to push.
|
|
if [ "$PROBE_PUSH" = "1" ] && [ "$ahead" -gt 0 ]; then
|
|
p " ${DIM}probing: git push origin main (bounded 45s — this publishes the commits above)…${N}"
|
|
start=$(date +%s)
|
|
out="$(timeout 45 git push origin main 2>&1)"; rc=$?
|
|
el=$(( $(date +%s) - start ))
|
|
if [ "$rc" = "0" ]; then
|
|
p " ${G}✓${N} push OK in ${el}s — signer responsive"
|
|
elif [ "$rc" = "124" ]; then
|
|
p " ${R}✗${N} push TIMED OUT after ${el}s — the ngit/Amber signer looks unresponsive"
|
|
p " ${DIM}fix: open Amber in the foreground, or: ngit account login --interactive${N}"
|
|
problems+=("signer unresponsive (push timed out)")
|
|
signature+="signer-timeout;"
|
|
else
|
|
p " ${R}✗${N} push failed (rc=$rc):"
|
|
printf '%s\n' "$out" | tail -3 | while IFS= read -r l; do p " ${DIM}$l${N}"; done
|
|
problems+=("push failed (rc=$rc)")
|
|
signature+="signer-fail;"
|
|
fi
|
|
elif [ "$ahead" -gt 0 ]; then
|
|
p " ${DIM}run with --probe-push to actually test signing (publishes the commits above)${N}"
|
|
fi
|
|
|
|
# ── Deploy state ─────────────────────────────────────────────────────────────
|
|
p ""
|
|
p "${B}▸ Deploy${N}"
|
|
deployed="$(cat "$SHA_FILE" 2>/dev/null || echo none)"
|
|
origin="$(git rev-parse origin/main 2>/dev/null || echo none)"
|
|
last_epoch="$(cat "$INTERVAL_FILE" 2>/dev/null || echo 0)"
|
|
case "$last_epoch" in ''|*[!0-9]*) last_epoch=0 ;; esac
|
|
|
|
if [ "$deployed" = "$origin" ] && [ "$origin" != "none" ]; then
|
|
if [ "$last_epoch" -gt 0 ]; then
|
|
p " ${G}✓${N} live is current (${origin:0:9}), deployed $(human_age $((now - last_epoch))) ago"
|
|
else
|
|
p " ${G}✓${N} live is current (${origin:0:9})"
|
|
fi
|
|
else
|
|
p " ${Y}●${N} ${B}PENDING${N}: origin/main ${origin:0:9} but last deployed ${deployed:0:9}"
|
|
if [ -f "$SKIP_FILE" ]; then
|
|
IFS='|' read -r sk_at sk_reason sk_sha sk_detail < "$SKIP_FILE"
|
|
sk_age=$(( now - ${sk_at:-0} ))
|
|
p " last skip: ${B}${sk_reason:-unknown}${N} ($(human_age "$sk_age") ago) ${DIM}${sk_detail}${N}"
|
|
case "${sk_reason:-}" in
|
|
throttled)
|
|
p " ${DIM}→ batching to save Netlify credits; it deploys automatically when the window elapses.${N}" ;;
|
|
dirty-tree)
|
|
p " ${R}→ tracked files are modified — COMMIT them or the deploy stays skipped.${N}"
|
|
problems+=("deploy blocked: working tree is dirty"); signature+="dirty;" ;;
|
|
smoke-failed)
|
|
p " ${R}→ the built bundle failed the smoke test (real breakage — check the journal).${N}"
|
|
problems+=("smoke test failed during deploy"); signature+="smoke;" ;;
|
|
diverged)
|
|
p " ${R}→ local main diverged from origin/main — a manual merge is needed.${N}"
|
|
problems+=("local main diverged from origin/main"); signature+="diverged;" ;;
|
|
fetch-failed)
|
|
p " ${Y}→ couldn't fetch origin (network/relay issue).${N}" ;;
|
|
non-site-only|dist-unchanged)
|
|
p " ${DIM}→ no deploy was needed (docs-only or byte-identical build).${N}" ;;
|
|
locked|"")
|
|
p " ${DIM}→ reason: ${sk_reason:-unknown}${N}" ;;
|
|
*)
|
|
p " ${DIM}→ reason: ${sk_reason}${N}" ;;
|
|
esac
|
|
else
|
|
p " ${Y}no skip reason recorded — the deploy timer may not be running.${N}"
|
|
problems+=("deploy pending with no recorded reason"); signature+="noreason;"
|
|
fi
|
|
fi
|
|
|
|
min_interval="${SKATESPOTS_DEPLOY_MIN_INTERVAL:-86400}"
|
|
case "$min_interval" in ''|*[!0-9]*) min_interval=0 ;; esac
|
|
if [ "$min_interval" -gt 0 ] && [ "$last_epoch" -gt 0 ]; then
|
|
remain=$(( min_interval - (now - last_epoch) ))
|
|
if [ "$remain" -gt 0 ]; then
|
|
p " ${DIM}throttle: next auto-deploy eligible in ~$(human_age "$remain") · force now: scripts/auto-deploy.sh --force${N}"
|
|
fi
|
|
fi
|
|
|
|
# ── Timer ────────────────────────────────────────────────────────────────────
|
|
p ""
|
|
p "${B}▸ Timer${N}"
|
|
timer_state="$(systemctl --user is-active skatespots-deploy.timer 2>/dev/null || echo unknown)"
|
|
if [ "$timer_state" = "active" ]; then
|
|
next_at="$(systemctl --user list-timers skatespots-deploy.timer --no-legend --no-pager 2>/dev/null | awk '{print $1" "$2" "$3}')"
|
|
p " ${G}✓${N} skatespots-deploy.timer active${next_at:+ ${DIM}next $next_at${N}}"
|
|
else
|
|
p " ${R}✗${N} skatespots-deploy.timer: ${timer_state}"
|
|
problems+=("deploy timer is not active")
|
|
signature+="timer;"
|
|
fi
|
|
if [ -f "$RUN_FILE" ]; then
|
|
run_age=$(( now - $(cat "$RUN_FILE" 2>/dev/null || echo 0) ))
|
|
if [ "$run_age" -gt 600 ]; then
|
|
p " ${Y}⚠${N} deploy script last ran $(human_age "$run_age") ago (it polls ~every 3m)"
|
|
problems+=("deploy script hasn't run in $(human_age "$run_age")")
|
|
signature+="stale-run;"
|
|
else
|
|
p " ${DIM}last run $(human_age "$run_age") ago${N}"
|
|
fi
|
|
fi
|
|
|
|
# ── Summary + optional notification ──────────────────────────────────────────
|
|
p ""
|
|
if [ ${#problems[@]} -eq 0 ]; then
|
|
p "${G}${B}All good.${N}"
|
|
else
|
|
p "${R}${B}Needs attention:${N}"
|
|
for prob in "${problems[@]}"; do p " • $prob"; done
|
|
fi
|
|
|
|
if [ "$NOTIFY" = "1" ]; then
|
|
if [ ${#problems[@]} -gt 0 ]; then
|
|
prev="$(cat "$NOTIFY_FILE" 2>/dev/null || true)"
|
|
if [ "$signature" != "$prev" ]; then
|
|
body="$(printf '%s; ' "${problems[@]}")"
|
|
if command -v notify-send >/dev/null 2>&1; then
|
|
notify-send -a SkateSpots "SkateSpots ops" "$body" 2>/dev/null || true
|
|
fi
|
|
printf '%s' "$signature" > "$NOTIFY_FILE"
|
|
fi
|
|
else
|
|
rm -f "$NOTIFY_FILE" # healthy → re-arm notifications
|
|
fi
|
|
fi
|
|
|
|
if [ "$QUIET" = "1" ] && [ ${#problems[@]} -eq 0 ]; then
|
|
exit 0
|
|
fi
|
|
printf '%s' "$REPORT"
|
|
|
|
[ ${#problems[@]} -eq 0 ] && exit 0 || exit 1
|