b6cc7797e6
A non-site-only skip advances the recorded sha without a production deploy, so 'deployed Nm ago' could read as if that sha was deployed. Say 'last production deploy Nm ago' instead.
222 lines
9.4 KiB
Bash
Executable file
222 lines
9.4 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
#
|
|
# ops-health.sh — "why isn't it live?" at a glance.
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Two silent failures cost real time: the ngit/Amber remote signer stalling
|
|
# mid-push, and the deploy timer skipping while origin/main had moved. Both were
|
|
# invisible. This reports everything needed to tell them apart, and can notify.
|
|
#
|
|
# scripts/ops-health.sh # read-only report
|
|
# scripts/ops-health.sh --probe-push # ALSO attempt a bounded real push
|
|
# scripts/ops-health.sh --notify # desktop-notify on actionable problems
|
|
# scripts/ops-health.sh --quiet # print/notify only when something's wrong
|
|
#
|
|
# `--probe-push` publishes any unpushed commits (that IS the probe — signing only
|
|
# happens when there's something to sign). Without it the script never writes to
|
|
# the remote.
|
|
#
|
|
# Exit: 0 healthy · 1 something needs attention · 2 bad usage.
|
|
set -uo pipefail
|
|
|
|
REPO="/home/sovereignsk8/SkateSpots"
|
|
# Overridable so the report can be tested against a fake state dir.
|
|
STATE_DIR="${SKATESPOTS_STATE_DIR:-$HOME/.local/state/skatespots-deploy}"
|
|
SHA_FILE="$STATE_DIR/last-deployed-sha"
|
|
INTERVAL_FILE="$STATE_DIR/last-deploy-epoch"
|
|
SKIP_FILE="$STATE_DIR/last-skip"
|
|
RUN_FILE="$STATE_DIR/last-run"
|
|
NOTIFY_FILE="$STATE_DIR/last-notified"
|
|
PENDING_FILE="$STATE_DIR/pending-since"
|
|
mkdir -p "$STATE_DIR"
|
|
|
|
PROBE_PUSH=0
|
|
NOTIFY=0
|
|
QUIET=0
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--probe-push) PROBE_PUSH=1 ;;
|
|
--notify) NOTIFY=1 ;;
|
|
--quiet) QUIET=1 ;;
|
|
-h|--help) sed -n '2,20p' "$0"; exit 0 ;;
|
|
*) echo "ops-health: unknown option: $arg" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
if [ -t 1 ]; then
|
|
B=$'\e[1m'; DIM=$'\e[2m'; R=$'\e[31m'; G=$'\e[32m'; Y=$'\e[33m'; N=$'\e[0m'
|
|
else
|
|
B=; DIM=; R=; G=; Y=; N=
|
|
fi
|
|
|
|
REPORT=""
|
|
p() { REPORT+="$1"$'\n'; }
|
|
|
|
problems=() # short, actionable strings (for --notify and the summary)
|
|
signature="" # stable dedupe key for notifications
|
|
|
|
human_age() {
|
|
local s="$1"
|
|
if [ "$s" -lt 60 ]; then echo "${s}s"
|
|
elif [ "$s" -lt 3600 ]; then echo "$((s / 60))m"
|
|
elif [ "$s" -lt 86400 ]; then echo "$((s / 3600))h"
|
|
else echo "$((s / 86400))d"; fi
|
|
}
|
|
|
|
cd "$REPO" 2>/dev/null || { echo "ops-health: repo not found: $REPO" >&2; exit 2; }
|
|
now=$(date +%s)
|
|
|
|
# ── Repo & push state ────────────────────────────────────────────────────────
|
|
p "${B}▸ Repo & push${N}"
|
|
ahead=$(git rev-list --count origin/main..HEAD 2>/dev/null || echo 0)
|
|
behind=$(git rev-list --count HEAD..origin/main 2>/dev/null || echo 0)
|
|
head_short=$(git rev-parse --short HEAD 2>/dev/null || echo '?')
|
|
origin_short=$(git rev-parse --short origin/main 2>/dev/null || echo '?')
|
|
|
|
if [ "$ahead" = "0" ] && [ "$behind" = "0" ]; then
|
|
p " ${G}✓${N} in sync with origin/main ($head_short)"
|
|
rm -f "$PENDING_FILE"
|
|
elif [ "$ahead" -gt 0 ]; then
|
|
p " ${Y}▲${N} ${B}${ahead}${N} unpushed commit(s) (HEAD $head_short vs origin $origin_short)"
|
|
[ ! -f "$PENDING_FILE" ] && date +%s > "$PENDING_FILE"
|
|
pend_age=$(( now - $(cat "$PENDING_FILE" 2>/dev/null || echo "$now") ))
|
|
p " ${DIM}waiting $(human_age "$pend_age") — push is the missing step${N}"
|
|
if [ "$pend_age" -gt 1200 ]; then
|
|
problems+=("$ahead commit(s) unpushed for $(human_age "$pend_age") — the signer may be stuck")
|
|
signature+="pending-stuck;"
|
|
fi
|
|
else
|
|
p " ${Y}▼${N} ${behind} commit(s) behind origin/main"
|
|
fi
|
|
|
|
# Optional bounded probe: signing only happens when there IS something to push.
|
|
if [ "$PROBE_PUSH" = "1" ] && [ "$ahead" -gt 0 ]; then
|
|
p " ${DIM}probing: git push origin main (bounded 45s — this publishes the commits above)…${N}"
|
|
start=$(date +%s)
|
|
out="$(timeout 45 git push origin main 2>&1)"; rc=$?
|
|
el=$(( $(date +%s) - start ))
|
|
if [ "$rc" = "0" ]; then
|
|
p " ${G}✓${N} push OK in ${el}s — signer responsive"
|
|
elif [ "$rc" = "124" ]; then
|
|
p " ${R}✗${N} push TIMED OUT after ${el}s — the ngit/Amber signer looks unresponsive"
|
|
p " ${DIM}fix: open Amber in the foreground, or: ngit account login --interactive${N}"
|
|
problems+=("signer unresponsive (push timed out)")
|
|
signature+="signer-timeout;"
|
|
else
|
|
p " ${R}✗${N} push failed (rc=$rc):"
|
|
printf '%s\n' "$out" | tail -3 | while IFS= read -r l; do p " ${DIM}$l${N}"; done
|
|
problems+=("push failed (rc=$rc)")
|
|
signature+="signer-fail;"
|
|
fi
|
|
elif [ "$ahead" -gt 0 ]; then
|
|
p " ${DIM}run with --probe-push to actually test signing (publishes the commits above)${N}"
|
|
fi
|
|
|
|
# ── Deploy state ─────────────────────────────────────────────────────────────
|
|
p ""
|
|
p "${B}▸ Deploy${N}"
|
|
deployed="$(cat "$SHA_FILE" 2>/dev/null || echo none)"
|
|
origin="$(git rev-parse origin/main 2>/dev/null || echo none)"
|
|
last_epoch="$(cat "$INTERVAL_FILE" 2>/dev/null || echo 0)"
|
|
case "$last_epoch" in ''|*[!0-9]*) last_epoch=0 ;; esac
|
|
|
|
if [ "$deployed" = "$origin" ] && [ "$origin" != "none" ]; then
|
|
if [ "$last_epoch" -gt 0 ]; then
|
|
p " ${G}✓${N} live is current (${origin:0:9}) — last production deploy $(human_age $((now - last_epoch))) ago"
|
|
else
|
|
p " ${G}✓${N} live is current (${origin:0:9})"
|
|
fi
|
|
else
|
|
p " ${Y}●${N} ${B}PENDING${N}: origin/main ${origin:0:9} but last deployed ${deployed:0:9}"
|
|
if [ -f "$SKIP_FILE" ]; then
|
|
IFS='|' read -r sk_at sk_reason sk_sha sk_detail < "$SKIP_FILE"
|
|
sk_age=$(( now - ${sk_at:-0} ))
|
|
p " last skip: ${B}${sk_reason:-unknown}${N} ($(human_age "$sk_age") ago) ${DIM}${sk_detail}${N}"
|
|
case "${sk_reason:-}" in
|
|
throttled)
|
|
p " ${DIM}→ batching to save Netlify credits; it deploys automatically when the window elapses.${N}" ;;
|
|
dirty-tree)
|
|
p " ${R}→ tracked files are modified — COMMIT them or the deploy stays skipped.${N}"
|
|
problems+=("deploy blocked: working tree is dirty"); signature+="dirty;" ;;
|
|
smoke-failed)
|
|
p " ${R}→ the built bundle failed the smoke test (real breakage — check the journal).${N}"
|
|
problems+=("smoke test failed during deploy"); signature+="smoke;" ;;
|
|
diverged)
|
|
p " ${R}→ local main diverged from origin/main — a manual merge is needed.${N}"
|
|
problems+=("local main diverged from origin/main"); signature+="diverged;" ;;
|
|
fetch-failed)
|
|
p " ${Y}→ couldn't fetch origin (network/relay issue).${N}" ;;
|
|
non-site-only|dist-unchanged)
|
|
p " ${DIM}→ no deploy was needed (docs-only or byte-identical build).${N}" ;;
|
|
locked|"")
|
|
p " ${DIM}→ reason: ${sk_reason:-unknown}${N}" ;;
|
|
*)
|
|
p " ${DIM}→ reason: ${sk_reason}${N}" ;;
|
|
esac
|
|
else
|
|
p " ${Y}no skip reason recorded — the deploy timer may not be running.${N}"
|
|
problems+=("deploy pending with no recorded reason"); signature+="noreason;"
|
|
fi
|
|
fi
|
|
|
|
min_interval="${SKATESPOTS_DEPLOY_MIN_INTERVAL:-86400}"
|
|
case "$min_interval" in ''|*[!0-9]*) min_interval=0 ;; esac
|
|
if [ "$min_interval" -gt 0 ] && [ "$last_epoch" -gt 0 ]; then
|
|
remain=$(( min_interval - (now - last_epoch) ))
|
|
if [ "$remain" -gt 0 ]; then
|
|
p " ${DIM}throttle: next auto-deploy eligible in ~$(human_age "$remain") · force now: scripts/auto-deploy.sh --force${N}"
|
|
fi
|
|
fi
|
|
|
|
# ── Timer ────────────────────────────────────────────────────────────────────
|
|
p ""
|
|
p "${B}▸ Timer${N}"
|
|
timer_state="$(systemctl --user is-active skatespots-deploy.timer 2>/dev/null || echo unknown)"
|
|
if [ "$timer_state" = "active" ]; then
|
|
next_at="$(systemctl --user list-timers skatespots-deploy.timer --no-legend --no-pager 2>/dev/null | awk '{print $1" "$2" "$3}')"
|
|
p " ${G}✓${N} skatespots-deploy.timer active${next_at:+ ${DIM}next $next_at${N}}"
|
|
else
|
|
p " ${R}✗${N} skatespots-deploy.timer: ${timer_state}"
|
|
problems+=("deploy timer is not active")
|
|
signature+="timer;"
|
|
fi
|
|
if [ -f "$RUN_FILE" ]; then
|
|
run_age=$(( now - $(cat "$RUN_FILE" 2>/dev/null || echo 0) ))
|
|
if [ "$run_age" -gt 600 ]; then
|
|
p " ${Y}⚠${N} deploy script last ran $(human_age "$run_age") ago (it polls ~every 3m)"
|
|
problems+=("deploy script hasn't run in $(human_age "$run_age")")
|
|
signature+="stale-run;"
|
|
else
|
|
p " ${DIM}last run $(human_age "$run_age") ago${N}"
|
|
fi
|
|
fi
|
|
|
|
# ── Summary + optional notification ──────────────────────────────────────────
|
|
p ""
|
|
if [ ${#problems[@]} -eq 0 ]; then
|
|
p "${G}${B}All good.${N}"
|
|
else
|
|
p "${R}${B}Needs attention:${N}"
|
|
for prob in "${problems[@]}"; do p " • $prob"; done
|
|
fi
|
|
|
|
if [ "$NOTIFY" = "1" ]; then
|
|
if [ ${#problems[@]} -gt 0 ]; then
|
|
prev="$(cat "$NOTIFY_FILE" 2>/dev/null || true)"
|
|
if [ "$signature" != "$prev" ]; then
|
|
body="$(printf '%s; ' "${problems[@]}")"
|
|
if command -v notify-send >/dev/null 2>&1; then
|
|
notify-send -a SkateSpots "SkateSpots ops" "$body" 2>/dev/null || true
|
|
fi
|
|
printf '%s' "$signature" > "$NOTIFY_FILE"
|
|
fi
|
|
else
|
|
rm -f "$NOTIFY_FILE" # healthy → re-arm notifications
|
|
fi
|
|
fi
|
|
|
|
if [ "$QUIET" = "1" ] && [ ${#problems[@]} -eq 0 ]; then
|
|
exit 0
|
|
fi
|
|
printf '%s' "$REPORT"
|
|
|
|
[ ${#problems[@]} -eq 0 ] && exit 0 || exit 1
|