chore(ops): pre-push/pre-deploy test gate + patch-only deps fallback

- scripts/preflight.sh: app+server tsc, eslint, i18n, vitest; installed as
  .githooks/pre-push (scripts/install-hooks.sh) and run by auto-deploy before
  Netlify. Failures record preflight-failed and ops-health flags them.
- update-deps.sh: if the blanket npm update fails, restore the lock and retry
  patch-level packages one at a time, keeping those that pass a fast gate; the
  final npm test + smoke still gates the result, otherwise revert + 3-day retry.
- release.sh/deps set SKIP_PREFLIGHT=1 (they run the full npm test themselves).
This commit is contained in:
SkateSpots 2026-10-27 09:23:07 +07:00
parent 16ad6ad882
commit 39c388a6d7
8 changed files with 195 additions and 5 deletions

6
.githooks/pre-push Executable file
View file

@ -0,0 +1,6 @@
#!/usr/bin/env bash
# Run the fast gate before every manual push. Skip with:
# SKIP_PREFLIGHT=1 git push (or git push --no-verify)
set -euo pipefail
[ "${SKIP_PREFLIGHT:-0}" = "1" ] && exit 0
exec "$(git rev-parse --show-toplevel)/scripts/preflight.sh"

View file

@ -2390,6 +2390,18 @@ Rules:
`scripts/systemd/`, and `.nvmrc` pins Node 22. A 2026-10-27 scan-only run `scripts/systemd/`, and `.nvmrc` pins Node 22. A 2026-10-27 scan-only run
reports **3 packages / 4 OSV findings** (0 critical; `xlsx` is write-only and reports **3 packages / 4 OSV findings** (0 critical; `xlsx` is write-only and
the rest is dev build tooling). the rest is dev build tooling).
- **Pre-push + pre-deploy gate:** `scripts/preflight.sh` runs the app + server
typechecks → ESLint → i18n → vitest. It is installed as `.githooks/pre-push`
via `scripts/install-hooks.sh`, and `auto-deploy.sh` runs it before every
Netlify deploy; a failure records `preflight-failed` and `ops-health.sh`
flags it. Skip one push with `SKIP_PREFLIGHT=1` (or `git push --no-verify`).
The dependency and release scripts set `SKIP_PREFLIGHT=1` because they already
run the full `npm test` themselves.
- **Patch-only dependency fallback:** if the blanket `npm update` fails, the
script restores the lock and retries the **patch-level** outdated packages one
at a time, keeping those that pass the fast gate and skipping the rest; a
final `npm test` + boot smoke gates the result. If both the full and fallback
paths fail, it restores the original tree and schedules the ~3-day retry.
### Community relay — self-hosted Newlay (2026-10-12) ### Community relay — self-hosted Newlay (2026-10-12)

View file

@ -135,6 +135,19 @@ if [ "$LOCK_HASH" != "$(cat "$LOCK_FILE" 2>/dev/null || true)" ]; then
echo "$LOCK_HASH" > "$LOCK_FILE" echo "$LOCK_HASH" > "$LOCK_FILE"
fi fi
# 4b. Fast gate before spending a deploy: typecheck (app + servers) + lint +
# i18n + unit tests. The build/smoke below still guard runtime.
if [ ! -d node_modules ]; then
log "node_modules missing — npm ci"
npm ci --silent
fi
log "preflight: typecheck + lint + i18n + tests…"
if ! "$REPO/scripts/preflight.sh"; then
log "PREFLIGHT FAILED — not deploying"
record_skip preflight-failed "the test gate failed"
exit 0
fi
# 5. Build the web bundle directly (skip the `npm i` in `npm run build`). # 5. Build the web bundle directly (skip the `npm i` in `npm run build`).
log "building…" log "building…"
./node_modules/.bin/vite build -l error ./node_modules/.bin/vite build -l error

9
scripts/install-hooks.sh Executable file
View file

@ -0,0 +1,9 @@
#!/usr/bin/env bash
#
# Install the repo's git hooks (currently the pre-push test gate).
set -euo pipefail
REPO="$(cd "$(dirname "$0")/.." && pwd)"
cd "$REPO"
git config core.hooksPath .githooks
chmod +x .githooks/pre-push scripts/preflight.sh
echo "pre-push hook installed (bypass: SKIP_PREFLIGHT=1 or git push --no-verify)"

View file

@ -149,6 +149,9 @@ else
smoke-failed) smoke-failed)
p " ${R}→ the built bundle failed the smoke test (real breakage — check the journal).${N}" p " ${R}→ the built bundle failed the smoke test (real breakage — check the journal).${N}"
problems+=("smoke test failed during deploy"); signature+="smoke;" ;; problems+=("smoke test failed during deploy"); signature+="smoke;" ;;
preflight-failed)
p " ${R}→ the test gate failed — fix typecheck/lint/i18n/tests before it can deploy.${N}"
problems+=("deploy blocked: preflight failed"); signature+="preflight;" ;;
diverged) diverged)
p " ${R}→ local main diverged from origin/main — a manual merge is needed.${N}" p " ${R}→ local main diverged from origin/main — a manual merge is needed.${N}"
problems+=("local main diverged from origin/main"); signature+="diverged;" ;; problems+=("local main diverged from origin/main"); signature+="diverged;" ;;

32
scripts/preflight.sh Executable file
View file

@ -0,0 +1,32 @@
#!/usr/bin/env bash
#
# Fast gate for manual pushes and the auto-deploy timer:
# typecheck (app + servers) → lint → i18n → unit tests. Does NOT build; the
# deploy script builds and boots the bundle afterwards.
#
# Run manually: scripts/preflight.sh
# Bypass once: SKIP_PREFLIGHT=1 git push (or git push --no-verify)
set -euo pipefail
REPO="$(cd "$(dirname "$0")/.." && pwd)"
cd "$REPO"
if [ -d "$HOME/.nvm/versions/node" ]; then
NODE_BIN="$(ls -d "$HOME"/.nvm/versions/node/*/bin 2>/dev/null | sort -V | tail -1)"
[ -n "$NODE_BIN" ] && export PATH="$NODE_BIN:$PATH"
fi
export PATH="$HOME/go/bin:$PATH"
[ -d node_modules ] || { echo "preflight: node_modules missing — run npm ci" >&2; exit 1; }
echo "preflight: typecheck"
npx tsc --noEmit
npx tsc -p cvm-server/tsconfig.json
npx tsc -p ai-proxy/tsconfig.json
npx tsc -p push-service/tsconfig.json
echo "preflight: lint"
npx eslint
echo "preflight: i18n"
node scripts/check-i18n.mjs
echo "preflight: unit tests"
npx vitest run --reporter=dot --silent
echo "preflight: OK"

View file

@ -130,6 +130,8 @@ log "committing the release…"
git add package.json package-lock.json release-notes.txt git add package.json package-lock.json release-notes.txt
git commit -m "release: $VERSION" >/dev/null git commit -m "release: $VERSION" >/dev/null
git tag -a "v$VERSION" -m "SkateSpots $VERSION" git tag -a "v$VERSION" -m "SkateSpots $VERSION"
# `npm test` already ran above; don't make the pre-push hook repeat it.
export SKIP_PREFLIGHT=1
git push origin main git push origin main
if ! git push origin "v$VERSION" 2>/dev/null; then if ! git push origin "v$VERSION" 2>/dev/null; then
log "warning: tag v$VERSION was created locally but the push failed" log "warning: tag v$VERSION was created locally but the push failed"

View file

@ -29,6 +29,8 @@ if [ -d "$HOME/.nvm/versions/node" ]; then
[ -n "$NODE_BIN" ] && export PATH="$NODE_BIN:$PATH" [ -n "$NODE_BIN" ] && export PATH="$NODE_BIN:$PATH"
fi fi
export PATH="$HOME/go/bin:$PATH" export PATH="$HOME/go/bin:$PATH"
# The job runs the full gate itself; don't make the pre-push hook repeat it.
export SKIP_PREFLIGHT=1
cd "$REPO" cd "$REPO"
INTERVAL_DAYS="${DEPS_INTERVAL_DAYS:-16}" INTERVAL_DAYS="${DEPS_INTERVAL_DAYS:-16}"
@ -38,6 +40,7 @@ LAST="$STATE_DIR/last-run"
REPORT="$STATE_DIR/report-$(LC_ALL=C date +%F).txt" REPORT="$STATE_DIR/report-$(LC_ALL=C date +%F).txt"
FAILURE_FILE="$STATE_DIR/last-failure" FAILURE_FILE="$STATE_DIR/last-failure"
FAILURE_DIR_BASE="$STATE_DIR/failures" FAILURE_DIR_BASE="$STATE_DIR/failures"
OUTDATED_JSON="$STATE_DIR/outdated-$(LC_ALL=C date +%F).json"
# OSV-Scanner (broader than npm audit: queries the OSV database for the lockfile). # OSV-Scanner (broader than npm audit: queries the OSV database for the lockfile).
# Installed with: go install github.com/google/osv-scanner/v2/cmd/osv-scanner@latest # Installed with: go install github.com/google/osv-scanner/v2/cmd/osv-scanner@latest
OSV_BIN="${OSV_BIN:-$HOME/go/bin/osv-scanner}" OSV_BIN="${OSV_BIN:-$HOME/go/bin/osv-scanner}"
@ -57,6 +60,68 @@ notify() {
} }
finish_timer() { date +%s > "$LAST"; } finish_timer() { date +%s > "$LAST"; }
# ── fallback helpers ────────────────────────────────────────────────────────
# Fast gate for the per-package fallback. The full `npm test` + smoke still runs
# once at the end; this keeps the fallback from taking an hour.
fast_gate() {
npx tsc --noEmit >/dev/null 2>&1 \
&& npx tsc -p cvm-server/tsconfig.json >/dev/null 2>&1 \
&& npx tsc -p ai-proxy/tsconfig.json >/dev/null 2>&1 \
&& npx tsc -p push-service/tsconfig.json >/dev/null 2>&1 \
&& node scripts/check-i18n.mjs >/dev/null 2>&1 \
&& npx vitest run --reporter=dot --silent >/dev/null 2>&1
}
# If the blanket `npm update` fails, retry only patch-level bumps one package at
# a time; keep each one that passes the fast gate and skip the rest. This keeps
# safe updates instead of reverting the whole batch. Requires OUTDATED_JSON
# captured before the full update.
patch_only_fallback() {
local tmp list pkg kept=() skipped=()
tmp="$(mktemp -d)"
cp package.json package-lock.json "$tmp"/ 2>/dev/null || true
list="$(node -e '
const fs = require("fs");
let j = {};
try { j = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); } catch {}
for (const [name, v] of Object.entries(j)) {
const cur = String(v.current || ""), want = String(v.wanted || "");
if (!cur || !want || cur === want) continue;
const a = cur.split("."), b = want.split(".");
if (a.length < 3 || b.length < 3) continue;
if (a[0] === b[0] && a[1] === b[1] && Number(b[2]) > Number(a[2])) console.log(name);
}
' "$OUTDATED_JSON" 2>/dev/null || true)"
if [ -z "$list" ]; then
log "fallback: no patch-only updates available"
rm -rf "$tmp"
return 1
fi
while IFS= read -r pkg; do
[ -n "$pkg" ] || continue
log "fallback: trying $pkg"
if npm update "$pkg" >/dev/null 2>&1 \
&& npm dedupe >/dev/null 2>&1 \
&& fast_gate; then
kept+=("$pkg")
# New baseline for the next package.
cp package.json package-lock.json "$tmp"/ 2>/dev/null || true
else
log "fallback: $pkg failed the fast gate — reverting it"
cp "$tmp/package.json" "$tmp/package-lock.json" . 2>/dev/null || true
npm i --silent >/dev/null 2>&1 || true
skipped+=("$pkg")
fi
done <<< "$list"
rm -rf "$tmp"
log "fallback: kept ${#kept[@]} (${kept[*]:-none}), skipped ${#skipped[@]} (${skipped[*]:-none})"
[ "${#kept[@]}" -gt 0 ]
}
# ── interval gate ─────────────────────────────────────────────────────────── # ── interval gate ───────────────────────────────────────────────────────────
if [ "$FORCE" -eq 0 ] && [ "$SCAN_ONLY" -eq 0 ] && [ -f "$LAST" ]; then if [ "$FORCE" -eq 0 ] && [ "$SCAN_ONLY" -eq 0 ] && [ -f "$LAST" ]; then
last=$(cat "$LAST" 2>/dev/null || echo 0) last=$(cat "$LAST" 2>/dev/null || echo 0)
@ -111,6 +176,9 @@ fi
cp package.json "$STATE_DIR/package.json.bak" 2>/dev/null || true cp package.json "$STATE_DIR/package.json.bak" 2>/dev/null || true
cp package-lock.json "$STATE_DIR/package-lock.json.bak" 2>/dev/null || true cp package-lock.json "$STATE_DIR/package-lock.json.bak" 2>/dev/null || true
# Snapshot the pre-update outdated list for the patch-only fallback.
npm outdated --json --depth=0 > "$OUTDATED_JSON" 2>/dev/null || true
# ── apply non-breaking updates ────────────────────────────────────────────── # ── apply non-breaking updates ──────────────────────────────────────────────
log "applying non-breaking updates (npm update + npm dedupe + npm audit fix)…" log "applying non-breaking updates (npm update + npm dedupe + npm audit fix)…"
npm update 2>&1 | tail -3 npm update 2>&1 | tail -3
@ -151,19 +219,64 @@ if npm test >"$STATE_DIR/test.log" 2>&1 && node scripts/smoke-test.mjs >"$STATE_
rm -f "$FAILURE_FILE" rm -f "$FAILURE_FILE"
log "done" log "done"
else else
log "tests FAILED — saving diagnostics and restoring the previous lockfile" log "full update FAILED — saving diagnostics and trying the patch-only fallback"
FAIL_DIR="$FAILURE_DIR_BASE/$(date +%F-%H%M)" FAIL_DIR="$FAILURE_DIR_BASE/$(date +%F-%H%M)"
mkdir -p "$FAIL_DIR" mkdir -p "$FAIL_DIR"
cp "$STATE_DIR/test.log" "$STATE_DIR/smoke.log" "$FAIL_DIR"/ 2>/dev/null || true cp "$STATE_DIR/test.log" "$STATE_DIR/smoke.log" "$FAIL_DIR"/ 2>/dev/null || true
cp package.json package-lock.json "$FAIL_DIR"/ 2>/dev/null || true cp package.json package-lock.json "$FAIL_DIR"/ 2>/dev/null || true
printf '%s\n' "tests failed at $(date -Is)" "see $FAIL_DIR/test.log" > "$FAIL_DIR/summary.txt" printf '%s\n' "full update failed at $(date -Is)" "see $FAIL_DIR/test.log" > "$FAIL_DIR/summary.txt"
echo "$(date -Is) tests failed — $FAIL_DIR" > "$FAILURE_FILE" echo "$(date -Is) full update failed — $FAIL_DIR" > "$FAILURE_FILE"
# Restore the original tree before trying the fallback.
cp "$STATE_DIR/package.json.bak" package.json 2>/dev/null || true
cp "$STATE_DIR/package-lock.json.bak" package-lock.json 2>/dev/null || true
npm i --silent >/dev/null 2>&1 || true
if patch_only_fallback \
&& npm test >"$STATE_DIR/test.log" 2>&1 \
&& node scripts/smoke-test.mjs >"$STATE_DIR/smoke.log" 2>&1; then
{
echo; echo "=== post-fallback npm audit ==="; npm audit 2>&1 || true
echo; echo "=== post-fallback osv-scanner ==="
if [ -x "$OSV_BIN" ]; then "$OSV_BIN" scan source -L package-lock.json 2>&1 || true; fi
} | tee -a "$REPORT"
OSV_SUMMARY="$(grep -E '^Total [0-9]+ packages affected' "$REPORT" | tail -1)"
[ -n "$OSV_SUMMARY" ] || OSV_SUMMARY="OSV scan: no summary"
if [ -n "$(git status --porcelain package.json package-lock.json)" ]; then
git add package.json package-lock.json
git commit -m "chore(deps): scheduled dependency update + audit ($(LC_ALL=C date +%F))" >/dev/null 2>&1 || true
if [ "$NO_PUSH" -eq 0 ]; then
if timeout 180 git push origin main >"$STATE_DIR/push.log" 2>&1; then
log "pushed patch-only dependency update"
notify "Dependency update applied via the patch-only fallback, tests pass, pushed. ${OSV_SUMMARY}"
else
log "push failed/skipped (see $STATE_DIR/push.log)"
notify "Patch-only dependency update committed locally but the push failed. ${OSV_SUMMARY}"
fi
else
log "committed locally (--no-push)"
notify "Patch-only dependency update applied and committed locally. ${OSV_SUMMARY}"
fi
else
log "patch-only fallback produced no package changes"
notify "Dependency scan complete: nothing safe to update. ${OSV_SUMMARY}"
fi
finish_timer
rm -f "$FAILURE_FILE"
log "done (patch-only fallback)"
exit 0
fi
# Both the full and the patch-only update failed: restore and retry in ~3 days.
mkdir -p "$FAIL_DIR/fallback"
cp "$STATE_DIR/test.log" "$STATE_DIR/smoke.log" "$FAIL_DIR/fallback"/ 2>/dev/null || true
cp package.json package-lock.json "$FAIL_DIR/fallback"/ 2>/dev/null || true
cp "$STATE_DIR/package.json.bak" package.json 2>/dev/null || true cp "$STATE_DIR/package.json.bak" package.json 2>/dev/null || true
cp "$STATE_DIR/package-lock.json.bak" package-lock.json 2>/dev/null || true cp "$STATE_DIR/package-lock.json.bak" package-lock.json 2>/dev/null || true
npm i --silent >/dev/null 2>&1 || true npm i --silent >/dev/null 2>&1 || true
# A failed run must NOT consume the whole interval: retry in ~3 days instead.
RETRY_DAYS=3; [ "$INTERVAL_DAYS" -gt 4 ] || RETRY_DAYS=1 RETRY_DAYS=3; [ "$INTERVAL_DAYS" -gt 4 ] || RETRY_DAYS=1
echo $(( $(date +%s) - (INTERVAL_DAYS - RETRY_DAYS) * 86400 )) > "$LAST" echo $(( $(date +%s) - (INTERVAL_DAYS - RETRY_DAYS) * 86400 )) > "$LAST"
notify "Dependency update REVERTED — tests failed (see $FAIL_DIR). ${OSV_SUMMARY}" notify "Dependency update REVERTED — full and patch fallback failed (see $FAIL_DIR). ${OSV_SUMMARY}"
exit 1 exit 1
fi fi