feat(ops): push-marked admin alerts, scan-change DMs, deploy/health alerts
- NIP-17 wraps now carry the app's ['skatespots','push'] marker on every admin copy (including the sender), so the push service wakes the phone with the app closed. Push test wraps 39fbbd4e…/c88c9f55… verified on the community relay. - The weekly scan-only job DMs only when the OSV summary changes; unchanged scans are logged, not messaged. - auto-deploy alerts on Netlify failure and on a post-deploy production URL health-check failure; deploy-failed/health-failed appear in ops-health and the daily report.
This commit is contained in:
SkateSpots
parent
b87e81545b
commit
bf28519e9e
6 changed files with 152 additions and 51 deletions
|
|
@ -2415,6 +2415,14 @@ Rules:
|
|||
- `scripts/send-ops-alert.sh "…"` sends a short alert; wired into
|
||||
`auto-deploy.sh` (preflight/smoke failure), `update-deps.sh` (full+fallback
|
||||
failure) and `ops-health.sh --notify` (deduped by problem signature).
|
||||
- **Push:** every admin copy (including the sender's) carries the app's
|
||||
`["skatespots","push"]` marker, so the push service wakes the phone with the
|
||||
app closed. A push test was sent 2026-10-27 (wraps `39fbbd4e…`/`c88c9f55…`).
|
||||
- **Scan noise:** the weekly scan-only DM is sent only when the OSV summary
|
||||
**changes**; unchanged scans are logged, not messaged.
|
||||
- **Deploy alerts:** `auto-deploy.sh` alerts on a Netlify failure and on a
|
||||
post-deploy `https://skate-spots.org/` health-check failure
|
||||
(`deploy-failed` / `health-failed` show in ops-health and the daily report).
|
||||
- `skatespots-ops-report.timer` runs daily at 07:00 (Persistent); install with
|
||||
`scripts/install-ops-report-timer.sh`. A test report was DMed to both admins
|
||||
on 2026-10-27 (kind 1059 on the community relay for each).
|
||||
|
|
|
|||
|
|
@ -187,10 +187,34 @@ fi
|
|||
|
||||
# 7b. Deploy to Netlify production.
|
||||
log "deploying to Netlify…"
|
||||
netlify deploy --prod --dir=dist
|
||||
if ! netlify deploy --prod --dir=dist; then
|
||||
log "NETLIFY DEPLOY FAILED"
|
||||
record_skip deploy-failed "netlify deploy failed"
|
||||
timeout 60 "$REPO/scripts/send-ops-alert.sh" "The website deploy failed at Netlify. The live site is unchanged or incomplete. Please check the Netlify dashboard." || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "$REMOTE_SHA" > "$SHA_FILE"
|
||||
echo "$DIST_HASH" > "$DIST_HASH_FILE"
|
||||
date +%s > "$INTERVAL_FILE"
|
||||
|
||||
# 7c. Production health check. The CDN can take a few seconds, so retry
|
||||
# briefly. If it never answers, leave a health-failed marker (the next tick
|
||||
# sees LAST_SHA == REMOTE_SHA and won't loop-redeploy) and alert.
|
||||
health_ok=0
|
||||
for _ in 1 2 3 4; do
|
||||
if curl -fsSL --max-time 20 https://skate-spots.org/ >/dev/null 2>&1; then
|
||||
health_ok=1
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
if [ "$health_ok" -ne 1 ]; then
|
||||
log "POST-DEPLOY HEALTH CHECK FAILED"
|
||||
record_skip health-failed "the production URL did not answer"
|
||||
timeout 60 "$REPO/scripts/send-ops-alert.sh" "The website deployed but did not answer its health check. Please check it." || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm -f "$SKIP_FILE"
|
||||
log "deployed ${REMOTE_SHA:0:9} ✓"
|
||||
|
|
|
|||
|
|
@ -152,6 +152,12 @@ else
|
|||
preflight-failed)
|
||||
p " ${R}→ the test gate failed — fix typecheck/lint/i18n/tests before it can deploy.${N}"
|
||||
problems+=("deploy blocked: preflight failed"); signature+="preflight;" ;;
|
||||
deploy-failed)
|
||||
p " ${R}→ Netlify rejected the deploy — check the Netlify dashboard.${N}"
|
||||
problems+=("Netlify deploy failed"); signature+="deploy-failed;" ;;
|
||||
health-failed)
|
||||
p " ${R}→ the production URL did not answer after the deploy.${N}"
|
||||
problems+=("production health check failed"); signature+="health-failed;" ;;
|
||||
diverged)
|
||||
p " ${R}→ local main diverged from origin/main — a manual merge is needed.${N}"
|
||||
problems+=("local main diverged from origin/main"); signature+="diverged;" ;;
|
||||
|
|
|
|||
|
|
@ -63,53 +63,61 @@ fi
|
|||
|
||||
# ── website ─────────────────────────────────────────────────────────────────
|
||||
web_line=""
|
||||
if [ "$deployed_sha" = "$origin_sha" ]; then
|
||||
skip_reason=""; sk_at=""; sk_sha=""; sk_detail=""
|
||||
if [ -f "$skip_file" ]; then
|
||||
IFS='|' read -r sk_at skip_reason sk_sha sk_detail < "$skip_file"
|
||||
fi
|
||||
|
||||
if [ "$skip_reason" = "health-failed" ]; then
|
||||
web_line="deployed but did not answer its health check"
|
||||
problems+=("The website deployed but did not answer its health check.")
|
||||
elif [ "$deployed_sha" = "$origin_sha" ]; then
|
||||
web_line="the website is up to date"
|
||||
else
|
||||
if [ -f "$skip_file" ]; then
|
||||
IFS='|' read -r sk_at sk_reason sk_sha sk_detail < "$skip_file"
|
||||
case "${sk_reason:-}" in
|
||||
throttled)
|
||||
# `sk_at` is the last CHECK time (the timer polls every 3 min); the
|
||||
# 24h window is measured from the last successful DEPLOY.
|
||||
case "${skip_reason:-}" in
|
||||
throttled)
|
||||
# `sk_at` is the last CHECK time (the timer polls every 3 min); the 24h
|
||||
# window is measured from the last successful DEPLOY.
|
||||
remain=$((86400 - (now - ${deploy_epoch:-0})))
|
||||
if [ "$remain" -gt 0 ]; then
|
||||
web_line="new changes are waiting; it will update automatically in about $(human_left "$remain")"
|
||||
else
|
||||
web_line="new changes are waiting; it should update at the next check"
|
||||
fi ;;
|
||||
non-site-only|dist-unchanged)
|
||||
web_line="up to date (the latest changes did not affect the website)" ;;
|
||||
dirty-tree)
|
||||
if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
|
||||
web_line="paused because the server has uncommitted changes"
|
||||
problems+=("The website deploy is paused because the server has uncommitted changes.")
|
||||
else
|
||||
remain=$((86400 - (now - ${deploy_epoch:-0})))
|
||||
if [ "$remain" -gt 0 ]; then
|
||||
web_line="new changes are waiting; it will update automatically in about $(human_left "$remain")"
|
||||
else
|
||||
web_line="new changes are waiting; it should update at the next check"
|
||||
fi ;;
|
||||
non-site-only|dist-unchanged)
|
||||
web_line="up to date (the latest changes did not affect the website)" ;;
|
||||
dirty-tree)
|
||||
if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
|
||||
web_line="paused because the server has uncommitted changes"
|
||||
problems+=("The website deploy is paused because the server has uncommitted changes.")
|
||||
else
|
||||
remain=$((86400 - (now - ${deploy_epoch:-0})))
|
||||
if [ "$remain" -gt 0 ]; then
|
||||
web_line="new changes are waiting; it will update automatically in about $(human_left "$remain")"
|
||||
else
|
||||
web_line="new changes are waiting; the next check will deploy them"
|
||||
fi
|
||||
fi ;;
|
||||
preflight-failed)
|
||||
web_line="paused because the tests failed"
|
||||
problems+=("The website deploy is paused because the tests failed.") ;;
|
||||
smoke-failed)
|
||||
web_line="paused because the built app did not start"
|
||||
problems+=("The website deploy is paused because the built app did not start.") ;;
|
||||
diverged)
|
||||
web_line="paused because the server code is out of sync"
|
||||
problems+=("The website deploy is paused because the server code is out of sync.") ;;
|
||||
fetch-failed)
|
||||
web_line="waiting because the server could not reach the code host"
|
||||
problems+=("The website deploy could not reach the code host.") ;;
|
||||
*)
|
||||
web_line="new changes are waiting (reason: ${sk_reason:-unknown})" ;;
|
||||
esac
|
||||
else
|
||||
web_line="new changes are waiting; the next check will deploy them"
|
||||
fi
|
||||
web_line="new changes are waiting; the next check will deploy them"
|
||||
fi
|
||||
fi ;;
|
||||
preflight-failed)
|
||||
web_line="paused because the tests failed"
|
||||
problems+=("The website deploy is paused because the tests failed.") ;;
|
||||
smoke-failed)
|
||||
web_line="paused because the built app did not start"
|
||||
problems+=("The website deploy is paused because the built app did not start.") ;;
|
||||
deploy-failed)
|
||||
web_line="the deploy failed at Netlify"
|
||||
problems+=("The website deploy failed at Netlify.") ;;
|
||||
diverged)
|
||||
web_line="paused because the server code is out of sync"
|
||||
problems+=("The website deploy is paused because the server code is out of sync.") ;;
|
||||
fetch-failed)
|
||||
web_line="waiting because the server could not reach the code host"
|
||||
problems+=("The website deploy could not reach the code host.") ;;
|
||||
"")
|
||||
web_line="new changes are waiting; the next check will deploy them" ;;
|
||||
*)
|
||||
web_line="new changes are waiting (reason: ${skip_reason})" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
deploy_timer="$(systemctl --user is-active skatespots-deploy.timer 2>/dev/null || echo unknown)"
|
||||
|
|
|
|||
|
|
@ -16,9 +16,10 @@ import { readFileSync } from 'node:fs';
|
|||
import { homedir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { SimplePool, useWebSocketImplementation } from 'nostr-tools/pool';
|
||||
import * as nip17 from 'nostr-tools/nip17';
|
||||
import * as nip44 from 'nostr-tools/nip44';
|
||||
import * as nip59 from 'nostr-tools/nip59';
|
||||
import * as nip19 from 'nostr-tools/nip19';
|
||||
import { getPublicKey } from 'nostr-tools/pure';
|
||||
import { finalizeEvent, generateSecretKey, getPublicKey } from 'nostr-tools/pure';
|
||||
import WebSocket from 'ws';
|
||||
import { ADMIN_PUBKEYS } from '../src/lib/admins';
|
||||
|
||||
|
|
@ -50,6 +51,55 @@ function withTimeout<T>(p: Promise<T>, ms: number): Promise<T> {
|
|||
]);
|
||||
}
|
||||
|
||||
const DM_KIND = 14;
|
||||
const GIFT_WRAP_KIND = 1059;
|
||||
/** Same plaintext marker the app's DM sender and push-service use. */
|
||||
const PUSH_MARKER = 'skatespots';
|
||||
|
||||
function randomPastTimestamp(): number {
|
||||
return Math.floor(Date.now() / 1000) - Math.floor(Math.random() * 2 * 24 * 60 * 60);
|
||||
}
|
||||
|
||||
/**
|
||||
* NIP-17 gift wraps, mirroring `useDirectMessages.ts` but adding the push
|
||||
* marker to every admin copy (including the sender's, so the founder's phone
|
||||
* also wakes for the daily report). The push service only reads the recipient
|
||||
* `p` tag + marker; it never decrypts.
|
||||
*/
|
||||
function createWraps(
|
||||
senderSecret: Uint8Array,
|
||||
senderPubkey: string,
|
||||
admins: string[],
|
||||
text: string,
|
||||
) {
|
||||
const rumor = nip59.createRumor(
|
||||
{
|
||||
kind: DM_KIND,
|
||||
created_at: Math.floor(Date.now() / 1000),
|
||||
tags: [...admins.map((pubkey) => ['p', pubkey]), ['subject', 'SkateSpots ops']],
|
||||
content: text,
|
||||
},
|
||||
senderSecret,
|
||||
);
|
||||
|
||||
const targets = [...new Set([senderPubkey, ...admins])];
|
||||
return targets.map((target) => {
|
||||
const seal = nip59.createSeal(rumor, senderSecret, target);
|
||||
const ephemeral = generateSecretKey();
|
||||
const conversationKey = nip44.v2.utils.getConversationKey(ephemeral, target);
|
||||
const content = nip44.v2.encrypt(JSON.stringify(seal), conversationKey);
|
||||
return finalizeEvent(
|
||||
{
|
||||
kind: GIFT_WRAP_KIND,
|
||||
created_at: randomPastTimestamp(),
|
||||
tags: [['p', target], [PUSH_MARKER, 'push']],
|
||||
content,
|
||||
},
|
||||
ephemeral,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const text = readFileSync(0, 'utf8').trim();
|
||||
if (!text) throw new Error('empty report on stdin');
|
||||
|
|
@ -60,11 +110,6 @@ async function main(): Promise<void> {
|
|||
const senderSecret = decoded.data;
|
||||
const senderPubkey = getPublicKey(senderSecret);
|
||||
|
||||
// `wrapManyEvents` adds the sender automatically; don't add the founder twice.
|
||||
const recipients = ADMIN_PUBKEYS
|
||||
.filter((pubkey) => pubkey !== senderPubkey)
|
||||
.map((publicKey) => ({ publicKey }));
|
||||
|
||||
const pool = new SimplePool();
|
||||
const authors = [...new Set([...ADMIN_PUBKEYS, senderPubkey])];
|
||||
|
||||
|
|
@ -101,7 +146,7 @@ async function main(): Promise<void> {
|
|||
}
|
||||
}
|
||||
|
||||
const wraps = nip17.wrapManyEvents(senderSecret, recipients, text, 'SkateSpots ops');
|
||||
const wraps = createWraps(senderSecret, senderPubkey, ADMIN_PUBKEYS, text);
|
||||
|
||||
if (process.env.OPS_DRY_RUN === '1') {
|
||||
console.log(`sender: ${nip19.npubEncode(senderPubkey)}`);
|
||||
|
|
|
|||
|
|
@ -163,7 +163,17 @@ ls -1dt "$FAILURE_DIR_BASE"/* 2>/dev/null | tail -n +9 | xargs -r rm -rf
|
|||
if [ "$SCAN_ONLY" -eq 1 ]; then
|
||||
log "scan-only; report: $REPORT"
|
||||
echo "$OSV_SUMMARY"
|
||||
notify "Vulnerability scan: $OSV_SUMMARY (report: $REPORT)"
|
||||
PREV_REPORT="$(ls -1t "$STATE_DIR"/report-*.txt 2>/dev/null | sed -n '2p' | head -1 || true)"
|
||||
PREV_OSV=""
|
||||
if [ -n "$PREV_REPORT" ]; then
|
||||
PREV_OSV="$(grep -E '^Total [0-9]+ packages affected' "$PREV_REPORT" | tail -1 || true)"
|
||||
fi
|
||||
if [ "$OSV_SUMMARY" != "$PREV_OSV" ]; then
|
||||
notify "Vulnerability scan changed: $OSV_SUMMARY (report: $REPORT)"
|
||||
timeout 90 "$REPO/scripts/send-ops-alert.sh" "Security scan changed. $OSV_SUMMARY. Check the server report for details." || true
|
||||
else
|
||||
log "scan unchanged since the previous report — not alerting"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue