From c01521ea3faa91ca8217a26dbe4cff4171fa18d7 Mon Sep 17 00:00:00 2001 From: SkateSpots Date: Mon, 28 Sep 2026 12:41:50 +0700 Subject: [PATCH] feat(backup): scoped weekly archive of the community relay dataset scripts/archive-relay.mjs snapshots the app's own content off goskatespots.nostr1.com to gzipped JSONL of the raw signed events, so a restore can republish them verbatim to any relay. scripts/restore-relay.mjs re-verifies signatures and publishes to an explicit --to (no default, to avoid replaying a snapshot onto the live relay). scripts/backup-relay.sh wraps it and pushes with restic when a repo is configured; a weekly systemd user timer runs it. Scoped on purpose: that relay is PUBLIC and carries general traffic (15.7k kind-0 profiles, 2k kind-30078 app-data from other clients, 408 foreign NIP-99 listings), so the archive keeps only the app's own kinds/tags plus contributor profiles, SkateSpots labels, and reports that reference archived content. ~114 events / 63 KiB. Not configured yet: offsite (restic.env) and a full restore drill. --- AGENTS.md | 35 +++ scripts/archive-relay.mjs | 235 +++++++++++++++++++++ scripts/backup-relay.sh | 45 ++++ scripts/install-archive-timer.sh | 16 ++ scripts/restic.env.example | 23 ++ scripts/restore-relay.mjs | 86 ++++++++ scripts/systemd/skatespots-archive.service | 6 + scripts/systemd/skatespots-archive.timer | 10 + 8 files changed, 456 insertions(+) create mode 100755 scripts/archive-relay.mjs create mode 100755 scripts/backup-relay.sh create mode 100755 scripts/install-archive-timer.sh create mode 100644 scripts/restic.env.example create mode 100755 scripts/restore-relay.mjs create mode 100644 scripts/systemd/skatespots-archive.service create mode 100644 scripts/systemd/skatespots-archive.timer diff --git a/AGENTS.md b/AGENTS.md index 2bb45d3..f23c647 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -623,6 +623,41 @@ Rules: - `zapstore.yaml` sets `release_source` → local APK, `release_notes` → `release-notes.txt`, `icon` → `public/images/icon-1024.png`. - Dry run (no upload): `SIGN_WITH='' zsp publish --offline zapstore.yaml`. +### Relay archive (dataset backup) +- `scripts/archive-relay.mjs` snapshots **the app's own content** off the + community relay to gzipped JSONL of the raw signed events (restorable to any + relay, signatures intact). Default dir `~/.backups/relay` (override with + `SKATESPOTS_ARCHIVE_DIR`), keeps the last 12, and writes `manifest-*.json` + + `latest.json` (counts by kind, author count, sha256, byte size). +- ⚠️ **`goskatespots.nostr1.com` is a PUBLIC relay with general traffic** + (verified 2026-09-28): 15.7k kind-0 profiles, mostly non-skaters; 2k kind-30078 + app-data (Armada/Plebs/Amethyst); 379 kind-30023 articles (Yakihonne); 408 + NIP-99 listings (Shopstr). So the archive is deliberately **scoped** — exclusive + kinds wholesale (30818/30017/30823/35xxx/…), shared kinds only with the app's + own tags (`t=skateboarding` for events + listings, `t=skatespots` collections, + `t=skatespotsnews` newsletters, `d=skatespots*` app data), `org.skatespots.*` + labels, reports/RSVPs that reference archived content, and kind-0 profiles for + archived authors only. Result: ~114 events / ~63 KiB. **Never archive the whole + relay** — it is strangers' data, and a restore would republish it. +- ⚠️ **Query ONE kind at a time, and stop on "no new events", not on a short + page.** A single filter covering many kinds comes back truncated (once kept + only 3 of 54 spots), and `querySync` can resolve early on a large page (lost + ~150 of 3.3k profiles). Paginate with `until` until a page adds nothing. +- `scripts/restore-relay.mjs --dry-run` re-verifies every + signature; `--to wss://relay` republishes. There is **no default target** — + restoring is deliberate, and the mistake to avoid is publishing a snapshot back + to the live relay. No full restore drill has been run yet (needs a throwaway + relay target). +- Offsite: `scripts/backup-relay.sh` runs the archive, then `restic backup` + + `forget --prune` **if** `~/.config/skatespots/restic.env` exists (template: + `scripts/restic.env.example`). **Not configured yet — local copies only**, so + the archive currently survives relay loss but not disk loss. +- Weekly systemd user timer: `scripts/install-archive-timer.sh` copies + `scripts/systemd/skatespots-archive.{service,timer}` into `~/.config/systemd/user` + and enables them. Logs: `journalctl --user -u skatespots-archive`. +- Commits touching only `scripts/`, `*.md`, `cvm-server/`, `skills/`, `.agents/` + are **skipped by the deploy timer** → no Netlify credits spent. + ### Git / ngit publishing - Push: `git push origin main` (handled by `git-remote-nostr`; publishes a signed repo-state event). - Login (once): `ngit account login --interactive` → scan the QR in Amber ("Add a new application → Connect app"). Plain `ngit account login` errors; the `--interactive` flag is required. diff --git a/scripts/archive-relay.mjs b/scripts/archive-relay.mjs new file mode 100755 index 0000000..4ac4155 --- /dev/null +++ b/scripts/archive-relay.mjs @@ -0,0 +1,235 @@ +#!/usr/bin/env node +// Archive the SkateSpots dataset off the community relay. +// +// A snapshot written as gzipped JSONL of the RAW signed events, so a restore can +// republish them verbatim (signatures intact) to any relay. The dataset is small, +// so a plain full snapshot is simpler and more trustworthy than incremental sync +// — and it is the whole disaster plan: if the relay (or its host) disappears, +// the community's content can be rebuilt anywhere. +// +// node scripts/archive-relay.mjs [--out DIR] [--relay URL] [--keep N] +// +// ⚠️ goskatespots.nostr1.com is a PUBLIC relay carrying general Nostr traffic +// (verified 2026-09-28): 15.7k kind-0 profiles (mostly non-skaters), 2k kind-30078 +// app-data events (Armada/Plebs/Amethyst), 379 long-form articles (Yakihonne), +// 408 NIP-99 listings (Shopstr)… almost none of it ours. Archiving the whole +// relay would mean taking custody of strangers' data AND republishing it on a +// restore. So this stays deliberately scoped to the app's own content: +// +// 1. exclusive kinds the app owns outright (30818 spots, 30017 shops, …); +// 2. shared kinds filtered by the app's OWN tags (e.g. `t=skateboarding` +// for events/listings, `t=skatespots` for collections, `t=skatespotsnews` +// for newsletters, `d=skatespots*` for app data); +// 3. `org.skatespots.*` labels and reports that reference archived content; +// 4. kind-0 profiles for archived authors only. +// DM kinds (4, 13, 1059, 1060) are never touched. +// +// Storage defaults to ~/.backups/relay (override with SKATESPOTS_ARCHIVE_DIR). +// Offsite: scripts/backup-relay.sh wraps this and pushes with restic when +// ~/.config/skatespots/restic.env is present. +// +// Keep the kind lists in sync with src/lib/types.ts / NIP.md when a kind ships. +import { createHash } from 'node:crypto'; +import { gzipSync } from 'node:zlib'; +import { mkdirSync, writeFileSync, readdirSync, statSync, unlinkSync } from 'node:fs'; +import { join } from 'node:path'; +import { homedir } from 'node:os'; +import { SimplePool, verifyEvent } from 'nostr-tools'; + +const DEFAULT_RELAY = 'wss://goskatespots.nostr1.com'; +const PAGE = 2000; // small pages: a big page can close early and come back short +const MAX_PAGES = 400; +const AUTHOR_BATCH = 150; // kind-0 lookups batched by author +const PROFILE_KIND = 0; + +// Kinds nothing else on the relay uses — keep every event of these. +const EXCLUSIVE_KINDS = [ + 2259, // spot status reports + 7103, // event updates + 9087, // crew endorsements + 24899, // spot chat + 30017, // skate shops + 30403, // marketplace offers + 30818, // skate spots + 30823, // tours + 31871, // trade attestations + 31872, // trade confirmation requests + 31873, // trusted-trader recommendations + 31969, // crew join requests + 32671, // Sasquatch Pro receipts + 36229, // crew invites + 36387, // collective join requests + ...Array.from({ length: 15 }, (_, i) => 35000 + i), // 35000–35014 role/crew profiles +]; + +const hasTag = (v) => (ev) => ev.tags.some(([n, x]) => n === 't' && x === v); +const dPrefix = (p) => (ev) => (ev.tags.find(([n]) => n === 'd')?.[1] ?? '').startsWith(p); +const labelNamespace = (ev) => + ev.tags.filter(([n]) => n === 'L').map(([, v]) => v ?? '').find((v) => v.startsWith('org.skatespots.')); + +// Shared kinds — keep only events carrying the app's own marker. +const FILTERED_KINDS = [ + { kind: 30003, keep: hasTag('skatespots') }, // spot collections + { kind: 30023, keep: hasTag('skatespotsnews') }, // newsletter issues + { kind: 30078, keep: dPrefix('skatespots') }, // app data (read-state, subs, …) + { kind: 30402, keep: hasTag('skateboarding') }, // marketplace listings + { kind: 31923, keep: hasTag('skateboarding') }, // calendar events + { kind: 9041, keep: hasTag('skateboarding') }, // zap goals + { kind: 4497, keep: hasTag('skatespots.cordn.crew-chat') }, // crew chat envelopes + { kind: 1985, keep: (ev) => !!labelNamespace(ev) }, // moderation / membership / vouch +]; + +// Kinds kept only when they reference archived content (matched after the above). +const REFERENCE_KINDS = [1984, 31925]; // NIP-56 reports, NIP-52 RSVPs + +// ── args ────────────────────────────────────────────────────────────────────── +function arg(name, fallback) { + const i = process.argv.indexOf(`--${name}`); + return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : fallback; +} + +const RELAY = arg('relay', DEFAULT_RELAY); +const OUT_DIR = arg('out', process.env.SKATESPOTS_ARCHIVE_DIR || join(homedir(), '.backups', 'relay')); +const KEEP = Math.max(1, Number(arg('keep', '12')) || 12); +const stamp = new Date().toISOString().replace(/[:.]/g, '-'); + +// ── fetch ───────────────────────────────────────────────────────────────────── +// Query ONE kind at a time. A single filter covering many kinds came back +// truncated (the relay caps a shared result window), which silently dropped most +// spots/listings/profiles. Stop only when a page brings nothing new: a short page +// can just mean the subscription closed early, with older events still available. +async function collect(pool, relay, filter, expectedKind) { + const seen = new Map(); + let until = Math.floor(Date.now() / 1000) + 120; // tolerate future-dated events + for (let page = 0; page < MAX_PAGES; page++) { + let batch; + try { + batch = await pool.querySync([relay], { ...filter, limit: PAGE, until }); + } catch { + break; // keep whatever this filter yielded rather than losing the run + } + if (!batch.length) break; + + let oldest = until; + let added = 0; + for (const ev of batch) { + if (expectedKind !== undefined && ev.kind !== expectedKind) continue; + if (seen.has(ev.id)) continue; + if (!verifyEvent(ev)) continue; // never archive an invalid signature + seen.set(ev.id, ev); + added += 1; + if (ev.created_at < oldest) oldest = ev.created_at; + } + if (added === 0) break; + until = oldest - 1; + } + return [...seen.values()]; +} + +const coordOf = (ev) => { + const d = ev.tags.find(([n]) => n === 'd')?.[1]; + return d !== undefined && ev.kind >= 30000 && ev.kind < 40000 ? `${ev.kind}:${ev.pubkey}:${d}` : null; +}; + +// ── run ─────────────────────────────────────────────────────────────────────── +const pool = new SimplePool(); +let events; +try { + const byId = new Map(); + const add = (evs) => evs.forEach((e) => byId.set(e.id, e)); + + // 1. Kinds the app owns outright. + for (const kind of EXCLUSIVE_KINDS) add(await collect(pool, RELAY, { kinds: [kind] }, kind)); + if (!byId.size) throw new Error(`no events returned by ${RELAY}`); + + // 2. Shared kinds, filtered by the app's own tags. + for (const { kind, keep } of FILTERED_KINDS) { + add((await collect(pool, RELAY, { kinds: [kind] }, kind)).filter(keep)); + } + + // 3. Kinds kept only when they point at what we already have. + const ids = new Set(byId.keys()); + const coords = new Set([...byId.values()].map(coordOf).filter(Boolean)); + for (const kind of REFERENCE_KINDS) { + add( + (await collect(pool, RELAY, { kinds: [kind] }, kind)).filter((ev) => + ev.tags.some(([n, v]) => (n === 'e' && ids.has(v)) || (n === 'a' && coords.has(v))), + ), + ); + } + + // 4. Profiles for archived authors only (not the relay's 15k strangers). + const authors = [...new Set([...byId.values()].map((e) => e.pubkey))]; + for (let i = 0; i < authors.length; i += AUTHOR_BATCH) { + const batch = authors.slice(i, i + AUTHOR_BATCH); + add(await collect(pool, RELAY, { kinds: [PROFILE_KIND], authors: batch }, PROFILE_KIND)); + } + + events = [...byId.values()].sort((a, b) => a.created_at - b.created_at); +} finally { + pool.close([RELAY]); +} + +// ── write ───────────────────────────────────────────────────────────────────── +const jsonl = events.map((e) => JSON.stringify(e)).join('\n') + '\n'; +const body = gzipSync(Buffer.from(jsonl, 'utf8'), { level: 9 }); +const sha256 = createHash('sha256').update(body).digest('hex'); + +const byKind = {}; +for (const e of events) byKind[e.kind] = (byKind[e.kind] || 0) + 1; + +const file = `relay-${stamp}.jsonl.gz`; +mkdirSync(OUT_DIR, { recursive: true }); +writeFileSync(join(OUT_DIR, file), body); + +const previous = readdirSync(OUT_DIR) + .filter((f) => f.startsWith('relay-') && f.endsWith('.jsonl.gz') && f !== file) + .sort() + .pop() ?? null; + +const manifest = { + relay: RELAY, + generatedAt: new Date().toISOString(), + file, + previous, + eventCount: events.length, + uniqueAuthors: new Set(events.map((e) => e.pubkey)).size, + byKind, + newestCreatedAt: events[events.length - 1].created_at, + oldestCreatedAt: events[0].created_at, + bytes: body.length, + sha256, + scope: { + exclusiveKinds: EXCLUSIVE_KINDS, + filtered: 'shared kinds kept only with the app tag (skateboarding / skatespots / skatespotsnews / d=skatespots*)', + referenceKinds: REFERENCE_KINDS, + profiles: 'kind 0 — authors of archived content only', + }, +}; +writeFileSync(join(OUT_DIR, `manifest-${stamp}.json`), JSON.stringify(manifest, null, 2) + '\n'); +writeFileSync(join(OUT_DIR, 'latest.json'), JSON.stringify(manifest, null, 2) + '\n'); + +// ── prune ───────────────────────────────────────────────────────────────────── +const archives = readdirSync(OUT_DIR) + .filter((f) => f.startsWith('relay-') && f.endsWith('.jsonl.gz')) + .sort(); +for (const old of archives.slice(0, Math.max(0, archives.length - KEEP))) { + unlinkSync(join(OUT_DIR, old)); + const m = join(OUT_DIR, `manifest-${old.replace(/^relay-/, '').replace(/\.jsonl\.gz$/, '')}.json`); + try { + if (statSync(m).isFile()) unlinkSync(m); + } catch { + /* manifest already gone */ + } +} + +const top = Object.entries(byKind) + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([k, n]) => `${k}:${n}`) + .join(' '); +console.log( + `[archive] ${events.length} events · ${manifest.uniqueAuthors} authors · ${(body.length / 1024).toFixed(0)} KiB · ${file}\n` + + `[archive] by kind — ${top}\n` + + `[archive] sha256 ${sha256.slice(0, 16)}… → ${OUT_DIR}`, +); diff --git a/scripts/backup-relay.sh b/scripts/backup-relay.sh new file mode 100755 index 0000000..52f419e --- /dev/null +++ b/scripts/backup-relay.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# +# Weekly SkateSpots relay archive + optional offsite copy. +# +# Driven by the systemd user timer `skatespots-archive.timer` (install with +# scripts/install-archive-timer.sh). Local archives land in +# ~/.backups/relay; if ~/.config/skatespots/restic.env exists, the whole +# directory is pushed to a restic repository (S3/R2/B2/SFTP) and old snapshots +# are pruned. Without that file the archive is local-only — still useful, but +# not a real offsite backup. +# +# Manual run / debug: +# scripts/backup-relay.sh +# journalctl --user -u skatespots-archive -n 50 +set -euo pipefail + +REPO="/home/sovereignsk8/SkateSpots" +# systemd runs with a minimal PATH; node lives under nvm here. +export PATH="/home/sovereignsk8/.nvm/versions/node/v22.23.1/bin:$PATH" + +ARCHIVE_DIR="${SKATESPOTS_ARCHIVE_DIR:-$HOME/.backups/relay}" +RESTIC_ENV="$HOME/.config/skatespots/restic.env" + +log() { echo "[$(date -Is)] $*"; } + +mkdir -p "$ARCHIVE_DIR" +cd "$REPO" + +log "archiving the community relay…" +node scripts/archive-relay.mjs + +if [ -f "$RESTIC_ENV" ]; then + # shellcheck disable=SC1090 + set -a; . "$RESTIC_ENV"; set +a + if [ -z "${RESTIC_REPOSITORY:-}" ]; then + log "restic.env present but RESTIC_REPOSITORY is empty — skipping offsite copy" + exit 0 + fi + log "pushing to restic $RESTIC_REPOSITORY…" + restic backup --tag skatespots-relay "$ARCHIVE_DIR" + restic forget --tag skatespots-relay --keep-last 12 --prune + log "offsite copy done" +else + log "no $RESTIC_ENV — archive kept local-only (set one up for a real offsite backup)" +fi diff --git a/scripts/install-archive-timer.sh b/scripts/install-archive-timer.sh new file mode 100755 index 0000000..2b04bd2 --- /dev/null +++ b/scripts/install-archive-timer.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Install the weekly relay-archive systemd user timer. +# scripts/install-archive-timer.sh +set -euo pipefail + +SRC="$(cd "$(dirname "$0")/systemd" && pwd)" +DEST="$HOME/.config/systemd/user" +mkdir -p "$DEST" + +cp "$SRC/skatespots-archive.service" "$DEST/" +cp "$SRC/skatespots-archive.timer" "$DEST/" + +systemctl --user daemon-reload +systemctl --user enable --now skatespots-archive.timer +systemctl --user list-timers skatespots-archive.timer --no-pager diff --git a/scripts/restic.env.example b/scripts/restic.env.example new file mode 100644 index 0000000..f18b9fc --- /dev/null +++ b/scripts/restic.env.example @@ -0,0 +1,23 @@ +# Template for the offsite backup target used by scripts/backup-relay.sh. +# Copy to ~/.config/skatespots/restic.env and fill in — that file is outside the +# repo and must never be committed. +# +# Any restic backend works. Examples: +# Backblaze B2: RESTIC_REPOSITORY=b2:my-bucket:skatespots/relay +# S3 / R2: RESTIC_REPOSITORY=s3:https://.r2.cloudflarestorage.com/ +# SFTP (host you already run): RESTIC_REPOSITORY=sftp:user@host:/srv/skatespots-backups +# +# One-time init, from this machine: +# set -a; . ~/.config/skatespots/restic.env; set +a +# restic init +# +# Then the weekly timer will keep it in sync (restic backup + forget --prune). + +RESTIC_REPOSITORY= +RESTIC_PASSWORD= + +# Backend credentials (only the ones your backend needs): +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= +# B2_ACCOUNT_ID= +# B2_ACCOUNT_KEY= diff --git a/scripts/restore-relay.mjs b/scripts/restore-relay.mjs new file mode 100755 index 0000000..a229d08 --- /dev/null +++ b/scripts/restore-relay.mjs @@ -0,0 +1,86 @@ +#!/usr/bin/env node +// Restore a SkateSpots archive (from scripts/archive-relay.mjs) to a relay. +// +// Publishes the RAW archived events verbatim — signatures are re-verified first, +// so anything tampered with is dropped. Addressable events are safe to republish: +// the newest `created_at` wins, so replaying a snapshot converges to the same +// state. This is the "prove you can get it back" half of the backup. +// +// node scripts/restore-relay.mjs --dry-run +// node scripts/restore-relay.mjs --to wss://your.relay +// +// There is deliberately NO default target: restoring is a deliberate act, and +// the common mistake is publishing a snapshot back to the live relay. Use +// --dry-run to preview, then point --to at the relay you actually want. +import { readFileSync } from 'node:fs'; +import { gunzipSync } from 'node:zlib'; +import { SimplePool, verifyEvent } from 'nostr-tools'; + +const args = process.argv.slice(2); +const file = args.find((a) => !a.startsWith('--')); +const toIndex = args.indexOf('--to'); +const TO = toIndex >= 0 ? args[toIndex + 1] : null; +const DRY = args.includes('--dry-run'); + +if (!file) { + console.error('usage: node scripts/restore-relay.mjs [--dry-run | --to wss://relay]'); + process.exit(2); +} +if (!TO && !DRY) { + console.error('[restore] refusing to publish without an explicit --to (use --dry-run to preview)'); + process.exit(2); +} + +const lines = gunzipSync(readFileSync(file)).toString('utf8').split('\n').filter(Boolean); +const events = []; +let invalid = 0; +for (const line of lines) { + try { + const ev = JSON.parse(line); + if (verifyEvent(ev)) events.push(ev); + else invalid += 1; + } catch { + invalid += 1; + } +} + +const byKind = {}; +for (const e of events) byKind[e.kind] = (byKind[e.kind] || 0) + 1; +console.log(`[restore] ${events.length} valid events (${invalid} invalid/ignored) from ${file}`); +console.log( + '[restore] kinds:', + Object.entries(byKind) + .sort((a, b) => b[1] - a[1]) + .map(([k, n]) => `${k}:${n}`) + .join(' '), +); + +if (DRY) { + console.log('[restore] dry run — nothing published'); + process.exit(0); +} + +const pool = new SimplePool(); +let ok = 0; +let fail = 0; +let index = 0; +const CONCURRENCY = 10; + +async function worker() { + while (index < events.length) { + const ev = events[index++]; + try { + const results = await Promise.allSettled(pool.publish([TO], ev)); + if (results.some((r) => r.status === 'fulfilled')) ok += 1; + else fail += 1; + } catch { + fail += 1; + } + if ((ok + fail) % 100 === 0) process.stdout.write(`\r[restore] ${ok + fail}/${events.length}…`); + } +} + +await Promise.all(Array.from({ length: Math.min(CONCURRENCY, events.length) }, worker)); +pool.close([TO]); +process.stdout.write(`\r[restore] done — ${ok} accepted, ${fail} failed \n`); +if (fail) process.exit(1); diff --git a/scripts/systemd/skatespots-archive.service b/scripts/systemd/skatespots-archive.service new file mode 100644 index 0000000..2d4810e --- /dev/null +++ b/scripts/systemd/skatespots-archive.service @@ -0,0 +1,6 @@ +[Unit] +Description=Archive the SkateSpots community relay dataset + +[Service] +Type=oneshot +ExecStart=%h/SkateSpots/scripts/backup-relay.sh diff --git a/scripts/systemd/skatespots-archive.timer b/scripts/systemd/skatespots-archive.timer new file mode 100644 index 0000000..271ebd1 --- /dev/null +++ b/scripts/systemd/skatespots-archive.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Weekly SkateSpots relay archive + +[Timer] +OnCalendar=weekly +Persistent=true +RandomizedDelaySec=1h + +[Install] +WantedBy=timers.target