6407310213
Point SKATE_RELAY, the default relay pools, the DM/CVM/zap relay arrays, the cvm-server + push-service defaults, and the archive default at wss://relay.skate-spots.org, keeping the legacy hosted relay as a fallback during the client-update window. Restored 115/115 archived events; the relay runs behind Caddy with TLS. Docs + AGENTS.md updated.
240 lines
11 KiB
JavaScript
Executable file
240 lines
11 KiB
JavaScript
Executable file
#!/usr/bin/env node
|
||
// Archive the SkateSpots dataset off the community relay.
|
||
//
|
||
// A snapshot written as gzipped JSONL of the RAW signed events, so a restore can
|
||
// republish them verbatim (signatures intact) to any relay. The dataset is small,
|
||
// so a plain full snapshot is simpler and more trustworthy than incremental sync
|
||
// — and it is the whole disaster plan: if the relay (or its host) disappears,
|
||
// the community's content can be rebuilt anywhere.
|
||
//
|
||
// node scripts/archive-relay.mjs [--out DIR] [--relay URL] [--keep N]
|
||
//
|
||
// The community relay is now our own Newlay instance (relay.skate-spots.org,
|
||
// 2026-10-12), which carries only app content. The LEGACY hosted relay
|
||
// (goskatespots.nostr1.com) was PUBLIC with general Nostr traffic (verified
|
||
// 2026-09-28): 15.7k kind-0 profiles, Armada/Plebs/Amethyst app-data, Yakihonne
|
||
// articles, Shopstr listings… almost none of it ours. Archiving wholesale would
|
||
// mean taking custody of strangers' data AND republishing it on a restore, so
|
||
// this stays deliberately scoped to the app's own content either way:
|
||
//
|
||
// 1. exclusive kinds the app owns outright (30818 spots, 30017 shops, …);
|
||
// 2. shared kinds filtered by the app's OWN tags (e.g. `t=skateboarding`
|
||
// for events/listings, `t=skatespots` for collections, `t=skatespotsnews`
|
||
// for newsletters, `d=skatespots*` for app data);
|
||
// 3. `org.skatespots.*` labels and reports that reference archived content;
|
||
// 4. kind-0 profiles for archived authors only.
|
||
// DM kinds (4, 13, 1059, 1060) are never touched.
|
||
//
|
||
// Storage defaults to ~/.backups/relay (override with SKATESPOTS_ARCHIVE_DIR).
|
||
// Offsite: scripts/backup-relay.sh wraps this and pushes with restic when
|
||
// ~/.config/skatespots/restic.env is present.
|
||
//
|
||
// Keep the kind lists in sync with src/lib/types.ts / NIP.md when a kind ships.
|
||
import { createHash } from 'node:crypto';
|
||
import { gzipSync } from 'node:zlib';
|
||
import { mkdirSync, writeFileSync, readdirSync, statSync, unlinkSync } from 'node:fs';
|
||
import { join } from 'node:path';
|
||
import { homedir } from 'node:os';
|
||
import { SimplePool, verifyEvent } from 'nostr-tools';
|
||
|
||
const DEFAULT_RELAY = 'wss://relay.skate-spots.org';
|
||
const PAGE = 2000; // small pages: a big page can close early and come back short
|
||
const MAX_PAGES = 400;
|
||
const AUTHOR_BATCH = 150; // kind-0 lookups batched by author
|
||
const PROFILE_KIND = 0;
|
||
|
||
// Kinds nothing else on the relay uses — keep every event of these.
|
||
const EXCLUSIVE_KINDS = [
|
||
2259, // spot status reports
|
||
7103, // event updates
|
||
1293, // witnessed skill badges (rider cards)
|
||
9087, // crew endorsements
|
||
24899, // spot chat
|
||
30017, // skate shops
|
||
30403, // marketplace offers
|
||
30818, // skate spots
|
||
30823, // tours
|
||
31871, // trade attestations
|
||
31872, // trade confirmation requests
|
||
31873, // trusted-trader recommendations
|
||
31969, // crew join requests
|
||
32271, // rider cards (addressable, d=rider-card)
|
||
32671, // Sasquatch Pro receipts
|
||
34967, // shipment status (addressable)
|
||
35679, // park access passes (addressable)
|
||
36229, // crew invites
|
||
36387, // collective join requests
|
||
...Array.from({ length: 15 }, (_, i) => 35000 + i), // 35000–35014 role/crew profiles
|
||
];
|
||
|
||
const hasTag = (v) => (ev) => ev.tags.some(([n, x]) => n === 't' && x === v);
|
||
const dPrefix = (p) => (ev) => (ev.tags.find(([n]) => n === 'd')?.[1] ?? '').startsWith(p);
|
||
const labelNamespace = (ev) =>
|
||
ev.tags.filter(([n]) => n === 'L').map(([, v]) => v ?? '').find((v) => v.startsWith('org.skatespots.'));
|
||
|
||
// Shared kinds — keep only events carrying the app's own marker.
|
||
const FILTERED_KINDS = [
|
||
{ kind: 30003, keep: hasTag('skatespots') }, // spot collections
|
||
{ kind: 30023, keep: hasTag('skatespotsnews') }, // newsletter issues
|
||
{ kind: 30078, keep: dPrefix('skatespots') }, // app data (read-state, subs, …)
|
||
{ kind: 30402, keep: hasTag('skateboarding') }, // marketplace listings
|
||
{ kind: 31923, keep: hasTag('skateboarding') }, // calendar events
|
||
{ kind: 9041, keep: hasTag('skateboarding') }, // zap goals
|
||
{ kind: 4497, keep: hasTag('skatespots.cordn.crew-chat') }, // crew chat envelopes
|
||
{ kind: 1985, keep: (ev) => !!labelNamespace(ev) }, // moderation / membership / vouch
|
||
];
|
||
|
||
// Kinds kept only when they reference archived content (matched after the above).
|
||
const REFERENCE_KINDS = [1984, 31925]; // NIP-56 reports, NIP-52 RSVPs
|
||
|
||
// ── args ──────────────────────────────────────────────────────────────────────
|
||
function arg(name, fallback) {
|
||
const i = process.argv.indexOf(`--${name}`);
|
||
return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : fallback;
|
||
}
|
||
|
||
const RELAY = arg('relay', DEFAULT_RELAY);
|
||
const OUT_DIR = arg('out', process.env.SKATESPOTS_ARCHIVE_DIR || join(homedir(), '.backups', 'relay'));
|
||
const KEEP = Math.max(1, Number(arg('keep', '12')) || 12);
|
||
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||
|
||
// ── fetch ─────────────────────────────────────────────────────────────────────
|
||
// Query ONE kind at a time. A single filter covering many kinds came back
|
||
// truncated (the relay caps a shared result window), which silently dropped most
|
||
// spots/listings/profiles. Stop only when a page brings nothing new: a short page
|
||
// can just mean the subscription closed early, with older events still available.
|
||
async function collect(pool, relay, filter, expectedKind) {
|
||
const seen = new Map();
|
||
let until = Math.floor(Date.now() / 1000) + 120; // tolerate future-dated events
|
||
for (let page = 0; page < MAX_PAGES; page++) {
|
||
let batch;
|
||
try {
|
||
batch = await pool.querySync([relay], { ...filter, limit: PAGE, until });
|
||
} catch {
|
||
break; // keep whatever this filter yielded rather than losing the run
|
||
}
|
||
if (!batch.length) break;
|
||
|
||
let oldest = until;
|
||
let added = 0;
|
||
for (const ev of batch) {
|
||
if (expectedKind !== undefined && ev.kind !== expectedKind) continue;
|
||
if (seen.has(ev.id)) continue;
|
||
if (!verifyEvent(ev)) continue; // never archive an invalid signature
|
||
seen.set(ev.id, ev);
|
||
added += 1;
|
||
if (ev.created_at < oldest) oldest = ev.created_at;
|
||
}
|
||
if (added === 0) break;
|
||
until = oldest - 1;
|
||
}
|
||
return [...seen.values()];
|
||
}
|
||
|
||
const coordOf = (ev) => {
|
||
const d = ev.tags.find(([n]) => n === 'd')?.[1];
|
||
return d !== undefined && ev.kind >= 30000 && ev.kind < 40000 ? `${ev.kind}:${ev.pubkey}:${d}` : null;
|
||
};
|
||
|
||
// ── run ───────────────────────────────────────────────────────────────────────
|
||
const pool = new SimplePool();
|
||
let events;
|
||
try {
|
||
const byId = new Map();
|
||
const add = (evs) => evs.forEach((e) => byId.set(e.id, e));
|
||
|
||
// 1. Kinds the app owns outright.
|
||
for (const kind of EXCLUSIVE_KINDS) add(await collect(pool, RELAY, { kinds: [kind] }, kind));
|
||
if (!byId.size) throw new Error(`no events returned by ${RELAY}`);
|
||
|
||
// 2. Shared kinds, filtered by the app's own tags.
|
||
for (const { kind, keep } of FILTERED_KINDS) {
|
||
add((await collect(pool, RELAY, { kinds: [kind] }, kind)).filter(keep));
|
||
}
|
||
|
||
// 3. Kinds kept only when they point at what we already have.
|
||
const ids = new Set(byId.keys());
|
||
const coords = new Set([...byId.values()].map(coordOf).filter(Boolean));
|
||
for (const kind of REFERENCE_KINDS) {
|
||
add(
|
||
(await collect(pool, RELAY, { kinds: [kind] }, kind)).filter((ev) =>
|
||
ev.tags.some(([n, v]) => (n === 'e' && ids.has(v)) || (n === 'a' && coords.has(v))),
|
||
),
|
||
);
|
||
}
|
||
|
||
// 4. Profiles for archived authors only (not the relay's 15k strangers).
|
||
const authors = [...new Set([...byId.values()].map((e) => e.pubkey))];
|
||
for (let i = 0; i < authors.length; i += AUTHOR_BATCH) {
|
||
const batch = authors.slice(i, i + AUTHOR_BATCH);
|
||
add(await collect(pool, RELAY, { kinds: [PROFILE_KIND], authors: batch }, PROFILE_KIND));
|
||
}
|
||
|
||
events = [...byId.values()].sort((a, b) => a.created_at - b.created_at);
|
||
} finally {
|
||
pool.close([RELAY]);
|
||
}
|
||
|
||
// ── write ─────────────────────────────────────────────────────────────────────
|
||
const jsonl = events.map((e) => JSON.stringify(e)).join('\n') + '\n';
|
||
const body = gzipSync(Buffer.from(jsonl, 'utf8'), { level: 9 });
|
||
const sha256 = createHash('sha256').update(body).digest('hex');
|
||
|
||
const byKind = {};
|
||
for (const e of events) byKind[e.kind] = (byKind[e.kind] || 0) + 1;
|
||
|
||
const file = `relay-${stamp}.jsonl.gz`;
|
||
mkdirSync(OUT_DIR, { recursive: true });
|
||
writeFileSync(join(OUT_DIR, file), body);
|
||
|
||
const previous = readdirSync(OUT_DIR)
|
||
.filter((f) => f.startsWith('relay-') && f.endsWith('.jsonl.gz') && f !== file)
|
||
.sort()
|
||
.pop() ?? null;
|
||
|
||
const manifest = {
|
||
relay: RELAY,
|
||
generatedAt: new Date().toISOString(),
|
||
file,
|
||
previous,
|
||
eventCount: events.length,
|
||
uniqueAuthors: new Set(events.map((e) => e.pubkey)).size,
|
||
byKind,
|
||
newestCreatedAt: events[events.length - 1].created_at,
|
||
oldestCreatedAt: events[0].created_at,
|
||
bytes: body.length,
|
||
sha256,
|
||
scope: {
|
||
exclusiveKinds: EXCLUSIVE_KINDS,
|
||
filtered: 'shared kinds kept only with the app tag (skateboarding / skatespots / skatespotsnews / d=skatespots*)',
|
||
referenceKinds: REFERENCE_KINDS,
|
||
profiles: 'kind 0 — authors of archived content only',
|
||
},
|
||
};
|
||
writeFileSync(join(OUT_DIR, `manifest-${stamp}.json`), JSON.stringify(manifest, null, 2) + '\n');
|
||
writeFileSync(join(OUT_DIR, 'latest.json'), JSON.stringify(manifest, null, 2) + '\n');
|
||
|
||
// ── prune ─────────────────────────────────────────────────────────────────────
|
||
const archives = readdirSync(OUT_DIR)
|
||
.filter((f) => f.startsWith('relay-') && f.endsWith('.jsonl.gz'))
|
||
.sort();
|
||
for (const old of archives.slice(0, Math.max(0, archives.length - KEEP))) {
|
||
unlinkSync(join(OUT_DIR, old));
|
||
const m = join(OUT_DIR, `manifest-${old.replace(/^relay-/, '').replace(/\.jsonl\.gz$/, '')}.json`);
|
||
try {
|
||
if (statSync(m).isFile()) unlinkSync(m);
|
||
} catch {
|
||
/* manifest already gone */
|
||
}
|
||
}
|
||
|
||
const top = Object.entries(byKind)
|
||
.sort((a, b) => b[1] - a[1])
|
||
.slice(0, 10)
|
||
.map(([k, n]) => `${k}:${n}`)
|
||
.join(' ');
|
||
console.log(
|
||
`[archive] ${events.length} events · ${manifest.uniqueAuthors} authors · ${(body.length / 1024).toFixed(0)} KiB · ${file}\n` +
|
||
`[archive] by kind — ${top}\n` +
|
||
`[archive] sha256 ${sha256.slice(0, 16)}… → ${OUT_DIR}`,
|
||
);
|