SkateSpots/scripts/archive-relay.mjs

#!/usr/bin/env node
// Archive the SkateSpots dataset off the community relay.
//
// A snapshot written as gzipped JSONL of the RAW signed events, so a restore can
// republish them verbatim (signatures intact) to any relay. The dataset is small,
// so a plain full snapshot is simpler and more trustworthy than incremental sync
// — and it is the whole disaster plan: if the relay (or its host) disappears,
// the community's content can be rebuilt anywhere.
//
//   node scripts/archive-relay.mjs [--out DIR] [--relay URL] [--keep N]
//
// The community relay is now our own Newlay instance (relay.skate-spots.org,
// 2026-10-12), which carries only app content. The LEGACY hosted relay
// (goskatespots.nostr1.com) was PUBLIC with general Nostr traffic (verified
// 2026-09-28): 15.7k kind-0 profiles, Armada/Plebs/Amethyst app-data, Yakihonne
// articles, Shopstr listings… almost none of it ours. Archiving wholesale would
// mean taking custody of strangers' data AND republishing it on a restore, so
// this stays deliberately scoped to the app's own content either way:
//
//   1. exclusive kinds the app owns outright (30818 spots, 30017 shops, …);
//   2. shared kinds filtered by the app's OWN tags (e.g. `t=skateboarding`
//      for events/listings, `t=skatespots` for collections, `t=skatespotsnews`
//      for newsletters, `d=skatespots*` for app data);
//   3. `org.skatespots.*` labels and reports that reference archived content;
//   4. kind-0 profiles for archived authors only.
// DM kinds (4, 13, 1059, 1060) are never touched.
//
// Storage defaults to ~/.backups/relay (override with SKATESPOTS_ARCHIVE_DIR).
// Offsite: scripts/backup-relay.sh wraps this and pushes with restic when
// ~/.config/skatespots/restic.env is present.
//
// Keep the kind lists in sync with src/lib/types.ts / NIP.md when a kind ships.
import { createHash } from 'node:crypto';
import { gzipSync } from 'node:zlib';
import { mkdirSync, writeFileSync, readdirSync, statSync, unlinkSync } from 'node:fs';
import { join } from 'node:path';
import { homedir } from 'node:os';
import { SimplePool, verifyEvent } from 'nostr-tools';

const DEFAULT_RELAY = 'wss://relay.skate-spots.org';
const PAGE = 2000; // small pages: a big page can close early and come back short
const MAX_PAGES = 400;
const AUTHOR_BATCH = 150; // kind-0 lookups batched by author
const PROFILE_KIND = 0;

// Kinds nothing else on the relay uses — keep every event of these.
const EXCLUSIVE_KINDS = [
  2259, // spot status reports
  7103, // event updates
  1293, // witnessed skill badges (rider cards)
  9087, // crew endorsements
  24899, // spot chat
  30017, // skate shops
  30403, // marketplace offers
  30818, // skate spots
  30823, // tours
  31871, // trade attestations
  31872, // trade confirmation requests
  31873, // trusted-trader recommendations
  31969, // crew join requests
  32271, // rider cards (addressable, d=rider-card)
  32671, // Sasquatch Pro receipts
  34967, // shipment status (addressable)
  35679, // park access passes (addressable)
  36229, // crew invites
  36387, // collective join requests
  ...Array.from({ length: 15 }, (_, i) => 35000 + i), // 35000–35014 role/crew profiles
];

const hasTag = (v) => (ev) => ev.tags.some(([n, x]) => n === 't' && x === v);
const dPrefix = (p) => (ev) => (ev.tags.find(([n]) => n === 'd')?.[1] ?? '').startsWith(p);
const labelNamespace = (ev) =>
  ev.tags.filter(([n]) => n === 'L').map(([, v]) => v ?? '').find((v) => v.startsWith('org.skatespots.'));

// Shared kinds — keep only events carrying the app's own marker.
const FILTERED_KINDS = [
  { kind: 30003, keep: hasTag('skatespots') }, // spot collections
  { kind: 30023, keep: hasTag('skatespotsnews') }, // newsletter issues
  { kind: 30078, keep: dPrefix('skatespots') }, // app data (read-state, subs, …)
  { kind: 30402, keep: hasTag('skateboarding') }, // marketplace listings
  { kind: 31923, keep: hasTag('skateboarding') }, // calendar events
  { kind: 9041, keep: hasTag('skateboarding') }, // zap goals
  { kind: 4497, keep: hasTag('skatespots.cordn.crew-chat') }, // crew chat envelopes
  { kind: 1985, keep: (ev) => !!labelNamespace(ev) }, // moderation / membership / vouch
];

// Kinds kept only when they reference archived content (matched after the above).
const REFERENCE_KINDS = [1984, 31925]; // NIP-56 reports, NIP-52 RSVPs

// ── args ──────────────────────────────────────────────────────────────────────
function arg(name, fallback) {
  const i = process.argv.indexOf(`--${name}`);
  return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : fallback;
}

const RELAY = arg('relay', DEFAULT_RELAY);
const OUT_DIR = arg('out', process.env.SKATESPOTS_ARCHIVE_DIR || join(homedir(), '.backups', 'relay'));
const KEEP = Math.max(1, Number(arg('keep', '12')) || 12);
const stamp = new Date().toISOString().replace(/[:.]/g, '-');

// ── fetch ─────────────────────────────────────────────────────────────────────
// Query ONE kind at a time. A single filter covering many kinds came back
// truncated (the relay caps a shared result window), which silently dropped most
// spots/listings/profiles. Stop only when a page brings nothing new: a short page
// can just mean the subscription closed early, with older events still available.
async function collect(pool, relay, filter, expectedKind) {
  const seen = new Map();
  let until = Math.floor(Date.now() / 1000) + 120; // tolerate future-dated events
  for (let page = 0; page < MAX_PAGES; page++) {
    let batch;
    try {
      batch = await pool.querySync([relay], { ...filter, limit: PAGE, until });
    } catch {
      break; // keep whatever this filter yielded rather than losing the run
    }
    if (!batch.length) break;

    let oldest = until;
    let added = 0;
    for (const ev of batch) {
      if (expectedKind !== undefined && ev.kind !== expectedKind) continue;
      if (seen.has(ev.id)) continue;
      if (!verifyEvent(ev)) continue; // never archive an invalid signature
      seen.set(ev.id, ev);
      added += 1;
      if (ev.created_at < oldest) oldest = ev.created_at;
    }
    if (added === 0) break;
    until = oldest - 1;
  }
  return [...seen.values()];
}

const coordOf = (ev) => {
  const d = ev.tags.find(([n]) => n === 'd')?.[1];
  return d !== undefined && ev.kind >= 30000 && ev.kind < 40000 ? `${ev.kind}:${ev.pubkey}:${d}` : null;
};

// ── run ───────────────────────────────────────────────────────────────────────
const pool = new SimplePool();
let events;
try {
  const byId = new Map();
  const add = (evs) => evs.forEach((e) => byId.set(e.id, e));

  // 1. Kinds the app owns outright.
  for (const kind of EXCLUSIVE_KINDS) add(await collect(pool, RELAY, { kinds: [kind] }, kind));
  if (!byId.size) throw new Error(`no events returned by ${RELAY}`);

  // 2. Shared kinds, filtered by the app's own tags.
  for (const { kind, keep } of FILTERED_KINDS) {
    add((await collect(pool, RELAY, { kinds: [kind] }, kind)).filter(keep));
  }

  // 3. Kinds kept only when they point at what we already have.
  const ids = new Set(byId.keys());
  const coords = new Set([...byId.values()].map(coordOf).filter(Boolean));
  for (const kind of REFERENCE_KINDS) {
    add(
      (await collect(pool, RELAY, { kinds: [kind] }, kind)).filter((ev) =>
        ev.tags.some(([n, v]) => (n === 'e' && ids.has(v)) || (n === 'a' && coords.has(v))),
      ),
    );
  }

  // 4. Profiles for archived authors only (not the relay's 15k strangers).
  const authors = [...new Set([...byId.values()].map((e) => e.pubkey))];
  for (let i = 0; i < authors.length; i += AUTHOR_BATCH) {
    const batch = authors.slice(i, i + AUTHOR_BATCH);
    add(await collect(pool, RELAY, { kinds: [PROFILE_KIND], authors: batch }, PROFILE_KIND));
  }

  events = [...byId.values()].sort((a, b) => a.created_at - b.created_at);
} finally {
  pool.close([RELAY]);
}

// ── write ─────────────────────────────────────────────────────────────────────
const jsonl = events.map((e) => JSON.stringify(e)).join('\n') + '\n';
const body = gzipSync(Buffer.from(jsonl, 'utf8'), { level: 9 });
const sha256 = createHash('sha256').update(body).digest('hex');

const byKind = {};
for (const e of events) byKind[e.kind] = (byKind[e.kind] || 0) + 1;

const file = `relay-${stamp}.jsonl.gz`;
mkdirSync(OUT_DIR, { recursive: true });
writeFileSync(join(OUT_DIR, file), body);

const previous = readdirSync(OUT_DIR)
  .filter((f) => f.startsWith('relay-') && f.endsWith('.jsonl.gz') && f !== file)
  .sort()
  .pop() ?? null;

const manifest = {
  relay: RELAY,
  generatedAt: new Date().toISOString(),
  file,
  previous,
  eventCount: events.length,
  uniqueAuthors: new Set(events.map((e) => e.pubkey)).size,
  byKind,
  newestCreatedAt: events[events.length - 1].created_at,
  oldestCreatedAt: events[0].created_at,
  bytes: body.length,
  sha256,
  scope: {
    exclusiveKinds: EXCLUSIVE_KINDS,
    filtered: 'shared kinds kept only with the app tag (skateboarding / skatespots / skatespotsnews / d=skatespots*)',
    referenceKinds: REFERENCE_KINDS,
    profiles: 'kind 0 — authors of archived content only',
  },
};
writeFileSync(join(OUT_DIR, `manifest-${stamp}.json`), JSON.stringify(manifest, null, 2) + '\n');
writeFileSync(join(OUT_DIR, 'latest.json'), JSON.stringify(manifest, null, 2) + '\n');

// ── prune ─────────────────────────────────────────────────────────────────────
const archives = readdirSync(OUT_DIR)
  .filter((f) => f.startsWith('relay-') && f.endsWith('.jsonl.gz'))
  .sort();
for (const old of archives.slice(0, Math.max(0, archives.length - KEEP))) {
  unlinkSync(join(OUT_DIR, old));
  const m = join(OUT_DIR, `manifest-${old.replace(/^relay-/, '').replace(/\.jsonl\.gz$/, '')}.json`);
  try {
    if (statSync(m).isFile()) unlinkSync(m);
  } catch {
    /* manifest already gone */
  }
}

const top = Object.entries(byKind)
  .sort((a, b) => b[1] - a[1])
  .slice(0, 10)
  .map(([k, n]) => `${k}:${n}`)
  .join(' ');
console.log(
  `[archive] ${events.length} events · ${manifest.uniqueAuthors} authors · ${(body.length / 1024).toFixed(0)} KiB · ${file}\n` +
    `[archive] by kind — ${top}\n` +
    `[archive] sha256 ${sha256.slice(0, 16)}… → ${OUT_DIR}`,
);