feat(media): image-only per-npub Blossom quotas and upload proxy
This commit is contained in:
parent
c4a1671f81
commit
a4f72a3d05
4 changed files with 265 additions and 2 deletions
|
|
@ -35,12 +35,15 @@ svc skatespots-relay.service
|
||||||
svc skatespots-ai.service
|
svc skatespots-ai.service
|
||||||
svc skatespots-push.service
|
svc skatespots-push.service
|
||||||
svc skatespots-cvm.service
|
svc skatespots-cvm.service
|
||||||
|
svc skatespots-quota.service
|
||||||
svc caddy.service
|
svc caddy.service
|
||||||
|
|
||||||
[ "$(code -H 'Accept: application/nostr+json' https://relay.skate-spots.org/)" != "200" ] \
|
[ "$(code -H 'Accept: application/nostr+json' https://relay.skate-spots.org/)" != "200" ] \
|
||||||
&& ISSUES+=("relay public TLS not answering 200")
|
&& ISSUES+=("relay public TLS not answering 200")
|
||||||
[ "$(code https://push.skate-spots.org/health)" != "200" ] \
|
[ "$(code https://push.skate-spots.org/health)" != "200" ] \
|
||||||
&& ISSUES+=("push /health not 200")
|
&& ISSUES+=("push /health not 200")
|
||||||
|
[ "$(code http://127.0.0.1:8790/health)" != "200" ] \
|
||||||
|
&& ISSUES+=("blossom quota proxy /health not 200")
|
||||||
|
|
||||||
disk=$(df --output=pcent / | tail -1 | tr -d ' %')
|
disk=$(df --output=pcent / | tail -1 | tr -d ' %')
|
||||||
[ "${disk:-0}" -ge 85 ] && ISSUES+=("disk ${disk}% used")
|
[ "${disk:-0}" -ge 85 ] && ISSUES+=("disk ${disk}% used")
|
||||||
|
|
@ -58,6 +61,13 @@ cordn_rejects=$(docker logs --since 15m newlay-newlay-1 2>&1 | grep -c 'cordn ra
|
||||||
cordn_announce=$(docker logs --since 20m newlay-newlay-1 2>&1 | grep -c 'cordn announce' || true)
|
cordn_announce=$(docker logs --since 20m newlay-newlay-1 2>&1 | grep -c 'cordn announce' || true)
|
||||||
[ "${cordn_announce:-0}" -eq 0 ] && ISSUES+=("cordn coordinator has not announced in 20m")
|
[ "${cordn_announce:-0}" -eq 0 ] && ISSUES+=("cordn coordinator has not announced in 20m")
|
||||||
|
|
||||||
|
# Blossom media budget (usage is persisted by the quota proxy).
|
||||||
|
quota_state=/home/ubuntu/skatespots-quota/usage.json
|
||||||
|
if [ -f "$quota_state" ]; then
|
||||||
|
total_bytes=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("totalBytes",0))' "$quota_state" 2>/dev/null || echo 0)
|
||||||
|
[ "${total_bytes:-0}" -ge 19327352832 ] && ISSUES+=("blossom media >18 GiB")
|
||||||
|
fi
|
||||||
|
|
||||||
# TLS expiry (Caddy auto-renews; this catches a broken renewal early).
|
# TLS expiry (Caddy auto-renews; this catches a broken renewal early).
|
||||||
expiry=$(echo | openssl s_client -servername relay.skate-spots.org -connect relay.skate-spots.org:443 2>/dev/null \
|
expiry=$(echo | openssl s_client -servername relay.skate-spots.org -connect relay.skate-spots.org:443 2>/dev/null \
|
||||||
| openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
| openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||||||
|
|
|
||||||
184
scripts/quota-proxy.mjs
Normal file
184
scripts/quota-proxy.mjs
Normal file
|
|
@ -0,0 +1,184 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* SkateSpots Blossom upload quota proxy.
|
||||||
|
*
|
||||||
|
* Caddy sends ONLY `PUT /upload` here; reads and all other Blossom routes go
|
||||||
|
* straight to Newlay. The proxy verifies the signed BUD-11 (kind 24242) token,
|
||||||
|
* enforces image-only + size, per-npub daily budgets and a generous per-IP
|
||||||
|
* backstop, then forwards the accepted bytes to Newlay.
|
||||||
|
*
|
||||||
|
* Primary quota key = npub (portable, fair across devices/IPs).
|
||||||
|
* Secondary key = client IP (anti-abuse / key-rotation backstop).
|
||||||
|
* Admin pubkeys get a higher npub budget, never an unlimited bypass.
|
||||||
|
*
|
||||||
|
* This is intentionally dependency-light: it reuses the nostr-tools already
|
||||||
|
* installed by the push service, so the relay host needs no new npm install.
|
||||||
|
*
|
||||||
|
* Environment:
|
||||||
|
* PORT=8790
|
||||||
|
* NOSTR_TOOLS_ROOT=/home/ubuntu/skatespots-push/
|
||||||
|
* QUOTA_STATE=/home/ubuntu/skatespots-quota/usage.json
|
||||||
|
* ADMIN_PUBKEYS=20e7...,caba...
|
||||||
|
* MAX_IMAGE_BYTES=10485760
|
||||||
|
* DAILY_BYTES=262144000 (250 MiB)
|
||||||
|
* DAILY_UPLOADS=50
|
||||||
|
* ADMIN_MULTIPLIER=3
|
||||||
|
* IP_HOURLY_BYTES=524288000 (500 MiB)
|
||||||
|
* IP_HOURLY_UPLOADS=100
|
||||||
|
* TOTAL_BYTES=21474836480 (20 GiB)
|
||||||
|
*/
|
||||||
|
import http from 'node:http';
|
||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
||||||
|
import { dirname, join } from 'node:path';
|
||||||
|
import { createRequire } from 'node:module';
|
||||||
|
|
||||||
|
const require = createRequire(process.env.NOSTR_TOOLS_ROOT || '/home/ubuntu/skatespots-push/');
|
||||||
|
const { verifyEvent } = require('nostr-tools');
|
||||||
|
|
||||||
|
const PORT = Number(process.env.PORT || 8790);
|
||||||
|
const UPSTREAM_PORT = Number(process.env.UPSTREAM_PORT || 7777);
|
||||||
|
const MAX_IMAGE_BYTES = Number(process.env.MAX_IMAGE_BYTES || 10 * 1024 * 1024);
|
||||||
|
const DAILY_BYTES = Number(process.env.DAILY_BYTES || 250 * 1024 * 1024);
|
||||||
|
const DAILY_UPLOADS = Number(process.env.DAILY_UPLOADS || 50);
|
||||||
|
const ADMIN_MULTIPLIER = Number(process.env.ADMIN_MULTIPLIER || 3);
|
||||||
|
const IP_HOURLY_BYTES = Number(process.env.IP_HOURLY_BYTES || 500 * 1024 * 1024);
|
||||||
|
const IP_HOURLY_UPLOADS = Number(process.env.IP_HOURLY_UPLOADS || 100);
|
||||||
|
const TOTAL_BYTES = Number(process.env.TOTAL_BYTES || 20 * 1024 * 1024 * 1024);
|
||||||
|
const STATE_PATH = process.env.QUOTA_STATE || '/home/ubuntu/skatespots-quota/usage.json';
|
||||||
|
const ADMIN_PUBKEYS = new Set(
|
||||||
|
String(process.env.ADMIN_PUBKEYS || '')
|
||||||
|
.split(',').map((v) => v.trim().toLowerCase()).filter(Boolean),
|
||||||
|
);
|
||||||
|
const IMAGE_TYPES = new Set([
|
||||||
|
'image/jpeg', 'image/png', 'image/webp', 'image/avif',
|
||||||
|
'image/heic', 'image/heif', 'image/gif',
|
||||||
|
]);
|
||||||
|
|
||||||
|
mkdirSync(dirname(STATE_PATH), { recursive: true });
|
||||||
|
|
||||||
|
let state = loadState();
|
||||||
|
function loadState() {
|
||||||
|
try {
|
||||||
|
return JSON.parse(readFileSync(STATE_PATH, 'utf8'));
|
||||||
|
} catch {
|
||||||
|
return { day: '', hour: '', totalBytes: 0, users: {}, ips: {} };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
function saveState() {
|
||||||
|
const tmp = `${STATE_PATH}.tmp`;
|
||||||
|
writeFileSync(tmp, JSON.stringify(state));
|
||||||
|
renameSync(tmp, STATE_PATH);
|
||||||
|
}
|
||||||
|
function rollWindows() {
|
||||||
|
const now = new Date();
|
||||||
|
const day = now.toISOString().slice(0, 10);
|
||||||
|
const hour = now.toISOString().slice(0, 13);
|
||||||
|
if (state.day !== day) { state.day = day; state.users = {}; }
|
||||||
|
if (state.hour !== hour) { state.hour = hour; state.ips = {}; }
|
||||||
|
}
|
||||||
|
function clientIp(req) {
|
||||||
|
const xff = String(req.headers['x-forwarded-for'] || '').split(',')[0].trim();
|
||||||
|
return xff || req.socket.remoteAddress || 'unknown';
|
||||||
|
}
|
||||||
|
function json(res, status, body) {
|
||||||
|
const payload = JSON.stringify(body);
|
||||||
|
res.writeHead(status, {
|
||||||
|
'content-type': 'application/json',
|
||||||
|
'content-length': Buffer.byteLength(payload),
|
||||||
|
'access-control-allow-origin': '*',
|
||||||
|
});
|
||||||
|
res.end(payload);
|
||||||
|
}
|
||||||
|
function readBody(req, limit) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const declared = Number(req.headers['content-length'] || 0);
|
||||||
|
if (declared > limit) { reject(Object.assign(new Error('too large'), { code: 413 })); return; }
|
||||||
|
const chunks = [];
|
||||||
|
let total = 0;
|
||||||
|
req.on('data', (chunk) => {
|
||||||
|
total += chunk.length;
|
||||||
|
if (total > limit) {
|
||||||
|
reject(Object.assign(new Error('too large'), { code: 413 }));
|
||||||
|
req.destroy();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
chunks.push(chunk);
|
||||||
|
});
|
||||||
|
req.on('end', () => resolve(Buffer.concat(chunks)));
|
||||||
|
req.on('error', reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
function decodeToken(header) {
|
||||||
|
const m = /^Nostr\s+(.+)$/i.exec(String(header || ''));
|
||||||
|
if (!m) return null;
|
||||||
|
try {
|
||||||
|
return JSON.parse(Buffer.from(m[1], 'base64').toString('utf8'));
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
function tag(event, name) {
|
||||||
|
return event.tags?.find(([n]) => n === name)?.[1];
|
||||||
|
}
|
||||||
|
function checkToken(event, body, contentType) {
|
||||||
|
if (!event || event.kind !== 24242) return 'Invalid upload authorization.';
|
||||||
|
if (!verifyEvent(event)) return 'Invalid upload signature.';
|
||||||
|
if (tag(event, 't') !== 'upload') return 'Invalid upload token type.';
|
||||||
|
const exp = Number(tag(event, 'expiration') || 0);
|
||||||
|
if (!exp || exp < Math.floor(Date.now() / 1000)) return 'Upload authorization expired.';
|
||||||
|
if (!contentType.startsWith('image/')) return 'Only images can be uploaded.';
|
||||||
|
const x = String(tag(event, 'x') || '').toLowerCase();
|
||||||
|
const size = Number(tag(event, 'size') || 0);
|
||||||
|
const digest = createHash('sha256').update(body).digest('hex');
|
||||||
|
if (!x || x !== digest || size !== body.length) return 'Upload token does not match the image.';
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
function checkQuota(pubkey, bytes, ip) {
|
||||||
|
const admin = ADMIN_PUBKEYS.has(pubkey);
|
||||||
|
const multiplier = admin ? ADMIN_MULTIPLIER : 1;
|
||||||
|
const user = state.users[pubkey] || (state.users[pubkey] = { uploads: 0, bytes: 0 });
|
||||||
|
if (user.uploads + 1 > DAILY_UPLOADS * multiplier) return 'Daily upload count reached. Try again tomorrow.';
|
||||||
|
if (user.bytes + bytes > DAILY_BYTES * multiplier) return 'Daily upload size reached. Try again tomorrow.';
|
||||||
|
const ipKey = createHash('sha256').update(ip).digest('hex').slice(0, 24);
|
||||||
|
const ipRow = state.ips[ipKey] || (state.ips[ipKey] = { uploads: 0, bytes: 0 });
|
||||||
|
if (ipRow.uploads + 1 > IP_HOURLY_UPLOADS) return 'Too many uploads from this connection. Try again later.';
|
||||||
|
if (ipRow.bytes + bytes > IP_HOURLY_BYTES) return 'Upload burst limit reached. Try again later.';
|
||||||
|
if (state.totalBytes + bytes > TOTAL_BYTES) return 'The relay media budget is full. Try again later.';
|
||||||
|
user.uploads += 1; user.bytes += bytes;
|
||||||
|
ipRow.uploads += 1; ipRow.bytes += bytes;
|
||||||
|
state.totalBytes += bytes;
|
||||||
|
saveState();
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
function forward(req, res, body) {
|
||||||
|
const headers = { ...req.headers, host: `127.0.0.1:${UPSTREAM_PORT}` };
|
||||||
|
if (body) headers['content-length'] = String(body.length);
|
||||||
|
const upstream = http.request({
|
||||||
|
hostname: '127.0.0.1', port: UPSTREAM_PORT, path: req.url, method: req.method, headers,
|
||||||
|
}, (up) => {
|
||||||
|
res.writeHead(up.statusCode || 502, up.headers);
|
||||||
|
up.pipe(res);
|
||||||
|
});
|
||||||
|
upstream.on('error', () => json(res, 502, { error: 'Media relay unavailable.' }));
|
||||||
|
if (body) upstream.end(body); else req.pipe(upstream);
|
||||||
|
}
|
||||||
|
const server = http.createServer(async (req, res) => {
|
||||||
|
if (req.url === '/health') { json(res, 200, { ok: true }); return; }
|
||||||
|
if (req.method !== 'PUT' || req.url !== '/upload') { forward(req, res); return; }
|
||||||
|
|
||||||
|
const contentType = String(req.headers['content-type'] || '').toLowerCase().split(';')[0];
|
||||||
|
if (!IMAGE_TYPES.has(contentType)) { json(res, 415, { error: 'Only JPEG, PNG, WebP, AVIF, HEIC or GIF images can be uploaded.' }); return; }
|
||||||
|
let body;
|
||||||
|
try { body = await readBody(req, MAX_IMAGE_BYTES); }
|
||||||
|
catch (err) { json(res, err?.code === 413 ? 413 : 400, { error: err?.code === 413 ? 'Image is too large. Max 10 MB.' : 'Could not read upload.' }); return; }
|
||||||
|
const event = decodeToken(req.headers.authorization);
|
||||||
|
const problem = checkToken(event, body, contentType);
|
||||||
|
if (problem) { json(res, 401, { error: problem }); return; }
|
||||||
|
rollWindows();
|
||||||
|
const quota = checkQuota(event.pubkey.toLowerCase(), body.length, clientIp(req));
|
||||||
|
if (quota) { json(res, 429, { error: quota, 'retry-after': '3600' }); return; }
|
||||||
|
console.log(`[quota] accepted ${event.pubkey.slice(0, 8)}… ${body.length} bytes ${contentType}`);
|
||||||
|
forward(req, res, body);
|
||||||
|
});
|
||||||
|
server.listen(PORT, '127.0.0.1', () => console.log(`[quota] listening on 127.0.0.1:${PORT}`));
|
||||||
28
scripts/systemd/skatespots-quota.service
Normal file
28
scripts/systemd/skatespots-quota.service
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
[Unit]
|
||||||
|
Description=SkateSpots Blossom upload quota proxy
|
||||||
|
After=network-online.target docker.service
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=ubuntu
|
||||||
|
WorkingDirectory=/home/ubuntu/skatespots-quota
|
||||||
|
ExecStart=/usr/bin/node /home/ubuntu/skatespots-quota/quota-proxy.mjs
|
||||||
|
Restart=always
|
||||||
|
RestartSec=2
|
||||||
|
Environment=PORT=8790
|
||||||
|
Environment=NOSTR_TOOLS_ROOT=/home/ubuntu/skatespots-push/
|
||||||
|
Environment=QUOTA_STATE=/home/ubuntu/skatespots-quota/usage.json
|
||||||
|
Environment=ADMIN_PUBKEYS=20e7c953439e66f3882ab96fe0faa8ba246d6fb12c03b0cb4e2dc4623b8bcb21,caba497f7beda0650d6b78ab211216e4d2ebabeec9ca6eb2a05cee8e55c6871b
|
||||||
|
Environment=MAX_IMAGE_BYTES=10485760
|
||||||
|
Environment=DAILY_BYTES=262144000
|
||||||
|
Environment=DAILY_UPLOADS=50
|
||||||
|
Environment=ADMIN_MULTIPLIER=3
|
||||||
|
Environment=IP_HOURLY_BYTES=524288000
|
||||||
|
Environment=IP_HOURLY_UPLOADS=100
|
||||||
|
Environment=TOTAL_BYTES=21474836480
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
|
@ -11,7 +11,41 @@ import { useCurrentUser } from "./useCurrentUser";
|
||||||
* mirror URL on the returned tags so photo redundancy is explicit in the event;
|
* mirror URL on the returned tags so photo redundancy is explicit in the event;
|
||||||
* the read side also derives it from the sha256 (lib/imageFallback).
|
* the read side also derives it from the sha256 (lib/imageFallback).
|
||||||
*/
|
*/
|
||||||
const UPLOAD_SERVERS = ['https://blossom.skate-spots.org/', 'https://blossom.primal.net/', 'https://blossom.band/'];
|
// Uploads go only through our own Blossom/quota proxy. The public mirrors are
|
||||||
|
// still used by the read-side image fallback for older blobs, but including them
|
||||||
|
// here would let a quota rejection silently fall through to third-party hosting.
|
||||||
|
const UPLOAD_SERVERS = ['https://blossom.skate-spots.org/'];
|
||||||
|
|
||||||
|
const MAX_IMAGE_BYTES = 10 * 1024 * 1024;
|
||||||
|
const IMAGE_TYPES = new Set([
|
||||||
|
'image/jpeg', 'image/png', 'image/webp', 'image/avif',
|
||||||
|
'image/heic', 'image/heif', 'image/gif',
|
||||||
|
]);
|
||||||
|
|
||||||
|
/** Unwrap the quota proxy's JSON error so forms can show a human message. */
|
||||||
|
function uploadErrorMessage(error: unknown): string {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
const match = /Blossom upload failed \(\d+\):\s*(\{.*\})/.exec(message);
|
||||||
|
if (match) {
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(match[1]) as { error?: string };
|
||||||
|
if (parsed.error) return parsed.error;
|
||||||
|
} catch {
|
||||||
|
// fall through to the original message
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateImage(file: File): void {
|
||||||
|
const type = (file.type || '').toLowerCase();
|
||||||
|
if (!IMAGE_TYPES.has(type)) {
|
||||||
|
throw new Error('Only JPEG, PNG, WebP, AVIF, HEIC or GIF images can be uploaded.');
|
||||||
|
}
|
||||||
|
if (file.size > MAX_IMAGE_BYTES) {
|
||||||
|
throw new Error('Image is too large. The maximum is 10 MB.');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function hostOf(url: string): string {
|
function hostOf(url: string): string {
|
||||||
try {
|
try {
|
||||||
|
|
@ -30,12 +64,19 @@ export function useUploadFile() {
|
||||||
throw new Error('Must be logged in to upload files');
|
throw new Error('Must be logged in to upload files');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
validateImage(file);
|
||||||
|
|
||||||
const uploader = new BlossomUploader({
|
const uploader = new BlossomUploader({
|
||||||
servers: UPLOAD_SERVERS,
|
servers: UPLOAD_SERVERS,
|
||||||
signer: user.signer,
|
signer: user.signer,
|
||||||
});
|
});
|
||||||
|
|
||||||
const tags = await uploader.upload(file);
|
let tags;
|
||||||
|
try {
|
||||||
|
tags = await uploader.upload(file);
|
||||||
|
} catch (error) {
|
||||||
|
throw new Error(uploadErrorMessage(error));
|
||||||
|
}
|
||||||
|
|
||||||
// Remember the mirror copy of the same blob. Blossom is content-addressed,
|
// Remember the mirror copy of the same blob. Blossom is content-addressed,
|
||||||
// so `<other-host>/<sha256>` is the same file.
|
// so `<other-host>/<sha256>` is the same file.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue