From e85b9aacb632df42d30ef6c7c5c82270d6a169c5 Mon Sep 17 00:00:00 2001 From: SkateSpots Date: Tue, 27 Oct 2026 13:06:04 +0700 Subject: [PATCH] feat(chat): all-language on-device translation (M2M100 + Basque Opus) - M2M100 418M (MIT, 617 MB q8) covers 23 app languages; Basque uses converted Opus eu-en/en-eu q8 packs; all self-hosted under relay.skate-spots.org/models. - Worker uses M2M100 directly and pivots Basque through English; UI targets all languages; old pilot restriction/note removed; memory-only + guard unchanged. - Verified en<->es, ca<->gl, hi<->en from the host. Basque packs are converted and hosted; the final MarianTokenizer/ONNX runtime check is still pending. - 138 tests, 24 locales, build and smoke green. SpotChat deliberately skipped. --- AGENTS.md | 49 +++++++---- src/components/chat/TranslateMessage.tsx | 20 ++--- src/components/chat/TranslationSettings.tsx | 1 - src/lib/translation.test.ts | 16 ++-- src/lib/translation.ts | 6 +- src/workers/translate.worker.ts | 91 ++++++++++++++------- 6 files changed, 109 insertions(+), 74 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f817742..fc83e18 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2432,7 +2432,7 @@ Rules: `~/.bash_history`; clear any open shell's in-memory history with `history -c; history -w` if it predates this. -### Message translation — on-device pilot live (2026-10-27) +### Message translation — on-device, all languages (2026-10-27) - **Detector (all 24):** `src/lib/detectLanguage.ts` — **tinyld** (MIT, lazy-loaded, local) + script rules (Devanagari→hi, Thai→th, kana→ja, Hangul→ko, Han→zh-CN/TW, Greek→el) + a weighted **Catalan/Galician/Basque/ @@ -2446,30 +2446,43 @@ Rules: **No Chrome Translator, no server, no AI proxy/OpenRouter, no third-party API.** Model files are public bytes; only the model is downloaded, never the message. -- **Pilot (English ↔ Spanish) is live:** - - Models: `Xenova/opus-mt-en-es` + `es-en` (Helsinki Opus-MT, ONNX q8) hosted - on the relay VPS at - `https://relay.skate-spots.org/models/Xenova//resolve/main/…` - (235 MB total; Caddy `handle_path /models/*` → `/srv/models`, CORS `*`; - backup `/etc/caddy/Caddyfile.bak-…`). Both directions verified from the - hosted files ("Hello, how are you?" → "Hola, ¿cómo estás?" and back). +- **All-language runtime (M2M100 + Basque Opus):** + - `Xenova/m2m100_418M` (MIT, **617 MB** q8) covers 23 app languages + (en/es/ca/gl/hi/th/ja/pt/fr/de/it/nl/pl/ru/ko/zh/uk/tr/vi/id/sv/cs/ro/el). + Basque is not in M2M100, so `Xenova/opus-mt-eu-en` + `en-eu` (Apache-2.0, + converted q8; ~374 MB each) pivot through English. + - Hosted at `https://relay.skate-spots.org/models/…` (Caddy + `handle_path /models/*` → `/srv/models`, CORS `*`; backup + `/etc/caddy/Caddyfile.bak-…`). + - **Verified from the host:** en↔es, ca↔gl, hi↔en. The Basque packs are + converted and hosted (ONNX q8 + generated `tokenizer.json`), but the final + in-browser MarianTokenizer/ONNX runtime verification is **still pending** + (a Marian/ONNX runtime error in the standalone check). - Runtime: **`@xenova/transformers` 2.17.2** + ONNX Runtime **1.14** WASM self-hosted in `public/ort/` (37 MB, copied from `onnxruntime-web`), loaded - from `../ort/`; worker `src/workers/translate.worker.ts`. + from `../ort/`; worker `src/workers/translate.worker.ts`. M2M100 is used + directly; Basque pivots eu→en→target / source→en→eu. - **Hard guard:** `src/lib/translationGuard.ts` permits only model - GET/HEAD (no query string) and same-origin `/ort/` assets. The message text + GET/HEAD (no query string) and same-origin `/ort/` assets. Message text only travels via `postMessage` to the local worker. `translation.test.ts` + `translationGuard.test.ts` (9 tests) prove `fetch` is never called with a sentinel message and every other request is blocked. - - UI: `src/components/chat/TranslateMessage.tsx` in DMs (`ConversationPage`) - and crew chat (`CrewChat`); `TranslationSettings` in Settings → Appearance + - UI: `TranslateMessage` in DMs (`ConversationPage`) and crew chat + (`CrewChat`); `TranslationSettings` in Settings → Appearance (auto-translate **off** by default). Translations are **memory-only**; - reload/logout clears them. First use downloads ~110 MB per direction with a - progress indicator. - - i18n: 12 new `translateUi` keys in en/es + all 24 `drift.ts` (check 100%). -- **Still to do:** add the remaining language pairs via the **English pivot** - (`xx→en→target`; Opus-MT packs on the same host), add SpotChat, and measure - speed/memory on the APK before a **1.7.2** release. + reload/logout clears them + terminates the worker. First use downloads the + multilingual pack with progress. 12 `translateUi` keys in en/es + all 24 + `drift.ts` (check 100%); the old “pilot” note is unused. + - **SpotChat was deliberately skipped** (owner request). + - **Provenance:** M2M100 is MIT open weights (Meta origin) but runs entirely + locally — nothing goes to Meta. Basque Opus is Apache-2.0. No Chrome, + server translation, AI proxy or third-party API. + - **Conversion:** Docker Python 3.11 + torch 2.4.1 + transformers 4.46.3 + + optimum 1.24; dynamic q8; `tokenizer.json` generated from the SentencePiece + models (needs protobuf + `sentencepiece_model_pb2.py`). +- **Still to do:** finish the Basque tokenizer/ONNX runtime fix, verify all 24 + pairs in the browser/APK, measure speed/memory, and ship **1.7.2** (the phone + is on 1.7.1). No SpotChat. - **Keyboard caveat:** a malicious OS keyboard or rooted device is outside the app’s control. Warn/recommend a trusted offline keyboard, and avoid the OS keyboard for **secrets** (custom secure input / QR / bunker/Amber, diff --git a/src/components/chat/TranslateMessage.tsx b/src/components/chat/TranslateMessage.tsx index 25bae18..bb2ab72 100644 --- a/src/components/chat/TranslateMessage.tsx +++ b/src/components/chat/TranslateMessage.tsx @@ -63,10 +63,8 @@ export function TranslateMessage({ text, messageId, isOwn = false, className }: const [progress, setProgress] = useState(null); const [error, setError] = useState(null); - const pilotTarget = language === 'en' || language === 'es'; - const handleTranslate = useCallback(async () => { - if (!text.trim() || !pilotTarget) return; + if (!text.trim()) return; setState('detecting'); setError(null); setProgress(null); @@ -105,13 +103,13 @@ export function TranslateMessage({ text, messageId, isOwn = false, className }: setState('error'); setError(t('translateUi.failed')); } - }, [text, pilotTarget, messageId, language, t]); + }, [text, messageId, language, t]); useEffect(() => { - if (!autoTranslate || isOwn || !pilotTarget) return; + if (!autoTranslate || isOwn) return; if (state !== 'idle' || !text.trim()) return; void handleTranslate(); - }, [autoTranslate, isOwn, pilotTarget, state, text, handleTranslate]); + }, [autoTranslate, isOwn, state, text, handleTranslate]); const body = state === 'done' && !showOriginal ? translated ?? text : text; @@ -120,22 +118,20 @@ export function TranslateMessage({ text, messageId, isOwn = false, className }:

{body}

{!isOwn && (
- {!pilotTarget && {t('translateUi.pilotNote')}} - - {pilotTarget && state === 'idle' && ( + {state === 'idle' && ( )} - {pilotTarget && (state === 'detecting' || state === 'loading') && ( + {(state === 'detecting' || state === 'loading') && ( {t('translateUi.translating')} {progress !== null ? ` ${Math.round(progress)}%` : '…'} )} - {pilotTarget && state === 'done' && ( + {state === 'done' && ( <>

{t('translateUi.privacy')}

-

{t('translateUi.pilotNote')}

); } diff --git a/src/lib/translation.test.ts b/src/lib/translation.test.ts index 6d8c95b..defe44d 100644 --- a/src/lib/translation.test.ts +++ b/src/lib/translation.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { clearTranslationMemory, translateText } from './translation'; +import { clearTranslationMemory, isTranslationPairSupported, translateText } from './translation'; class FakeWorker { onmessage: ((event: MessageEvent) => void) | null = null; @@ -59,14 +59,10 @@ describe('translateText', () => { expect(result).toBe('same language'); }); - it('rejects pairs outside the pilot', async () => { - await expect( - translateText({ - id: 'message-3', - text: 'hola', - source: 'ca', - target: 'en', - }), - ).rejects.toThrow('UNSUPPORTED'); + it('supports all app language pairs (M2M100 + Basque Opus)', () => { + expect(isTranslationPairSupported('ca', 'en')).toBe(true); + expect(isTranslationPairSupported('eu', 'en')).toBe(true); + expect(isTranslationPairSupported('hi', 'es')).toBe(true); + expect(isTranslationPairSupported('en', 'en')).toBe(false); }); }); diff --git a/src/lib/translation.ts b/src/lib/translation.ts index 85cf5df..6e20300 100644 --- a/src/lib/translation.ts +++ b/src/lib/translation.ts @@ -41,19 +41,19 @@ interface PendingRequest { timer: ReturnType; } -const MODEL_TIMEOUT_MS = 300_000; // first use downloads a ~110 MB model +const MODEL_TIMEOUT_MS = 900_000; // first use downloads the multilingual model (~617 MB) const memoryCache = new Map(); const pending = new Map(); let worker: Worker | null = null; let sequence = 0; -/** The pilot supports English ↔ Spanish. */ +/** All app language pairs are supported (M2M100 + Opus Basque). */ export function isTranslationPairSupported( source: Language | null | undefined, target: Language, ): boolean { if (!source) return false; - return (source === 'en' && target === 'es') || (source === 'es' && target === 'en'); + return source !== target; } function rejectAll(error: Error): void { diff --git a/src/workers/translate.worker.ts b/src/workers/translate.worker.ts index 189a72b..234451b 100644 --- a/src/workers/translate.worker.ts +++ b/src/workers/translate.worker.ts @@ -1,10 +1,12 @@ /** - * On-device translation worker. + * On-device translation worker (all app languages). * + * - One multilingual model (M2M100, MIT) covers every app language except + * Basque. Basque uses a small Opus-MT eu↔en pair and pivots through English. * - Model files come from the self-hosted relay (public ONNX bytes). * - Message text is received via postMessage and never put into a URL/body. - * - A hard network guard blocks every request except the model host - * (GET/HEAD) and same-origin ONNX runtime assets. + * - The network guard blocks every request except the model host (GET/HEAD) + * and same-origin ONNX runtime assets. * - No logging of message text; errors are returned as codes only. */ import { env, pipeline } from '@xenova/transformers'; @@ -14,13 +16,15 @@ const MODEL_ORIGIN = 'https://relay.skate-spots.org'; const MODEL_PREFIX = '/models/'; const APP_ORIGIN = self.location.origin; +const M2M100 = 'Xenova/m2m100_418M'; +const EU_EN = 'Xenova/opus-mt-eu-en'; +const EN_EU = 'Xenova/opus-mt-en-eu'; + env.allowLocalModels = false; env.allowRemoteModels = true; env.remoteHost = `${MODEL_ORIGIN}${MODEL_PREFIX}`; env.useBrowserCache = true; -// Self-hosted ONNX runtime (public/ort/). Keeping this same-origin avoids the -// default CDN and lets the network guard stay strict. const onnxWasm = (env.backends.onnx as unknown as { wasm?: { wasmPaths?: string; numThreads?: number }; } | undefined)?.wasm; @@ -39,11 +43,6 @@ globalThis.fetch = createGuardedFetch( }, ); -const MODELS: Record = { - 'en-es': 'Xenova/opus-mt-en-es', - 'es-en': 'Xenova/opus-mt-es-en', -}; - type Incoming = { type: 'translate'; id: number; @@ -55,7 +54,7 @@ type Incoming = { type Outgoing = | { type: 'progress'; id: number; status?: string; progress?: number; file?: string } | { type: 'result'; id: number; text: string } - | { type: 'error'; id: number; code: 'UNSUPPORTED' | 'FAILED' }; + | { type: 'error'; id: number; code: 'FAILED' }; const ctx = self as unknown as { postMessage: (message: Outgoing) => void; @@ -65,6 +64,14 @@ const ctx = self as unknown as { // eslint-disable-next-line @typescript-eslint/no-explicit-any const translators = new Map>(); +/** Collapse UI variants onto the model's language tokens. */ +function baseLanguage(code: string): string { + if (code === 'pt-PT') return 'pt'; + if (code === 'ko-KP') return 'ko'; + if (code === 'zh-CN' || code === 'zh-TW') return 'zh'; + return code; +} + // eslint-disable-next-line @typescript-eslint/no-explicit-any function getTranslator(modelId: string, requestId: number): Promise { const existing = translators.get(modelId); @@ -92,31 +99,55 @@ function getTranslator(modelId: string, requestId: number): Promise { return loading; } +async function run( + modelId: string, + text: string, + requestId: number, + options: Record = {}, +): Promise { + const translator = await getTranslator(modelId, requestId); + const output = await translator(text, { max_new_tokens: 512, ...options }); + const translated = Array.isArray(output) + ? output[0]?.translation_text + : output?.translation_text; + if (typeof translated !== 'string' || !translated) { + throw new Error('empty translation'); + } + return translated; +} + +async function translate( + text: string, + source: string, + target: string, + requestId: number, +): Promise { + const s = baseLanguage(source); + const t = baseLanguage(target); + if (s === t) return text; + + // Basque is not in M2M100: use Opus eu↔en and pivot through English. + if (s === 'eu' || t === 'eu') { + if (s === 'eu' && t === 'en') return run(EU_EN, text, requestId); + if (s === 'en' && t === 'eu') return run(EN_EU, text, requestId); + if (s === 'eu') { + const english = await run(EU_EN, text, requestId); + return run(M2M100, english, requestId, { src_lang: 'en', tgt_lang: t }); + } + const english = await run(M2M100, text, requestId, { src_lang: s, tgt_lang: 'en' }); + return run(EN_EU, english, requestId); + } + + return run(M2M100, text, requestId, { src_lang: s, tgt_lang: t }); +} + ctx.onmessage = (event) => { const data = event.data; if (!data || data.type !== 'translate' || typeof data.id !== 'number') return; - const pair = `${data.source}-${data.target}`; - const model = MODELS[pair]; - if (!model) { - ctx.postMessage({ type: 'error', id: data.id, code: 'UNSUPPORTED' }); - return; - } - if (data.source === data.target) { - ctx.postMessage({ type: 'result', id: data.id, text: data.text }); - return; - } - void (async () => { try { - const translator = await getTranslator(model, data.id); - const output = await translator(data.text, { max_new_tokens: 512 }); - const translated = Array.isArray(output) - ? output[0]?.translation_text - : output?.translation_text; - if (typeof translated !== 'string' || !translated) { - throw new Error('empty translation'); - } + const translated = await translate(data.text, data.source, data.target, data.id); ctx.postMessage({ type: 'result', id: data.id, text: translated }); } catch { // Never forward the underlying error: it may contain message text.