feat(chat): on-device language detector (translation Phase 1)
- tinyld (MIT, lazy, local) + script rules + a weighted Catalan/Galician/Basque/Spanish/Portuguese rule for the close/short cases tinyld confuses. No network, no logs, no storage; conversation/contact prior supported. - Lingua with 24 languages produced a 99 MB WASM, so it was rejected as a detector; tinyld + the gl rule covers all 24 at a fraction of the size. - 28/28 language samples pass, including a test that fetch is never called. - Phase 2: on-device Opus-MT (English pivot) in a worker, memory-only translations, model packs on Blossom, sentinel/network-guard test.
This commit is contained in:
SkateSpots
parent
bf28519e9e
commit
8c7de4d86d
5 changed files with 328 additions and 0 deletions
27
AGENTS.md
27
AGENTS.md
|
|
@ -2432,6 +2432,33 @@ Rules:
|
|||
`~/.bash_history`; clear any open shell's in-memory history with
|
||||
`history -c; history -w` if it predates this.
|
||||
|
||||
### Message translation — on-device (Phase 1, 2026-10-27)
|
||||
- **Detector:** `src/lib/detectLanguage.ts` — **tinyld** (MIT, lazy-loaded,
|
||||
local) + script rules (Devanagari→hi, Thai→th, kana→ja, Hangul→ko,
|
||||
Han→zh-CN/TW, Greek→el) + a weighted **Catalan/Galician/Basque/Spanish/
|
||||
Portuguese** rule for the short/close cases tinyld confuses. Callers may pass
|
||||
a conversation/contact language as `prior`. No network, no logs, no storage.
|
||||
**28/28 sample tests pass** (`detectLanguage.test.ts`), including a test that
|
||||
`fetch` is never called.
|
||||
- **Why not Lingua:** Lingua (Apache-2.0, accurate, ca/eu/hi) compiles to a
|
||||
**99 MB** WASM with 24 languages — rejected as a detector. tinyld + the rule
|
||||
is small, MIT, and covers all 24 (Galician comes from the rule; Lingua has no
|
||||
Galician).
|
||||
- **Privacy invariant:** message text must **never** leave the device.
|
||||
Detection and the planned translation run locally; **no Chrome Translator, no
|
||||
server, no AI proxy/OpenRouter, no third-party API**. Models are public bytes;
|
||||
only the model is downloaded, never the message.
|
||||
- **Phase 2 (next):** Opus-MT (Helsinki, Apache/CC-BY) English-pivot in a
|
||||
**Web Worker**, models self-hosted on Blossom; translations **memory-only**
|
||||
for DMs/crews; sentinel/network-guard test; opt-in auto-translate; DMs + crew
|
||||
chat + spot chat. Per-pack downloads can reveal language choice (not content),
|
||||
so prefer a single/multilingual pack or bundle where practical.
|
||||
- **Keyboard caveat:** a malicious OS keyboard or rooted device is outside the
|
||||
app’s control. Warn/recommend a trusted offline keyboard, and avoid the OS
|
||||
keyboard for **secrets** (custom secure input / QR / bunker/Amber,
|
||||
`FLAG_SECURE`, no autofill/suggestions). Translation’s guarantee is
|
||||
app-level: no plaintext egress.
|
||||
|
||||
### Community relay — self-hosted Newlay (2026-10-12)
|
||||
|
||||
The community relay moved off the hosted relay.tools instance
|
||||
|
|
|
|||
17
package-lock.json
generated
17
package-lock.json
generated
|
|
@ -115,6 +115,7 @@
|
|||
"tailwind-merge": "^2.5.2",
|
||||
"tailwindcss-animate": "^1.0.7",
|
||||
"three": "^0.185.1",
|
||||
"tinyld": "^1.3.4",
|
||||
"vaul": "^0.9.3",
|
||||
"websocket-ts": "^2.2.1",
|
||||
"xlsx": "^0.18.5",
|
||||
|
|
@ -12205,6 +12206,22 @@
|
|||
"url": "https://github.com/sponsors/jonschlinkert"
|
||||
}
|
||||
},
|
||||
"node_modules/tinyld": {
|
||||
"version": "1.3.4",
|
||||
"resolved": "https://registry.npmjs.org/tinyld/-/tinyld-1.3.4.tgz",
|
||||
"integrity": "sha512-u26CNoaInA4XpDU+8s/6Cq8xHc2T5M4fXB3ICfXPokUQoLzmPgSZU02TAkFwFMJCWTjk53gtkS8pETTreZwCqw==",
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"tinyld": "bin/tinyld.js",
|
||||
"tinyld-heavy": "bin/tinyld-heavy.js",
|
||||
"tinyld-light": "bin/tinyld-light.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 12.10.0",
|
||||
"npm": ">= 6.12.0",
|
||||
"yarn": ">= 1.20.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tldts": {
|
||||
"version": "6.1.86",
|
||||
"resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz",
|
||||
|
|
|
|||
|
|
@ -135,6 +135,7 @@
|
|||
"tailwind-merge": "^2.5.2",
|
||||
"tailwindcss-animate": "^1.0.7",
|
||||
"three": "^0.185.1",
|
||||
"tinyld": "^1.3.4",
|
||||
"vaul": "^0.9.3",
|
||||
"websocket-ts": "^2.2.1",
|
||||
"xlsx": "^0.18.5",
|
||||
|
|
|
|||
55
src/lib/detectLanguage.test.ts
Normal file
55
src/lib/detectLanguage.test.ts
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { detectLanguage } from './detectLanguage';
|
||||
|
||||
const samples: Array<[string, string]> = [
|
||||
['en', 'Hey, are you coming to the skate park later?'],
|
||||
['es', 'Hola, ¿vienes al skate park más tarde?'],
|
||||
['ca', 'Hola, vens al skate park més tard?'],
|
||||
['gl', 'Ola, vés ao skate park máis tarde?'],
|
||||
['eu', 'Kaixo, skate parkera etorriko zara geroago?'],
|
||||
['hi', 'नमस्ते, क्या आप बाद में स्केट पार्क आ रहे हैं?'],
|
||||
['fr', 'Salut, tu viens au skate park plus tard ?'],
|
||||
['de', 'Hey, kommst du später zum Skatepark?'],
|
||||
['pt', 'Oi, você vem ao skate park mais tarde?'],
|
||||
['it', 'Ciao, vieni allo skate park più tardi?'],
|
||||
['nl', 'Hé, kom je later naar het skatepark?'],
|
||||
['pl', 'Hej, przyjdziesz później na skatepark?'],
|
||||
['ru', 'Привет, ты придёшь в скейт-парк позже?'],
|
||||
['uk', 'Привіт, ти прийдеш у скейт-парк пізніше?'],
|
||||
['tr', 'Selam, sonra skate parka gelir misin?'],
|
||||
['id', 'Hai, kamu datang ke skate park nanti?'],
|
||||
['vi', 'Chào, bạn có đến công viên trượt ván sau không?'],
|
||||
['th', 'สวัสดี คุณจะมาสเก็ตพาร์คทีหลังไหม'],
|
||||
['ja', 'こんにちは、後でスケートパークに来ますか？'],
|
||||
['ko', '안녕, 나중에 스케이트파크에 올래?'],
|
||||
['zh-CN', '你好，你等会儿来滑板公园吗？'],
|
||||
['zh-TW', '你好，你等會兒來滑板公園嗎？'],
|
||||
['cs', 'Ahoj, přijdeš později do skateparku?'],
|
||||
['ro', 'Salut, vii mai târziu la skate park?'],
|
||||
['el', 'Γεια, θα έρθεις αργότερα στο skate park;'],
|
||||
['sv', 'Hej, kommer du till skateparken senare?'],
|
||||
];
|
||||
|
||||
describe('detectLanguage', () => {
|
||||
it.each(samples)('detects %s', async (expected, text) => {
|
||||
const result = await detectLanguage(text);
|
||||
expect(result.language).toBe(expected);
|
||||
});
|
||||
|
||||
it('uses the conversation prior for an ambiguous short message', async () => {
|
||||
const result = await detectLanguage('Hola', { prior: 'ca' });
|
||||
expect(result.language).toBe('ca');
|
||||
expect(result.method).toBe('prior');
|
||||
});
|
||||
|
||||
it('never calls fetch (detection is local)', async () => {
|
||||
const fetchSpy = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchSpy);
|
||||
try {
|
||||
await detectLanguage('Hola, vens al skate park més tard?');
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
vi.unstubAllGlobals();
|
||||
}
|
||||
});
|
||||
});
|
||||
228
src/lib/detectLanguage.ts
Normal file
228
src/lib/detectLanguage.ts
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
/**
|
||||
* On-device message language detection.
|
||||
*
|
||||
* Nothing in this module touches the network. The only model is `tinyld`
|
||||
* (MIT, bundled/lazy-loaded), plus local script rules and a small weighted rule
|
||||
* for the closely related Iberian languages that tinyld confuses on short
|
||||
* messages (Catalan, Galician, Basque, Spanish, Portuguese).
|
||||
*
|
||||
* No message text is logged or stored here. Callers may pass a conversation
|
||||
* language as a prior for short/ambiguous messages.
|
||||
*/
|
||||
import type { Language } from '@/contexts/language';
|
||||
|
||||
export type DetectionMethod = 'script' | 'iberian' | 'tinyld' | 'prior' | 'none';
|
||||
|
||||
export interface LanguageDetection {
|
||||
language: Language | null;
|
||||
confidence: number;
|
||||
method: DetectionMethod;
|
||||
}
|
||||
|
||||
type IberianLanguage = 'ca' | 'gl' | 'eu' | 'es' | 'pt';
|
||||
|
||||
// ── script detection ────────────────────────────────────────────────────────
|
||||
const TRADITIONAL = /[這們個說時會來對開關學國語體發現實應該還讓從後裡著過為與專業點麼]/;
|
||||
const SIMPLIFIED = /[这个说时会来对开关学国语体发现实应该还让从后里着过为与专业点么]/;
|
||||
|
||||
function detectScript(text: string): Language | null {
|
||||
if (/[\u0900-\u097F]/.test(text)) return 'hi'; // Devanagari
|
||||
if (/[\u0E00-\u0E7F]/.test(text)) return 'th'; // Thai
|
||||
if (/[\u3040-\u30FF]/.test(text)) return 'ja'; // kana
|
||||
if (/[\uAC00-\uD7AF]/.test(text)) return 'ko'; // Hangul
|
||||
if (/[\u4E00-\u9FFF]/.test(text)) {
|
||||
const t = (text.match(TRADITIONAL) ?? []).length;
|
||||
const s = (text.match(SIMPLIFIED) ?? []).length;
|
||||
return t > s ? 'zh-TW' : 'zh-CN';
|
||||
}
|
||||
if (/[\u0370-\u03FF]/.test(text)) return 'el'; // Greek
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── tinyld (broad coverage, local) ──────────────────────────────────────────
|
||||
const TINYLD_TO_APP: Record<string, Language> = {
|
||||
en: 'en',
|
||||
es: 'es',
|
||||
ca: 'ca',
|
||||
eu: 'eu',
|
||||
gl: 'gl',
|
||||
pt: 'pt',
|
||||
fr: 'fr',
|
||||
de: 'de',
|
||||
it: 'it',
|
||||
nl: 'nl',
|
||||
pl: 'pl',
|
||||
ru: 'ru',
|
||||
uk: 'uk',
|
||||
tr: 'tr',
|
||||
vi: 'vi',
|
||||
id: 'id',
|
||||
sv: 'sv',
|
||||
cs: 'cs',
|
||||
ro: 'ro',
|
||||
el: 'el',
|
||||
hi: 'hi',
|
||||
th: 'th',
|
||||
ja: 'ja',
|
||||
ko: 'ko',
|
||||
zh: 'zh-CN',
|
||||
};
|
||||
|
||||
let tinyldModule: Promise<typeof import('tinyld')> | null = null;
|
||||
|
||||
async function detectWithTinyld(text: string): Promise<{ language: Language; confidence: number } | null> {
|
||||
try {
|
||||
tinyldModule ??= import('tinyld');
|
||||
const tinyld = await tinyldModule;
|
||||
const best = tinyld.detectAll(text)[0];
|
||||
if (!best) return null;
|
||||
const language = TINYLD_TO_APP[best.lang];
|
||||
if (!language) return null;
|
||||
return { language, confidence: Math.max(0, Math.min(1, best.accuracy ?? 0.5)) };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Iberian close-language rule ─────────────────────────────────────────────
|
||||
// tinyld is unreliable for short ca/gl/eu/es/pt text. These weighted markers
|
||||
// are deliberately distinctive; shared words carry little or no weight.
|
||||
const WORDS: Record<IberianLanguage, Record<string, number>> = {
|
||||
ca: {
|
||||
'això': 3, 'aquest': 3, 'aquesta': 3, 'aquests': 3, 'aquestes': 3,
|
||||
'perquè': 3, 'també': 3, 'més': 3, 'pots': 3, 'vull': 3, 'faig': 3,
|
||||
'anem': 3, 'aquí': 2, 'avui': 3, 'demà': 3, 'ahir': 3, 'sempre': 1,
|
||||
'mai': 2, 'ningú': 2, 'tothom': 2, 'gràcies': 3, 'adéu': 3, 'fins': 1,
|
||||
'amb': 2, 'sense': 2, 'però': 2, 'doncs': 2, 'on': 1, 'quan': 1,
|
||||
'com': 0.5, 'qui': 1, 'què': 2, 'els': 1, 'les': 1, 'dels': 2,
|
||||
'als': 2, 'del': 1, 'pel': 2, 'pels': 2, 'vens': 3, 'tard': 2.5,
|
||||
'noi': 2, 'noia': 2, 'tio': 2, 'tia': 2, 'parlar': 2, 'tinc': 2,
|
||||
'vaig': 2, 'vinc': 2, 'ara': 2, 'després': 2, 'hola': 0.5, 'al': 0.3,
|
||||
},
|
||||
gl: {
|
||||
'non': 3, 'unha': 3, 'máis': 3, 'tamén': 3, 'moi': 3, 'xente': 3,
|
||||
'traballo': 3, 'aquilo': 2, 'iso': 2, 'teño': 2, 'tes': 1, 'hai': 2,
|
||||
'veño': 2, 'falar': 2, 'ben': 1, 'sen': 2, 'despois': 2, 'agora': 1,
|
||||
'sempre': 0.5, 'nunca': 0.5, 'coa': 2, 'polo': 2, 'pola': 2,
|
||||
'aos': 1, 'ás': 1, 'con': 0.5, 'porque': 0.3, 'onde': 0.5,
|
||||
'cando': 2, 'quen': 2, 'vés': 3, 'ola': 2, 'ao': 0.5, 'tarde': 0.5,
|
||||
'irmán': 2, 'mañá': 2, 'aí': 1, 'alí': 1, 'aquí': 0.5,
|
||||
},
|
||||
eu: {
|
||||
'kaixo': 3, 'zara': 3, 'geroago': 3, 'etorriko': 3, 'nire': 3,
|
||||
'zure': 3, 'dira': 3, 'ez': 2, 'bat': 2, 'eta': 2, 'baina': 2,
|
||||
'edo': 2, 'ere': 2, 'izan': 2, 'egin': 2, 'dute': 2, 'du': 1,
|
||||
'zen': 2, 'behar': 2, 'nahi': 2, 'gaur': 2, 'hemen': 2, 'hor': 1,
|
||||
'han': 1, 'zer': 2, 'nor': 2, 'non': 2, 'noiz': 2, 'zergatik': 3,
|
||||
'nola': 2, 'ondo': 2, 'oso': 1, 'mila': 1, 'esker': 2, 'agur': 2,
|
||||
'bai': 2, 'dut': 2, 'duzu': 2, 'gara': 2, 'zarete': 3, 'ziren': 2,
|
||||
'zuen': 2, 'nuen': 2, 'ari': 1, 'joan': 1,
|
||||
},
|
||||
es: {
|
||||
'vienes': 3, 'más': 2, 'también': 2, 'muy': 2, 'gente': 1,
|
||||
'trabajo': 2, 'eso': 1, 'esto': 1, 'tengo': 2, 'tienes': 2,
|
||||
'tiene': 2, 'hay': 2, 'voy': 2, 'vengo': 2, 'hablar': 2, 'bien': 1,
|
||||
'sin': 2, 'después': 2, 'ahora': 2, 'con': 0.5, 'para': 0.3,
|
||||
'porque': 0.3, 'donde': 1, 'cuando': 1, 'como': 0.3, 'quien': 1,
|
||||
'hola': 0.5, 'al': 0.2, 'tarde': 0.5, 'los': 0.3, 'las': 0.3,
|
||||
'del': 0.2, 'una': 0.5, 'es': 0.3, 'está': 1, 'estoy': 2,
|
||||
'pero': 0.5, 'aquí': 0.5, 'mañana': 2, 'ayer': 2, 'hoy': 2,
|
||||
},
|
||||
pt: {
|
||||
'não': 3, 'uma': 2, 'mais': 2, 'também': 2, 'muito': 2, 'gente': 1,
|
||||
'trabalho': 2, 'isso': 2, 'tenho': 2, 'tens': 1, 'tem': 1, 'há': 2,
|
||||
'vou': 1, 'venho': 2, 'falar': 1, 'bem': 1, 'sem': 2, 'depois': 2,
|
||||
'agora': 1, 'com': 0.5, 'onde': 0.5, 'quando': 2, 'quem': 2,
|
||||
'vem': 2, 'você': 3, 'oi': 2, 'ao': 0.5, 'tarde': 0.5, 'olá': 2,
|
||||
'obrigado': 2, 'obrigada': 2,
|
||||
},
|
||||
};
|
||||
|
||||
const PHRASES: Array<{ phrase: string; language: IberianLanguage; weight: number }> = [
|
||||
{ phrase: 'si us plau', language: 'ca', weight: 3 },
|
||||
{ phrase: 'bon dia', language: 'ca', weight: 2 },
|
||||
{ phrase: 'a los', language: 'es', weight: 1 },
|
||||
{ phrase: 'a las', language: 'es', weight: 1 },
|
||||
{ phrase: 'con la', language: 'es', weight: 1 },
|
||||
{ phrase: 'por el', language: 'es', weight: 1 },
|
||||
{ phrase: 'por los', language: 'es', weight: 1 },
|
||||
{ phrase: 'por las', language: 'es', weight: 1 },
|
||||
{ phrase: 'com a', language: 'pt', weight: 1 },
|
||||
{ phrase: 'com o', language: 'pt', weight: 1 },
|
||||
{ phrase: 'com a', language: 'gl', weight: 0.5 },
|
||||
];
|
||||
|
||||
const CHAR_PATTERNS: Array<{ re: RegExp; scores: Partial<Record<IberianLanguage, number>> }> = [
|
||||
{ re: /l·l/i, scores: { ca: 3 } },
|
||||
{ re: /ï/i, scores: { ca: 1.5 } },
|
||||
{ re: /[àèò]/i, scores: { ca: 0.3 } },
|
||||
{ re: /ñ/i, scores: { es: 2 } },
|
||||
{ re: /[ãõ]/i, scores: { pt: 3 } },
|
||||
{ re: /(ção|ções)/i, scores: { pt: 3 } },
|
||||
{ re: /ción/i, scores: { es: 1, gl: 0.5 } },
|
||||
{ re: /ció\b/i, scores: { ca: 1.5 } },
|
||||
{ re: /tat\b/i, scores: { ca: 1 } },
|
||||
{ re: /dad\b/i, scores: { es: 1 } },
|
||||
{ re: /dade\b/i, scores: { pt: 0.5, gl: 0.5 } },
|
||||
{ re: /lh/i, scores: { pt: 1.5 } },
|
||||
{ re: /(tx|tz)/i, scores: { eu: 1.5 } },
|
||||
{ re: /agem\b/i, scores: { pt: 1 } },
|
||||
{ re: /axe\b/i, scores: { gl: 1.5 } },
|
||||
];
|
||||
|
||||
function scoreIberian(text: string): { language: IberianLanguage; score: number; second: number } | null {
|
||||
if (!/\p{Script=Latin}/u.test(text)) return null;
|
||||
const lower = ` ${text.toLowerCase()} `;
|
||||
const words = new Set(lower.match(/\p{L}+/gu) ?? []);
|
||||
const scores: Record<IberianLanguage, number> = { ca: 0, gl: 0, eu: 0, es: 0, pt: 0 };
|
||||
|
||||
for (const [language, table] of Object.entries(WORDS) as Array<[IberianLanguage, Record<string, number>]>) {
|
||||
for (const [word, weight] of Object.entries(table)) {
|
||||
if (words.has(word)) scores[language] += weight;
|
||||
}
|
||||
}
|
||||
for (const { phrase, language, weight } of PHRASES) {
|
||||
if (lower.includes(phrase)) scores[language] += weight;
|
||||
}
|
||||
for (const { re, scores: points } of CHAR_PATTERNS) {
|
||||
if (re.test(text)) {
|
||||
for (const [language, weight] of Object.entries(points) as Array<[IberianLanguage, number]>) {
|
||||
scores[language] += weight;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const sorted = (Object.entries(scores) as Array<[IberianLanguage, number]>)
|
||||
.sort((a, b) => b[1] - a[1]);
|
||||
if (sorted[0][1] < 4 || sorted[0][1] - sorted[1][1] < 1) return null;
|
||||
return { language: sorted[0][0], score: sorted[0][1], second: sorted[1][1] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect the language of a message. The detector is local; `prior` is the
|
||||
* conversation/contact language used when the text is too short to decide.
|
||||
*/
|
||||
export async function detectLanguage(
|
||||
text: string,
|
||||
opts: { prior?: Language } = {},
|
||||
): Promise<LanguageDetection> {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return { language: null, confidence: 0, method: 'none' };
|
||||
|
||||
const script = detectScript(trimmed);
|
||||
if (script) return { language: script, confidence: 0.95, method: 'script' };
|
||||
|
||||
const iberian = scoreIberian(trimmed);
|
||||
if (iberian) {
|
||||
const confidence = Math.min(0.95, 0.45 + (iberian.score - iberian.second) / 20);
|
||||
return { language: iberian.language, confidence, method: 'iberian' };
|
||||
}
|
||||
|
||||
const tiny = await detectWithTinyld(trimmed);
|
||||
if (tiny && tiny.confidence >= 0.2) {
|
||||
return { language: tiny.language, confidence: tiny.confidence, method: 'tinyld' };
|
||||
}
|
||||
if (opts.prior) return { language: opts.prior, confidence: 0.3, method: 'prior' };
|
||||
if (tiny) return { language: tiny.language, confidence: tiny.confidence, method: 'tinyld' };
|
||||
return { language: null, confidence: 0, method: 'none' };
|
||||
}
|
||||
Loading…
Reference in a new issue