8c7de4d86d
- tinyld (MIT, lazy, local) + script rules + a weighted Catalan/Galician/Basque/Spanish/Portuguese rule for the close/short cases tinyld confuses. No network, no logs, no storage; conversation/contact prior supported. - Lingua with 24 languages produced a 99 MB WASM, so it was rejected as a detector; tinyld + the gl rule covers all 24 at a fraction of the size. - 28/28 language samples pass, including a test that fetch is never called. - Phase 2: on-device Opus-MT (English pivot) in a worker, memory-only translations, model packs on Blossom, sentinel/network-guard test.
228 lines
9.4 KiB
TypeScript
228 lines
9.4 KiB
TypeScript
/**
|
|
* On-device message language detection.
|
|
*
|
|
* Nothing in this module touches the network. The only model is `tinyld`
|
|
* (MIT, bundled/lazy-loaded), plus local script rules and a small weighted rule
|
|
* for the closely related Iberian languages that tinyld confuses on short
|
|
* messages (Catalan, Galician, Basque, Spanish, Portuguese).
|
|
*
|
|
* No message text is logged or stored here. Callers may pass a conversation
|
|
* language as a prior for short/ambiguous messages.
|
|
*/
|
|
import type { Language } from '@/contexts/language';
|
|
|
|
export type DetectionMethod = 'script' | 'iberian' | 'tinyld' | 'prior' | 'none';
|
|
|
|
export interface LanguageDetection {
|
|
language: Language | null;
|
|
confidence: number;
|
|
method: DetectionMethod;
|
|
}
|
|
|
|
type IberianLanguage = 'ca' | 'gl' | 'eu' | 'es' | 'pt';
|
|
|
|
// ── script detection ────────────────────────────────────────────────────────
|
|
const TRADITIONAL = /[這們個說時會來對開關學國語體發現實應該還讓從後裡著過為與專業點麼]/;
|
|
const SIMPLIFIED = /[这个说时会来对开关学国语体发现实应该还让从后里着过为与专业点么]/;
|
|
|
|
function detectScript(text: string): Language | null {
|
|
if (/[\u0900-\u097F]/.test(text)) return 'hi'; // Devanagari
|
|
if (/[\u0E00-\u0E7F]/.test(text)) return 'th'; // Thai
|
|
if (/[\u3040-\u30FF]/.test(text)) return 'ja'; // kana
|
|
if (/[\uAC00-\uD7AF]/.test(text)) return 'ko'; // Hangul
|
|
if (/[\u4E00-\u9FFF]/.test(text)) {
|
|
const t = (text.match(TRADITIONAL) ?? []).length;
|
|
const s = (text.match(SIMPLIFIED) ?? []).length;
|
|
return t > s ? 'zh-TW' : 'zh-CN';
|
|
}
|
|
if (/[\u0370-\u03FF]/.test(text)) return 'el'; // Greek
|
|
return null;
|
|
}
|
|
|
|
// ── tinyld (broad coverage, local) ──────────────────────────────────────────
|
|
const TINYLD_TO_APP: Record<string, Language> = {
|
|
en: 'en',
|
|
es: 'es',
|
|
ca: 'ca',
|
|
eu: 'eu',
|
|
gl: 'gl',
|
|
pt: 'pt',
|
|
fr: 'fr',
|
|
de: 'de',
|
|
it: 'it',
|
|
nl: 'nl',
|
|
pl: 'pl',
|
|
ru: 'ru',
|
|
uk: 'uk',
|
|
tr: 'tr',
|
|
vi: 'vi',
|
|
id: 'id',
|
|
sv: 'sv',
|
|
cs: 'cs',
|
|
ro: 'ro',
|
|
el: 'el',
|
|
hi: 'hi',
|
|
th: 'th',
|
|
ja: 'ja',
|
|
ko: 'ko',
|
|
zh: 'zh-CN',
|
|
};
|
|
|
|
let tinyldModule: Promise<typeof import('tinyld')> | null = null;
|
|
|
|
async function detectWithTinyld(text: string): Promise<{ language: Language; confidence: number } | null> {
|
|
try {
|
|
tinyldModule ??= import('tinyld');
|
|
const tinyld = await tinyldModule;
|
|
const best = tinyld.detectAll(text)[0];
|
|
if (!best) return null;
|
|
const language = TINYLD_TO_APP[best.lang];
|
|
if (!language) return null;
|
|
return { language, confidence: Math.max(0, Math.min(1, best.accuracy ?? 0.5)) };
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// ── Iberian close-language rule ─────────────────────────────────────────────
|
|
// tinyld is unreliable for short ca/gl/eu/es/pt text. These weighted markers
|
|
// are deliberately distinctive; shared words carry little or no weight.
|
|
const WORDS: Record<IberianLanguage, Record<string, number>> = {
|
|
ca: {
|
|
'això': 3, 'aquest': 3, 'aquesta': 3, 'aquests': 3, 'aquestes': 3,
|
|
'perquè': 3, 'també': 3, 'més': 3, 'pots': 3, 'vull': 3, 'faig': 3,
|
|
'anem': 3, 'aquí': 2, 'avui': 3, 'demà': 3, 'ahir': 3, 'sempre': 1,
|
|
'mai': 2, 'ningú': 2, 'tothom': 2, 'gràcies': 3, 'adéu': 3, 'fins': 1,
|
|
'amb': 2, 'sense': 2, 'però': 2, 'doncs': 2, 'on': 1, 'quan': 1,
|
|
'com': 0.5, 'qui': 1, 'què': 2, 'els': 1, 'les': 1, 'dels': 2,
|
|
'als': 2, 'del': 1, 'pel': 2, 'pels': 2, 'vens': 3, 'tard': 2.5,
|
|
'noi': 2, 'noia': 2, 'tio': 2, 'tia': 2, 'parlar': 2, 'tinc': 2,
|
|
'vaig': 2, 'vinc': 2, 'ara': 2, 'després': 2, 'hola': 0.5, 'al': 0.3,
|
|
},
|
|
gl: {
|
|
'non': 3, 'unha': 3, 'máis': 3, 'tamén': 3, 'moi': 3, 'xente': 3,
|
|
'traballo': 3, 'aquilo': 2, 'iso': 2, 'teño': 2, 'tes': 1, 'hai': 2,
|
|
'veño': 2, 'falar': 2, 'ben': 1, 'sen': 2, 'despois': 2, 'agora': 1,
|
|
'sempre': 0.5, 'nunca': 0.5, 'coa': 2, 'polo': 2, 'pola': 2,
|
|
'aos': 1, 'ás': 1, 'con': 0.5, 'porque': 0.3, 'onde': 0.5,
|
|
'cando': 2, 'quen': 2, 'vés': 3, 'ola': 2, 'ao': 0.5, 'tarde': 0.5,
|
|
'irmán': 2, 'mañá': 2, 'aí': 1, 'alí': 1, 'aquí': 0.5,
|
|
},
|
|
eu: {
|
|
'kaixo': 3, 'zara': 3, 'geroago': 3, 'etorriko': 3, 'nire': 3,
|
|
'zure': 3, 'dira': 3, 'ez': 2, 'bat': 2, 'eta': 2, 'baina': 2,
|
|
'edo': 2, 'ere': 2, 'izan': 2, 'egin': 2, 'dute': 2, 'du': 1,
|
|
'zen': 2, 'behar': 2, 'nahi': 2, 'gaur': 2, 'hemen': 2, 'hor': 1,
|
|
'han': 1, 'zer': 2, 'nor': 2, 'non': 2, 'noiz': 2, 'zergatik': 3,
|
|
'nola': 2, 'ondo': 2, 'oso': 1, 'mila': 1, 'esker': 2, 'agur': 2,
|
|
'bai': 2, 'dut': 2, 'duzu': 2, 'gara': 2, 'zarete': 3, 'ziren': 2,
|
|
'zuen': 2, 'nuen': 2, 'ari': 1, 'joan': 1,
|
|
},
|
|
es: {
|
|
'vienes': 3, 'más': 2, 'también': 2, 'muy': 2, 'gente': 1,
|
|
'trabajo': 2, 'eso': 1, 'esto': 1, 'tengo': 2, 'tienes': 2,
|
|
'tiene': 2, 'hay': 2, 'voy': 2, 'vengo': 2, 'hablar': 2, 'bien': 1,
|
|
'sin': 2, 'después': 2, 'ahora': 2, 'con': 0.5, 'para': 0.3,
|
|
'porque': 0.3, 'donde': 1, 'cuando': 1, 'como': 0.3, 'quien': 1,
|
|
'hola': 0.5, 'al': 0.2, 'tarde': 0.5, 'los': 0.3, 'las': 0.3,
|
|
'del': 0.2, 'una': 0.5, 'es': 0.3, 'está': 1, 'estoy': 2,
|
|
'pero': 0.5, 'aquí': 0.5, 'mañana': 2, 'ayer': 2, 'hoy': 2,
|
|
},
|
|
pt: {
|
|
'não': 3, 'uma': 2, 'mais': 2, 'também': 2, 'muito': 2, 'gente': 1,
|
|
'trabalho': 2, 'isso': 2, 'tenho': 2, 'tens': 1, 'tem': 1, 'há': 2,
|
|
'vou': 1, 'venho': 2, 'falar': 1, 'bem': 1, 'sem': 2, 'depois': 2,
|
|
'agora': 1, 'com': 0.5, 'onde': 0.5, 'quando': 2, 'quem': 2,
|
|
'vem': 2, 'você': 3, 'oi': 2, 'ao': 0.5, 'tarde': 0.5, 'olá': 2,
|
|
'obrigado': 2, 'obrigada': 2,
|
|
},
|
|
};
|
|
|
|
const PHRASES: Array<{ phrase: string; language: IberianLanguage; weight: number }> = [
|
|
{ phrase: 'si us plau', language: 'ca', weight: 3 },
|
|
{ phrase: 'bon dia', language: 'ca', weight: 2 },
|
|
{ phrase: 'a los', language: 'es', weight: 1 },
|
|
{ phrase: 'a las', language: 'es', weight: 1 },
|
|
{ phrase: 'con la', language: 'es', weight: 1 },
|
|
{ phrase: 'por el', language: 'es', weight: 1 },
|
|
{ phrase: 'por los', language: 'es', weight: 1 },
|
|
{ phrase: 'por las', language: 'es', weight: 1 },
|
|
{ phrase: 'com a', language: 'pt', weight: 1 },
|
|
{ phrase: 'com o', language: 'pt', weight: 1 },
|
|
{ phrase: 'com a', language: 'gl', weight: 0.5 },
|
|
];
|
|
|
|
const CHAR_PATTERNS: Array<{ re: RegExp; scores: Partial<Record<IberianLanguage, number>> }> = [
|
|
{ re: /l·l/i, scores: { ca: 3 } },
|
|
{ re: /ï/i, scores: { ca: 1.5 } },
|
|
{ re: /[àèò]/i, scores: { ca: 0.3 } },
|
|
{ re: /ñ/i, scores: { es: 2 } },
|
|
{ re: /[ãõ]/i, scores: { pt: 3 } },
|
|
{ re: /(ção|ções)/i, scores: { pt: 3 } },
|
|
{ re: /ción/i, scores: { es: 1, gl: 0.5 } },
|
|
{ re: /ció\b/i, scores: { ca: 1.5 } },
|
|
{ re: /tat\b/i, scores: { ca: 1 } },
|
|
{ re: /dad\b/i, scores: { es: 1 } },
|
|
{ re: /dade\b/i, scores: { pt: 0.5, gl: 0.5 } },
|
|
{ re: /lh/i, scores: { pt: 1.5 } },
|
|
{ re: /(tx|tz)/i, scores: { eu: 1.5 } },
|
|
{ re: /agem\b/i, scores: { pt: 1 } },
|
|
{ re: /axe\b/i, scores: { gl: 1.5 } },
|
|
];
|
|
|
|
function scoreIberian(text: string): { language: IberianLanguage; score: number; second: number } | null {
|
|
if (!/\p{Script=Latin}/u.test(text)) return null;
|
|
const lower = ` ${text.toLowerCase()} `;
|
|
const words = new Set(lower.match(/\p{L}+/gu) ?? []);
|
|
const scores: Record<IberianLanguage, number> = { ca: 0, gl: 0, eu: 0, es: 0, pt: 0 };
|
|
|
|
for (const [language, table] of Object.entries(WORDS) as Array<[IberianLanguage, Record<string, number>]>) {
|
|
for (const [word, weight] of Object.entries(table)) {
|
|
if (words.has(word)) scores[language] += weight;
|
|
}
|
|
}
|
|
for (const { phrase, language, weight } of PHRASES) {
|
|
if (lower.includes(phrase)) scores[language] += weight;
|
|
}
|
|
for (const { re, scores: points } of CHAR_PATTERNS) {
|
|
if (re.test(text)) {
|
|
for (const [language, weight] of Object.entries(points) as Array<[IberianLanguage, number]>) {
|
|
scores[language] += weight;
|
|
}
|
|
}
|
|
}
|
|
|
|
const sorted = (Object.entries(scores) as Array<[IberianLanguage, number]>)
|
|
.sort((a, b) => b[1] - a[1]);
|
|
if (sorted[0][1] < 4 || sorted[0][1] - sorted[1][1] < 1) return null;
|
|
return { language: sorted[0][0], score: sorted[0][1], second: sorted[1][1] };
|
|
}
|
|
|
|
/**
|
|
* Detect the language of a message. The detector is local; `prior` is the
|
|
* conversation/contact language used when the text is too short to decide.
|
|
*/
|
|
export async function detectLanguage(
|
|
text: string,
|
|
opts: { prior?: Language } = {},
|
|
): Promise<LanguageDetection> {
|
|
const trimmed = text.trim();
|
|
if (!trimmed) return { language: null, confidence: 0, method: 'none' };
|
|
|
|
const script = detectScript(trimmed);
|
|
if (script) return { language: script, confidence: 0.95, method: 'script' };
|
|
|
|
const iberian = scoreIberian(trimmed);
|
|
if (iberian) {
|
|
const confidence = Math.min(0.95, 0.45 + (iberian.score - iberian.second) / 20);
|
|
return { language: iberian.language, confidence, method: 'iberian' };
|
|
}
|
|
|
|
const tiny = await detectWithTinyld(trimmed);
|
|
if (tiny && tiny.confidence >= 0.2) {
|
|
return { language: tiny.language, confidence: tiny.confidence, method: 'tinyld' };
|
|
}
|
|
if (opts.prior) return { language: opts.prior, confidence: 0.3, method: 'prior' };
|
|
if (tiny) return { language: tiny.language, confidence: tiny.confidence, method: 'tinyld' };
|
|
return { language: null, confidence: 0, method: 'none' };
|
|
}
|