/** * On-device message language detection. * * Nothing in this module touches the network. The only model is `tinyld` * (MIT, bundled/lazy-loaded), plus local script rules and a small weighted rule * for the closely related Iberian languages that tinyld confuses on short * messages (Catalan, Galician, Basque, Spanish, Portuguese). * * No message text is logged or stored here. Callers may pass a conversation * language as a prior for short/ambiguous messages. */ import type { Language } from '@/contexts/language'; export type DetectionMethod = 'script' | 'iberian' | 'tinyld' | 'prior' | 'none'; export interface LanguageDetection { language: Language | null; confidence: number; method: DetectionMethod; } type IberianLanguage = 'ca' | 'gl' | 'eu' | 'es' | 'pt'; // ── script detection ──────────────────────────────────────────────────────── const TRADITIONAL = /[這們個說時會來對開關學國語體發現實應該還讓從後裡著過為與專業點麼]/; const SIMPLIFIED = /[这个说时会来对开关学国语体发现实应该还让从后里着过为与专业点么]/; function detectScript(text: string): Language | null { if (/[\u0900-\u097F]/.test(text)) return 'hi'; // Devanagari if (/[\u0E00-\u0E7F]/.test(text)) return 'th'; // Thai if (/[\u3040-\u30FF]/.test(text)) return 'ja'; // kana if (/[\uAC00-\uD7AF]/.test(text)) return 'ko'; // Hangul if (/[\u4E00-\u9FFF]/.test(text)) { const t = (text.match(TRADITIONAL) ?? []).length; const s = (text.match(SIMPLIFIED) ?? []).length; return t > s ? 'zh-TW' : 'zh-CN'; } if (/[\u0370-\u03FF]/.test(text)) return 'el'; // Greek return null; } // ── tinyld (broad coverage, local) ────────────────────────────────────────── const TINYLD_TO_APP: Record = { en: 'en', es: 'es', ca: 'ca', eu: 'eu', gl: 'gl', pt: 'pt', fr: 'fr', de: 'de', it: 'it', nl: 'nl', pl: 'pl', ru: 'ru', uk: 'uk', tr: 'tr', vi: 'vi', id: 'id', sv: 'sv', cs: 'cs', ro: 'ro', el: 'el', hi: 'hi', th: 'th', ja: 'ja', ko: 'ko', zh: 'zh-CN', }; let tinyldModule: Promise | null = null; async function detectWithTinyld(text: string): Promise<{ language: Language; confidence: number } | null> { try { tinyldModule ??= import('tinyld'); const tinyld = await tinyldModule; const best = tinyld.detectAll(text)[0]; if (!best) return null; const language = TINYLD_TO_APP[best.lang]; if (!language) return null; return { language, confidence: Math.max(0, Math.min(1, best.accuracy ?? 0.5)) }; } catch { return null; } } // ── Iberian close-language rule ───────────────────────────────────────────── // tinyld is unreliable for short ca/gl/eu/es/pt text. These weighted markers // are deliberately distinctive; shared words carry little or no weight. const WORDS: Record> = { ca: { 'això': 3, 'aquest': 3, 'aquesta': 3, 'aquests': 3, 'aquestes': 3, 'perquè': 3, 'també': 3, 'més': 3, 'pots': 3, 'vull': 3, 'faig': 3, 'anem': 3, 'aquí': 2, 'avui': 3, 'demà': 3, 'ahir': 3, 'sempre': 1, 'mai': 2, 'ningú': 2, 'tothom': 2, 'gràcies': 3, 'adéu': 3, 'fins': 1, 'amb': 2, 'sense': 2, 'però': 2, 'doncs': 2, 'on': 1, 'quan': 1, 'com': 0.5, 'qui': 1, 'què': 2, 'els': 1, 'les': 1, 'dels': 2, 'als': 2, 'del': 1, 'pel': 2, 'pels': 2, 'vens': 3, 'tard': 2.5, 'noi': 2, 'noia': 2, 'tio': 2, 'tia': 2, 'parlar': 2, 'tinc': 2, 'vaig': 2, 'vinc': 2, 'ara': 2, 'després': 2, 'hola': 0.5, 'al': 0.3, }, gl: { 'non': 3, 'unha': 3, 'máis': 3, 'tamén': 3, 'moi': 3, 'xente': 3, 'traballo': 3, 'aquilo': 2, 'iso': 2, 'teño': 2, 'tes': 1, 'hai': 2, 'veño': 2, 'falar': 2, 'ben': 1, 'sen': 2, 'despois': 2, 'agora': 1, 'sempre': 0.5, 'nunca': 0.5, 'coa': 2, 'polo': 2, 'pola': 2, 'aos': 1, 'ás': 1, 'con': 0.5, 'porque': 0.3, 'onde': 0.5, 'cando': 2, 'quen': 2, 'vés': 3, 'ola': 2, 'ao': 0.5, 'tarde': 0.5, 'irmán': 2, 'mañá': 2, 'aí': 1, 'alí': 1, 'aquí': 0.5, }, eu: { 'kaixo': 3, 'zara': 3, 'geroago': 3, 'etorriko': 3, 'nire': 3, 'zure': 3, 'dira': 3, 'ez': 2, 'bat': 2, 'eta': 2, 'baina': 2, 'edo': 2, 'ere': 2, 'izan': 2, 'egin': 2, 'dute': 2, 'du': 1, 'zen': 2, 'behar': 2, 'nahi': 2, 'gaur': 2, 'hemen': 2, 'hor': 1, 'han': 1, 'zer': 2, 'nor': 2, 'non': 2, 'noiz': 2, 'zergatik': 3, 'nola': 2, 'ondo': 2, 'oso': 1, 'mila': 1, 'esker': 2, 'agur': 2, 'bai': 2, 'dut': 2, 'duzu': 2, 'gara': 2, 'zarete': 3, 'ziren': 2, 'zuen': 2, 'nuen': 2, 'ari': 1, 'joan': 1, }, es: { 'vienes': 3, 'más': 2, 'también': 2, 'muy': 2, 'gente': 1, 'trabajo': 2, 'eso': 1, 'esto': 1, 'tengo': 2, 'tienes': 2, 'tiene': 2, 'hay': 2, 'voy': 2, 'vengo': 2, 'hablar': 2, 'bien': 1, 'sin': 2, 'después': 2, 'ahora': 2, 'con': 0.5, 'para': 0.3, 'porque': 0.3, 'donde': 1, 'cuando': 1, 'como': 0.3, 'quien': 1, 'hola': 0.5, 'al': 0.2, 'tarde': 0.5, 'los': 0.3, 'las': 0.3, 'del': 0.2, 'una': 0.5, 'es': 0.3, 'está': 1, 'estoy': 2, 'pero': 0.5, 'aquí': 0.5, 'mañana': 2, 'ayer': 2, 'hoy': 2, }, pt: { 'não': 3, 'uma': 2, 'mais': 2, 'também': 2, 'muito': 2, 'gente': 1, 'trabalho': 2, 'isso': 2, 'tenho': 2, 'tens': 1, 'tem': 1, 'há': 2, 'vou': 1, 'venho': 2, 'falar': 1, 'bem': 1, 'sem': 2, 'depois': 2, 'agora': 1, 'com': 0.5, 'onde': 0.5, 'quando': 2, 'quem': 2, 'vem': 2, 'você': 3, 'oi': 2, 'ao': 0.5, 'tarde': 0.5, 'olá': 2, 'obrigado': 2, 'obrigada': 2, }, }; const PHRASES: Array<{ phrase: string; language: IberianLanguage; weight: number }> = [ { phrase: 'si us plau', language: 'ca', weight: 3 }, { phrase: 'bon dia', language: 'ca', weight: 2 }, { phrase: 'a los', language: 'es', weight: 1 }, { phrase: 'a las', language: 'es', weight: 1 }, { phrase: 'con la', language: 'es', weight: 1 }, { phrase: 'por el', language: 'es', weight: 1 }, { phrase: 'por los', language: 'es', weight: 1 }, { phrase: 'por las', language: 'es', weight: 1 }, { phrase: 'com a', language: 'pt', weight: 1 }, { phrase: 'com o', language: 'pt', weight: 1 }, { phrase: 'com a', language: 'gl', weight: 0.5 }, ]; const CHAR_PATTERNS: Array<{ re: RegExp; scores: Partial> }> = [ { re: /l·l/i, scores: { ca: 3 } }, { re: /ï/i, scores: { ca: 1.5 } }, { re: /[àèò]/i, scores: { ca: 0.3 } }, { re: /ñ/i, scores: { es: 2 } }, { re: /[ãõ]/i, scores: { pt: 3 } }, { re: /(ção|ções)/i, scores: { pt: 3 } }, { re: /ción/i, scores: { es: 1, gl: 0.5 } }, { re: /ció\b/i, scores: { ca: 1.5 } }, { re: /tat\b/i, scores: { ca: 1 } }, { re: /dad\b/i, scores: { es: 1 } }, { re: /dade\b/i, scores: { pt: 0.5, gl: 0.5 } }, { re: /lh/i, scores: { pt: 1.5 } }, { re: /(tx|tz)/i, scores: { eu: 1.5 } }, { re: /agem\b/i, scores: { pt: 1 } }, { re: /axe\b/i, scores: { gl: 1.5 } }, ]; function scoreIberian(text: string): { language: IberianLanguage; score: number; second: number } | null { if (!/\p{Script=Latin}/u.test(text)) return null; const lower = ` ${text.toLowerCase()} `; const words = new Set(lower.match(/\p{L}+/gu) ?? []); const scores: Record = { ca: 0, gl: 0, eu: 0, es: 0, pt: 0 }; for (const [language, table] of Object.entries(WORDS) as Array<[IberianLanguage, Record]>) { for (const [word, weight] of Object.entries(table)) { if (words.has(word)) scores[language] += weight; } } for (const { phrase, language, weight } of PHRASES) { if (lower.includes(phrase)) scores[language] += weight; } for (const { re, scores: points } of CHAR_PATTERNS) { if (re.test(text)) { for (const [language, weight] of Object.entries(points) as Array<[IberianLanguage, number]>) { scores[language] += weight; } } } const sorted = (Object.entries(scores) as Array<[IberianLanguage, number]>) .sort((a, b) => b[1] - a[1]); if (sorted[0][1] < 4 || sorted[0][1] - sorted[1][1] < 1) return null; return { language: sorted[0][0], score: sorted[0][1], second: sorted[1][1] }; } /** * Detect the language of a message. The detector is local; `prior` is the * conversation/contact language used when the text is too short to decide. */ export async function detectLanguage( text: string, opts: { prior?: Language } = {}, ): Promise { const trimmed = text.trim(); if (!trimmed) return { language: null, confidence: 0, method: 'none' }; const script = detectScript(trimmed); if (script) return { language: script, confidence: 0.95, method: 'script' }; const iberian = scoreIberian(trimmed); if (iberian) { const confidence = Math.min(0.95, 0.45 + (iberian.score - iberian.second) / 20); return { language: iberian.language, confidence, method: 'iberian' }; } const tiny = await detectWithTinyld(trimmed); if (tiny && tiny.confidence >= 0.2) { return { language: tiny.language, confidence: tiny.confidence, method: 'tinyld' }; } if (opts.prior) return { language: opts.prior, confidence: 0.3, method: 'prior' }; if (tiny) return { language: tiny.language, confidence: tiny.confidence, method: 'tinyld' }; return { language: null, confidence: 0, method: 'none' }; }