From 8a5ac47a195bf17750166db21e4302f42b974bc3 Mon Sep 17 00:00:00 2001 From: SkateSpots Date: Tue, 27 Oct 2026 16:32:29 +0700 Subject: [PATCH] fix(translation): repair Basque tokenizer IDs and evict stale cache --- src/lib/translationModelCache.test.ts | 76 +++++++++++++++++++++++++++ src/lib/translationModelCache.ts | 44 ++++++++++++++++ src/workers/translate.worker.ts | 10 ++++ 3 files changed, 130 insertions(+) create mode 100644 src/lib/translationModelCache.test.ts create mode 100644 src/lib/translationModelCache.ts diff --git a/src/lib/translationModelCache.test.ts b/src/lib/translationModelCache.test.ts new file mode 100644 index 0000000..5dd46ad --- /dev/null +++ b/src/lib/translationModelCache.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + clearStaleBasqueTokenizers, + isBasqueTokenizerUrl, + TRANSFORMERS_CACHE_NAME, +} from './translationModelCache'; + +const EU_TOKENIZER = + 'https://relay.skate-spots.org/models/Xenova/opus-mt-eu-en/resolve/main/tokenizer.json'; +const EN_TOKENIZER = + 'https://relay.skate-spots.org/models/Xenova/opus-mt-en-eu/resolve/main/tokenizer.json'; + +describe('translationModelCache', () => { + it('matches both Basque tokenizer URLs only', () => { + expect(isBasqueTokenizerUrl(EU_TOKENIZER)).toBe(true); + expect(isBasqueTokenizerUrl(EN_TOKENIZER)).toBe(true); + expect( + isBasqueTokenizerUrl( + 'https://relay.skate-spots.org/models/Xenova/opus-mt-eu-en/resolve/main/config.json', + ), + ).toBe(false); + expect( + isBasqueTokenizerUrl( + 'https://relay.skate-spots.org/models/Xenova/opus-mt-eu-en/resolve/main/onnx/encoder_model_quantized.onnx', + ), + ).toBe(false); + expect( + isBasqueTokenizerUrl( + 'https://relay.skate-spots.org/models/Xenova/m2m100_418M/resolve/main/tokenizer.json', + ), + ).toBe(false); + expect(isBasqueTokenizerUrl('not a url')).toBe(false); + }); + + it('deletes only the cached Basque tokenizer entries', async () => { + const deleted: string[] = []; + const requests = [ + { url: EU_TOKENIZER }, + { url: EN_TOKENIZER }, + { + url: 'https://relay.skate-spots.org/models/Xenova/opus-mt-eu-en/resolve/main/config.json', + }, + { url: 'https://relay.skate-spots.org/models/Xenova/m2m100_418M/resolve/main/tokenizer.json' }, + ]; + const open = vi.fn(async (name: string) => { + expect(name).toBe(TRANSFORMERS_CACHE_NAME); + return { + keys: async () => requests, + delete: async (request: { url: string }) => { + deleted.push(request.url); + return true; + }, + }; + }); + + const removed = await clearStaleBasqueTokenizers(cacheStorageStub(open)); + + expect(removed).toBe(2); + expect(deleted).toEqual([EU_TOKENIZER, EN_TOKENIZER]); + }); + + it('returns 0 when Cache Storage is unavailable', async () => { + await expect(clearStaleBasqueTokenizers(undefined)).resolves.toBe(0); + }); + + it('returns 0 when opening the cache fails', async () => { + const open = vi.fn(async () => { + throw new Error('blocked'); + }); + await expect(clearStaleBasqueTokenizers(cacheStorageStub(open))).resolves.toBe(0); + }); +}); + +function cacheStorageStub(open: (name: string) => Promise) { + return { open } as unknown as Pick; +} diff --git a/src/lib/translationModelCache.ts b/src/lib/translationModelCache.ts new file mode 100644 index 0000000..4f7cd7a --- /dev/null +++ b/src/lib/translationModelCache.ts @@ -0,0 +1,44 @@ +/** + * One-time cleanup for the Basque Opus tokenizers. + * + * The first hosted conversion wrote `tokenizer.json` with sentencepiece IDs + * instead of the Marian model's `vocab.json` IDs. That made the model loop + * (`Ezin…`) because it never recognised ``. The hosted file is fixed, but + * transformers.js caches model files in Cache Storage under the URL, so devices + * that already translated Basque would keep serving the broken tokenizer. + * + * We delete only the two `tokenizer.json` entries; the large ONNX weights stay + * cached and are reused. + */ + +export const TRANSFORMERS_CACHE_NAME = 'transformers-cache'; + +const BASQUE_TOKENIZER = /\/opus-mt-(?:eu-en|en-eu)\/resolve\/[^/]+\/tokenizer\.json$/; + +/** True for the Basque Opus tokenizer URLs (both directions). */ +export function isBasqueTokenizerUrl(url: string): boolean { + try { + return BASQUE_TOKENIZER.test(new URL(url).pathname); + } catch { + return false; + } +} + +/** + * Delete cached Basque tokenizer responses. Returns how many entries were + * removed. Safe when Cache Storage is unavailable (private mode, old browser). + */ +export async function clearStaleBasqueTokenizers( + cacheStorage: Pick | undefined = globalThis.caches, +): Promise { + if (!cacheStorage) return 0; + try { + const cache = await cacheStorage.open(TRANSFORMERS_CACHE_NAME); + const requests = await cache.keys(); + const stale = requests.filter((request) => isBasqueTokenizerUrl(request.url)); + await Promise.all(stale.map((request) => cache.delete(request))); + return stale.length; + } catch { + return 0; + } +} diff --git a/src/workers/translate.worker.ts b/src/workers/translate.worker.ts index 234451b..72b0916 100644 --- a/src/workers/translate.worker.ts +++ b/src/workers/translate.worker.ts @@ -11,6 +11,7 @@ */ import { env, pipeline } from '@xenova/transformers'; import { createGuardedFetch } from '@/lib/translationGuard'; +import { clearStaleBasqueTokenizers } from '@/lib/translationModelCache'; const MODEL_ORIGIN = 'https://relay.skate-spots.org'; const MODEL_PREFIX = '/models/'; @@ -63,6 +64,14 @@ const ctx = self as unknown as { // eslint-disable-next-line @typescript-eslint/no-explicit-any const translators = new Map>(); +let basqueCacheChecked = false; + +/** Evict the pre-fix Basque tokenizer from browser cache exactly once. */ +async function ensureBasqueCache(): Promise { + if (basqueCacheChecked) return; + basqueCacheChecked = true; + await clearStaleBasqueTokenizers(); +} /** Collapse UI variants onto the model's language tokens. */ function baseLanguage(code: string): string { @@ -128,6 +137,7 @@ async function translate( // Basque is not in M2M100: use Opus eu↔en and pivot through English. if (s === 'eu' || t === 'eu') { + await ensureBasqueCache(); if (s === 'eu' && t === 'en') return run(EU_EN, text, requestId); if (s === 'en' && t === 'eu') return run(EN_EU, text, requestId); if (s === 'eu') {