fix(translation): repair Basque tokenizer IDs and evict stale cache

src/lib/translationModelCache.test.ts

@@ -0,0 +1,76 @@
+import { describe, expect, it, vi } from 'vitest';
+import {
+  clearStaleBasqueTokenizers,
+  isBasqueTokenizerUrl,
+  TRANSFORMERS_CACHE_NAME,
+} from './translationModelCache';
+
+const EU_TOKENIZER =
+  'https://relay.skate-spots.org/models/Xenova/opus-mt-eu-en/resolve/main/tokenizer.json';
+const EN_TOKENIZER =
+  'https://relay.skate-spots.org/models/Xenova/opus-mt-en-eu/resolve/main/tokenizer.json';
+
+describe('translationModelCache', () => {
+  it('matches both Basque tokenizer URLs only', () => {
+    expect(isBasqueTokenizerUrl(EU_TOKENIZER)).toBe(true);
+    expect(isBasqueTokenizerUrl(EN_TOKENIZER)).toBe(true);
+    expect(
+      isBasqueTokenizerUrl(
+        'https://relay.skate-spots.org/models/Xenova/opus-mt-eu-en/resolve/main/config.json',
+      ),
+    ).toBe(false);
+    expect(
+      isBasqueTokenizerUrl(
+        'https://relay.skate-spots.org/models/Xenova/opus-mt-eu-en/resolve/main/onnx/encoder_model_quantized.onnx',
+      ),
+    ).toBe(false);
+    expect(
+      isBasqueTokenizerUrl(
+        'https://relay.skate-spots.org/models/Xenova/m2m100_418M/resolve/main/tokenizer.json',
+      ),
+    ).toBe(false);
+    expect(isBasqueTokenizerUrl('not a url')).toBe(false);
+  });
+
+  it('deletes only the cached Basque tokenizer entries', async () => {
+    const deleted: string[] = [];
+    const requests = [
+      { url: EU_TOKENIZER },
+      { url: EN_TOKENIZER },
+      {
+        url: 'https://relay.skate-spots.org/models/Xenova/opus-mt-eu-en/resolve/main/config.json',
+      },
+      { url: 'https://relay.skate-spots.org/models/Xenova/m2m100_418M/resolve/main/tokenizer.json' },
+    ];
+    const open = vi.fn(async (name: string) => {
+      expect(name).toBe(TRANSFORMERS_CACHE_NAME);
+      return {
+        keys: async () => requests,
+        delete: async (request: { url: string }) => {
+          deleted.push(request.url);
+          return true;
+        },
+      };
+    });
+
+    const removed = await clearStaleBasqueTokenizers(cacheStorageStub(open));
+
+    expect(removed).toBe(2);
+    expect(deleted).toEqual([EU_TOKENIZER, EN_TOKENIZER]);
+  });
+
+  it('returns 0 when Cache Storage is unavailable', async () => {
+    await expect(clearStaleBasqueTokenizers(undefined)).resolves.toBe(0);
+  });
+
+  it('returns 0 when opening the cache fails', async () => {
+    const open = vi.fn(async () => {
+      throw new Error('blocked');
+    });
+    await expect(clearStaleBasqueTokenizers(cacheStorageStub(open))).resolves.toBe(0);
+  });
+});
+
+function cacheStorageStub(open: (name: string) => Promise<unknown>) {
+  return { open } as unknown as Pick<CacheStorage, 'open'>;
+}

src/lib/translationModelCache.ts

@@ -0,0 +1,44 @@
+/**
+ * One-time cleanup for the Basque Opus tokenizers.
+ *
+ * The first hosted conversion wrote `tokenizer.json` with sentencepiece IDs
+ * instead of the Marian model's `vocab.json` IDs. That made the model loop
+ * (`Ezin…`) because it never recognised `</s>`. The hosted file is fixed, but
+ * transformers.js caches model files in Cache Storage under the URL, so devices
+ * that already translated Basque would keep serving the broken tokenizer.
+ *
+ * We delete only the two `tokenizer.json` entries; the large ONNX weights stay
+ * cached and are reused.
+ */
+
+export const TRANSFORMERS_CACHE_NAME = 'transformers-cache';
+
+const BASQUE_TOKENIZER = /\/opus-mt-(?:eu-en|en-eu)\/resolve\/[^/]+\/tokenizer\.json$/;
+
+/** True for the Basque Opus tokenizer URLs (both directions). */
+export function isBasqueTokenizerUrl(url: string): boolean {
+  try {
+    return BASQUE_TOKENIZER.test(new URL(url).pathname);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Delete cached Basque tokenizer responses. Returns how many entries were
+ * removed. Safe when Cache Storage is unavailable (private mode, old browser).
+ */
+export async function clearStaleBasqueTokenizers(
+  cacheStorage: Pick<CacheStorage, 'open'> | undefined = globalThis.caches,
+): Promise<number> {
+  if (!cacheStorage) return 0;
+  try {
+    const cache = await cacheStorage.open(TRANSFORMERS_CACHE_NAME);
+    const requests = await cache.keys();
+    const stale = requests.filter((request) => isBasqueTokenizerUrl(request.url));
+    await Promise.all(stale.map((request) => cache.delete(request)));
+    return stale.length;
+  } catch {
+    return 0;
+  }
+}

src/workers/translate.worker.ts

@@ -11,6 +11,7 @@
  */
 import { env, pipeline } from '@xenova/transformers';
 import { createGuardedFetch } from '@/lib/translationGuard';
+import { clearStaleBasqueTokenizers } from '@/lib/translationModelCache';
 
 const MODEL_ORIGIN = 'https://relay.skate-spots.org';
 const MODEL_PREFIX = '/models/';
@@ -63,6 +64,14 @@ const ctx = self as unknown as {
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const translators = new Map<string, Promise<any>>();
+let basqueCacheChecked = false;
+
+/** Evict the pre-fix Basque tokenizer from browser cache exactly once. */
+async function ensureBasqueCache(): Promise<void> {
+  if (basqueCacheChecked) return;
+  basqueCacheChecked = true;
+  await clearStaleBasqueTokenizers();
+}
 
 /** Collapse UI variants onto the model's language tokens. */
 function baseLanguage(code: string): string {
@@ -128,6 +137,7 @@ async function translate(
 
   // Basque is not in M2M100: use Opus eu↔en and pivot through English.
   if (s === 'eu' || t === 'eu') {
+    await ensureBasqueCache();
     if (s === 'eu' && t === 'en') return run(EU_EN, text, requestId);
     if (s === 'en' && t === 'eu') return run(EN_EU, text, requestId);
     if (s === 'eu') {