From d7bcc35d512a2e67e21e41b0afa7146b0c193244 Mon Sep 17 00:00:00 2001 From: SkateSpots Date: Sun, 27 Sep 2026 11:02:56 +0700 Subject: [PATCH] feat(sasquatch): stream the first-pass answer + drop dead Pollinations model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - sasquatchBrain: new streamSasquatchAI — OpenAI-style SSE via OpenRouter (if the key is set) then Pollinations; falls back to the non-streaming caller only when nothing at all was delivered, so blocked-stream environments still answer. - SasquatchGuide: stream the first pass into a live bubble (raw JSON tool calls are suppressed so partial JSON never shows; prose-then-tool output is dropped), finalise the bubble in place, show a blinking cursor, and hide the generic 'thinking...' bubble while a reply streams. - Drop Pollinations 'mistral' (now a 404) leaving the live 'openai' slug. --- AGENTS.md | 5 +- src/components/SasquatchGuide.tsx | 69 ++++++++++--- src/lib/sasquatchBrain.ts | 155 +++++++++++++++++++++++++++++- 3 files changed, 216 insertions(+), 13 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 45eaf86..3b16a4f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,7 +107,10 @@ session with **no other memory of this project**. So: `VITE_OPENROUTER_API_KEY` is opt-in, so production runs on the Pollinations fallback. `sasquatchBrain.ts` caps each request (15 s) and the whole attempt (35 s); the system prompt is a real `system` message; relay results are - signature-verified (`verifyEvent`) before use. + signature-verified (`verifyEvent`) before use. The first pass **streams** + (`streamSasquatchAI`, OpenAI-style SSE) into a live bubble — raw JSON tool + calls are suppressed so partial JSON never renders. Pollinations' `mistral` + slug is dead (404); only `openai` (gpt-oss-20b) is live. - **ContextVM (CVM) server** (`cvm-server/`, Bun): standalone MCP-over-Nostr endpoint for *external* agents. Runs on this machine as a **system** service (`/etc/systemd/system/skatespots-cvm.service`, `sudo systemctl restart diff --git a/src/components/SasquatchGuide.tsx b/src/components/SasquatchGuide.tsx index a5fdc0b..b241206 100644 --- a/src/components/SasquatchGuide.tsx +++ b/src/components/SasquatchGuide.tsx @@ -20,7 +20,7 @@ import { useSasquatchPro } from '@/hooks/useSasquatchPro'; import { SasquatchProDialog } from '@/components/SasquatchProDialog'; import { useSasquatchActions, type SasquatchDraft } from '@/hooks/useSasquatchActions'; import { SasquatchActionCard } from '@/components/SasquatchActionCard'; -import { callSasquatchAI } from '@/lib/sasquatchBrain'; +import { callSasquatchAI, streamSasquatchAI } from '@/lib/sasquatchBrain'; import { answerFromGlossary } from '@/lib/glossary'; import { TripItineraryCard } from '@/components/TripItineraryCard'; import { buildItinerary, type Itinerary, type PlannerSpot, type PlannerEvent, type PlannerCoach, type PlannerCrew, type WeatherLite } from '@/lib/tripPlanner'; @@ -85,6 +85,7 @@ function parseWeatherLite(json: string): WeatherLite | undefined { // AI calls route through the shared Sasquatch brain (sasquatchBrain.ts), which // handles the free-model fallback list + rate-limit backoff/retry in one place. const callAI = callSasquatchAI; +const streamAI = streamSasquatchAI; export type SasquatchContext = 'home' | 'onboarding' | 'map' | 'lab'; @@ -958,6 +959,8 @@ export function SasquatchGuide({ message, onDismiss }: SasquatchGuideProps) { let accumulated = ''; let aiSucceeded = false; let usedCVM = false; + let liveTs = 0; // timestamp of the streaming bubble (0 = none) + let streamSuppressed = false; // output looks like a JSON tool call — never render it try { // The system prompt goes in a real `system` message rather than being // smuggled into a user turn — the model then treats the relay/derived @@ -972,8 +975,33 @@ export function SasquatchGuide({ message, onDismiss }: SasquatchGuideProps) { ...history.slice(0, -1).map((m) => ({ role: m.role, content: m.content })), ]; - // First pass — AI either answers directly or outputs a JSON tool call - const aiText = await callAI(historyWithContext); + // First pass — AI either answers directly or outputs a JSON tool call. + // Streamed so the answer appears as it's written; raw JSON tool calls are + // suppressed so partial JSON never shows up as text. + const aiText = await streamAI(historyWithContext, 400, { + onDelta: (cumulative) => { + if (streamSuppressed) return; + if (cumulative.trimStart().startsWith('{')) { + streamSuppressed = true; + return; + } + if (!liveTs) { + liveTs = Date.now(); + appendMessage({ role: 'sasquatch', text: cumulative, timestamp: liveTs, isStreaming: true }); + } else { + const ts = liveTs; + setChatMessages((prev) => prev.map((m) => (m.timestamp === ts ? { ...m, text: cumulative } : m))); + } + }, + }); + + // If the model wrote prose then switched to a tool call, drop that bubble + // so the real result replaces it cleanly. + if (liveTs && /\{[\s\S]*"tool"\s*:\s*"\w+"[\s\S]*\}/.test(aiText)) { + const ts = liveTs; + setChatMessages((prev) => prev.filter((m) => m.timestamp !== ts)); + liveTs = 0; + } // Parse tool call — look for {"tool":"...","args":{...}} anywhere in the response let toolCall: { tool: string; args: Record } | null = null; @@ -1197,15 +1225,32 @@ Instructions: } setStreamingText(''); - appendMessage({ - role: 'sasquatch', - text: accumulated || "Hmm, couldn't process that. Try rephrasing!", - timestamp: Date.now(), - isLive: usedCVM, - }); + if (liveTs) { + // Finalise the streamed bubble in place instead of adding a duplicate. + const ts = liveTs; + const finalText = accumulated || aiText || "Hmm, couldn't process that. Try rephrasing!"; + setChatMessages((prev) => + prev.map((m) => + m.timestamp === ts ? { ...m, text: finalText, isStreaming: false, isLive: usedCVM } : m, + ), + ); + } else { + appendMessage({ + role: 'sasquatch', + text: accumulated || "Hmm, couldn't process that. Try rephrasing!", + timestamp: Date.now(), + isLive: usedCVM, + }); + } aiSucceeded = true; } catch (aiErr) { console.warn('[Sasquatch] AI unavailable, using static FAQ:', aiErr); + // Drop any half-streamed bubble so the fallback renders cleanly. + if (liveTs) { + const ts = liveTs; + setChatMessages((prev) => prev.filter((m) => m.timestamp !== ts)); + liveTs = 0; + } } if (!aiSucceeded) { @@ -1358,6 +1403,8 @@ Instructions: }, [shouldShowPanel, isMinimized]); const inputDisabled = isThinking; + // A reply is arriving token-by-token — hide the generic "thinking…" bubble. + const hasStreamingMsg = chatMessages.some((m) => m.isStreaming); // Build a fallback generic message for pages with no specific context const activeMessage = message ?? { @@ -1591,7 +1638,7 @@ Instructions: )} -
+
{msg.isStreaming && ▍}
{msg.itinerary && } {msg.draft && (
@@ -1608,7 +1655,7 @@ Instructions: ))} {/* Thinking indicator */} - {(isThinking || streamingText) && ( + {((isThinking && !hasStreamingMsg) || streamingText) && (
🦧 Sasquatch
{streamingText ? ( diff --git a/src/lib/sasquatchBrain.ts b/src/lib/sasquatchBrain.ts index 0bc6ed1..b27398e 100644 --- a/src/lib/sasquatchBrain.ts +++ b/src/lib/sasquatchBrain.ts @@ -43,7 +43,9 @@ const TOTAL_TIMEOUT_MS = 35000; // Independent of OpenRouter, so if OpenRouter's shared free key is fully // rate-limited we still have a second brain to fall back to. No API key needed. const POLLINATIONS_URL = 'https://text.pollinations.ai/openai'; -const POLLINATIONS_MODELS = ['openai', 'mistral']; +// `mistral` was removed upstream (404 on the legacy endpoint), so keep only +// live slugs. `openai` currently maps to gpt-oss-20b server-side. +const POLLINATIONS_MODELS = ['openai']; // Only try the top few OpenRouter models before failing over to the keyless // backup. Grinding through all 8 on a busy free tier is the #1 cause of a @@ -166,6 +168,157 @@ export async function callSasquatchAI( throw new Error('All AI providers failed (OpenRouter + backup).'); } +// ─── Streaming variant ─────────────────────────────────────────────────────── +// Same provider order + budget as callSasquatchAI, but reads the OpenAI-style +// SSE stream and reports the growing text via `onDelta`. Used for the first +// pass so answers appear as they're written. Falls back to the non-streaming +// caller only if nothing at all was delivered. + +/** Result of one provider's streaming attempt. */ +interface StreamAttempt { + text: string; + /** True once at least one delta reached `onDelta`. */ + delivered: boolean; +} + +async function streamOpenAiCompatible( + url: string, + headers: Record, + body: Record, + onDelta: ((cumulative: string) => void) | undefined, + remainingMs: () => number, +): Promise { + const left = remainingMs(); + if (left <= 1000) return { text: '', delivered: false }; + + const response = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...headers }, + body: JSON.stringify({ ...body, stream: true }), + signal: AbortSignal.timeout(Math.min(PER_REQUEST_TIMEOUT_MS, left)), + }); + if (!response.ok || !response.body) return { text: '', delivered: false }; + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let sseBuffer = ''; + let text = ''; + let delivered = false; + + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + sseBuffer += decoder.decode(value, { stream: true }); + + let newline = sseBuffer.indexOf('\n'); + while (newline !== -1) { + const line = sseBuffer.slice(0, newline).trim(); + sseBuffer = sseBuffer.slice(newline + 1); + if (line.startsWith('data:')) { + const payload = line.slice(5).trim(); + if (payload && payload !== '[DONE]') { + try { + const json = JSON.parse(payload); + const delta: string = json.choices?.[0]?.delta?.content ?? ''; + if (delta) { + text += delta; + delivered = true; + onDelta?.(text); + } + } catch { + // Keep-alive / partial line — ignore. + } + } + } + newline = sseBuffer.indexOf('\n'); + } + } + + return { text, delivered }; +} + +async function streamOpenRouter( + messages: ChatTurn[], + maxTokens: number, + onDelta: ((cumulative: string) => void) | undefined, + remainingMs: () => number, +): Promise { + if (!OPENROUTER_API_KEY) return { text: '', delivered: false }; + const models = FREE_MODELS.slice(0, MAX_OPENROUTER_MODELS); + for (const model of models) { + if (remainingMs() <= 1000) break; + try { + const attempt = await streamOpenAiCompatible( + OPENROUTER_URL, + { + Authorization: `Bearer ${OPENROUTER_API_KEY}`, + 'HTTP-Referer': 'https://skate-spots.org', + 'X-Title': 'SkateSpots', + }, + { model, messages, max_tokens: maxTokens }, + onDelta, + remainingMs, + ); + if (attempt.text || attempt.delivered) return attempt; + } catch (err) { + console.warn(`[Sasquatch] OpenRouter stream ${model} threw:`, err); + } + } + return { text: '', delivered: false }; +} + +async function streamPollinations( + messages: ChatTurn[], + maxTokens: number, + onDelta: ((cumulative: string) => void) | undefined, + remainingMs: () => number, +): Promise { + for (const model of POLLINATIONS_MODELS) { + if (remainingMs() <= 1000) break; + try { + const attempt = await streamOpenAiCompatible( + POLLINATIONS_URL, + {}, + { model, messages, max_tokens: maxTokens }, + onDelta, + remainingMs, + ); + if (attempt.text || attempt.delivered) return attempt; + } catch (err) { + console.warn(`[Sasquatch] Pollinations stream ${model} threw:`, err); + } + } + return { text: '', delivered: false }; +} + +/** + * Streaming version of {@link callSasquatchAI}. Reports the growing response + * via `onDelta` (always the *cumulative* text, so the caller can just replace + * the bubble) and resolves with the full text. Only if nothing was streamed at + * all does it fall back to the non-streaming path, so the user still gets an + * answer on environments that block streaming. + */ +export async function streamSasquatchAI( + messages: ChatTurn[], + maxTokens = 400, + handlers: { onDelta?: (cumulative: string) => void } = {}, +): Promise { + const deadline = Date.now() + TOTAL_TIMEOUT_MS; + const remainingMs = () => deadline - Date.now(); + + const primary = await streamOpenRouter(messages, maxTokens, handlers.onDelta, remainingMs); + if (primary.text) return primary.text; + + const backup = await streamPollinations(messages, maxTokens, handlers.onDelta, remainingMs); + if (backup.text) return backup.text; + + // A partial answer was already shown — don't restart with a fresh call. + if (primary.delivered || backup.delivered) return primary.text || backup.text; + + console.warn('[Sasquatch] Streaming unavailable — falling back to non-streaming call'); + return callSasquatchAI(messages, maxTokens); +} + /** Parse a `{"tool":"name","args":{…}}` tool call out of an AI response, if any. */ export function parseToolCall(aiText: string): { tool: string; args: Record } | null { const jsonMatch = aiText.match(/\{[\s\S]*"tool"\s*:\s*"(\w+)"[\s\S]*\}/);