feat(sasquatch): stream the first-pass answer + drop dead Pollinations model
- sasquatchBrain: new streamSasquatchAI — OpenAI-style SSE via OpenRouter (if the key is set) then Pollinations; falls back to the non-streaming caller only when nothing at all was delivered, so blocked-stream environments still answer. - SasquatchGuide: stream the first pass into a live bubble (raw JSON tool calls are suppressed so partial JSON never shows; prose-then-tool output is dropped), finalise the bubble in place, show a blinking cursor, and hide the generic 'thinking...' bubble while a reply streams. - Drop Pollinations 'mistral' (now a 404) leaving the live 'openai' slug.
This commit is contained in:
parent
fdf33d2c45
commit
d7bcc35d51
3 changed files with 216 additions and 13 deletions
|
|
@ -107,7 +107,10 @@ session with **no other memory of this project**. So:
|
||||||
`VITE_OPENROUTER_API_KEY` is opt-in, so production runs on the Pollinations
|
`VITE_OPENROUTER_API_KEY` is opt-in, so production runs on the Pollinations
|
||||||
fallback. `sasquatchBrain.ts` caps each request (15 s) and the whole attempt
|
fallback. `sasquatchBrain.ts` caps each request (15 s) and the whole attempt
|
||||||
(35 s); the system prompt is a real `system` message; relay results are
|
(35 s); the system prompt is a real `system` message; relay results are
|
||||||
signature-verified (`verifyEvent`) before use.
|
signature-verified (`verifyEvent`) before use. The first pass **streams**
|
||||||
|
(`streamSasquatchAI`, OpenAI-style SSE) into a live bubble — raw JSON tool
|
||||||
|
calls are suppressed so partial JSON never renders. Pollinations' `mistral`
|
||||||
|
slug is dead (404); only `openai` (gpt-oss-20b) is live.
|
||||||
- **ContextVM (CVM) server** (`cvm-server/`, Bun): standalone MCP-over-Nostr
|
- **ContextVM (CVM) server** (`cvm-server/`, Bun): standalone MCP-over-Nostr
|
||||||
endpoint for *external* agents. Runs on this machine as a **system** service
|
endpoint for *external* agents. Runs on this machine as a **system** service
|
||||||
(`/etc/systemd/system/skatespots-cvm.service`, `sudo systemctl restart
|
(`/etc/systemd/system/skatespots-cvm.service`, `sudo systemctl restart
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ import { useSasquatchPro } from '@/hooks/useSasquatchPro';
|
||||||
import { SasquatchProDialog } from '@/components/SasquatchProDialog';
|
import { SasquatchProDialog } from '@/components/SasquatchProDialog';
|
||||||
import { useSasquatchActions, type SasquatchDraft } from '@/hooks/useSasquatchActions';
|
import { useSasquatchActions, type SasquatchDraft } from '@/hooks/useSasquatchActions';
|
||||||
import { SasquatchActionCard } from '@/components/SasquatchActionCard';
|
import { SasquatchActionCard } from '@/components/SasquatchActionCard';
|
||||||
import { callSasquatchAI } from '@/lib/sasquatchBrain';
|
import { callSasquatchAI, streamSasquatchAI } from '@/lib/sasquatchBrain';
|
||||||
import { answerFromGlossary } from '@/lib/glossary';
|
import { answerFromGlossary } from '@/lib/glossary';
|
||||||
import { TripItineraryCard } from '@/components/TripItineraryCard';
|
import { TripItineraryCard } from '@/components/TripItineraryCard';
|
||||||
import { buildItinerary, type Itinerary, type PlannerSpot, type PlannerEvent, type PlannerCoach, type PlannerCrew, type WeatherLite } from '@/lib/tripPlanner';
|
import { buildItinerary, type Itinerary, type PlannerSpot, type PlannerEvent, type PlannerCoach, type PlannerCrew, type WeatherLite } from '@/lib/tripPlanner';
|
||||||
|
|
@ -85,6 +85,7 @@ function parseWeatherLite(json: string): WeatherLite | undefined {
|
||||||
// AI calls route through the shared Sasquatch brain (sasquatchBrain.ts), which
|
// AI calls route through the shared Sasquatch brain (sasquatchBrain.ts), which
|
||||||
// handles the free-model fallback list + rate-limit backoff/retry in one place.
|
// handles the free-model fallback list + rate-limit backoff/retry in one place.
|
||||||
const callAI = callSasquatchAI;
|
const callAI = callSasquatchAI;
|
||||||
|
const streamAI = streamSasquatchAI;
|
||||||
|
|
||||||
export type SasquatchContext = 'home' | 'onboarding' | 'map' | 'lab';
|
export type SasquatchContext = 'home' | 'onboarding' | 'map' | 'lab';
|
||||||
|
|
||||||
|
|
@ -958,6 +959,8 @@ export function SasquatchGuide({ message, onDismiss }: SasquatchGuideProps) {
|
||||||
let accumulated = '';
|
let accumulated = '';
|
||||||
let aiSucceeded = false;
|
let aiSucceeded = false;
|
||||||
let usedCVM = false;
|
let usedCVM = false;
|
||||||
|
let liveTs = 0; // timestamp of the streaming bubble (0 = none)
|
||||||
|
let streamSuppressed = false; // output looks like a JSON tool call — never render it
|
||||||
try {
|
try {
|
||||||
// The system prompt goes in a real `system` message rather than being
|
// The system prompt goes in a real `system` message rather than being
|
||||||
// smuggled into a user turn — the model then treats the relay/derived
|
// smuggled into a user turn — the model then treats the relay/derived
|
||||||
|
|
@ -972,8 +975,33 @@ export function SasquatchGuide({ message, onDismiss }: SasquatchGuideProps) {
|
||||||
...history.slice(0, -1).map((m) => ({ role: m.role, content: m.content })),
|
...history.slice(0, -1).map((m) => ({ role: m.role, content: m.content })),
|
||||||
];
|
];
|
||||||
|
|
||||||
// First pass — AI either answers directly or outputs a JSON tool call
|
// First pass — AI either answers directly or outputs a JSON tool call.
|
||||||
const aiText = await callAI(historyWithContext);
|
// Streamed so the answer appears as it's written; raw JSON tool calls are
|
||||||
|
// suppressed so partial JSON never shows up as text.
|
||||||
|
const aiText = await streamAI(historyWithContext, 400, {
|
||||||
|
onDelta: (cumulative) => {
|
||||||
|
if (streamSuppressed) return;
|
||||||
|
if (cumulative.trimStart().startsWith('{')) {
|
||||||
|
streamSuppressed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!liveTs) {
|
||||||
|
liveTs = Date.now();
|
||||||
|
appendMessage({ role: 'sasquatch', text: cumulative, timestamp: liveTs, isStreaming: true });
|
||||||
|
} else {
|
||||||
|
const ts = liveTs;
|
||||||
|
setChatMessages((prev) => prev.map((m) => (m.timestamp === ts ? { ...m, text: cumulative } : m)));
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// If the model wrote prose then switched to a tool call, drop that bubble
|
||||||
|
// so the real result replaces it cleanly.
|
||||||
|
if (liveTs && /\{[\s\S]*"tool"\s*:\s*"\w+"[\s\S]*\}/.test(aiText)) {
|
||||||
|
const ts = liveTs;
|
||||||
|
setChatMessages((prev) => prev.filter((m) => m.timestamp !== ts));
|
||||||
|
liveTs = 0;
|
||||||
|
}
|
||||||
|
|
||||||
// Parse tool call — look for {"tool":"...","args":{...}} anywhere in the response
|
// Parse tool call — look for {"tool":"...","args":{...}} anywhere in the response
|
||||||
let toolCall: { tool: string; args: Record<string, unknown> } | null = null;
|
let toolCall: { tool: string; args: Record<string, unknown> } | null = null;
|
||||||
|
|
@ -1197,15 +1225,32 @@ Instructions:
|
||||||
}
|
}
|
||||||
|
|
||||||
setStreamingText('');
|
setStreamingText('');
|
||||||
|
if (liveTs) {
|
||||||
|
// Finalise the streamed bubble in place instead of adding a duplicate.
|
||||||
|
const ts = liveTs;
|
||||||
|
const finalText = accumulated || aiText || "Hmm, couldn't process that. Try rephrasing!";
|
||||||
|
setChatMessages((prev) =>
|
||||||
|
prev.map((m) =>
|
||||||
|
m.timestamp === ts ? { ...m, text: finalText, isStreaming: false, isLive: usedCVM } : m,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
appendMessage({
|
appendMessage({
|
||||||
role: 'sasquatch',
|
role: 'sasquatch',
|
||||||
text: accumulated || "Hmm, couldn't process that. Try rephrasing!",
|
text: accumulated || "Hmm, couldn't process that. Try rephrasing!",
|
||||||
timestamp: Date.now(),
|
timestamp: Date.now(),
|
||||||
isLive: usedCVM,
|
isLive: usedCVM,
|
||||||
});
|
});
|
||||||
|
}
|
||||||
aiSucceeded = true;
|
aiSucceeded = true;
|
||||||
} catch (aiErr) {
|
} catch (aiErr) {
|
||||||
console.warn('[Sasquatch] AI unavailable, using static FAQ:', aiErr);
|
console.warn('[Sasquatch] AI unavailable, using static FAQ:', aiErr);
|
||||||
|
// Drop any half-streamed bubble so the fallback renders cleanly.
|
||||||
|
if (liveTs) {
|
||||||
|
const ts = liveTs;
|
||||||
|
setChatMessages((prev) => prev.filter((m) => m.timestamp !== ts));
|
||||||
|
liveTs = 0;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!aiSucceeded) {
|
if (!aiSucceeded) {
|
||||||
|
|
@ -1358,6 +1403,8 @@ Instructions:
|
||||||
}, [shouldShowPanel, isMinimized]);
|
}, [shouldShowPanel, isMinimized]);
|
||||||
|
|
||||||
const inputDisabled = isThinking;
|
const inputDisabled = isThinking;
|
||||||
|
// A reply is arriving token-by-token — hide the generic "thinking…" bubble.
|
||||||
|
const hasStreamingMsg = chatMessages.some((m) => m.isStreaming);
|
||||||
|
|
||||||
// Build a fallback generic message for pages with no specific context
|
// Build a fallback generic message for pages with no specific context
|
||||||
const activeMessage = message ?? {
|
const activeMessage = message ?? {
|
||||||
|
|
@ -1591,7 +1638,7 @@ Instructions:
|
||||||
</span>
|
</span>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
<div className="whitespace-pre-wrap"><ChatText text={msg.text} /></div>
|
<div className="whitespace-pre-wrap"><ChatText text={msg.text} />{msg.isStreaming && <span className="ml-0.5 inline-block animate-pulse text-[#ccff00]">▍</span>}</div>
|
||||||
{msg.itinerary && <TripItineraryCard itinerary={msg.itinerary} />}
|
{msg.itinerary && <TripItineraryCard itinerary={msg.itinerary} />}
|
||||||
{msg.draft && (
|
{msg.draft && (
|
||||||
<div className="mt-2">
|
<div className="mt-2">
|
||||||
|
|
@ -1608,7 +1655,7 @@ Instructions:
|
||||||
))}
|
))}
|
||||||
|
|
||||||
{/* Thinking indicator */}
|
{/* Thinking indicator */}
|
||||||
{(isThinking || streamingText) && (
|
{((isThinking && !hasStreamingMsg) || streamingText) && (
|
||||||
<div className="bg-[#ccff00]/5 text-gray-300 mr-4 text-[11px] p-2 rounded-lg border border-[#ccff00]/10">
|
<div className="bg-[#ccff00]/5 text-gray-300 mr-4 text-[11px] p-2 rounded-lg border border-[#ccff00]/10">
|
||||||
<div className="font-bold text-[9px] mb-1 opacity-60">🦧 Sasquatch</div>
|
<div className="font-bold text-[9px] mb-1 opacity-60">🦧 Sasquatch</div>
|
||||||
{streamingText ? (
|
{streamingText ? (
|
||||||
|
|
|
||||||
|
|
@ -43,7 +43,9 @@ const TOTAL_TIMEOUT_MS = 35000;
|
||||||
// Independent of OpenRouter, so if OpenRouter's shared free key is fully
|
// Independent of OpenRouter, so if OpenRouter's shared free key is fully
|
||||||
// rate-limited we still have a second brain to fall back to. No API key needed.
|
// rate-limited we still have a second brain to fall back to. No API key needed.
|
||||||
const POLLINATIONS_URL = 'https://text.pollinations.ai/openai';
|
const POLLINATIONS_URL = 'https://text.pollinations.ai/openai';
|
||||||
const POLLINATIONS_MODELS = ['openai', 'mistral'];
|
// `mistral` was removed upstream (404 on the legacy endpoint), so keep only
|
||||||
|
// live slugs. `openai` currently maps to gpt-oss-20b server-side.
|
||||||
|
const POLLINATIONS_MODELS = ['openai'];
|
||||||
|
|
||||||
// Only try the top few OpenRouter models before failing over to the keyless
|
// Only try the top few OpenRouter models before failing over to the keyless
|
||||||
// backup. Grinding through all 8 on a busy free tier is the #1 cause of a
|
// backup. Grinding through all 8 on a busy free tier is the #1 cause of a
|
||||||
|
|
@ -166,6 +168,157 @@ export async function callSasquatchAI(
|
||||||
throw new Error('All AI providers failed (OpenRouter + backup).');
|
throw new Error('All AI providers failed (OpenRouter + backup).');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─── Streaming variant ───────────────────────────────────────────────────────
|
||||||
|
// Same provider order + budget as callSasquatchAI, but reads the OpenAI-style
|
||||||
|
// SSE stream and reports the growing text via `onDelta`. Used for the first
|
||||||
|
// pass so answers appear as they're written. Falls back to the non-streaming
|
||||||
|
// caller only if nothing at all was delivered.
|
||||||
|
|
||||||
|
/** Result of one provider's streaming attempt. */
|
||||||
|
interface StreamAttempt {
|
||||||
|
text: string;
|
||||||
|
/** True once at least one delta reached `onDelta`. */
|
||||||
|
delivered: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function streamOpenAiCompatible(
|
||||||
|
url: string,
|
||||||
|
headers: Record<string, string>,
|
||||||
|
body: Record<string, unknown>,
|
||||||
|
onDelta: ((cumulative: string) => void) | undefined,
|
||||||
|
remainingMs: () => number,
|
||||||
|
): Promise<StreamAttempt> {
|
||||||
|
const left = remainingMs();
|
||||||
|
if (left <= 1000) return { text: '', delivered: false };
|
||||||
|
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json', ...headers },
|
||||||
|
body: JSON.stringify({ ...body, stream: true }),
|
||||||
|
signal: AbortSignal.timeout(Math.min(PER_REQUEST_TIMEOUT_MS, left)),
|
||||||
|
});
|
||||||
|
if (!response.ok || !response.body) return { text: '', delivered: false };
|
||||||
|
|
||||||
|
const reader = response.body.getReader();
|
||||||
|
const decoder = new TextDecoder();
|
||||||
|
let sseBuffer = '';
|
||||||
|
let text = '';
|
||||||
|
let delivered = false;
|
||||||
|
|
||||||
|
for (;;) {
|
||||||
|
const { done, value } = await reader.read();
|
||||||
|
if (done) break;
|
||||||
|
sseBuffer += decoder.decode(value, { stream: true });
|
||||||
|
|
||||||
|
let newline = sseBuffer.indexOf('\n');
|
||||||
|
while (newline !== -1) {
|
||||||
|
const line = sseBuffer.slice(0, newline).trim();
|
||||||
|
sseBuffer = sseBuffer.slice(newline + 1);
|
||||||
|
if (line.startsWith('data:')) {
|
||||||
|
const payload = line.slice(5).trim();
|
||||||
|
if (payload && payload !== '[DONE]') {
|
||||||
|
try {
|
||||||
|
const json = JSON.parse(payload);
|
||||||
|
const delta: string = json.choices?.[0]?.delta?.content ?? '';
|
||||||
|
if (delta) {
|
||||||
|
text += delta;
|
||||||
|
delivered = true;
|
||||||
|
onDelta?.(text);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Keep-alive / partial line — ignore.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
newline = sseBuffer.indexOf('\n');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { text, delivered };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function streamOpenRouter(
|
||||||
|
messages: ChatTurn[],
|
||||||
|
maxTokens: number,
|
||||||
|
onDelta: ((cumulative: string) => void) | undefined,
|
||||||
|
remainingMs: () => number,
|
||||||
|
): Promise<StreamAttempt> {
|
||||||
|
if (!OPENROUTER_API_KEY) return { text: '', delivered: false };
|
||||||
|
const models = FREE_MODELS.slice(0, MAX_OPENROUTER_MODELS);
|
||||||
|
for (const model of models) {
|
||||||
|
if (remainingMs() <= 1000) break;
|
||||||
|
try {
|
||||||
|
const attempt = await streamOpenAiCompatible(
|
||||||
|
OPENROUTER_URL,
|
||||||
|
{
|
||||||
|
Authorization: `Bearer ${OPENROUTER_API_KEY}`,
|
||||||
|
'HTTP-Referer': 'https://skate-spots.org',
|
||||||
|
'X-Title': 'SkateSpots',
|
||||||
|
},
|
||||||
|
{ model, messages, max_tokens: maxTokens },
|
||||||
|
onDelta,
|
||||||
|
remainingMs,
|
||||||
|
);
|
||||||
|
if (attempt.text || attempt.delivered) return attempt;
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(`[Sasquatch] OpenRouter stream ${model} threw:`, err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { text: '', delivered: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function streamPollinations(
|
||||||
|
messages: ChatTurn[],
|
||||||
|
maxTokens: number,
|
||||||
|
onDelta: ((cumulative: string) => void) | undefined,
|
||||||
|
remainingMs: () => number,
|
||||||
|
): Promise<StreamAttempt> {
|
||||||
|
for (const model of POLLINATIONS_MODELS) {
|
||||||
|
if (remainingMs() <= 1000) break;
|
||||||
|
try {
|
||||||
|
const attempt = await streamOpenAiCompatible(
|
||||||
|
POLLINATIONS_URL,
|
||||||
|
{},
|
||||||
|
{ model, messages, max_tokens: maxTokens },
|
||||||
|
onDelta,
|
||||||
|
remainingMs,
|
||||||
|
);
|
||||||
|
if (attempt.text || attempt.delivered) return attempt;
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(`[Sasquatch] Pollinations stream ${model} threw:`, err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { text: '', delivered: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Streaming version of {@link callSasquatchAI}. Reports the growing response
|
||||||
|
* via `onDelta` (always the *cumulative* text, so the caller can just replace
|
||||||
|
* the bubble) and resolves with the full text. Only if nothing was streamed at
|
||||||
|
* all does it fall back to the non-streaming path, so the user still gets an
|
||||||
|
* answer on environments that block streaming.
|
||||||
|
*/
|
||||||
|
export async function streamSasquatchAI(
|
||||||
|
messages: ChatTurn[],
|
||||||
|
maxTokens = 400,
|
||||||
|
handlers: { onDelta?: (cumulative: string) => void } = {},
|
||||||
|
): Promise<string> {
|
||||||
|
const deadline = Date.now() + TOTAL_TIMEOUT_MS;
|
||||||
|
const remainingMs = () => deadline - Date.now();
|
||||||
|
|
||||||
|
const primary = await streamOpenRouter(messages, maxTokens, handlers.onDelta, remainingMs);
|
||||||
|
if (primary.text) return primary.text;
|
||||||
|
|
||||||
|
const backup = await streamPollinations(messages, maxTokens, handlers.onDelta, remainingMs);
|
||||||
|
if (backup.text) return backup.text;
|
||||||
|
|
||||||
|
// A partial answer was already shown — don't restart with a fresh call.
|
||||||
|
if (primary.delivered || backup.delivered) return primary.text || backup.text;
|
||||||
|
|
||||||
|
console.warn('[Sasquatch] Streaming unavailable — falling back to non-streaming call');
|
||||||
|
return callSasquatchAI(messages, maxTokens);
|
||||||
|
}
|
||||||
|
|
||||||
/** Parse a `{"tool":"name","args":{…}}` tool call out of an AI response, if any. */
|
/** Parse a `{"tool":"name","args":{…}}` tool call out of an AI response, if any. */
|
||||||
export function parseToolCall(aiText: string): { tool: string; args: Record<string, unknown> } | null {
|
export function parseToolCall(aiText: string): { tool: string; args: Record<string, unknown> } | null {
|
||||||
const jsonMatch = aiText.match(/\{[\s\S]*"tool"\s*:\s*"(\w+)"[\s\S]*\}/);
|
const jsonMatch = aiText.match(/\{[\s\S]*"tool"\s*:\s*"(\w+)"[\s\S]*\}/);
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue