feat(sasquatch): stream the first-pass answer + drop dead Pollinations model
- sasquatchBrain: new streamSasquatchAI — OpenAI-style SSE via OpenRouter (if the key is set) then Pollinations; falls back to the non-streaming caller only when nothing at all was delivered, so blocked-stream environments still answer. - SasquatchGuide: stream the first pass into a live bubble (raw JSON tool calls are suppressed so partial JSON never shows; prose-then-tool output is dropped), finalise the bubble in place, show a blinking cursor, and hide the generic 'thinking...' bubble while a reply streams. - Drop Pollinations 'mistral' (now a 404) leaving the live 'openai' slug.
This commit is contained in:
parent
fdf33d2c45
commit
d7bcc35d51
3 changed files with 216 additions and 13 deletions
|
|
@ -107,7 +107,10 @@ session with **no other memory of this project**. So:
|
|||
`VITE_OPENROUTER_API_KEY` is opt-in, so production runs on the Pollinations
|
||||
fallback. `sasquatchBrain.ts` caps each request (15 s) and the whole attempt
|
||||
(35 s); the system prompt is a real `system` message; relay results are
|
||||
signature-verified (`verifyEvent`) before use.
|
||||
signature-verified (`verifyEvent`) before use. The first pass **streams**
|
||||
(`streamSasquatchAI`, OpenAI-style SSE) into a live bubble — raw JSON tool
|
||||
calls are suppressed so partial JSON never renders. Pollinations' `mistral`
|
||||
slug is dead (404); only `openai` (gpt-oss-20b) is live.
|
||||
- **ContextVM (CVM) server** (`cvm-server/`, Bun): standalone MCP-over-Nostr
|
||||
endpoint for *external* agents. Runs on this machine as a **system** service
|
||||
(`/etc/systemd/system/skatespots-cvm.service`, `sudo systemctl restart
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import { useSasquatchPro } from '@/hooks/useSasquatchPro';
|
|||
import { SasquatchProDialog } from '@/components/SasquatchProDialog';
|
||||
import { useSasquatchActions, type SasquatchDraft } from '@/hooks/useSasquatchActions';
|
||||
import { SasquatchActionCard } from '@/components/SasquatchActionCard';
|
||||
import { callSasquatchAI } from '@/lib/sasquatchBrain';
|
||||
import { callSasquatchAI, streamSasquatchAI } from '@/lib/sasquatchBrain';
|
||||
import { answerFromGlossary } from '@/lib/glossary';
|
||||
import { TripItineraryCard } from '@/components/TripItineraryCard';
|
||||
import { buildItinerary, type Itinerary, type PlannerSpot, type PlannerEvent, type PlannerCoach, type PlannerCrew, type WeatherLite } from '@/lib/tripPlanner';
|
||||
|
|
@ -85,6 +85,7 @@ function parseWeatherLite(json: string): WeatherLite | undefined {
|
|||
// AI calls route through the shared Sasquatch brain (sasquatchBrain.ts), which
|
||||
// handles the free-model fallback list + rate-limit backoff/retry in one place.
|
||||
const callAI = callSasquatchAI;
|
||||
const streamAI = streamSasquatchAI;
|
||||
|
||||
export type SasquatchContext = 'home' | 'onboarding' | 'map' | 'lab';
|
||||
|
||||
|
|
@ -958,6 +959,8 @@ export function SasquatchGuide({ message, onDismiss }: SasquatchGuideProps) {
|
|||
let accumulated = '';
|
||||
let aiSucceeded = false;
|
||||
let usedCVM = false;
|
||||
let liveTs = 0; // timestamp of the streaming bubble (0 = none)
|
||||
let streamSuppressed = false; // output looks like a JSON tool call — never render it
|
||||
try {
|
||||
// The system prompt goes in a real `system` message rather than being
|
||||
// smuggled into a user turn — the model then treats the relay/derived
|
||||
|
|
@ -972,8 +975,33 @@ export function SasquatchGuide({ message, onDismiss }: SasquatchGuideProps) {
|
|||
...history.slice(0, -1).map((m) => ({ role: m.role, content: m.content })),
|
||||
];
|
||||
|
||||
// First pass — AI either answers directly or outputs a JSON tool call
|
||||
const aiText = await callAI(historyWithContext);
|
||||
// First pass — AI either answers directly or outputs a JSON tool call.
|
||||
// Streamed so the answer appears as it's written; raw JSON tool calls are
|
||||
// suppressed so partial JSON never shows up as text.
|
||||
const aiText = await streamAI(historyWithContext, 400, {
|
||||
onDelta: (cumulative) => {
|
||||
if (streamSuppressed) return;
|
||||
if (cumulative.trimStart().startsWith('{')) {
|
||||
streamSuppressed = true;
|
||||
return;
|
||||
}
|
||||
if (!liveTs) {
|
||||
liveTs = Date.now();
|
||||
appendMessage({ role: 'sasquatch', text: cumulative, timestamp: liveTs, isStreaming: true });
|
||||
} else {
|
||||
const ts = liveTs;
|
||||
setChatMessages((prev) => prev.map((m) => (m.timestamp === ts ? { ...m, text: cumulative } : m)));
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// If the model wrote prose then switched to a tool call, drop that bubble
|
||||
// so the real result replaces it cleanly.
|
||||
if (liveTs && /\{[\s\S]*"tool"\s*:\s*"\w+"[\s\S]*\}/.test(aiText)) {
|
||||
const ts = liveTs;
|
||||
setChatMessages((prev) => prev.filter((m) => m.timestamp !== ts));
|
||||
liveTs = 0;
|
||||
}
|
||||
|
||||
// Parse tool call — look for {"tool":"...","args":{...}} anywhere in the response
|
||||
let toolCall: { tool: string; args: Record<string, unknown> } | null = null;
|
||||
|
|
@ -1197,15 +1225,32 @@ Instructions:
|
|||
}
|
||||
|
||||
setStreamingText('');
|
||||
appendMessage({
|
||||
role: 'sasquatch',
|
||||
text: accumulated || "Hmm, couldn't process that. Try rephrasing!",
|
||||
timestamp: Date.now(),
|
||||
isLive: usedCVM,
|
||||
});
|
||||
if (liveTs) {
|
||||
// Finalise the streamed bubble in place instead of adding a duplicate.
|
||||
const ts = liveTs;
|
||||
const finalText = accumulated || aiText || "Hmm, couldn't process that. Try rephrasing!";
|
||||
setChatMessages((prev) =>
|
||||
prev.map((m) =>
|
||||
m.timestamp === ts ? { ...m, text: finalText, isStreaming: false, isLive: usedCVM } : m,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
appendMessage({
|
||||
role: 'sasquatch',
|
||||
text: accumulated || "Hmm, couldn't process that. Try rephrasing!",
|
||||
timestamp: Date.now(),
|
||||
isLive: usedCVM,
|
||||
});
|
||||
}
|
||||
aiSucceeded = true;
|
||||
} catch (aiErr) {
|
||||
console.warn('[Sasquatch] AI unavailable, using static FAQ:', aiErr);
|
||||
// Drop any half-streamed bubble so the fallback renders cleanly.
|
||||
if (liveTs) {
|
||||
const ts = liveTs;
|
||||
setChatMessages((prev) => prev.filter((m) => m.timestamp !== ts));
|
||||
liveTs = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (!aiSucceeded) {
|
||||
|
|
@ -1358,6 +1403,8 @@ Instructions:
|
|||
}, [shouldShowPanel, isMinimized]);
|
||||
|
||||
const inputDisabled = isThinking;
|
||||
// A reply is arriving token-by-token — hide the generic "thinking…" bubble.
|
||||
const hasStreamingMsg = chatMessages.some((m) => m.isStreaming);
|
||||
|
||||
// Build a fallback generic message for pages with no specific context
|
||||
const activeMessage = message ?? {
|
||||
|
|
@ -1591,7 +1638,7 @@ Instructions:
|
|||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="whitespace-pre-wrap"><ChatText text={msg.text} /></div>
|
||||
<div className="whitespace-pre-wrap"><ChatText text={msg.text} />{msg.isStreaming && <span className="ml-0.5 inline-block animate-pulse text-[#ccff00]">▍</span>}</div>
|
||||
{msg.itinerary && <TripItineraryCard itinerary={msg.itinerary} />}
|
||||
{msg.draft && (
|
||||
<div className="mt-2">
|
||||
|
|
@ -1608,7 +1655,7 @@ Instructions:
|
|||
))}
|
||||
|
||||
{/* Thinking indicator */}
|
||||
{(isThinking || streamingText) && (
|
||||
{((isThinking && !hasStreamingMsg) || streamingText) && (
|
||||
<div className="bg-[#ccff00]/5 text-gray-300 mr-4 text-[11px] p-2 rounded-lg border border-[#ccff00]/10">
|
||||
<div className="font-bold text-[9px] mb-1 opacity-60">🦧 Sasquatch</div>
|
||||
{streamingText ? (
|
||||
|
|
|
|||
|
|
@ -43,7 +43,9 @@ const TOTAL_TIMEOUT_MS = 35000;
|
|||
// Independent of OpenRouter, so if OpenRouter's shared free key is fully
|
||||
// rate-limited we still have a second brain to fall back to. No API key needed.
|
||||
const POLLINATIONS_URL = 'https://text.pollinations.ai/openai';
|
||||
const POLLINATIONS_MODELS = ['openai', 'mistral'];
|
||||
// `mistral` was removed upstream (404 on the legacy endpoint), so keep only
|
||||
// live slugs. `openai` currently maps to gpt-oss-20b server-side.
|
||||
const POLLINATIONS_MODELS = ['openai'];
|
||||
|
||||
// Only try the top few OpenRouter models before failing over to the keyless
|
||||
// backup. Grinding through all 8 on a busy free tier is the #1 cause of a
|
||||
|
|
@ -166,6 +168,157 @@ export async function callSasquatchAI(
|
|||
throw new Error('All AI providers failed (OpenRouter + backup).');
|
||||
}
|
||||
|
||||
// ─── Streaming variant ───────────────────────────────────────────────────────
|
||||
// Same provider order + budget as callSasquatchAI, but reads the OpenAI-style
|
||||
// SSE stream and reports the growing text via `onDelta`. Used for the first
|
||||
// pass so answers appear as they're written. Falls back to the non-streaming
|
||||
// caller only if nothing at all was delivered.
|
||||
|
||||
/** Result of one provider's streaming attempt. */
|
||||
interface StreamAttempt {
|
||||
text: string;
|
||||
/** True once at least one delta reached `onDelta`. */
|
||||
delivered: boolean;
|
||||
}
|
||||
|
||||
async function streamOpenAiCompatible(
|
||||
url: string,
|
||||
headers: Record<string, string>,
|
||||
body: Record<string, unknown>,
|
||||
onDelta: ((cumulative: string) => void) | undefined,
|
||||
remainingMs: () => number,
|
||||
): Promise<StreamAttempt> {
|
||||
const left = remainingMs();
|
||||
if (left <= 1000) return { text: '', delivered: false };
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', ...headers },
|
||||
body: JSON.stringify({ ...body, stream: true }),
|
||||
signal: AbortSignal.timeout(Math.min(PER_REQUEST_TIMEOUT_MS, left)),
|
||||
});
|
||||
if (!response.ok || !response.body) return { text: '', delivered: false };
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let sseBuffer = '';
|
||||
let text = '';
|
||||
let delivered = false;
|
||||
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
sseBuffer += decoder.decode(value, { stream: true });
|
||||
|
||||
let newline = sseBuffer.indexOf('\n');
|
||||
while (newline !== -1) {
|
||||
const line = sseBuffer.slice(0, newline).trim();
|
||||
sseBuffer = sseBuffer.slice(newline + 1);
|
||||
if (line.startsWith('data:')) {
|
||||
const payload = line.slice(5).trim();
|
||||
if (payload && payload !== '[DONE]') {
|
||||
try {
|
||||
const json = JSON.parse(payload);
|
||||
const delta: string = json.choices?.[0]?.delta?.content ?? '';
|
||||
if (delta) {
|
||||
text += delta;
|
||||
delivered = true;
|
||||
onDelta?.(text);
|
||||
}
|
||||
} catch {
|
||||
// Keep-alive / partial line — ignore.
|
||||
}
|
||||
}
|
||||
}
|
||||
newline = sseBuffer.indexOf('\n');
|
||||
}
|
||||
}
|
||||
|
||||
return { text, delivered };
|
||||
}
|
||||
|
||||
async function streamOpenRouter(
|
||||
messages: ChatTurn[],
|
||||
maxTokens: number,
|
||||
onDelta: ((cumulative: string) => void) | undefined,
|
||||
remainingMs: () => number,
|
||||
): Promise<StreamAttempt> {
|
||||
if (!OPENROUTER_API_KEY) return { text: '', delivered: false };
|
||||
const models = FREE_MODELS.slice(0, MAX_OPENROUTER_MODELS);
|
||||
for (const model of models) {
|
||||
if (remainingMs() <= 1000) break;
|
||||
try {
|
||||
const attempt = await streamOpenAiCompatible(
|
||||
OPENROUTER_URL,
|
||||
{
|
||||
Authorization: `Bearer ${OPENROUTER_API_KEY}`,
|
||||
'HTTP-Referer': 'https://skate-spots.org',
|
||||
'X-Title': 'SkateSpots',
|
||||
},
|
||||
{ model, messages, max_tokens: maxTokens },
|
||||
onDelta,
|
||||
remainingMs,
|
||||
);
|
||||
if (attempt.text || attempt.delivered) return attempt;
|
||||
} catch (err) {
|
||||
console.warn(`[Sasquatch] OpenRouter stream ${model} threw:`, err);
|
||||
}
|
||||
}
|
||||
return { text: '', delivered: false };
|
||||
}
|
||||
|
||||
async function streamPollinations(
|
||||
messages: ChatTurn[],
|
||||
maxTokens: number,
|
||||
onDelta: ((cumulative: string) => void) | undefined,
|
||||
remainingMs: () => number,
|
||||
): Promise<StreamAttempt> {
|
||||
for (const model of POLLINATIONS_MODELS) {
|
||||
if (remainingMs() <= 1000) break;
|
||||
try {
|
||||
const attempt = await streamOpenAiCompatible(
|
||||
POLLINATIONS_URL,
|
||||
{},
|
||||
{ model, messages, max_tokens: maxTokens },
|
||||
onDelta,
|
||||
remainingMs,
|
||||
);
|
||||
if (attempt.text || attempt.delivered) return attempt;
|
||||
} catch (err) {
|
||||
console.warn(`[Sasquatch] Pollinations stream ${model} threw:`, err);
|
||||
}
|
||||
}
|
||||
return { text: '', delivered: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming version of {@link callSasquatchAI}. Reports the growing response
|
||||
* via `onDelta` (always the *cumulative* text, so the caller can just replace
|
||||
* the bubble) and resolves with the full text. Only if nothing was streamed at
|
||||
* all does it fall back to the non-streaming path, so the user still gets an
|
||||
* answer on environments that block streaming.
|
||||
*/
|
||||
export async function streamSasquatchAI(
|
||||
messages: ChatTurn[],
|
||||
maxTokens = 400,
|
||||
handlers: { onDelta?: (cumulative: string) => void } = {},
|
||||
): Promise<string> {
|
||||
const deadline = Date.now() + TOTAL_TIMEOUT_MS;
|
||||
const remainingMs = () => deadline - Date.now();
|
||||
|
||||
const primary = await streamOpenRouter(messages, maxTokens, handlers.onDelta, remainingMs);
|
||||
if (primary.text) return primary.text;
|
||||
|
||||
const backup = await streamPollinations(messages, maxTokens, handlers.onDelta, remainingMs);
|
||||
if (backup.text) return backup.text;
|
||||
|
||||
// A partial answer was already shown — don't restart with a fresh call.
|
||||
if (primary.delivered || backup.delivered) return primary.text || backup.text;
|
||||
|
||||
console.warn('[Sasquatch] Streaming unavailable — falling back to non-streaming call');
|
||||
return callSasquatchAI(messages, maxTokens);
|
||||
}
|
||||
|
||||
/** Parse a `{"tool":"name","args":{…}}` tool call out of an AI response, if any. */
|
||||
export function parseToolCall(aiText: string): { tool: string; args: Record<string, unknown> } | null {
|
||||
const jsonMatch = aiText.match(/\{[\s\S]*"tool"\s*:\s*"(\w+)"[\s\S]*\}/);
|
||||
|
|
|
|||
Loading…
Reference in a new issue