import { streamChatWithTools, resolveModel, DEFAULT_MAIN_MODEL, type LlmMessage, type OpenAIToolSchema, } from "../llm"; import { safeErrorMessage } from "../safeError"; import { createServerSupabase } from "../supabase"; import { buildUserMcpTools, type McpToolEvent, } from "../mcpConnectors"; import { COURTLISTENER_TOOLS, type CaseCitationEvent, type CourtlistenerToolEvent, } from "./tools/courtlistenerTools"; import { type DocStore, type DocIndex, type TabularCellStore, type WorkflowStore, type ToolCall, type AskInputsEvent, type EditAnnotation, devLog, } from "./types"; import { TOOLS, WORKFLOW_TOOLS } from "./tools/toolSchemas"; import { parseCitationsWithDiagnostics, parsePartialCitationObjects, createCitation, CITATIONS_OPEN_TAG, } from "./citations"; import { runToolCalls, type CourtlistenerTurnState, } from "./tools/toolDispatcher"; import { type TurnEditState, type TurnReadState, } from "./tools/documentOps"; export type AssistantEvent = | { type: "reasoning"; text: string } | AskInputsEvent | { type: "ask_inputs_response"; responses: { id: string; kind: "choice" | "documents"; question?: string; answer?: string; filenames?: string[]; skipped?: boolean; }[]; } | { type: "doc_read"; filename: string; document_id?: string } | { type: "doc_find"; filename: string; query: string; total_matches: number; } | { type: "doc_created"; filename: string; download_url: string; document_id?: string; version_id?: string; version_number?: number | null; } | { type: "doc_download"; filename: string; download_url: string } | { type: "doc_replicated"; /** Source document being copied. */ filename: string; count: number; copies: { new_filename: string; document_id: string; version_id: string; }[]; } | { type: "workflow_applied"; workflow_id: string; title: string } | { type: "doc_edited"; filename: string; document_id: string; version_id: string; /** Per-document monotonic Vn; null if backend couldn't determine it. */ version_number: number | null; download_url: string; annotations: EditAnnotation[]; } | CaseCitationEvent | CourtlistenerToolEvent | McpToolEvent | { type: "case_opinions"; cluster_id: number; case: unknown } | { type: "content"; text: string } | { type: "error"; message: string }; export class AssistantStreamError extends Error { fullText: string; events: AssistantEvent[]; constructor(message: string, fullText: string, events: AssistantEvent[]) { super(message); this.name = "AssistantStreamError"; this.fullText = fullText; this.events = events; } } export class AssistantStreamAbortError extends AssistantStreamError { constructor(fullText: string, events: AssistantEvent[]) { super("Stream aborted.", fullText, events); this.name = "AbortError"; } } class AssistantStreamAskInputsPause extends Error { constructor() { super("Waiting for user input."); this.name = "AssistantStreamAskInputsPause"; } } export function isAbortError(error: unknown): boolean { if (!error || typeof error !== "object") return false; const record = error as { name?: unknown; message?: unknown }; return ( record.name === "AbortError" || record.message === "Stream aborted." ); } function throwIfAborted(signal?: AbortSignal) { if (!signal?.aborted) return; const err = new Error("Stream aborted."); err.name = "AbortError"; throw err; } export async function runLLMStream(params: { apiMessages: unknown[]; docStore: DocStore; docIndex: DocIndex; userId: string; db: ReturnType; write: (s: string) => void; extraTools?: unknown[]; includeResearchTools?: boolean; workflowStore?: WorkflowStore; tabularStore?: TabularCellStore; buildCitations?: (fullText: string) => unknown[]; model?: string; apiKeys?: import("../llm").UserApiKeys; signal?: AbortSignal; /** * If set, generate_docx will attach created docs to this project so * they appear in the project sidebar. Leave null for general chats — * generated docs still get persisted, but as standalone documents. */ projectId?: string | null; }): Promise<{ fullText: string; events: AssistantEvent[]; citations: unknown[]; }> { const { apiMessages, docStore, docIndex, userId, db, write, extraTools, includeResearchTools = true, workflowStore, tabularStore, buildCitations, model, apiKeys, signal, projectId, } = params; const researchTools = includeResearchTools ? COURTLISTENER_TOOLS : []; const mcpTools = await buildUserMcpTools(userId, db); const baseTools = [...TOOLS, ...researchTools, ...WORKFLOW_TOOLS]; const activeTools = extraTools?.length ? [...baseTools, ...mcpTools, ...extraTools] : [...baseTools, ...mcpTools]; // Extract system prompt; pass remaining turns to the adapter as // plain user/assistant messages. const rawMsgs = apiMessages as { role: string; content: string | null }[]; const systemPrompt = rawMsgs[0]?.role === "system" ? (rawMsgs[0].content ?? "") : ""; const chatMessages: LlmMessage[] = rawMsgs .filter((m) => m.role !== "system") .map((m) => ({ role: m.role === "assistant" ? "assistant" : "user", content: m.content ?? "", })); const events: AssistantEvent[] = []; // One assistant turn produces at most one document_versions row per // edited doc. `runToolCalls` fires once per tool-call batch; the model // may emit multiple batches in a single turn, so this map persists // across batches to let subsequent edit_document calls overwrite the // turn's existing version instead of creating a new one. const turnEditState: TurnEditState = new Map(); // Suppress repeated full-document reads for the same document/version in // one assistant response. The guard is invalidated when edit_document // changes that document so a post-edit verification read can still happen. const turnReadState: TurnReadState = new Map(); const courtlistenerTurnState: CourtlistenerTurnState = { casesByClusterId: new Map(), }; let fullText = ""; let iterText = ""; let iterVisibleText = ""; let iterReasoning = ""; let visibleTailBuffer = ""; let citationsOpenSeen = false; let streamingCitationsBuffer = ""; let streamedCitationCount = 0; const emitCitationStreamSnapshot = ( status: "started" | "partial", citations: unknown[], ) => { if (buildCitations) return; write(`data: ${JSON.stringify({ type: "citations", status, citations })}\n\n`); }; const streamHiddenCitationContent = (delta: string) => { if (buildCitations || !delta) return; streamingCitationsBuffer += delta; const partial = parsePartialCitationObjects(streamingCitationsBuffer); if (partial.length <= streamedCitationCount) return; streamedCitationCount = partial.length; const citations = partial.map((c) => createCitation( c, docIndex, courtlistenerTurnState.casesByClusterId, ), ); emitCitationStreamSnapshot("partial", citations); }; const streamVisibleContent = (delta: string) => { if (!delta) return; if (citationsOpenSeen) { streamHiddenCitationContent(delta); return; } const combined = visibleTailBuffer + delta; const markerIdx = combined.indexOf(CITATIONS_OPEN_TAG); if (markerIdx >= 0) { const visible = combined.slice(0, markerIdx); if (visible) { iterVisibleText += visible; write( `data: ${JSON.stringify({ type: "content_delta", text: visible })}\n\n`, ); } visibleTailBuffer = ""; citationsOpenSeen = true; streamingCitationsBuffer = ""; streamedCitationCount = 0; emitCitationStreamSnapshot("started", []); streamHiddenCitationContent( combined.slice(markerIdx + CITATIONS_OPEN_TAG.length), ); return; } const keep = Math.min(CITATIONS_OPEN_TAG.length - 1, combined.length); const visible = combined.slice(0, combined.length - keep); visibleTailBuffer = combined.slice(combined.length - keep); if (visible) { iterVisibleText += visible; write( `data: ${JSON.stringify({ type: "content_delta", text: visible })}\n\n`, ); } }; const flushVisibleTail = (opts: { emit?: boolean } = {}) => { const emit = opts.emit ?? true; if (citationsOpenSeen || !visibleTailBuffer) { visibleTailBuffer = ""; return; } iterVisibleText += visibleTailBuffer; if (emit) { write( `data: ${JSON.stringify({ type: "content_delta", text: visibleTailBuffer })}\n\n`, ); } visibleTailBuffer = ""; }; const flushText = (opts: { emit?: boolean } = {}) => { if (!iterText) return; fullText += iterText; flushVisibleTail(opts); if (iterVisibleText) { events.push({ type: "content", text: iterVisibleText }); } iterText = ""; iterVisibleText = ""; visibleTailBuffer = ""; citationsOpenSeen = false; streamingCitationsBuffer = ""; streamedCitationCount = 0; }; const flushPartialTurn = (opts: { emit?: boolean } = {}) => { flushText(opts); if (iterReasoning) { events.push({ type: "reasoning", text: iterReasoning }); iterReasoning = ""; } }; const selectedModel = resolveModel(model, DEFAULT_MAIN_MODEL); try { throwIfAborted(signal); await streamChatWithTools({ model: selectedModel, systemPrompt, messages: chatMessages, tools: activeTools as OpenAIToolSchema[], maxIterations: 10, apiKeys, enableThinking: true, abortSignal: signal, callbacks: { onContentDelta: (delta) => { iterText += delta; streamVisibleContent(delta); }, onReasoningDelta: (delta) => { iterReasoning += delta; write( `data: ${JSON.stringify({ type: "reasoning_delta", text: delta })}\n\n`, ); }, onReasoningBlockEnd: () => { if (!iterReasoning) return; events.push({ type: "reasoning", text: iterReasoning }); write(`data: ${JSON.stringify({ type: "reasoning_block_end" })}\n\n`); iterReasoning = ""; }, // Fires after Claude's turn ends with stop_reason=tool_use, before // the tool actually runs. Flushes any buffered assistant text so // it's emitted in chronological order, then signals the client so // it can open a fresh PreResponseWrapper (shows "Working…") while // the tool executes — avoids the dead gap between message_stop // and the first tool-specific event. onToolCallStart: (call) => { flushText(); write( `data: ${JSON.stringify({ type: "tool_call_start", name: call.name, })}\n\n`, ); }, }, runTools: async (calls) => { throwIfAborted(signal); // Emit any text the model produced before this tool turn so the // UI sees it before the tool results stream in. flushText(); const toolCalls: ToolCall[] = calls.map((c) => ({ id: c.id, function: { name: c.name, arguments: JSON.stringify(c.input), }, })); const { toolResults, docsRead, docsFound, docsCreated, docsReplicated, workflowsApplied, docsEdited, askInputsEvents, courtlistenerEvents, caseCitationEvents, mcpEvents, } = await runToolCalls( toolCalls, docStore, userId, db, write, workflowStore, tabularStore, docIndex, turnEditState, turnReadState, projectId, courtlistenerTurnState, apiKeys, ); throwIfAborted(signal); for (const r of docsRead) { events.push({ type: "doc_read", filename: r.filename, document_id: r.document_id, }); } for (const f of docsFound) { events.push({ type: "doc_find", filename: f.filename, query: f.query, total_matches: f.total_matches, }); } for (const dl of docsCreated) { events.push({ type: "doc_created", filename: dl.filename, download_url: dl.download_url, document_id: dl.document_id, version_id: dl.version_id, version_number: dl.version_number ?? null, }); } for (const r of docsReplicated) { events.push({ type: "doc_replicated", filename: r.filename, count: r.count, copies: r.copies, }); } for (const wf of workflowsApplied) { events.push({ type: "workflow_applied", workflow_id: wf.workflow_id, title: wf.title, }); } for (const e of docsEdited) { events.push({ type: "doc_edited", filename: e.filename, document_id: e.document_id, version_id: e.version_id, version_number: e.version_number, download_url: e.download_url, annotations: e.annotations, }); } for (const askInputsEvent of askInputsEvents) { write(`data: ${JSON.stringify(askInputsEvent)}\n\n`); events.push(askInputsEvent); } for (const event of courtlistenerEvents) { events.push(event); } for (const event of mcpEvents) { events.push(event); } for (const event of caseCitationEvents) { events.push(event); } if (askInputsEvents.length > 0) { throw new AssistantStreamAskInputsPause(); } // Index alignment would break if any tool branch skips its // push (unhandled tool name, disabled store, guard failure). // Each tool_result already carries its tool_call_id, so key off // that directly — and fall back to an error result for any // tool_use that didn't produce one, so Claude's next request // has a tool_result for every tool_use it sent. const resultByCallId = new Map(); for (const r of toolResults) { const row = r as { tool_call_id: string; content?: unknown }; resultByCallId.set(row.tool_call_id, String(row.content ?? "")); } return toolCalls.map((c) => ({ tool_use_id: c.id, content: resultByCallId.get(c.id) ?? JSON.stringify({ error: `Tool '${c.function.name}' is not available.`, }), })); }, }); } catch (err) { if (err instanceof AssistantStreamAskInputsPause) { // The ask_inputs event has already been emitted and persisted in `events`. // Stop this assistant turn here so the model does not add redundant // prose telling the user to answer the picker or attach documents. } else if (isAbortError(err)) { flushPartialTurn({ emit: false }); throw new AssistantStreamAbortError(fullText, events); } else { flushPartialTurn(); const message = safeErrorMessage(err, "Stream error"); events.push({ type: "error", message }); throw new AssistantStreamError(message, fullText, events); } } flushText(); // Parse and emit citations from block const { citations: parsedCitations, diagnostics: citationDiagnostics } = parseCitationsWithDiagnostics(fullText); const citations = buildCitations ? buildCitations(fullText) : parsedCitations.map((c) => createCitation( c, docIndex, courtlistenerTurnState.casesByClusterId, ), ); devLog("[chat/stream] final citations", { hasCitationsBlock: citationDiagnostics.hasBlock, citationsBlockLength: citationDiagnostics.rawLength, parseError: citationDiagnostics.error, parsedCitationCount: parsedCitations.length, emittedCitationCount: citations.length, usedCustomCitationBuilder: !!buildCitations, }); write( `data: ${JSON.stringify({ type: "citations", status: "final", citations })}\n\n`, ); write("data: [DONE]\n\n"); return { fullText, events, citations }; }