diff --git a/surfsense_backend/app/agents/new_chat/system_prompt.py b/surfsense_backend/app/agents/new_chat/system_prompt.py index 91b4eee08..61a8fbdd6 100644 --- a/surfsense_backend/app/agents/new_chat/system_prompt.py +++ b/surfsense_backend/app/agents/new_chat/system_prompt.py @@ -64,14 +64,18 @@ You have access to the following tools: - The preview card will automatically be displayed in the chat. 4. display_image: Display an image in the chat with metadata. - - Use this tool when you want to show an image to the user. + - Use this tool when you want to show an image from a URL to the user. - This displays the image with an optional title, description, and source attribution. - Common use cases: * Showing an image from a URL mentioned in the conversation * Displaying a diagram, chart, or illustration you're referencing * Showing visual examples when explaining concepts + - IMPORTANT: Do NOT use this tool for user-uploaded image attachments! + * User attachments are already visible in the chat UI - the user can see them + * This tool requires a valid HTTP/HTTPS URL, not a local file path + * When a user uploads an image, just analyze it and respond - don't try to display it again - Args: - - src: The URL of the image to display (must be a valid HTTP/HTTPS image URL) + - src: The URL of the image to display (must be a valid HTTP/HTTPS image URL, not a local path) - alt: Alternative text describing the image (for accessibility) - title: Optional title to display below the image - description: Optional description providing context about the image diff --git a/surfsense_backend/app/tasks/chat/stream_new_chat.py b/surfsense_backend/app/tasks/chat/stream_new_chat.py index 2038e85dc..aff6fa32b 100644 --- a/surfsense_backend/app/tasks/chat/stream_new_chat.py +++ b/surfsense_backend/app/tasks/chat/stream_new_chat.py @@ -255,13 +255,59 @@ async def stream_new_chat( # Initial thinking step - analyzing the request analyze_step_id = next_thinking_step_id() last_active_step_id = analyze_step_id - last_active_step_title = "Understanding your request" - last_active_step_items = [ - f"Processing: {user_query[:80]}{'...' if len(user_query) > 80 else ''}" - ] + + # Determine step title and action verb based on context + if attachments and mentioned_documents: + last_active_step_title = "Analyzing your content" + action_verb = "Reading" + elif attachments: + last_active_step_title = "Reading your content" + action_verb = "Reading" + elif mentioned_documents: + last_active_step_title = "Analyzing referenced content" + action_verb = "Analyzing" + else: + last_active_step_title = "Understanding your request" + action_verb = "Processing" + + # Build the message with inline context about attachments/documents + processing_parts = [] + + # Add the user query + query_text = user_query[:80] + ("..." if len(user_query) > 80 else "") + processing_parts.append(query_text) + + # Add file attachment names inline + if attachments: + attachment_names = [] + for attachment in attachments: + name = attachment.name + if len(name) > 30: + name = name[:27] + "..." + attachment_names.append(name) + if len(attachment_names) == 1: + processing_parts.append(f"[{attachment_names[0]}]") + else: + processing_parts.append(f"[{len(attachment_names)} files]") + + # Add mentioned document names inline + if mentioned_documents: + doc_names = [] + for doc in mentioned_documents: + title = doc.title + if len(title) > 30: + title = title[:27] + "..." + doc_names.append(title) + if len(doc_names) == 1: + processing_parts.append(f"[{doc_names[0]}]") + else: + processing_parts.append(f"[{len(doc_names)} documents]") + + last_active_step_items = [f"{action_verb}: {' '.join(processing_parts)}"] + yield streaming_service.format_thinking_step( step_id=analyze_step_id, - title="Understanding your request", + title=last_active_step_title, status="in_progress", items=last_active_step_items, ) @@ -369,13 +415,13 @@ async def stream_new_chat( if isinstance(tool_input, dict) else "" ) - last_active_step_title = "Displaying image" + last_active_step_title = "Analyzing the image" last_active_step_items = [ - f"Image: {title[:50] if title else src[:50]}{'...' if len(title or src) > 50 else ''}" + f"Analyzing: {title[:50] if title else src[:50]}{'...' if len(title or src) > 50 else ''}" ] yield streaming_service.format_thinking_step( step_id=tool_step_id, - title="Displaying image", + title="Analyzing the image", status="in_progress", items=last_active_step_items, ) @@ -471,7 +517,7 @@ async def stream_new_chat( else str(tool_input) ) yield streaming_service.format_terminal_info( - f"Displaying image: {src[:60]}{'...' if len(src) > 60 else ''}", + f"Analyzing image: {src[:60]}{'...' if len(src) > 60 else ''}", "info", ) elif tool_name == "scrape_webpage": @@ -575,20 +621,20 @@ async def stream_new_chat( items=completed_items, ) elif tool_name == "display_image": - # Build completion items for image display + # Build completion items for image analysis if isinstance(tool_output, dict): title = tool_output.get("title", "") alt = tool_output.get("alt", "Image") display_name = title or alt completed_items = [ *last_active_step_items, - f"Showing: {display_name[:50]}{'...' if len(display_name) > 50 else ''}", + f"Analyzed: {display_name[:50]}{'...' if len(display_name) > 50 else ''}", ] else: - completed_items = [*last_active_step_items, "Image displayed"] + completed_items = [*last_active_step_items, "Image analyzed"] yield streaming_service.format_thinking_step( step_id=original_step_id, - title="Displaying image", + title="Analyzing the image", status="completed", items=completed_items, ) @@ -744,7 +790,7 @@ async def stream_new_chat( "alt", "Image" ) yield streaming_service.format_terminal_info( - f"Image displayed: {title[:40]}{'...' if len(title) > 40 else ''}", + f"Image analyzed: {title[:40]}{'...' if len(title) > 40 else ''}", "success", ) elif tool_name == "scrape_webpage": diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index 4a344ec36..9687d2b4d 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -171,6 +171,14 @@ export default function NewChatPage() { const initializeThread = useCallback(async () => { setIsInitializing(true); + // Reset all state when switching between chats to prevent stale data + setMessages([]); + setThreadId(null); + setMessageThinkingSteps(new Map()); + setMentionedDocumentIds([]); + setMentionedDocuments([]); + setMessageDocumentsMap({}); + try { if (urlChatId > 0) { // Thread exists - load messages @@ -219,7 +227,7 @@ export default function NewChatPage() { } finally { setIsInitializing(false); } - }, [urlChatId, setMessageDocumentsMap]); + }, [urlChatId, setMessageDocumentsMap, setMentionedDocumentIds, setMentionedDocuments]); // Initialize on mount useEffect(() => { @@ -238,6 +246,13 @@ export default function NewChatPage() { // Handle new message from user const onNew = useCallback( async (message: AppendMessage) => { + // Abort any previous streaming request to prevent race conditions + // when user sends a second query while the first is still streaming + if (abortControllerRef.current) { + abortControllerRef.current.abort(); + abortControllerRef.current = null; + } + // Extract user query text from content parts let userQuery = ""; for (const part of message.content) { @@ -297,6 +312,8 @@ export default function NewChatPage() { role: "user", content: message.content, createdAt: new Date(), + // Include attachments so they can be displayed + attachments: message.attachments || [], }; setMessages((prev) => [...prev, userMessage]); diff --git a/surfsense_web/components/assistant-ui/attachment.tsx b/surfsense_web/components/assistant-ui/attachment.tsx index dfb63cbf3..152f54127 100644 --- a/surfsense_web/components/assistant-ui/attachment.tsx +++ b/surfsense_web/components/assistant-ui/attachment.tsx @@ -41,13 +41,23 @@ const useAttachmentSrc = () => { const { file, src } = useAssistantState( useShallow(({ attachment }): { file?: File; src?: string } => { if (!attachment || attachment.type !== "image") return {}; + + // First priority: use File object if available (for new uploads) if (attachment.file) return { file: attachment.file }; - // Only try to filter if content is an array (standard assistant-ui format) - // Our custom ChatAttachment has content as a string, so skip this - if (Array.isArray(attachment.content)) { - const src = attachment.content.filter((c) => c.type === "image")[0]?.image; - if (src) return { src }; + + // Second priority: use stored imageDataUrl (for persisted messages) + // This is stored in our custom ChatAttachment interface + const customAttachment = attachment as { imageDataUrl?: string }; + if (customAttachment.imageDataUrl) { + return { src: customAttachment.imageDataUrl }; } + + // Third priority: try to extract from content array (standard assistant-ui format) + if (Array.isArray(attachment.content)) { + const contentSrc = attachment.content.filter((c) => c.type === "image")[0]?.image; + if (contentSrc) return { src: contentSrc }; + } + return {}; }) ); @@ -218,11 +228,77 @@ const AttachmentRemove: FC = () => { ); }; +/** + * Image attachment with preview thumbnail (click to expand) + */ +const MessageImageAttachment: FC = () => { + const attachmentName = useAssistantState(({ attachment }) => attachment?.name || "Image"); + const src = useAttachmentSrc(); + + if (!src) return null; + + return ( + +
+ {attachmentName} + {/* Hover overlay with filename */} +
+
+ + {attachmentName} + +
+
+
+
+ ); +}; + +/** + * Document/file attachment as chip (similar to mentioned documents) + */ +const MessageDocumentAttachment: FC = () => { + const attachmentName = useAssistantState(({ attachment }) => attachment?.name || "Attachment"); + + return ( + + + + {attachmentName} + + + ); +}; + +/** + * Attachment component for user messages + * Shows image preview for images, chip for documents + */ +const MessageAttachmentChip: FC = () => { + const isImage = useAssistantState(({ attachment }) => attachment?.type === "image"); + + if (isImage) { + return ; + } + + return ; +}; + export const UserMessageAttachments: FC = () => { return ( -
- -
+ ); }; diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index a2b1e0425..1d4ebed6d 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -426,6 +426,9 @@ const Composer: FC = () => { // Check if thread is empty (new chat) const isThreadEmpty = useAssistantState(({ thread }) => thread.isEmpty); + // Check if thread is currently running (streaming response) + const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning); + // Auto-focus editor when on new chat page useEffect(() => { if (isThreadEmpty && !hasAutoFocusedRef.current && editorRef.current) { @@ -504,6 +507,10 @@ const Composer: FC = () => { // Handle submit from inline editor (Enter key) const handleSubmit = useCallback(() => { + // Prevent sending while a response is still streaming + if (isThreadRunning) { + return; + } if (!showDocumentPopover) { composerRuntime.send(); // Clear the editor after sending @@ -511,7 +518,7 @@ const Composer: FC = () => { setMentionedDocuments([]); setMentionedDocumentIds([]); } - }, [showDocumentPopover, composerRuntime, setMentionedDocuments, setMentionedDocumentIds]); + }, [showDocumentPopover, isThreadRunning, composerRuntime, setMentionedDocuments, setMentionedDocumentIds]); // Handle document removal from inline editor const handleDocumentRemove = useCallback( @@ -973,19 +980,23 @@ const UserMessage: FC = () => { const messageId = useAssistantState(({ message }) => message?.id); const messageDocumentsMap = useAtomValue(messageDocumentsMapAtom); const mentionedDocs = messageId ? messageDocumentsMap[messageId] : undefined; + const hasAttachments = useAssistantState( + ({ message }) => message?.attachments && message.attachments.length > 0 + ); return ( - -
- {/* Display mentioned documents as chips */} - {mentionedDocs && mentionedDocs.length > 0 && ( -
- {mentionedDocs.map((doc) => ( + {/* Display attachments and mentioned documents */} + {(hasAttachments || (mentionedDocs && mentionedDocs.length > 0)) && ( +
+ {/* Attachments (images show as thumbnails, documents as chips) */} + + {/* Mentioned documents as chips */} + {mentionedDocs?.map((doc) => ( { + const timer = setTimeout(() => setIsVisible(true), delay); + return () => clearTimeout(timer); + }, [delay]); + + return isVisible; +} + +/** + * Get file icon based on file extension (all icons are muted/gray) + */ +function getFileIcon(name: string): React.ReactNode { + const ext = name.split(".").pop()?.toLowerCase() || ""; + + // PDF / Word documents + if (ext === "pdf" || ["doc", "docx"].includes(ext)) { + return ; + } + // Spreadsheets + if (["xls", "xlsx", "csv"].includes(ext)) { + return ; + } + // Images + if (["png", "jpg", "jpeg", "gif", "webp", "svg"].includes(ext)) { + return ; + } + // Audio + if (["mp3", "wav", "m4a", "ogg", "webm"].includes(ext)) { + return ; + } + // Video + if (["mp4", "mov", "avi", "mkv"].includes(ext)) { + return ; + } + // Code files + if (["js", "ts", "tsx", "jsx", "py", "html", "css", "json", "md"].includes(ext)) { + return ; + } + // Default + return ; +} + +/** + * Compact attachment tile component - matches the chat UI style + */ +const AttachmentTile: React.FC<{ name: string }> = ({ name }) => { + const icon = getFileIcon(name); + + return ( + + {icon} + {name} + + ); +}; + +/** + * Parse text and render bracketed items (like [filename.pdf]) as styled tiles + */ +function parseAndRenderWithBadges(text: string): React.ReactNode { + // Match patterns like [filename.ext] or [N files] or [N documents] + const regex = /\[([^\]]+)\]/g; + const matches = Array.from(text.matchAll(regex)); + + if (matches.length === 0) { + return text; + } + + const parts: React.ReactNode[] = []; + let lastIndex = 0; + + for (const match of matches) { + const matchIndex = match.index ?? 0; + + // Add text before the match + if (matchIndex > lastIndex) { + parts.push(text.slice(lastIndex, matchIndex)); + } + + const content = match[1]; + + // Render as a compact tile matching chat UI style with file-type colors + parts.push(); + + lastIndex = matchIndex + match[0].length; + } + + // Add remaining text + if (lastIndex < text.length) { + parts.push(text.slice(lastIndex)); + } + + return parts; +} + export type ChainOfThoughtItemProps = React.ComponentProps<"div">; export const ChainOfThoughtItem = ({ children, className, ...props }: ChainOfThoughtItemProps) => ( -
- {children} +
+ {typeof children === "string" ? parseAndRenderWithBadges(children) : children}
); @@ -80,9 +186,28 @@ export const ChainOfThoughtContent = ({ {...props} >
-
+ {/* Animated vertical connection line */} +
-
{children}
+
+ {React.Children.map(children, (child, index) => { + const key = React.isValidElement(child) ? child.key : `cot-item-${index}`; + return ( +
+ {child} +
+ ); + })} +
); @@ -98,14 +223,19 @@ export function ChainOfThought({ children, className }: ChainOfThoughtProps) { return (
- {childrenArray.map((child, index) => ( - - {React.isValidElement(child) && - React.cloneElement(child as React.ReactElement, { - isLast: index === childrenArray.length - 1, - })} - - ))} + {childrenArray.map((child, index) => { + // React.Children.toArray assigns stable keys to each child + const key = React.isValidElement(child) ? child.key : `cot-step-${index}`; + return ( + + {React.isValidElement(child) && + React.cloneElement(child as React.ReactElement, { + isLast: index === childrenArray.length - 1, + stepIndex: index, + })} + + ); + })}
); } @@ -114,19 +244,42 @@ export type ChainOfThoughtStepProps = { children: React.ReactNode; className?: string; isLast?: boolean; + /** Index of the step for staggered animation */ + stepIndex?: number; }; export const ChainOfThoughtStep = ({ children, className, isLast = false, + stepIndex = 0, ...props }: ChainOfThoughtStepProps & React.ComponentProps) => { + // Staggered entrance animation based on step index + const isVisible = useEntranceAnimation(stepIndex * 50); + return ( - + {children} + {/* Animated connection line to next step */}
-
+
); diff --git a/surfsense_web/lib/chat/attachment-adapter.ts b/surfsense_web/lib/chat/attachment-adapter.ts index b31af9116..f084af411 100644 --- a/surfsense_web/lib/chat/attachment-adapter.ts +++ b/surfsense_web/lib/chat/attachment-adapter.ts @@ -60,9 +60,11 @@ interface ProcessAttachmentResponse { /** * Extended CompleteAttachment with our custom extractedContent field * We store the extracted text in a custom field so we can access it in onNew + * For images, we also store the data URL so it can be displayed after persistence */ export interface ChatAttachment extends CompleteAttachment { extractedContent: string; + imageDataUrl?: string; // Base64 data URL for images (persists across page reloads) } /** @@ -118,6 +120,21 @@ async function processAttachment(file: File): Promise // Store processed results for the send() method const processedAttachments = new Map(); +// Store image data URLs for attachments (so they persist after File objects are lost) +const imageDataUrls = new Map(); + +/** + * Convert a File to a data URL (base64) for images + */ +async function fileToDataUrl(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result as string); + reader.onerror = reject; + reader.readAsDataURL(file); + }); +} + /** * Create the attachment adapter for assistant-ui * @@ -170,6 +187,12 @@ export function createAttachmentAdapter(): AttachmentAdapter { } as PendingAttachment; try { + // For images, convert to data URL so we can display them after persistence + if (attachmentType === "image") { + const dataUrl = await fileToDataUrl(file); + imageDataUrls.set(id, dataUrl); + } + // Process the file through the backend ETL service const result = await processAttachment(file); @@ -204,10 +227,14 @@ export function createAttachmentAdapter(): AttachmentAdapter { */ async send(pendingAttachment: PendingAttachment): Promise { const result = processedAttachments.get(pendingAttachment.id); + const imageDataUrl = imageDataUrls.get(pendingAttachment.id); if (result) { // Clean up stored result processedAttachments.delete(pendingAttachment.id); + if (imageDataUrl) { + imageDataUrls.delete(pendingAttachment.id); + } return { id: result.id, @@ -222,6 +249,7 @@ export function createAttachmentAdapter(): AttachmentAdapter { }, ], extractedContent: result.content, + imageDataUrl, // Store data URL for images so they can be displayed after persistence }; } @@ -238,6 +266,7 @@ export function createAttachmentAdapter(): AttachmentAdapter { status: { type: "complete" }, content: [], extractedContent: "", + imageDataUrl, // Still include data URL if available }; },