diff --git a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx index eceb46231..d95aab6e8 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx @@ -6,6 +6,7 @@ import { useTranslations } from "next-intl"; import type React from "react"; import { useCallback, useEffect, useRef, useState } from "react"; import { toast } from "sonner"; +import { pendingUserImageDataUrlsAtom } from "@/atoms/chat/pending-user-images.atom"; import { myAccessAtom } from "@/atoms/members/members-query.atoms"; import { updateLLMPreferencesMutationAtom } from "@/atoms/new-llm-config/new-llm-config-mutation.atoms"; import { @@ -33,6 +34,7 @@ export function DashboardClientLayout({ const pathname = usePathname(); const { search_space_id } = useParams(); const setActiveSearchSpaceIdState = useSetAtom(activeSearchSpaceIdAtom); + const setPendingUserImageUrls = useSetAtom(pendingUserImageDataUrlsAtom); const { data: preferences = {}, @@ -142,6 +144,14 @@ export function DashboardClientLayout({ const electronAPI = useElectronAPI(); + useEffect(() => { + if (!electronAPI?.onChatScreenCapture) return; + return electronAPI.onChatScreenCapture((dataUrl: string) => { + if (typeof dataUrl !== "string" || !dataUrl.startsWith("data:image/")) return; + setPendingUserImageUrls((prev) => [...prev, dataUrl]); + }); + }, [electronAPI, setPendingUserImageUrls]); + useEffect(() => { const activeSeacrhSpaceId = typeof search_space_id === "string" diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index 62332d2c4..fe23cb2c7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -26,6 +26,7 @@ import { messageDocumentsMapAtom, sidebarSelectedDocumentsAtom, } from "@/atoms/chat/mentioned-documents.atom"; +import { pendingUserImageDataUrlsAtom } from "@/atoms/chat/pending-user-images.atom"; import { clearPlanOwnerRegistry, // extractWriteTodosFromContent, @@ -45,8 +46,8 @@ import { } from "@/components/assistant-ui/token-usage-context"; import { useChatSessionStateSync } from "@/hooks/use-chat-session-state"; import { useMessagesSync } from "@/hooks/use-messages-sync"; -import { documentsApiService } from "@/lib/apis/documents-api.service"; import { getAgentFilesystemSelection } from "@/lib/agent-filesystem"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; import { getBearerToken } from "@/lib/auth-utils"; import { convertToThreadMessage } from "@/lib/chat/message-utils"; import { @@ -76,6 +77,7 @@ import { type ThreadListResponse, type ThreadRecord, } from "@/lib/chat/thread-persistence"; +import { extractUserTurnForNewChatApi } from "@/lib/chat/user-turn-api-parts"; import { NotFoundError } from "@/lib/error"; import { trackChatCreated, @@ -231,6 +233,8 @@ export default function NewChatPage() { const updateChatTabTitle = useSetAtom(updateChatTabTitleAtom); const removeChatTab = useSetAtom(removeChatTabAtom); const setAgentCreatedDocuments = useSetAtom(agentCreatedDocumentsAtom); + const pendingUserImageUrls = useAtomValue(pendingUserImageDataUrlsAtom); + const setPendingUserImageUrls = useSetAtom(pendingUserImageDataUrlsAtom); // Get current user for author info in shared chats const { data: currentUser } = useAtomValue(currentUserAtom); @@ -494,18 +498,13 @@ export default function NewChatPage() { abortControllerRef.current = null; } - // Extract user query text from content parts - let userQuery = ""; - for (const part of message.content) { - if (part.type === "text") { - userQuery += part.text; - } - } + const urlsSnapshot = [...pendingUserImageUrls]; + setPendingUserImageUrls([]); + const { userQuery, userImages } = extractUserTurnForNewChatApi(message, urlsSnapshot); - if (!userQuery.trim()) return; + if (!userQuery.trim() && userImages.length === 0) return; - // Check if podcast is already generating - if (isPodcastGenerating() && looksLikePodcastRequest(userQuery)) { + if (userQuery.trim() && isPodcastGenerating() && looksLikePodcastRequest(userQuery)) { toast.warning("A podcast is already being generated."); return; } @@ -560,10 +559,27 @@ export default function NewChatPage() { } : undefined; + const existingImageUrls = new Set( + message.content + .filter( + (p): p is { type: "image"; image: string } => + typeof p === "object" && + p !== null && + "type" in p && + p.type === "image" && + "image" in p + ) + .map((p) => p.image) + ); + const extraImageParts = urlsSnapshot + .filter((u) => !existingImageUrls.has(u)) + .map((image) => ({ type: "image" as const, image })); + const userDisplayContent = [...message.content, ...extraImageParts]; + const userMessage: ThreadMessageLike = { id: userMsgId, role: "user", - content: message.content, + content: userDisplayContent, createdAt: new Date(), metadata: authorMetadata, }; @@ -571,7 +587,7 @@ export default function NewChatPage() { // Track message sent trackChatMessageSent(searchSpaceId, currentThreadId, { - hasAttachments: false, + hasAttachments: userImages.length > 0, hasMentionedDocuments: mentionedDocumentIds.surfsense_doc_ids.length > 0 || mentionedDocumentIds.document_ids.length > 0, @@ -596,7 +612,7 @@ export default function NewChatPage() { })); } - const persistContent: unknown[] = [...message.content]; + const persistContent: unknown[] = [...userDisplayContent]; if (allMentionedDocs.length > 0) { persistContent.push({ @@ -661,8 +677,7 @@ export default function NewChatPage() { const selection = await getAgentFilesystemSelection(); if ( selection.filesystem_mode === "desktop_local_folder" && - (!selection.local_filesystem_mounts || - selection.local_filesystem_mounts.length === 0) + (!selection.local_filesystem_mounts || selection.local_filesystem_mounts.length === 0) ) { toast.error("Select a local folder before using Local Folder mode."); return; @@ -711,6 +726,7 @@ export default function NewChatPage() { ? mentionedDocumentIds.surfsense_doc_ids : undefined, disabled_tools: disabledTools.length > 0 ? disabledTools : undefined, + ...(userImages.length > 0 ? { user_images: userImages } : {}), }), signal: controller.signal, }); @@ -842,14 +858,7 @@ export default function NewChatPage() { }); } else { const tcId = `interrupt-${action.name}`; - addToolCall( - contentPartsState, - toolsWithUI, - tcId, - action.name, - action.args, - true - ); + addToolCall(contentPartsState, toolsWithUI, tcId, action.name, action.args, true); updateToolCall(contentPartsState, tcId, { result: { __interrupt__: true, ...interruptData }, }); @@ -989,6 +998,9 @@ export default function NewChatPage() { disabledTools, updateChatTabTitle, tokenUsageStore, + pendingUserImageUrls, + setPendingUserImageUrls, + toolsWithUI, ] ); @@ -1189,14 +1201,7 @@ export default function NewChatPage() { }); } else { const tcId = `interrupt-${action.name}`; - addToolCall( - contentPartsState, - toolsWithUI, - tcId, - action.name, - action.args, - true - ); + addToolCall(contentPartsState, toolsWithUI, tcId, action.name, action.args, true); updateToolCall(contentPartsState, tcId, { result: { __interrupt__: true, @@ -1261,7 +1266,7 @@ export default function NewChatPage() { abortControllerRef.current = null; } }, - [pendingInterrupt, messages, searchSpaceId, tokenUsageStore] + [pendingInterrupt, messages, searchSpaceId, tokenUsageStore, toolsWithUI] ); useEffect(() => { @@ -1588,7 +1593,7 @@ export default function NewChatPage() { abortControllerRef.current = null; } }, - [threadId, searchSpaceId, messages, disabledTools, tokenUsageStore] + [threadId, searchSpaceId, messages, disabledTools, tokenUsageStore, toolsWithUI] ); // Handle editing a message - truncates history and regenerates with new query diff --git a/surfsense_web/atoms/chat/pending-user-images.atom.ts b/surfsense_web/atoms/chat/pending-user-images.atom.ts new file mode 100644 index 000000000..6898e745d --- /dev/null +++ b/surfsense_web/atoms/chat/pending-user-images.atom.ts @@ -0,0 +1,3 @@ +import { atom } from "jotai"; + +export const pendingUserImageDataUrlsAtom = atom([]); diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index 2ec422fbf..6862662f2 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -16,6 +16,7 @@ import { ChevronUp, Clipboard, Dot, + Camera, Globe, Plus, Settings2, @@ -40,6 +41,7 @@ import { mentionedDocumentsAtom, sidebarSelectedDocumentsAtom, } from "@/atoms/chat/mentioned-documents.atom"; +import { pendingUserImageDataUrlsAtom } from "@/atoms/chat/pending-user-images.atom"; import { connectorDialogOpenAtom } from "@/atoms/connector-dialog/connector-dialog.atoms"; import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; import { documentsSidebarOpenAtom } from "@/atoms/documents/ui.atoms"; @@ -89,6 +91,7 @@ import { useBatchCommentsPreload } from "@/hooks/use-comments"; import { useCommentsSync } from "@/hooks/use-comments-sync"; import { useMediaQuery } from "@/hooks/use-media-query"; import { useElectronAPI } from "@/hooks/use-platform"; +import { captureDisplayToPngDataUrl } from "@/lib/chat/display-media-capture"; import { SLIDEOUT_PANEL_OPENED_EVENT } from "@/lib/layout-events"; import { cn } from "@/lib/utils"; @@ -295,6 +298,32 @@ const ConnectToolsBanner: FC<{ isThreadEmpty: boolean }> = ({ isThreadEmpty }) = ); }; +const PendingScreenImageStrip: FC = () => { + const [urls, setUrls] = useAtom(pendingUserImageDataUrlsAtom); + if (urls.length === 0) return null; + return ( +
+ {urls.map((url, index) => ( +
+ {/* biome-ignore lint/performance/noImgElement: data URL thumbnails from capture */} + + +
+ ))} +
+ ); +}; + const ClipboardChip: FC<{ text: string; onDismiss: () => void }> = ({ text, onDismiss }) => { const [expanded, setExpanded] = useState(false); const isLong = text.length > 120; @@ -702,6 +731,7 @@ const Composer: FC = () => { )}
+ {clipboardInitialText && ( = ({ isBlockedByOtherUser = false }, [] ); + const pendingScreenImages = useAtomValue(pendingUserImageDataUrlsAtom); + const setPendingScreenImages = useSetAtom(pendingUserImageDataUrlsAtom); + const electronAPI = useElectronAPI(); + const isComposerTextEmpty = useAuiState(({ composer }) => { const text = composer.text?.trim() || ""; return text.length === 0; }); - const isComposerEmpty = isComposerTextEmpty && mentionedDocuments.length === 0; + const isComposerEmpty = + isComposerTextEmpty && mentionedDocuments.length === 0 && pendingScreenImages.length === 0; + + const handleScreenCapture = useCallback(async () => { + const url = await captureDisplayToPngDataUrl(); + if (url) setPendingScreenImages((prev) => [...prev, url]); + }, [setPendingScreenImages]); const { data: userConfigs } = useAtomValue(newLLMConfigsAtom); const { data: globalConfigs } = useAtomValue(globalNewLLMConfigsAtom); @@ -1201,6 +1241,20 @@ const ComposerAction: FC = ({ isBlockedByOtherUser = false
)}
+ {/* Electron: native shortcut → pending images; skip in-webview getDisplayMedia. */} + {!electronAPI && ( + void handleScreenCapture()} + > + + + )} !thread.isRunning}> = ({ isBlockedByOtherUser = false : !hasModelConfigured ? "Please select a model from the header to start chatting" : isComposerEmpty - ? "Enter a message to send" + ? "Enter a message or add a screenshot to send" : "Send message" } side="bottom" diff --git a/surfsense_web/lib/chat/display-media-capture.ts b/surfsense_web/lib/chat/display-media-capture.ts new file mode 100644 index 000000000..c2fb69aae --- /dev/null +++ b/surfsense_web/lib/chat/display-media-capture.ts @@ -0,0 +1,120 @@ +/** `getDisplayMedia` → single PNG frame (data URL). */ +function getImageCaptureCtor(): + | (new ( + track: MediaStreamTrack + ) => { grabFrame: () => Promise }) + | undefined { + if (typeof window === "undefined") return undefined; + const IC = ( + window as unknown as { + ImageCapture?: new (track: MediaStreamTrack) => { grabFrame: () => Promise }; + } + ).ImageCapture; + return typeof IC === "function" ? IC : undefined; +} + +function stopAllTracks(stream: MediaStream): void { + for (const t of stream.getTracks()) { + t.stop(); + } +} + +async function captureTrackToPngDataUrl( + track: MediaStreamTrack, + stream: MediaStream +): Promise { + const ImageCtor = getImageCaptureCtor(); + if (ImageCtor !== undefined) { + try { + const ic = new ImageCtor(track); + const bitmap = await ic.grabFrame(); + try { + const canvas = document.createElement("canvas"); + canvas.width = bitmap.width; + canvas.height = bitmap.height; + const ctx = canvas.getContext("2d"); + if (!ctx) { + stopAllTracks(stream); + return null; + } + ctx.drawImage(bitmap, 0, 0); + stopAllTracks(stream); + return canvas.toDataURL("image/png"); + } finally { + if ("close" in bitmap && typeof bitmap.close === "function") { + bitmap.close(); + } + } + } catch { + /* fall through to