From 6ecd75fbbbc834fafce36b0c8ffb367b89d1beea Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 6 Apr 2026 21:32:49 -0700 Subject: [PATCH 01/47] refactor: simplify HeroSection component and enhance UI with new features - Removed dynamic import of HeroCarousel and replaced it with a static layout. - Introduced new TAB_ITEMS for showcasing features with descriptions and media. - Enhanced the layout and styling for better responsiveness and visual appeal. - Cleaned up unused code and improved overall readability of the component. --- .../components/homepage/hero-section.tsx | 636 +++++++++--------- 1 file changed, 313 insertions(+), 323 deletions(-) diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index 299cf1032..1bb28e770 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -1,39 +1,22 @@ "use client"; import { AnimatePresence, motion } from "motion/react"; -import dynamic from "next/dynamic"; +import { Monitor } from "lucide-react"; import Link from "next/link"; -import type React from "react"; -import { useEffect, useRef, useState } from "react"; +import React, { useCallback, useEffect, useRef, useState, memo } from "react"; import Balancer from "react-wrap-balancer"; import { AUTH_TYPE, BACKEND_URL } from "@/lib/env-config"; import { trackLoginAttempt } from "@/lib/posthog/events"; import { cn } from "@/lib/utils"; +import { + ExpandedMediaOverlay, + useExpandedMedia, +} from "@/components/ui/expanded-gif-overlay"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip"; -const HeroCarousel = dynamic( - () => import("@/components/ui/hero-carousel").then((m) => ({ default: m.HeroCarousel })), - { - ssr: false, - loading: () => ( -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- ), - } -); - -// Official Google "G" logo with brand colors const GoogleLogo = ({ className }: { className?: string }) => ( ( ); -function useIsDesktop(breakpoint = 1024) { - const [isDesktop, setIsDesktop] = useState(false); - useEffect(() => { - const mql = window.matchMedia(`(min-width: ${breakpoint}px)`); - setIsDesktop(mql.matches); - const handler = (e: MediaQueryListEvent) => setIsDesktop(e.matches); - mql.addEventListener("change", handler); - return () => mql.removeEventListener("change", handler); - }, [breakpoint]); - return isDesktop; -} +const TAB_ITEMS = [ + { + title: "Connect & Sync", + description: + "Connect data sources like Notion, Drive and Gmail. Automatically sync to keep them updated.", + src: "/homepage/hero_tutorial/ConnectorFlowGif.mp4", + featured: true, + }, + { + title: "Upload Documents", + description: "Upload documents directly, from images to massive PDFs.", + src: "/homepage/hero_tutorial/DocUploadGif.mp4", + featured: true, + }, + { + title: "Search & Citation", + description: + "Ask questions and get cited responses from your knowledge base.", + src: "/homepage/hero_tutorial/BSNCGif.mp4", + featured: false, + }, + { + title: "Document Q&A", + description: "Mention specific documents in chat for targeted answers.", + src: "/homepage/hero_tutorial/BQnaGif_compressed.mp4", + featured: false, + }, + { + title: "Reports", + description: "Generate reports from your sources in many formats.", + src: "/homepage/hero_tutorial/ReportGenGif_compressed.mp4", + featured: false, + }, + { + title: "Podcasts", + description: "Turn anything into a podcast in under 20 seconds.", + src: "/homepage/hero_tutorial/PodcastGenGif.mp4", + featured: false, + }, + { + title: "Image Generation", + description: + "Generate high-quality images easily from your conversations.", + src: "/homepage/hero_tutorial/ImageGenGif.mp4", + featured: false, + }, + { + title: "Collaborative Chat", + description: + "Collaborate on AI-powered conversations in realtime with your team.", + src: "/homepage/hero_realtime/RealTimeChatGif.mp4", + featured: false, + }, + { + title: "Comments", + description: "Add comments and tag teammates on any message.", + src: "/homepage/hero_realtime/RealTimeCommentsFlow.mp4", + featured: false, + }, + { + title: "Video Generation", + description: + "Create short videos with AI-generated visuals and narration from your sources.", + src: "/homepage/hero_tutorial/video_gen_surf.mp4", + featured: false, + }, +] as const; export function HeroSection() { - const containerRef = useRef(null); - const parentRef = useRef(null); - const isDesktop = useIsDesktop(); - return ( -
- - {isDesktop && ( - <> - - - - - - )} +
+
+

+ NotebookLM for Teams +

+
+
+

+ An open source, privacy focused alternative to NotebookLM for teams with no data limits. +

-

-
-
- NotebookLM for Teams +
+ +
+
-

-

- Connect any LLM to your internal knowledge sources and chat with it in real time alongside - your team. -

-
- - {/* */} -
-
- +
); @@ -158,193 +156,155 @@ function GetStartedButton() { if (isGoogleAuth) { return ( - - {/* Animated gradient background on hover */} - - {/* Google logo with subtle animation */} - - - - Continue with Google - + + Continue with Google + ); } return ( - - - Get Started - - + + Get Started + ); } -const BackgroundGrids = () => { - return ( -
-
- - -
-
- - -
-
- - -
-
- - -
-
- ); -}; +const BrowserWindow = () => { + const [selectedIndex, setSelectedIndex] = useState(0); + const selectedItem = TAB_ITEMS[selectedIndex]; + const intervalRef = useRef(null); + const { expanded, open, close } = useExpandedMedia(); -const CollisionMechanism = ({ - parentRef, - beamOptions = {}, -}: { - parentRef: React.RefObject; - beamOptions?: { - initialX?: number; - translateX?: number; - initialY?: number; - translateY?: number; - rotate?: number; - className?: string; - duration?: number; - delay?: number; - repeatDelay?: number; - }; -}) => { - const beamRef = useRef(null); - const [collision, setCollision] = useState<{ - detected: boolean; - coordinates: { x: number; y: number } | null; - }>({ detected: false, coordinates: null }); - const [beamKey, setBeamKey] = useState(0); - const [cycleCollisionDetected, setCycleCollisionDetected] = useState(false); + const startInterval = useCallback(() => { + if (intervalRef.current) { + clearInterval(intervalRef.current); + } + intervalRef.current = setInterval(() => { + setSelectedIndex((prev) => (prev + 1) % TAB_ITEMS.length); + }, 10000); + }, []); useEffect(() => { - const checkCollision = () => { - if (beamRef.current && parentRef.current && !cycleCollisionDetected) { - const beamRect = beamRef.current.getBoundingClientRect(); - const parentRect = parentRef.current.getBoundingClientRect(); - const rightEdge = parentRect.right; - - if (beamRect.right >= rightEdge - 20) { - const relativeX = parentRect.width - 20; - const relativeY = beamRect.top - parentRect.top + beamRect.height / 2; - - setCollision({ - detected: true, - coordinates: { x: relativeX, y: relativeY }, - }); - setCycleCollisionDetected(true); - if (beamRef.current) { - beamRef.current.style.opacity = "0"; - } - } - } - }; - - const animationInterval = setInterval(checkCollision, 100); - - return () => clearInterval(animationInterval); - }, [cycleCollisionDetected, parentRef]); - - useEffect(() => { - if (!collision.detected || !collision.coordinates) return; - - const timer1 = setTimeout(() => { - setCollision({ detected: false, coordinates: null }); - setCycleCollisionDetected(false); - if (beamRef.current) { - beamRef.current.style.opacity = "1"; - } - }, 2000); - - const timer2 = setTimeout(() => { - setBeamKey((prevKey) => prevKey + 1); - }, 2000); - + startInterval(); return () => { - clearTimeout(timer1); - clearTimeout(timer2); + if (intervalRef.current) { + clearInterval(intervalRef.current); + } }; - }, [collision]); + }, [startInterval]); + + const handleTabClick = (index: number) => { + setSelectedIndex(index); + startInterval(); + }; return ( <> - + +
+
+
+
+
+
+
+ {TAB_ITEMS.map((item, index) => ( + + + {index !== TAB_ITEMS.length - 1 && ( +
+ )} + + ))} +
+
+
+ + +
+
+

+ {selectedItem.title} +

+

+ {selectedItem.description} +

+
+
+ {/* biome-ignore lint/a11y/useKeyWithClickEvents: wrapper for video expand */} +
+ +
+ + +
+ + - {collision.detected && collision.coordinates && ( - )} @@ -352,62 +312,92 @@ const CollisionMechanism = ({ ); }; -const Explosion = ({ ...props }: React.HTMLProps) => { - const spans = Array.from({ length: 20 }, (_, index) => ({ - id: index, - initialX: 0, - initialY: 0, - directionX: Math.floor(Math.random() * 80 - 40), - directionY: Math.floor(Math.random() * -50 - 10), - })); +const TabVideo = memo(function TabVideo({ src }: { src: string }) { + const videoRef = useRef(null); + const [hasLoaded, setHasLoaded] = useState(false); + + useEffect(() => { + setHasLoaded(false); + const video = videoRef.current; + if (!video) return; + video.currentTime = 0; + video.play().catch(() => { }); + }, [src]); + + const handleCanPlay = useCallback(() => { + setHasLoaded(true); + }, []); return ( -
- - {spans.map((span) => ( - - ))} +
+
+ ); +} diff --git a/surfsense_web/app/desktop/permissions/page.tsx b/surfsense_web/app/desktop/permissions/page.tsx index 6c08e35b5..178b6a533 100644 --- a/surfsense_web/app/desktop/permissions/page.tsx +++ b/surfsense_web/app/desktop/permissions/page.tsx @@ -5,6 +5,7 @@ import { useRouter } from "next/navigation"; import { Logo } from "@/components/Logo"; import { Button } from "@/components/ui/button"; import { Spinner } from "@/components/ui/spinner"; +import { useElectronAPI } from "@/hooks/use-platform"; type PermissionStatus = "authorized" | "denied" | "not determined" | "restricted" | "limited"; @@ -57,19 +58,18 @@ function StatusBadge({ status }: { status: PermissionStatus }) { export default function DesktopPermissionsPage() { const router = useRouter(); + const api = useElectronAPI(); const [permissions, setPermissions] = useState(null); - const [isElectron, setIsElectron] = useState(false); useEffect(() => { - if (!window.electronAPI) return; - setIsElectron(true); + if (!api) return; let interval: ReturnType | null = null; const isResolved = (s: string) => s === "authorized" || s === "restricted"; const poll = async () => { - const status = await window.electronAPI!.getPermissionsStatus(); + const status = await api.getPermissionsStatus(); setPermissions(status); if (isResolved(status.accessibility) && isResolved(status.screenRecording)) { @@ -80,9 +80,9 @@ export default function DesktopPermissionsPage() { poll(); interval = setInterval(poll, 2000); return () => { if (interval) clearInterval(interval); }; - }, []); + }, [api]); - if (!isElectron) { + if (!api) { return (

This page is only available in the desktop app.

@@ -102,15 +102,15 @@ export default function DesktopPermissionsPage() { const handleRequest = async (action: string) => { if (action === "requestScreenRecording") { - await window.electronAPI!.requestScreenRecording(); + await api.requestScreenRecording(); } else if (action === "requestAccessibility") { - await window.electronAPI!.requestAccessibility(); + await api.requestAccessibility(); } }; const handleContinue = () => { if (allGranted) { - window.electronAPI!.restartApp(); + api.restartApp(); } }; diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index 097047bb1..fb83e2113 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -1,6 +1,7 @@ "use client"; import { useCallback, useEffect, useRef, useState } from "react"; +import { useElectronAPI } from "@/hooks/use-platform"; import { getBearerToken, ensureTokensFromElectron } from "@/lib/auth-utils"; type SSEEvent = @@ -34,26 +35,27 @@ function friendlyError(raw: string | number): string { const AUTO_DISMISS_MS = 3000; export default function SuggestionPage() { + const api = useElectronAPI(); const [suggestion, setSuggestion] = useState(""); const [isLoading, setIsLoading] = useState(true); - const [isDesktop, setIsDesktop] = useState(true); const [error, setError] = useState(null); const abortRef = useRef(null); + const isDesktop = !!api?.onAutocompleteContext; + useEffect(() => { - if (!window.electronAPI?.onAutocompleteContext) { - setIsDesktop(false); + if (!api?.onAutocompleteContext) { setIsLoading(false); } - }, []); + }, [api]); useEffect(() => { if (!error) return; const timer = setTimeout(() => { - window.electronAPI?.dismissSuggestion?.(); + api?.dismissSuggestion?.(); }, AUTO_DISMISS_MS); return () => clearTimeout(timer); - }, [error]); + }, [error, api]); const fetchSuggestion = useCallback( async (screenshot: string, searchSpaceId: string, appName?: string, windowTitle?: string) => { @@ -153,9 +155,9 @@ export default function SuggestionPage() { ); useEffect(() => { - if (!window.electronAPI?.onAutocompleteContext) return; + if (!api?.onAutocompleteContext) return; - const cleanup = window.electronAPI.onAutocompleteContext((data) => { + const cleanup = api.onAutocompleteContext((data) => { const searchSpaceId = data.searchSpaceId || "1"; if (data.screenshot) { fetchSuggestion(data.screenshot, searchSpaceId, data.appName, data.windowTitle); @@ -163,7 +165,7 @@ export default function SuggestionPage() { }); return cleanup; - }, [fetchSuggestion]); + }, [fetchSuggestion, api]); if (!isDesktop) { return ( @@ -197,12 +199,12 @@ export default function SuggestionPage() { const handleAccept = () => { if (suggestion) { - window.electronAPI?.acceptSuggestion?.(suggestion); + api?.acceptSuggestion?.(suggestion); } }; const handleDismiss = () => { - window.electronAPI?.dismissSuggestion?.(); + api?.dismissSuggestion?.(); }; if (!suggestion) return null; diff --git a/surfsense_web/app/layout.tsx b/surfsense_web/app/layout.tsx index 784fd3bcf..8ebb0e848 100644 --- a/surfsense_web/app/layout.tsx +++ b/surfsense_web/app/layout.tsx @@ -10,6 +10,7 @@ import { ZeroProvider } from "@/components/providers/ZeroProvider"; import { ThemeProvider } from "@/components/theme/theme-provider"; import { Toaster } from "@/components/ui/sonner"; import { LocaleProvider } from "@/contexts/LocaleContext"; +import { PlatformProvider } from "@/contexts/platform-context"; import { ReactQueryClientProvider } from "@/lib/query-client/query-client.provider"; import { cn } from "@/lib/utils"; @@ -139,15 +140,17 @@ export default function RootLayout({ disableTransitionOnChange defaultTheme="system" > - - - - {children} - - - - - + + + + + {children} + + + + + + diff --git a/surfsense_web/components/UserDropdown.tsx b/surfsense_web/components/UserDropdown.tsx index 197db6287..19dceb06b 100644 --- a/surfsense_web/components/UserDropdown.tsx +++ b/surfsense_web/components/UserDropdown.tsx @@ -15,7 +15,7 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { Spinner } from "@/components/ui/spinner"; -import { logout } from "@/lib/auth-utils"; +import { getLoginPath, logout } from "@/lib/auth-utils"; import { resetUser, trackLogout } from "@/lib/posthog/events"; export function UserDropdown({ @@ -33,22 +33,19 @@ export function UserDropdown({ if (isLoggingOut) return; setIsLoggingOut(true); try { - // Track logout event and reset PostHog identity trackLogout(); resetUser(); - // Revoke refresh token on server and clear all tokens from localStorage await logout(); if (typeof window !== "undefined") { - window.location.href = "/"; + window.location.href = getLoginPath(); } } catch (error) { console.error("Error during logout:", error); - // Even if there's an error, try to clear tokens and redirect await logout(); if (typeof window !== "undefined") { - window.location.href = "/"; + window.location.href = getLoginPath(); } } }; diff --git a/surfsense_web/components/assistant-ui/assistant-message.tsx b/surfsense_web/components/assistant-ui/assistant-message.tsx index 0dcaf6350..d0cada0bd 100644 --- a/surfsense_web/components/assistant-ui/assistant-message.tsx +++ b/surfsense_web/components/assistant-ui/assistant-message.tsx @@ -87,6 +87,7 @@ import { } from "@/components/ui/drawer"; import { useComments } from "@/hooks/use-comments"; import { useMediaQuery } from "@/hooks/use-media-query"; +import { useElectronAPI } from "@/hooks/use-platform"; import { cn } from "@/lib/utils"; // Dynamically import video presentation tool to avoid loading Babel and Remotion in main bundle @@ -463,16 +464,17 @@ export const AssistantMessage: FC = () => { const AssistantActionBar: FC = () => { const isLast = useAuiState((s) => s.message.isLast); const aui = useAui(); + const api = useElectronAPI(); const [quickAskMode, setQuickAskMode] = useState(""); useEffect(() => { - if (!isLast || !window.electronAPI?.getQuickAskMode) return; - window.electronAPI.getQuickAskMode().then((mode) => { + if (!isLast || !api?.getQuickAskMode) return; + api.getQuickAskMode().then((mode) => { if (mode) setQuickAskMode(mode); }); - }, [isLast]); + }, [isLast, api]); - const isTransform = isLast && !!window.electronAPI?.replaceText && quickAskMode === "transform"; + const isTransform = isLast && !!api?.replaceText && quickAskMode === "transform"; return ( { type="button" onClick={() => { const text = aui.message().getCopyText(); - window.electronAPI?.replaceText(text); + api?.replaceText(text); }} className="ml-1 inline-flex items-center gap-1.5 rounded-md bg-primary px-3 py-1.5 text-xs font-medium text-primary-foreground transition-colors hover:bg-primary/90" > diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx index 3e8aad620..4a97863fb 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx @@ -3,6 +3,7 @@ import type { FC } from "react"; import { EnumConnectorName } from "@/contracts/enums/connector"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; +import { usePlatform } from "@/hooks/use-platform"; import { isSelfHosted } from "@/lib/env-config"; import { ConnectorCard } from "../components/connector-card"; import { @@ -74,9 +75,8 @@ export const AllConnectorsTab: FC = ({ onManage, onViewAccountsList, }) => { - // Check if self-hosted mode (for showing self-hosted only connectors) const selfHosted = isSelfHosted(); - const isDesktop = typeof window !== "undefined" && !!window.electronAPI; + const { isDesktop } = usePlatform(); const matchesSearch = (title: string, description: string) => title.toLowerCase().includes(searchQuery.toLowerCase()) || diff --git a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx index af7a8397c..2d55f4d20 100644 --- a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx +++ b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx @@ -24,6 +24,7 @@ export interface MentionedDocument { export interface InlineMentionEditorRef { focus: () => void; clear: () => void; + setText: (text: string) => void; getText: () => string; getMentionedDocuments: () => MentionedDocument[]; insertDocumentChip: (doc: Pick) => void; @@ -397,6 +398,19 @@ export const InlineMentionEditor = forwardRef { + if (!editorRef.current) return; + editorRef.current.innerText = text; + const empty = text.length === 0; + setIsEmpty(empty); + onChange?.(text, Array.from(mentionedDocs.values())); + focusAtEnd(); + }, + [focusAtEnd, onChange, mentionedDocs] + ); + const setDocumentChipStatus = useCallback( ( docId: number, @@ -469,6 +483,7 @@ export const InlineMentionEditor = forwardRef ({ focus: () => editorRef.current?.focus(), clear, + setText, getText, getMentionedDocuments, insertDocumentChip, diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index 597fcce39..7d8765399 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -89,6 +89,7 @@ import type { Document } from "@/contracts/types/document.types"; import { useBatchCommentsPreload } from "@/hooks/use-comments"; import { useCommentsSync } from "@/hooks/use-comments-sync"; import { useMediaQuery } from "@/hooks/use-media-query"; +import { useElectronAPI } from "@/hooks/use-platform"; import { cn } from "@/lib/utils"; /** Placeholder texts that cycle in new chats when input is empty */ @@ -362,18 +363,19 @@ const Composer: FC = () => { }; }, []); + const electronAPI = useElectronAPI(); const [clipboardInitialText, setClipboardInitialText] = useState(); const clipboardLoadedRef = useRef(false); useEffect(() => { - if (!window.electronAPI || clipboardLoadedRef.current) return; + if (!electronAPI || clipboardLoadedRef.current) return; clipboardLoadedRef.current = true; - window.electronAPI.getQuickAskText().then((text) => { + electronAPI.getQuickAskText().then((text) => { if (text) { setClipboardInitialText(text); setShowPromptPicker(true); } }); - }, []); + }, [electronAPI]); const isThreadEmpty = useAuiState(({ thread }) => thread.isEmpty); const isThreadRunning = useAuiState(({ thread }) => thread.isRunning); @@ -504,34 +506,28 @@ const Composer: FC = () => { : userText ? `${action.prompt}\n\n${userText}` : action.prompt; + editorRef.current?.setText(finalPrompt); aui.composer().setText(finalPrompt); - aui.composer().send(); - editorRef.current?.clear(); setShowPromptPicker(false); setActionQuery(""); - setMentionedDocuments([]); - setSidebarDocs([]); }, - [actionQuery, aui, setMentionedDocuments, setSidebarDocs] + [actionQuery, aui] ); const handleQuickAskSelect = useCallback( (action: { name: string; prompt: string; mode: "transform" | "explore" }) => { if (!clipboardInitialText) return; - window.electronAPI?.setQuickAskMode(action.mode); + electronAPI?.setQuickAskMode(action.mode); const finalPrompt = action.prompt.includes("{selection}") ? action.prompt.replace("{selection}", () => clipboardInitialText) : `${action.prompt}\n\n${clipboardInitialText}`; + editorRef.current?.setText(finalPrompt); aui.composer().setText(finalPrompt); - aui.composer().send(); - editorRef.current?.clear(); setShowPromptPicker(false); setActionQuery(""); setClipboardInitialText(undefined); - setMentionedDocuments([]); - setSidebarDocs([]); }, - [clipboardInitialText, aui, setMentionedDocuments, setSidebarDocs] + [clipboardInitialText, electronAPI, aui] ); // Keyboard navigation for document/action picker (arrow keys, Enter, Escape) diff --git a/surfsense_web/components/desktop/shortcut-recorder.tsx b/surfsense_web/components/desktop/shortcut-recorder.tsx new file mode 100644 index 000000000..0c0012002 --- /dev/null +++ b/surfsense_web/components/desktop/shortcut-recorder.tsx @@ -0,0 +1,168 @@ +"use client"; + +import { RotateCcw } from "lucide-react"; +import { useCallback, useRef, useState } from "react"; +import { Button } from "@/components/ui/button"; +import { cn } from "@/lib/utils"; + +// --------------------------------------------------------------------------- +// Accelerator ↔ display helpers +// --------------------------------------------------------------------------- + +export function keyEventToAccelerator(e: React.KeyboardEvent): string | null { + const parts: string[] = []; + if (e.ctrlKey || e.metaKey) parts.push("CommandOrControl"); + if (e.altKey) parts.push("Alt"); + if (e.shiftKey) parts.push("Shift"); + + const key = e.key; + if (["Control", "Meta", "Alt", "Shift"].includes(key)) return null; + + if (key === " ") parts.push("Space"); + else if (key.length === 1) parts.push(key.toUpperCase()); + else parts.push(key); + + if (parts.length < 2) return null; + return parts.join("+"); +} + +export function acceleratorToDisplay(accel: string): string[] { + if (!accel) return []; + return accel.split("+").map((part) => { + if (part === "CommandOrControl") return "Ctrl"; + if (part === "Space") return "Space"; + return part; + }); +} + +export const DEFAULT_SHORTCUTS = { + quickAsk: "CommandOrControl+Alt+S", + autocomplete: "CommandOrControl+Shift+Space", +}; + +// --------------------------------------------------------------------------- +// Kbd pill component +// --------------------------------------------------------------------------- + +export function Kbd({ + keys, + className, +}: { + keys: string[]; + className?: string; +}) { + return ( + + {keys.map((key) => ( + 3 && "px-2" + )} + > + {key} + + ))} + + ); +} + +// --------------------------------------------------------------------------- +// Shortcut recorder component +// --------------------------------------------------------------------------- + +export function ShortcutRecorder({ + value, + onChange, + onReset, + defaultValue, + label, + description, + icon: Icon, +}: { + value: string; + onChange: (accelerator: string) => void; + onReset: () => void; + defaultValue: string; + label: string; + description: string; + icon: React.ElementType; +}) { + const [recording, setRecording] = useState(false); + const inputRef = useRef(null); + + const handleKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (!recording) return; + e.preventDefault(); + e.stopPropagation(); + + if (e.key === "Escape") { + setRecording(false); + return; + } + + const accel = keyEventToAccelerator(e); + if (accel) { + onChange(accel); + setRecording(false); + } + }, + [recording, onChange] + ); + + const displayKeys = acceleratorToDisplay(value); + const isDefault = value === defaultValue; + + return ( +
+
+
+ +
+
+

{label}

+

+ {description} +

+
+
+ +
+ {!isDefault && ( + + )} + +
+
+ ); +} diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index 6138b67fb..380ffa656 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -55,7 +55,7 @@ import { useInbox } from "@/hooks/use-inbox"; import { useIsMobile } from "@/hooks/use-mobile"; import { notificationsApiService } from "@/lib/apis/notifications-api.service"; import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service"; -import { logout } from "@/lib/auth-utils"; +import { getLoginPath, logout } from "@/lib/auth-utils"; import { deleteThread, fetchThreads, updateThread } from "@/lib/chat/thread-persistence"; import { resetUser, trackLogout } from "@/lib/posthog/events"; import { cacheKeys } from "@/lib/query-client/cache-keys"; @@ -600,12 +600,12 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid await logout(); if (typeof window !== "undefined") { - router.push("/"); + router.push(getLoginPath()); } } catch (error) { console.error("Error during logout:", error); await logout(); - router.push("/"); + router.push(getLoginPath()); } }, [router]); diff --git a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx index aa409e179..f19b20971 100644 --- a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx @@ -41,6 +41,7 @@ import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import type { DocumentTypeEnum } from "@/contracts/types/document.types"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { useMediaQuery } from "@/hooks/use-media-query"; +import { useElectronAPI } from "@/hooks/use-platform"; import { documentsApiService } from "@/lib/apis/documents-api.service"; import { foldersApiService } from "@/lib/apis/folders-api.service"; import { authenticatedFetch } from "@/lib/auth-utils"; @@ -84,6 +85,7 @@ export function DocumentsSidebar({ const tSidebar = useTranslations("sidebar"); const params = useParams(); const isMobile = !useMediaQuery("(min-width: 640px)"); + const electronAPI = useElectronAPI(); const searchSpaceId = Number(params.search_space_id); const setConnectorDialogOpen = useSetAtom(connectorDialogOpenAtom); const setRightPanelCollapsed = useSetAtom(rightPanelCollapsedAtom); @@ -97,11 +99,11 @@ export function DocumentsSidebar({ const [watchedFolderIds, setWatchedFolderIds] = useState>(new Set()); useEffect(() => { - const api = typeof window !== "undefined" ? window.electronAPI : null; - if (!api?.getWatchedFolders) return; + if (!electronAPI?.getWatchedFolders) return; + const api = electronAPI; async function loadWatchedIds() { - const folders = await api!.getWatchedFolders(); + const folders = await api.getWatchedFolders(); if (folders.length === 0) { try { @@ -109,7 +111,7 @@ export function DocumentsSidebar({ for (const bf of backendFolders) { const meta = bf.metadata as Record | null; if (!meta?.watched || !meta.folder_path) continue; - await api!.addWatchedFolder({ + await api.addWatchedFolder({ path: meta.folder_path as string, name: bf.name, rootFolderId: bf.id, @@ -119,7 +121,7 @@ export function DocumentsSidebar({ active: true, }); } - const recovered = await api!.getWatchedFolders(); + const recovered = await api.getWatchedFolders(); const ids = new Set( recovered.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number) ); @@ -137,7 +139,7 @@ export function DocumentsSidebar({ } loadWatchedIds(); - }, [searchSpaceId]); + }, [searchSpaceId, electronAPI]); const { mutateAsync: deleteDocumentMutation } = useAtomValue(deleteDocumentMutationAtom); const [sidebarDocs, setSidebarDocs] = useAtom(sidebarSelectedDocumentsAtom); @@ -276,10 +278,9 @@ export function DocumentsSidebar({ const handleRescanFolder = useCallback( async (folder: FolderDisplay) => { - const api = window.electronAPI; - if (!api) return; + if (!electronAPI) return; - const watchedFolders = await api.getWatchedFolders(); + const watchedFolders = await electronAPI.getWatchedFolders(); const matched = watchedFolders.find((wf) => wf.rootFolderId === folder.id); if (!matched) { toast.error("This folder is not being watched"); @@ -298,28 +299,27 @@ export function DocumentsSidebar({ toast.error((err as Error)?.message || "Failed to re-scan folder"); } }, - [searchSpaceId] + [searchSpaceId, electronAPI] ); const handleStopWatching = useCallback(async (folder: FolderDisplay) => { - const api = window.electronAPI; - if (!api) return; + if (!electronAPI) return; - const watchedFolders = await api.getWatchedFolders(); + const watchedFolders = await electronAPI.getWatchedFolders(); const matched = watchedFolders.find((wf) => wf.rootFolderId === folder.id); if (!matched) { toast.error("This folder is not being watched"); return; } - await api.removeWatchedFolder(matched.path); + await electronAPI.removeWatchedFolder(matched.path); try { await foldersApiService.stopWatching(folder.id); } catch (err) { console.error("[DocumentsSidebar] Failed to clear watched metadata:", err); } toast.success(`Stopped watching: ${matched.name}`); - }, []); + }, [electronAPI]); const handleRenameFolder = useCallback(async (folder: FolderDisplay, newName: string) => { try { @@ -333,12 +333,11 @@ export function DocumentsSidebar({ const handleDeleteFolder = useCallback(async (folder: FolderDisplay) => { if (!confirm(`Delete folder "${folder.name}" and all its contents?`)) return; try { - const api = window.electronAPI; - if (api) { - const watchedFolders = await api.getWatchedFolders(); + if (electronAPI) { + const watchedFolders = await electronAPI.getWatchedFolders(); const matched = watchedFolders.find((wf) => wf.rootFolderId === folder.id); if (matched) { - await api.removeWatchedFolder(matched.path); + await electronAPI.removeWatchedFolder(matched.path); } } await foldersApiService.deleteFolder(folder.id); @@ -346,7 +345,7 @@ export function DocumentsSidebar({ } catch (e: unknown) { toast.error((e as Error)?.message || "Failed to delete folder"); } - }, []); + }, [electronAPI]); const handleMoveFolder = useCallback( (folder: FolderDisplay) => { diff --git a/surfsense_web/components/platform-gate.tsx b/surfsense_web/components/platform-gate.tsx new file mode 100644 index 000000000..6908c6d32 --- /dev/null +++ b/surfsense_web/components/platform-gate.tsx @@ -0,0 +1,16 @@ +"use client"; + +import type { ReactNode } from "react"; +import { usePlatform } from "@/hooks/use-platform"; + +export function DesktopOnly({ children }: { children: ReactNode }) { + const { isDesktop } = usePlatform(); + if (!isDesktop) return null; + return <>{children}; +} + +export function WebOnly({ children }: { children: ReactNode }) { + const { isWeb } = usePlatform(); + if (!isWeb) return null; + return <>{children}; +} diff --git a/surfsense_web/components/settings/user-settings-dialog.tsx b/surfsense_web/components/settings/user-settings-dialog.tsx index b74ff973b..919b08174 100644 --- a/surfsense_web/components/settings/user-settings-dialog.tsx +++ b/surfsense_web/components/settings/user-settings-dialog.tsx @@ -3,6 +3,7 @@ import { useAtom } from "jotai"; import { Globe, KeyRound, Monitor, Receipt, Sparkles, User } from "lucide-react"; import { useTranslations } from "next-intl"; +import { useMemo } from "react"; import { ApiKeyContent } from "@/app/dashboard/[search_space_id]/user-settings/components/ApiKeyContent"; import { CommunityPromptsContent } from "@/app/dashboard/[search_space_id]/user-settings/components/CommunityPromptsContent"; import { ProfileContent } from "@/app/dashboard/[search_space_id]/user-settings/components/ProfileContent"; @@ -11,37 +12,42 @@ import { PurchaseHistoryContent } from "@/app/dashboard/[search_space_id]/user-s import { DesktopContent } from "@/app/dashboard/[search_space_id]/user-settings/components/DesktopContent"; import { userSettingsDialogAtom } from "@/atoms/settings/settings-dialog.atoms"; import { SettingsDialog } from "@/components/settings/settings-dialog"; +import { usePlatform } from "@/hooks/use-platform"; export function UserSettingsDialog() { const t = useTranslations("userSettings"); const [state, setState] = useAtom(userSettingsDialogAtom); + const { isDesktop } = usePlatform(); - const navItems = [ - { value: "profile", label: t("profile_nav_label"), icon: }, - { - value: "api-key", - label: t("api_key_nav_label"), - icon: , - }, - { - value: "prompts", - label: "My Prompts", - icon: , - }, - { - value: "community-prompts", - label: "Community Prompts", - icon: , - }, - { - value: "purchases", - label: "Purchase History", - icon: , - }, - ...(typeof window !== "undefined" && window.electronAPI - ? [{ value: "desktop", label: "Desktop", icon: }] - : []), - ]; + const navItems = useMemo( + () => [ + { value: "profile", label: t("profile_nav_label"), icon: }, + { + value: "api-key", + label: t("api_key_nav_label"), + icon: , + }, + { + value: "prompts", + label: "My Prompts", + icon: , + }, + { + value: "community-prompts", + label: "Community Prompts", + icon: , + }, + { + value: "purchases", + label: "Purchase History", + icon: , + }, + ...(isDesktop + ? [{ value: "desktop", label: "Desktop", icon: }] + : []), + ], + [t, isDesktop] + ); return ( (null); const [watchFolder, setWatchFolder] = useState(true); const [folderSubmitting, setFolderSubmitting] = useState(false); - const isElectron = typeof window !== "undefined" && !!window.electronAPI?.browseFiles; + const isElectron = !!electronAPI?.browseFiles; const acceptedFileTypes = useMemo(() => { const etlService = process.env.NEXT_PUBLIC_ETL_SERVICE; @@ -216,33 +218,31 @@ export function DocumentUploadTab({ }, []); const handleBrowseFiles = useCallback(async () => { - const api = window.electronAPI; - if (!api?.browseFiles) return; + if (!electronAPI?.browseFiles) return; - const paths = await api.browseFiles(); + const paths = await electronAPI.browseFiles(); if (!paths || paths.length === 0) return; setSelectedFolder(null); - const fileDataList = await api.readLocalFiles(paths); + const fileDataList = await electronAPI.readLocalFiles(paths); const newFiles: FileWithId[] = fileDataList.map((fd) => ({ id: crypto.randomUUID?.() ?? `file-${Date.now()}-${Math.random().toString(36)}`, file: new File([fd.data], fd.name, { type: fd.mimeType }), })); setFiles((prev) => [...prev, ...newFiles]); - }, []); + }, [electronAPI]); const handleBrowseFolder = useCallback(async () => { - const api = window.electronAPI; - if (!api?.selectFolder) return; + if (!electronAPI?.selectFolder) return; - const folderPath = await api.selectFolder(); + const folderPath = await electronAPI.selectFolder(); if (!folderPath) return; const folderName = folderPath.split("/").pop() || folderPath.split("\\").pop() || folderPath; setFiles([]); setSelectedFolder({ path: folderPath, name: folderName }); setWatchFolder(true); - }, []); + }, [electronAPI]); const handleFolderChange = useCallback( (e: ChangeEvent) => { @@ -287,9 +287,7 @@ export function DocumentUploadTab({ ); const handleFolderSubmit = useCallback(async () => { - if (!selectedFolder) return; - const api = window.electronAPI; - if (!api) return; + if (!selectedFolder || !electronAPI) return; setFolderSubmitting(true); try { @@ -304,7 +302,7 @@ export function DocumentUploadTab({ const rootFolderId = (result as { root_folder_id?: number })?.root_folder_id ?? null; if (watchFolder) { - await api.addWatchedFolder({ + await electronAPI.addWatchedFolder({ path: selectedFolder.path, name: selectedFolder.name, excludePatterns: [ @@ -332,7 +330,7 @@ export function DocumentUploadTab({ } finally { setFolderSubmitting(false); } - }, [selectedFolder, watchFolder, searchSpaceId, shouldSummarize, onSuccess]); + }, [selectedFolder, watchFolder, searchSpaceId, shouldSummarize, onSuccess, electronAPI]); const handleUpload = async () => { setUploadProgress(0); diff --git a/surfsense_web/contexts/platform-context.tsx b/surfsense_web/contexts/platform-context.tsx new file mode 100644 index 000000000..bb3e3800d --- /dev/null +++ b/surfsense_web/contexts/platform-context.tsx @@ -0,0 +1,31 @@ +"use client"; + +import { createContext, useEffect, useState, type ReactNode } from "react"; + +export interface PlatformContextValue { + isDesktop: boolean; + isWeb: boolean; + electronAPI: ElectronAPI | null; +} + +const SSR_VALUE: PlatformContextValue = { + isDesktop: false, + isWeb: false, + electronAPI: null, +}; + +export const PlatformContext = createContext(SSR_VALUE); + +export function PlatformProvider({ children }: { children: ReactNode }) { + const [value, setValue] = useState(SSR_VALUE); + + useEffect(() => { + const api = window.electronAPI ?? null; + const isDesktop = !!api; + setValue({ isDesktop, isWeb: !isDesktop, electronAPI: api }); + }, []); + + return ( + {children} + ); +} diff --git a/surfsense_web/hooks/use-folder-sync.ts b/surfsense_web/hooks/use-folder-sync.ts index ef3326556..847d0081b 100644 --- a/surfsense_web/hooks/use-folder-sync.ts +++ b/surfsense_web/hooks/use-folder-sync.ts @@ -1,6 +1,7 @@ "use client"; import { useEffect, useRef } from "react"; +import { useElectronAPI } from "@/hooks/use-platform"; import { documentsApiService } from "@/lib/apis/documents-api.service"; interface FileChangedEvent { @@ -29,6 +30,7 @@ interface BatchItem { } export function useFolderSync() { + const electronAPI = useElectronAPI(); const queueRef = useRef([]); const processingRef = useRef(false); const debounceTimers = useRef>>(new Map()); @@ -49,9 +51,8 @@ export function useFolderSync() { target_file_paths: batch.filePaths, root_folder_id: batch.rootFolderId, }); - const api = typeof window !== "undefined" ? window.electronAPI : null; - if (api?.acknowledgeFileEvents && batch.ackIds.length > 0) { - await api.acknowledgeFileEvents(batch.ackIds); + if (electronAPI?.acknowledgeFileEvents && batch.ackIds.length > 0) { + await electronAPI.acknowledgeFileEvents(batch.ackIds); } } catch (err) { console.error("[FolderSync] Failed to trigger batch re-index:", err); @@ -117,25 +118,22 @@ export function useFolderSync() { useEffect(() => { isMountedRef.current = true; - const api = typeof window !== "undefined" ? window.electronAPI : null; - if (!api?.onFileChanged) { + if (!electronAPI?.onFileChanged) { return () => { isMountedRef.current = false; }; } - // Signal to main process that the renderer is ready to receive events - api.signalRendererReady?.(); + electronAPI.signalRendererReady?.(); - // Drain durable outbox first so events survive renderer startup gaps and restarts - void api.getPendingFileEvents?.().then((pendingEvents) => { + void electronAPI.getPendingFileEvents?.().then((pendingEvents) => { if (!isMountedRef.current || !pendingEvents?.length) return; for (const event of pendingEvents) { enqueueWithDebounce(event); } }); - const cleanup = api.onFileChanged((event: FileChangedEvent) => { + const cleanup = electronAPI.onFileChanged((event: FileChangedEvent) => { enqueueWithDebounce(event); }); @@ -149,5 +147,5 @@ export function useFolderSync() { pendingByFolder.current.clear(); firstEventTime.current.clear(); }; - }, []); + }, [electronAPI]); } diff --git a/surfsense_web/hooks/use-platform.ts b/surfsense_web/hooks/use-platform.ts new file mode 100644 index 000000000..dc1f7e914 --- /dev/null +++ b/surfsense_web/hooks/use-platform.ts @@ -0,0 +1,12 @@ +import { useContext } from "react"; +import { PlatformContext, type PlatformContextValue } from "@/contexts/platform-context"; + +export function usePlatform(): Pick { + const { isDesktop, isWeb } = useContext(PlatformContext); + return { isDesktop, isWeb }; +} + +export function useElectronAPI(): ElectronAPI | null { + const { electronAPI } = useContext(PlatformContext); + return electronAPI; +} diff --git a/surfsense_web/lib/auth-utils.ts b/surfsense_web/lib/auth-utils.ts index f7d1c5b09..d66934c3b 100644 --- a/surfsense_web/lib/auth-utils.ts +++ b/surfsense_web/lib/auth-utils.ts @@ -15,6 +15,7 @@ const PUBLIC_ROUTE_PREFIXES = [ "/login", "/register", "/auth", + "/desktop/login", "/docs", "/public", "/invite", @@ -34,6 +35,11 @@ export function isPublicRoute(pathname: string): boolean { return PUBLIC_ROUTE_PREFIXES.some((prefix) => pathname.startsWith(prefix)); } +export function getLoginPath(): string { + if (typeof window !== "undefined" && window.electronAPI) return "/desktop/login"; + return "/login"; +} + /** * Clears tokens and optionally redirects to login. * Call this when a 401 response is received. @@ -55,7 +61,7 @@ export function handleUnauthorized(): void { if (!excludedPaths.includes(pathname)) { localStorage.setItem(REDIRECT_PATH_KEY, currentPath); } - window.location.href = "/login"; + window.location.href = getLoginPath(); } } @@ -221,13 +227,12 @@ export function redirectToLogin(): void { const currentPath = window.location.pathname + window.location.search + window.location.hash; // Don't save auth-related paths or home page - const excludedPaths = ["/auth", "/auth/callback", "/", "/login", "/register"]; + const excludedPaths = ["/auth", "/auth/callback", "/", "/login", "/register", "/desktop/login"]; if (!excludedPaths.includes(window.location.pathname)) { localStorage.setItem(REDIRECT_PATH_KEY, currentPath); } - // Redirect to login page - window.location.href = "/login"; + window.location.href = getLoginPath(); } /** diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index 5e45635a2..615b861ea 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -81,6 +81,9 @@ interface ElectronAPI { // Auth token sync across windows getAuthTokens: () => Promise<{ bearer: string; refresh: string } | null>; setAuthTokens: (bearer: string, refresh: string) => Promise; + // Keyboard shortcut configuration + getShortcuts: () => Promise<{ quickAsk: string; autocomplete: string }>; + setShortcuts: (config: Partial<{ quickAsk: string; autocomplete: string }>) => Promise<{ quickAsk: string; autocomplete: string }>; } declare global { From bb1dcd32b6b89ca8bcfed3c606ff227759614904 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 7 Apr 2026 02:49:24 -0700 Subject: [PATCH 06/47] feat: enhance vision autocomplete service and UI feedback - Optimized the vision autocomplete service by starting the SSE stream immediately and deriving KB search queries directly from window titles. - Refactored the service to run KB filesystem pre-computation and agent graph compilation in parallel, improving performance. - Updated the SuggestionPage component to handle new agent step data, displaying progress indicators for each step. - Enhanced the CSS for the suggestion tooltip and agent activity indicators, improving the user interface and experience. --- .../app/agents/autocomplete/__init__.py | 11 + .../agents/autocomplete/autocomplete_agent.py | 429 ++++++++++++++++++ .../services/vision_autocomplete_service.py | 258 ++++------- surfsense_web/app/desktop/suggestion/page.tsx | 68 ++- .../app/desktop/suggestion/suggestion.css | 114 ++++- .../components/assistant-ui/thread.tsx | 34 +- 6 files changed, 686 insertions(+), 228 deletions(-) create mode 100644 surfsense_backend/app/agents/autocomplete/__init__.py create mode 100644 surfsense_backend/app/agents/autocomplete/autocomplete_agent.py diff --git a/surfsense_backend/app/agents/autocomplete/__init__.py b/surfsense_backend/app/agents/autocomplete/__init__.py new file mode 100644 index 000000000..55d7a692d --- /dev/null +++ b/surfsense_backend/app/agents/autocomplete/__init__.py @@ -0,0 +1,11 @@ +"""Agent-based vision autocomplete with scoped filesystem exploration.""" + +from app.agents.autocomplete.autocomplete_agent import ( + create_autocomplete_agent, + stream_autocomplete_agent, +) + +__all__ = [ + "create_autocomplete_agent", + "stream_autocomplete_agent", +] diff --git a/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py b/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py new file mode 100644 index 000000000..928a133cc --- /dev/null +++ b/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py @@ -0,0 +1,429 @@ +"""Vision autocomplete agent with scoped filesystem exploration. + +Converts the stateless single-shot vision autocomplete into an agent that +seeds a virtual filesystem from KB search results and lets the vision LLM +explore documents via ``ls``, ``read_file``, ``glob``, ``grep``, etc. +before generating the final completion. + +Performance: KB search and agent graph compilation run in parallel so +the only sequential latency is KB-search (or agent compile, whichever is +slower) + the agent's LLM turns. There is no separate "query extraction" +LLM call — the window title is used directly as the KB search query. +""" + +from __future__ import annotations + +import asyncio +import logging +import uuid +from typing import Any, AsyncGenerator + +from deepagents.graph import BASE_AGENT_PROMPT +from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware +from langchain.agents import create_agent +from langchain_anthropic.middleware import AnthropicPromptCachingMiddleware +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, ToolMessage + +from app.agents.new_chat.middleware.filesystem import SurfSenseFilesystemMiddleware +from app.agents.new_chat.middleware.knowledge_search import ( + build_scoped_filesystem, + search_knowledge_base, +) +from app.services.new_streaming_service import VercelStreamingService + +logger = logging.getLogger(__name__) + +KB_TOP_K = 10 + +# --------------------------------------------------------------------------- +# System prompt +# --------------------------------------------------------------------------- + +AUTOCOMPLETE_SYSTEM_PROMPT = """You are a smart writing assistant that analyzes the user's screen to draft or complete text. + +You will receive a screenshot of the user's screen. Your PRIMARY source of truth is the screenshot itself — the visual context determines what to write. + +Your job: +1. Analyze the ENTIRE screenshot to understand what the user is working on (email thread, chat conversation, document, code editor, form, etc.). +2. Identify the text area where the user will type. +3. Generate the text the user most likely wants to write based on the visual context. + +You also have access to the user's knowledge base documents via filesystem tools. However: +- ONLY consult the knowledge base if the screenshot clearly involves a topic where your KB documents are DIRECTLY relevant (e.g., the user is writing about a specific project/topic that matches a document title). +- Do NOT explore documents just because they exist. Most autocomplete requests can be answered purely from the screenshot. +- If you do read a document, only incorporate information that is 100% relevant to what the user is typing RIGHT NOW. Do not add extra details, background, or tangential information from the KB. +- Keep your output SHORT — autocomplete should feel like a natural continuation, not an essay. + +Key behavior: +- If the text area is EMPTY, draft a concise response or message based on what you see on screen (e.g., reply to an email, respond to a chat message, continue a document). +- If the text area already has text, continue it naturally — typically just a sentence or two. + +Rules: +- Output ONLY the text to be inserted. No quotes, no explanations, no meta-commentary. +- Be CONCISE. Prefer a single paragraph or a few sentences. Autocomplete is a quick assist, not a full draft. +- Match the tone and formality of the surrounding context. +- If the screen shows code, write code. If it shows a casual chat, be casual. If it shows a formal email, be formal. +- Do NOT describe the screenshot or explain your reasoning. +- Do NOT cite or reference documents explicitly — just let the knowledge inform your writing naturally. +- If you cannot determine what to write, output nothing. + +## Filesystem Tools `ls`, `read_file`, `write_file`, `edit_file`, `glob`, `grep` + +All file paths must start with a `/`. +- ls: list files and directories at a given path. +- read_file: read a file from the filesystem. +- write_file: create a temporary file in the session (not persisted). +- edit_file: edit a file in the session (not persisted for /documents/ files). +- glob: find files matching a pattern (e.g., "**/*.xml"). +- grep: search for text within files. + +## When to Use Filesystem Tools + +BEFORE reaching for any tool, ask yourself: "Can I write a good completion purely from the screenshot?" If yes, just write it — do NOT explore the KB. + +Only use tools when: +- The user is clearly writing about a specific topic that likely has detailed information in their KB. +- You need a specific fact, name, number, or reference that the screenshot doesn't provide. + +When you do use tools, be surgical: +- Check the `ls` output first. If no document title looks relevant, stop — do not read files just to see what's there. +- If a title looks relevant, read only the `` (first ~20 lines) and jump to matched chunks. Do not read entire documents. +- Extract only the specific information you need and move on to generating the completion. + +## Reading Documents Efficiently + +Documents are formatted as XML. Each document contains: +- `` — title, type, URL, etc. +- `` — a table of every chunk with its **line range** and a + `matched="true"` flag for chunks that matched the search query. +- `` — the actual chunks in original document order. + +**Workflow**: read the first ~20 lines to see the ``, identify +chunks marked `matched="true"`, then use `read_file(path, offset=, +limit=)` to jump directly to those sections.""" + +APP_CONTEXT_BLOCK = """ + +The user is currently working in "{app_name}" (window: "{window_title}"). Use this to understand the type of application and adapt your tone and format accordingly.""" + + +def _build_autocomplete_system_prompt(app_name: str, window_title: str) -> str: + prompt = AUTOCOMPLETE_SYSTEM_PROMPT + if app_name: + prompt += APP_CONTEXT_BLOCK.format(app_name=app_name, window_title=window_title) + return prompt + + +# --------------------------------------------------------------------------- +# Pre-compute KB filesystem (runs in parallel with agent compilation) +# --------------------------------------------------------------------------- + + +class _KBResult: + """Container for pre-computed KB filesystem results.""" + __slots__ = ("files", "ls_ai_msg", "ls_tool_msg") + + def __init__( + self, + files: dict[str, Any] | None = None, + ls_ai_msg: AIMessage | None = None, + ls_tool_msg: ToolMessage | None = None, + ) -> None: + self.files = files + self.ls_ai_msg = ls_ai_msg + self.ls_tool_msg = ls_tool_msg + + @property + def has_documents(self) -> bool: + return bool(self.files) + + +async def precompute_kb_filesystem( + search_space_id: int, + query: str, + top_k: int = KB_TOP_K, +) -> _KBResult: + """Search the KB and build the scoped filesystem outside the agent. + + This is designed to be called via ``asyncio.gather`` alongside agent + graph compilation so the two run concurrently. + """ + if not query: + return _KBResult() + + try: + search_results = await search_knowledge_base( + query=query, + search_space_id=search_space_id, + top_k=top_k, + ) + + if not search_results: + return _KBResult() + + new_files, _ = await build_scoped_filesystem( + documents=search_results, + search_space_id=search_space_id, + ) + + if not new_files: + return _KBResult() + + doc_paths = [ + p for p, v in new_files.items() + if p.startswith("/documents/") and v is not None + ] + tool_call_id = f"auto_ls_{uuid.uuid4().hex[:12]}" + ai_msg = AIMessage( + content="", + tool_calls=[{"name": "ls", "args": {"path": "/documents"}, "id": tool_call_id}], + ) + tool_msg = ToolMessage( + content=str(doc_paths) if doc_paths else "No documents found.", + tool_call_id=tool_call_id, + ) + return _KBResult(files=new_files, ls_ai_msg=ai_msg, ls_tool_msg=tool_msg) + + except Exception: + logger.warning("KB pre-computation failed, proceeding without KB", exc_info=True) + return _KBResult() + + +# --------------------------------------------------------------------------- +# Filesystem middleware — no save_document, no persistence +# --------------------------------------------------------------------------- + + +class AutocompleteFilesystemMiddleware(SurfSenseFilesystemMiddleware): + """Filesystem middleware for autocomplete — read-only exploration only. + + Strips ``save_document`` (permanent KB persistence) and passes + ``search_space_id=None`` so ``write_file`` / ``edit_file`` stay ephemeral. + """ + + def __init__(self) -> None: + super().__init__(search_space_id=None, created_by_id=None) + self.tools = [t for t in self.tools if t.name != "save_document"] + + +# --------------------------------------------------------------------------- +# Agent factory +# --------------------------------------------------------------------------- + + +async def _compile_agent( + llm: BaseChatModel, + app_name: str, + window_title: str, +) -> Any: + """Compile the agent graph (CPU-bound, runs in a thread).""" + system_prompt = _build_autocomplete_system_prompt(app_name, window_title) + final_system_prompt = system_prompt + "\n\n" + BASE_AGENT_PROMPT + + middleware = [ + AutocompleteFilesystemMiddleware(), + PatchToolCallsMiddleware(), + AnthropicPromptCachingMiddleware(unsupported_model_behavior="ignore"), + ] + + agent = await asyncio.to_thread( + create_agent, + llm, + system_prompt=final_system_prompt, + tools=[], + middleware=middleware, + ) + return agent.with_config({"recursion_limit": 200}) + + +async def create_autocomplete_agent( + llm: BaseChatModel, + *, + search_space_id: int, + kb_query: str, + app_name: str = "", + window_title: str = "", +) -> tuple[Any, _KBResult]: + """Create the autocomplete agent and pre-compute KB in parallel. + + Returns ``(agent, kb_result)`` so the caller can inject the pre-computed + filesystem into the agent's initial state without any middleware delay. + """ + agent, kb = await asyncio.gather( + _compile_agent(llm, app_name, window_title), + precompute_kb_filesystem(search_space_id, kb_query), + ) + return agent, kb + + +# --------------------------------------------------------------------------- +# Streaming helper +# --------------------------------------------------------------------------- + + +async def stream_autocomplete_agent( + agent: Any, + input_data: dict[str, Any], + streaming_service: VercelStreamingService, + *, + emit_message_start: bool = True, +) -> AsyncGenerator[str, None]: + """Stream agent events as Vercel SSE, with thinking steps for tool calls. + + When ``emit_message_start`` is False the caller has already sent the + ``message_start`` event (e.g. to show preparation steps before the agent + runs). + """ + thread_id = uuid.uuid4().hex + config = {"configurable": {"thread_id": thread_id}} + + current_text_id: str | None = None + active_tool_depth = 0 + thinking_step_counter = 0 + tool_step_ids: dict[str, str] = {} + step_titles: dict[str, str] = {} + completed_step_ids: set[str] = set() + last_active_step_id: str | None = None + + def next_thinking_step_id() -> str: + nonlocal thinking_step_counter + thinking_step_counter += 1 + return f"autocomplete-step-{thinking_step_counter}" + + def complete_current_step() -> str | None: + nonlocal last_active_step_id + if last_active_step_id and last_active_step_id not in completed_step_ids: + completed_step_ids.add(last_active_step_id) + title = step_titles.get(last_active_step_id, "Done") + event = streaming_service.format_thinking_step( + step_id=last_active_step_id, + title=title, + status="complete", + ) + last_active_step_id = None + return event + return None + + if emit_message_start: + yield streaming_service.format_message_start() + + # Emit an initial "Generating completion" step so the UI immediately + # shows activity once the agent starts its first LLM call. + gen_step_id = next_thinking_step_id() + last_active_step_id = gen_step_id + step_titles[gen_step_id] = "Generating completion" + yield streaming_service.format_thinking_step( + step_id=gen_step_id, + title="Generating completion", + status="in_progress", + ) + + try: + async for event in agent.astream_events(input_data, config=config, version="v2"): + event_type = event.get("event", "") + + if event_type == "on_chat_model_stream": + if active_tool_depth > 0: + continue + if "surfsense:internal" in event.get("tags", []): + continue + chunk = event.get("data", {}).get("chunk") + if chunk and hasattr(chunk, "content"): + content = chunk.content + if content and isinstance(content, str): + if current_text_id is None: + step_event = complete_current_step() + if step_event: + yield step_event + current_text_id = streaming_service.generate_text_id() + yield streaming_service.format_text_start(current_text_id) + yield streaming_service.format_text_delta(current_text_id, content) + + elif event_type == "on_tool_start": + active_tool_depth += 1 + tool_name = event.get("name", "unknown_tool") + run_id = event.get("run_id", "") + tool_input = event.get("data", {}).get("input", {}) + + if current_text_id is not None: + yield streaming_service.format_text_end(current_text_id) + current_text_id = None + + step_event = complete_current_step() + if step_event: + yield step_event + + tool_step_id = next_thinking_step_id() + tool_step_ids[run_id] = tool_step_id + last_active_step_id = tool_step_id + + title, items = _describe_tool_call(tool_name, tool_input) + step_titles[tool_step_id] = title + yield streaming_service.format_thinking_step( + step_id=tool_step_id, + title=title, + status="in_progress", + items=items, + ) + + elif event_type == "on_tool_end": + active_tool_depth = max(0, active_tool_depth - 1) + run_id = event.get("run_id", "") + step_id = tool_step_ids.pop(run_id, None) + if step_id and step_id not in completed_step_ids: + completed_step_ids.add(step_id) + title = step_titles.get(step_id, "Done") + yield streaming_service.format_thinking_step( + step_id=step_id, + title=title, + status="complete", + ) + if last_active_step_id == step_id: + last_active_step_id = None + + if current_text_id is not None: + yield streaming_service.format_text_end(current_text_id) + step_event = complete_current_step() + if step_event: + yield step_event + + yield streaming_service.format_finish() + yield streaming_service.format_done() + + except Exception as e: + logger.error(f"Autocomplete agent streaming error: {e}", exc_info=True) + if current_text_id is not None: + yield streaming_service.format_text_end(current_text_id) + yield streaming_service.format_error("Autocomplete failed. Please try again.") + yield streaming_service.format_done() + + +def _describe_tool_call(tool_name: str, tool_input: Any) -> tuple[str, list[str]]: + """Return a human-readable (title, items) for a tool call thinking step.""" + inp = tool_input if isinstance(tool_input, dict) else {} + if tool_name == "ls": + path = inp.get("path", "/") + return "Listing files", [path] + if tool_name == "read_file": + fp = inp.get("file_path", "") + display = fp if len(fp) <= 80 else "…" + fp[-77:] + return "Reading file", [display] + if tool_name == "write_file": + fp = inp.get("file_path", "") + display = fp if len(fp) <= 80 else "…" + fp[-77:] + return "Writing file", [display] + if tool_name == "edit_file": + fp = inp.get("file_path", "") + display = fp if len(fp) <= 80 else "…" + fp[-77:] + return "Editing file", [display] + if tool_name == "glob": + pat = inp.get("pattern", "") + base = inp.get("path", "/") + return "Searching files", [f"{pat} in {base}"] + if tool_name == "grep": + pat = inp.get("pattern", "") + path = inp.get("path", "") + display_pat = pat[:60] + ("…" if len(pat) > 60 else "") + return "Searching content", [f'"{display_pat}"' + (f" in {path}" if path else "")] + return f"Using {tool_name}", [] diff --git a/surfsense_backend/app/services/vision_autocomplete_service.py b/surfsense_backend/app/services/vision_autocomplete_service.py index f24a5c848..7d16c5864 100644 --- a/surfsense_backend/app/services/vision_autocomplete_service.py +++ b/surfsense_backend/app/services/vision_autocomplete_service.py @@ -1,139 +1,40 @@ +"""Vision autocomplete service — agent-based with scoped filesystem. + +Optimized pipeline: +1. Start the SSE stream immediately so the UI shows progress. +2. Derive a KB search query from window_title (no separate LLM call). +3. Run KB filesystem pre-computation and agent graph compilation in PARALLEL. +4. Inject pre-computed KB files as initial state and stream the agent. +""" + import logging from typing import AsyncGenerator -from langchain_core.messages import HumanMessage, SystemMessage +from langchain_core.messages import HumanMessage from sqlalchemy.ext.asyncio import AsyncSession -from app.retriever.chunks_hybrid_search import ChucksHybridSearchRetriever +from app.agents.autocomplete import create_autocomplete_agent, stream_autocomplete_agent from app.services.llm_service import get_vision_llm from app.services.new_streaming_service import VercelStreamingService logger = logging.getLogger(__name__) -KB_TOP_K = 5 -KB_MAX_CHARS = 4000 - -EXTRACT_QUERY_PROMPT = """Look at this screenshot and describe in 1-2 short sentences what the user is working on and what topic they need to write about. Be specific about the subject matter. Output ONLY the description, nothing else.""" - -EXTRACT_QUERY_PROMPT_WITH_APP = """The user is currently in the application "{app_name}" with the window titled "{window_title}". - -Look at this screenshot and describe in 1-2 short sentences what the user is working on and what topic they need to write about. Be specific about the subject matter. Output ONLY the description, nothing else.""" - -VISION_SYSTEM_PROMPT = """You are a smart writing assistant that analyzes the user's screen to draft or complete text. - -You will receive a screenshot of the user's screen. Your job: -1. Analyze the ENTIRE screenshot to understand what the user is working on (email thread, chat conversation, document, code editor, form, etc.). -2. Identify the text area where the user will type. -3. Based on the full visual context, generate the text the user most likely wants to write. - -Key behavior: -- If the text area is EMPTY, draft a full response or message based on what you see on screen (e.g., reply to an email, respond to a chat message, continue a document). -- If the text area already has text, continue it naturally. - -Rules: -- Output ONLY the text to be inserted. No quotes, no explanations, no meta-commentary. -- Be concise but complete — a full thought, not a fragment. -- Match the tone and formality of the surrounding context. -- If the screen shows code, write code. If it shows a casual chat, be casual. If it shows a formal email, be formal. -- Do NOT describe the screenshot or explain your reasoning. -- If you cannot determine what to write, output nothing.""" - -APP_CONTEXT_BLOCK = """ - -The user is currently working in "{app_name}" (window: "{window_title}"). Use this to understand the type of application and adapt your tone and format accordingly.""" - -KB_CONTEXT_BLOCK = """ - -You also have access to the user's knowledge base documents below. Use them to write more accurate, informed, and contextually relevant text. Do NOT cite or reference the documents explicitly — just let the knowledge inform your writing naturally. - - -{kb_context} -""" +PREP_STEP_ID = "autocomplete-prep" -def _build_system_prompt(app_name: str, window_title: str, kb_context: str) -> str: - """Assemble the system prompt from optional context blocks.""" - prompt = VISION_SYSTEM_PROMPT - if app_name: - prompt += APP_CONTEXT_BLOCK.format(app_name=app_name, window_title=window_title) - if kb_context: - prompt += KB_CONTEXT_BLOCK.format(kb_context=kb_context) - return prompt +def _derive_kb_query(app_name: str, window_title: str) -> str: + parts = [p for p in (window_title, app_name) if p] + return " ".join(parts) def _is_vision_unsupported_error(e: Exception) -> bool: - """Check if an exception indicates the model doesn't support vision/images.""" msg = str(e).lower() return "content must be a string" in msg or "does not support image" in msg -async def _extract_query_from_screenshot( - llm, screenshot_data_url: str, - app_name: str = "", window_title: str = "", -) -> str | None: - """Ask the Vision LLM to describe what the user is working on. - - Raises vision-unsupported errors so the caller can return a - friendly message immediately instead of retrying with astream. - """ - if app_name: - prompt_text = EXTRACT_QUERY_PROMPT_WITH_APP.format( - app_name=app_name, window_title=window_title, - ) - else: - prompt_text = EXTRACT_QUERY_PROMPT - - try: - response = await llm.ainvoke([ - HumanMessage(content=[ - {"type": "text", "text": prompt_text}, - {"type": "image_url", "image_url": {"url": screenshot_data_url}}, - ]), - ]) - query = response.content.strip() if hasattr(response, "content") else "" - return query if query else None - except Exception as e: - if _is_vision_unsupported_error(e): - raise - logger.warning(f"Failed to extract query from screenshot: {e}") - return None - - -async def _search_knowledge_base( - session: AsyncSession, search_space_id: int, query: str -) -> str: - """Search the KB and return formatted context string.""" - try: - retriever = ChucksHybridSearchRetriever(session) - results = await retriever.hybrid_search( - query_text=query, - top_k=KB_TOP_K, - search_space_id=search_space_id, - ) - - if not results: - return "" - - parts: list[str] = [] - char_count = 0 - for doc in results: - title = doc.get("document", {}).get("title", "Untitled") - for chunk in doc.get("chunks", []): - content = chunk.get("content", "").strip() - if not content: - continue - entry = f"[{title}]\n{content}" - if char_count + len(entry) > KB_MAX_CHARS: - break - parts.append(entry) - char_count += len(entry) - if char_count >= KB_MAX_CHARS: - break - - return "\n\n---\n\n".join(parts) - except Exception as e: - logger.warning(f"KB search failed, proceeding without context: {e}") - return "" +# --------------------------------------------------------------------------- +# Main entry point +# --------------------------------------------------------------------------- async def stream_vision_autocomplete( @@ -144,13 +45,7 @@ async def stream_vision_autocomplete( app_name: str = "", window_title: str = "", ) -> AsyncGenerator[str, None]: - """Analyze a screenshot with the vision LLM and stream a text completion. - - Pipeline: - 1. Extract a search query from the screenshot (non-streaming) - 2. Search the knowledge base for relevant context - 3. Stream the final completion with screenshot + KB + app context - """ + """Analyze a screenshot with a vision-LLM agent and stream a text completion.""" streaming = VercelStreamingService() vision_error_msg = ( "The selected model does not support vision. " @@ -164,62 +59,89 @@ async def stream_vision_autocomplete( yield streaming.format_done() return - kb_context = "" + # Start SSE stream immediately so the UI has something to show + yield streaming.format_message_start() + + kb_query = _derive_kb_query(app_name, window_title) + + # Show a preparation step while KB search + agent compile run + yield streaming.format_thinking_step( + step_id=PREP_STEP_ID, + title="Searching knowledge base", + status="in_progress", + items=[kb_query] if kb_query else [], + ) + try: - query = await _extract_query_from_screenshot( - llm, screenshot_data_url, app_name=app_name, window_title=window_title, + agent, kb = await create_autocomplete_agent( + llm, + search_space_id=search_space_id, + kb_query=kb_query, + app_name=app_name, + window_title=window_title, ) except Exception as e: - logger.warning(f"Vision autocomplete: selected model does not support vision: {e}") - yield streaming.format_message_start() - yield streaming.format_error(vision_error_msg) + if _is_vision_unsupported_error(e): + logger.warning("Vision autocomplete: model does not support vision: %s", e) + yield streaming.format_error(vision_error_msg) + yield streaming.format_done() + return + logger.error("Failed to create autocomplete agent: %s", e, exc_info=True) + yield streaming.format_error("Autocomplete failed. Please try again.") yield streaming.format_done() return - if query: - kb_context = await _search_knowledge_base(session, search_space_id, query) + has_kb = kb.has_documents + doc_count = len(kb.files) if has_kb else 0 # type: ignore[arg-type] - system_prompt = _build_system_prompt(app_name, window_title, kb_context) + yield streaming.format_thinking_step( + step_id=PREP_STEP_ID, + title="Searching knowledge base", + status="complete", + items=[f"Found {doc_count} document{'s' if doc_count != 1 else ''}"] if kb_query else ["Skipped"], + ) - messages = [ - SystemMessage(content=system_prompt), - HumanMessage(content=[ - { - "type": "text", - "text": "Analyze this screenshot. Understand the full context of what the user is working on, then generate the text they most likely want to write in the active text area.", - }, - { - "type": "image_url", - "image_url": {"url": screenshot_data_url}, - }, - ]), - ] + # Build agent input with pre-computed KB as initial state + if has_kb: + instruction = ( + "Analyze this screenshot, then explore the knowledge base documents " + "listed above — read the chunk index of any document whose title " + "looks relevant and check matched chunks for useful facts. " + "Finally, generate a concise autocomplete for the active text area, " + "enhanced with any relevant KB information you found." + ) + else: + instruction = ( + "Analyze this screenshot and generate a concise autocomplete " + "for the active text area based on what you see." + ) - text_started = False - text_id = "" + user_message = HumanMessage(content=[ + {"type": "text", "text": instruction}, + {"type": "image_url", "image_url": {"url": screenshot_data_url}}, + ]) + + input_data: dict = {"messages": [user_message]} + + if has_kb: + input_data["files"] = kb.files + input_data["messages"] = [kb.ls_ai_msg, kb.ls_tool_msg, user_message] + logger.info("Autocomplete: injected %d KB files into agent initial state", doc_count) + else: + logger.info("Autocomplete: no KB documents found, proceeding with screenshot only") + + # Stream the agent (message_start already sent above) try: - yield streaming.format_message_start() - text_id = streaming.generate_text_id() - yield streaming.format_text_start(text_id) - text_started = True - - async for chunk in llm.astream(messages): - token = chunk.content if hasattr(chunk, "content") else str(chunk) - if token: - yield streaming.format_text_delta(text_id, token) - - yield streaming.format_text_end(text_id) - yield streaming.format_finish() - yield streaming.format_done() - + async for sse in stream_autocomplete_agent( + agent, input_data, streaming, emit_message_start=False, + ): + yield sse except Exception as e: - if text_started: - yield streaming.format_text_end(text_id) - if _is_vision_unsupported_error(e): - logger.warning(f"Vision autocomplete: selected model does not support vision: {e}") + logger.warning("Vision autocomplete: model does not support vision: %s", e) yield streaming.format_error(vision_error_msg) + yield streaming.format_done() else: - logger.error(f"Vision autocomplete streaming error: {e}", exc_info=True) + logger.error("Vision autocomplete streaming error: %s", e, exc_info=True) yield streaming.format_error("Autocomplete failed. Please try again.") - yield streaming.format_done() + yield streaming.format_done() diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index fb83e2113..42ce025a8 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -10,7 +10,18 @@ type SSEEvent = | { type: "text-end"; id: string } | { type: "start"; messageId: string } | { type: "finish" } - | { type: "error"; errorText: string }; + | { type: "error"; errorText: string } + | { + type: "data-thinking-step"; + data: { id: string; title: string; status: string; items: string[] }; + }; + +interface AgentStep { + id: string; + title: string; + status: string; + items: string[]; +} function friendlyError(raw: string | number): string { if (typeof raw === "number") { @@ -34,11 +45,24 @@ function friendlyError(raw: string | number): string { const AUTO_DISMISS_MS = 3000; +function StepIcon({ status }: { status: string }) { + if (status === "complete") { + return ( + + + + + ); + } + return ; +} + export default function SuggestionPage() { const api = useElectronAPI(); const [suggestion, setSuggestion] = useState(""); const [isLoading, setIsLoading] = useState(true); const [error, setError] = useState(null); + const [steps, setSteps] = useState([]); const abortRef = useRef(null); const isDesktop = !!api?.onAutocompleteContext; @@ -66,6 +90,7 @@ export default function SuggestionPage() { setIsLoading(true); setSuggestion(""); setError(null); + setSteps([]); let token = getBearerToken(); if (!token) { @@ -137,6 +162,17 @@ export default function SuggestionPage() { setSuggestion((prev) => prev + parsed.delta); } else if (parsed.type === "error") { setError(friendlyError(parsed.errorText)); + } else if (parsed.type === "data-thinking-step") { + const { id, title, status, items } = parsed.data; + setSteps((prev) => { + const existing = prev.findIndex((s) => s.id === id); + if (existing >= 0) { + const updated = [...prev]; + updated[existing] = { id, title, status, items }; + return updated; + } + return [...prev, { id, title, status, items }]; + }); } } catch { continue; @@ -185,13 +221,33 @@ export default function SuggestionPage() { ); } - if (isLoading && !suggestion) { + const showLoading = isLoading && !suggestion; + + if (showLoading) { return (
-
- - - +
+ {steps.length === 0 && ( +
+ + Preparing… +
+ )} + {steps.length > 0 && ( +
+ {steps.map((step) => ( +
+ + + {step.title} + {step.items.length > 0 && ( + · {step.items[0]} + )} + +
+ ))} +
+ )}
); diff --git a/surfsense_web/app/desktop/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css index 62f4d2ea7..d2213fefd 100644 --- a/surfsense_web/app/desktop/suggestion/suggestion.css +++ b/surfsense_web/app/desktop/suggestion/suggestion.css @@ -19,13 +19,21 @@ body:has(.suggestion-body) { } .suggestion-tooltip { + box-sizing: border-box; background: #1e1e1e; border: 1px solid #3c3c3c; border-radius: 8px; padding: 8px 12px; margin: 4px; max-width: 400px; + /* MAX_HEIGHT in suggestion-window.ts is 400px. Subtract 8px for margin + (4px * 2) so the tooltip + margin fits within the Electron window. + box-sizing: border-box ensures padding + border are included. */ + max-height: 392px; box-shadow: 0 4px 16px rgba(0, 0, 0, 0.5); + display: flex; + flex-direction: column; + overflow: hidden; } .suggestion-text { @@ -35,6 +43,26 @@ body:has(.suggestion-body) { margin: 0 0 6px 0; word-wrap: break-word; white-space: pre-wrap; + overflow-y: auto; + flex: 1 1 auto; + min-height: 0; +} + +.suggestion-text::-webkit-scrollbar { + width: 5px; +} + +.suggestion-text::-webkit-scrollbar-track { + background: transparent; +} + +.suggestion-text::-webkit-scrollbar-thumb { + background: #555; + border-radius: 3px; +} + +.suggestion-text::-webkit-scrollbar-thumb:hover { + background: #777; } .suggestion-actions { @@ -43,6 +71,7 @@ body:has(.suggestion-body) { gap: 4px; border-top: 1px solid #2a2a2a; padding-top: 6px; + flex-shrink: 0; } .suggestion-btn { @@ -86,36 +115,77 @@ body:has(.suggestion-body) { font-size: 12px; } -.suggestion-loading { +/* --- Agent activity indicator --- */ + +.agent-activity { display: flex; - gap: 5px; + flex-direction: column; + gap: 4px; + overflow-y: auto; + max-height: 340px; +} + +.activity-initial { + display: flex; + align-items: center; + gap: 8px; padding: 2px 0; - justify-content: center; } -.suggestion-dot { - width: 4px; - height: 4px; +.activity-label { + color: #a1a1aa; + font-size: 12px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.activity-steps { + display: flex; + flex-direction: column; + gap: 3px; +} + +.activity-step { + display: flex; + align-items: center; + gap: 6px; + min-height: 18px; +} + +.step-label { + color: #d4d4d4; + font-size: 12px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.step-detail { + color: #71717a; + font-size: 11px; +} + +/* Spinner (in_progress) */ +.step-spinner { + width: 14px; + height: 14px; + flex-shrink: 0; + border: 1.5px solid #3f3f46; + border-top-color: #a78bfa; border-radius: 50%; - background: #666; - animation: suggestion-pulse 1.2s infinite ease-in-out; + animation: step-spin 0.7s linear infinite; } -.suggestion-dot:nth-child(2) { - animation-delay: 0.15s; +/* Checkmark icon (complete) */ +.step-icon { + width: 14px; + height: 14px; + flex-shrink: 0; } -.suggestion-dot:nth-child(3) { - animation-delay: 0.3s; -} - -@keyframes suggestion-pulse { - 0%, 80%, 100% { - opacity: 0.3; - transform: scale(0.8); - } - 40% { - opacity: 1; - transform: scale(1.1); +@keyframes step-spin { + to { + transform: rotate(360deg); } } diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index 7d8765399..6c8c619b2 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -92,15 +92,7 @@ import { useMediaQuery } from "@/hooks/use-media-query"; import { useElectronAPI } from "@/hooks/use-platform"; import { cn } from "@/lib/utils"; -/** Placeholder texts that cycle in new chats when input is empty */ -const CYCLING_PLACEHOLDERS = [ - "Ask SurfSense anything or @mention docs", - "Generate a podcast from my vacation ideas in Notion", - "Sum up last week's meeting notes from Drive in a bulleted list", - "Give me a brief overview of the most urgent tickets in Jira and Linear", - "Briefly, what are today's top ten important emails and calendar events?", - "Check if this week's Slack messages reference any GitHub issues", -]; +const COMPOSER_PLACEHOLDER = "Ask anything · Type / for prompts · Type @ to mention docs"; export const Thread: FC = () => { return ; @@ -380,29 +372,7 @@ const Composer: FC = () => { const isThreadEmpty = useAuiState(({ thread }) => thread.isEmpty); const isThreadRunning = useAuiState(({ thread }) => thread.isRunning); - // Cycling placeholder state - only cycles in new chats - const [placeholderIndex, setPlaceholderIndex] = useState(0); - - // Cycle through placeholders every 4 seconds when thread is empty (new chat) - useEffect(() => { - // Only cycle when thread is empty (new chat) - if (!isThreadEmpty) { - // Reset to first placeholder when chat becomes active - setPlaceholderIndex(0); - return; - } - - const intervalId = setInterval(() => { - setPlaceholderIndex((prev) => (prev + 1) % CYCLING_PLACEHOLDERS.length); - }, 6000); - - return () => clearInterval(intervalId); - }, [isThreadEmpty]); - - // Compute current placeholder - only cycle in new chats - const currentPlaceholder = isThreadEmpty - ? CYCLING_PLACEHOLDERS[placeholderIndex] - : CYCLING_PLACEHOLDERS[0]; + const currentPlaceholder = COMPOSER_PLACEHOLDER; // Live collaboration state const { data: currentUser } = useAtomValue(currentUserAtom); From 91ea293fa214bd1290f14c0f125a6ca66c4a875a Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 7 Apr 2026 03:10:06 -0700 Subject: [PATCH 07/47] chore: linting --- .../agents/autocomplete/autocomplete_agent.py | 27 +++- .../services/vision_autocomplete_service.py | 27 +++- .../components/DesktopContent.tsx | 36 ++--- surfsense_web/app/dashboard/layout.tsx | 2 +- surfsense_web/app/desktop/login/page.tsx | 54 ++----- .../app/desktop/permissions/page.tsx | 5 +- surfsense_web/app/desktop/suggestion/page.tsx | 17 +- .../app/desktop/suggestion/suggestion.css | 148 +++++++++--------- .../components/desktop/shortcut-recorder.tsx | 16 +- .../components/homepage/hero-section.tsx | 139 ++++++++-------- .../layout/ui/sidebar/DocumentsSidebar.tsx | 66 ++++---- .../components/sources/DocumentUploadTab.tsx | 2 +- surfsense_web/contexts/platform-context.tsx | 6 +- surfsense_web/types/window.d.ts | 4 +- 14 files changed, 285 insertions(+), 264 deletions(-) diff --git a/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py b/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py index 928a133cc..c6a071b0f 100644 --- a/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py +++ b/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py @@ -16,7 +16,8 @@ from __future__ import annotations import asyncio import logging import uuid -from typing import Any, AsyncGenerator +from collections.abc import AsyncGenerator +from typing import Any from deepagents.graph import BASE_AGENT_PROMPT from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware @@ -122,6 +123,7 @@ def _build_autocomplete_system_prompt(app_name: str, window_title: str) -> str: class _KBResult: """Container for pre-computed KB filesystem results.""" + __slots__ = ("files", "ls_ai_msg", "ls_tool_msg") def __init__( @@ -171,13 +173,16 @@ async def precompute_kb_filesystem( return _KBResult() doc_paths = [ - p for p, v in new_files.items() + p + for p, v in new_files.items() if p.startswith("/documents/") and v is not None ] tool_call_id = f"auto_ls_{uuid.uuid4().hex[:12]}" ai_msg = AIMessage( content="", - tool_calls=[{"name": "ls", "args": {"path": "/documents"}, "id": tool_call_id}], + tool_calls=[ + {"name": "ls", "args": {"path": "/documents"}, "id": tool_call_id} + ], ) tool_msg = ToolMessage( content=str(doc_paths) if doc_paths else "No documents found.", @@ -186,7 +191,9 @@ async def precompute_kb_filesystem( return _KBResult(files=new_files, ls_ai_msg=ai_msg, ls_tool_msg=tool_msg) except Exception: - logger.warning("KB pre-computation failed, proceeding without KB", exc_info=True) + logger.warning( + "KB pre-computation failed, proceeding without KB", exc_info=True + ) return _KBResult() @@ -320,7 +327,9 @@ async def stream_autocomplete_agent( ) try: - async for event in agent.astream_events(input_data, config=config, version="v2"): + async for event in agent.astream_events( + input_data, config=config, version="v2" + ): event_type = event.get("event", "") if event_type == "on_chat_model_stream": @@ -338,7 +347,9 @@ async def stream_autocomplete_agent( yield step_event current_text_id = streaming_service.generate_text_id() yield streaming_service.format_text_start(current_text_id) - yield streaming_service.format_text_delta(current_text_id, content) + yield streaming_service.format_text_delta( + current_text_id, content + ) elif event_type == "on_tool_start": active_tool_depth += 1 @@ -425,5 +436,7 @@ def _describe_tool_call(tool_name: str, tool_input: Any) -> tuple[str, list[str] pat = inp.get("pattern", "") path = inp.get("path", "") display_pat = pat[:60] + ("…" if len(pat) > 60 else "") - return "Searching content", [f'"{display_pat}"' + (f" in {path}" if path else "")] + return "Searching content", [ + f'"{display_pat}"' + (f" in {path}" if path else "") + ] return f"Using {tool_name}", [] diff --git a/surfsense_backend/app/services/vision_autocomplete_service.py b/surfsense_backend/app/services/vision_autocomplete_service.py index 2c2cd65d2..c28962b31 100644 --- a/surfsense_backend/app/services/vision_autocomplete_service.py +++ b/surfsense_backend/app/services/vision_autocomplete_service.py @@ -98,7 +98,9 @@ async def stream_vision_autocomplete( step_id=PREP_STEP_ID, title="Searching knowledge base", status="complete", - items=[f"Found {doc_count} document{'s' if doc_count != 1 else ''}"] if kb_query else ["Skipped"], + items=[f"Found {doc_count} document{'s' if doc_count != 1 else ''}"] + if kb_query + else ["Skipped"], ) # Build agent input with pre-computed KB as initial state @@ -116,24 +118,33 @@ async def stream_vision_autocomplete( "for the active text area based on what you see." ) - user_message = HumanMessage(content=[ - {"type": "text", "text": instruction}, - {"type": "image_url", "image_url": {"url": screenshot_data_url}}, - ]) + user_message = HumanMessage( + content=[ + {"type": "text", "text": instruction}, + {"type": "image_url", "image_url": {"url": screenshot_data_url}}, + ] + ) input_data: dict = {"messages": [user_message]} if has_kb: input_data["files"] = kb.files input_data["messages"] = [kb.ls_ai_msg, kb.ls_tool_msg, user_message] - logger.info("Autocomplete: injected %d KB files into agent initial state", doc_count) + logger.info( + "Autocomplete: injected %d KB files into agent initial state", doc_count + ) else: - logger.info("Autocomplete: no KB documents found, proceeding with screenshot only") + logger.info( + "Autocomplete: no KB documents found, proceeding with screenshot only" + ) # Stream the agent (message_start already sent above) try: async for sse in stream_autocomplete_agent( - agent, input_data, streaming, emit_message_start=False, + agent, + input_data, + streaming, + emit_message_start=False, ): yield sse except Exception as e: diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx index 07b746a19..a2f9da0f8 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx @@ -3,10 +3,7 @@ import { Clipboard, Sparkles } from "lucide-react"; import { useCallback, useEffect, useState } from "react"; import { toast } from "sonner"; -import { - DEFAULT_SHORTCUTS, - ShortcutRecorder, -} from "@/components/desktop/shortcut-recorder"; +import { DEFAULT_SHORTCUTS, ShortcutRecorder } from "@/components/desktop/shortcut-recorder"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Label } from "@/components/ui/label"; import { Spinner } from "@/components/ui/spinner"; @@ -29,22 +26,23 @@ export function DesktopContent() { let mounted = true; - Promise.all([ - api.getAutocompleteEnabled(), - api.getShortcuts?.() ?? Promise.resolve(null), - ]).then(([autoEnabled, config]) => { - if (!mounted) return; - setEnabled(autoEnabled); - if (config) setShortcuts(config); - setLoading(false); - setShortcutsLoaded(true); - }).catch(() => { - if (!mounted) return; - setLoading(false); - setShortcutsLoaded(true); - }); + Promise.all([api.getAutocompleteEnabled(), api.getShortcuts?.() ?? Promise.resolve(null)]) + .then(([autoEnabled, config]) => { + if (!mounted) return; + setEnabled(autoEnabled); + if (config) setShortcuts(config); + setLoading(false); + setShortcutsLoaded(true); + }) + .catch(() => { + if (!mounted) return; + setLoading(false); + setShortcutsLoaded(true); + }); - return () => { mounted = false; }; + return () => { + mounted = false; + }; }, [api]); if (!api) { diff --git a/surfsense_web/app/dashboard/layout.tsx b/surfsense_web/app/dashboard/layout.tsx index 25bea5467..1f5481b15 100644 --- a/surfsense_web/app/dashboard/layout.tsx +++ b/surfsense_web/app/dashboard/layout.tsx @@ -3,7 +3,7 @@ import { useEffect, useState } from "react"; import { USER_QUERY_KEY } from "@/atoms/user/user-query.atoms"; import { useGlobalLoadingEffect } from "@/hooks/use-global-loading"; -import { getBearerToken, ensureTokensFromElectron, redirectToLogin } from "@/lib/auth-utils"; +import { ensureTokensFromElectron, getBearerToken, redirectToLogin } from "@/lib/auth-utils"; import { queryClient } from "@/lib/query-client/client"; interface DashboardLayoutProps { diff --git a/surfsense_web/app/desktop/login/page.tsx b/surfsense_web/app/desktop/login/page.tsx index 529577b59..c81e284ba 100644 --- a/surfsense_web/app/desktop/login/page.tsx +++ b/surfsense_web/app/desktop/login/page.tsx @@ -2,30 +2,15 @@ import { IconBrandGoogleFilled } from "@tabler/icons-react"; import { useAtom } from "jotai"; -import { - Eye, - EyeOff, - Keyboard, - Clipboard, - Sparkles, -} from "lucide-react"; +import { Clipboard, Eye, EyeOff, Keyboard, Sparkles } from "lucide-react"; import Image from "next/image"; import { useRouter } from "next/navigation"; import { useCallback, useEffect, useState } from "react"; import { toast } from "sonner"; import { loginMutationAtom } from "@/atoms/auth/auth-mutation.atoms"; -import { - DEFAULT_SHORTCUTS, - ShortcutRecorder, -} from "@/components/desktop/shortcut-recorder"; +import { DEFAULT_SHORTCUTS, ShortcutRecorder } from "@/components/desktop/shortcut-recorder"; import { Button } from "@/components/ui/button"; -import { - Card, - CardContent, - CardDescription, - CardHeader, - CardTitle, -} from "@/components/ui/card"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Separator } from "@/components/ui/separator"; @@ -38,8 +23,7 @@ const isGoogleAuth = AUTH_TYPE === "GOOGLE"; export default function DesktopLoginPage() { const router = useRouter(); const api = useElectronAPI(); - const [{ mutateAsync: login, isPending: isLoggingIn }] = - useAtom(loginMutationAtom); + const [{ mutateAsync: login, isPending: isLoggingIn }] = useAtom(loginMutationAtom); const [email, setEmail] = useState(""); const [password, setPassword] = useState(""); @@ -54,10 +38,13 @@ export default function DesktopLoginPage() { setShortcutsLoaded(true); return; } - api.getShortcuts().then((config) => { - if (config) setShortcuts(config); - setShortcutsLoaded(true); - }).catch(() => setShortcutsLoaded(true)); + api + .getShortcuts() + .then((config) => { + if (config) setShortcuts(config); + setShortcutsLoaded(true); + }) + .catch(() => setShortcutsLoaded(true)); }, [api]); const updateShortcut = useCallback( @@ -118,8 +105,7 @@ export default function DesktopLoginPage() {
@@ -135,9 +121,7 @@ export default function DesktopLoginPage() { priority /> Welcome to SurfSense Desktop App - - Configure your shortcuts, then sign in to get started. - + Configure your shortcuts, then sign in to get started. @@ -181,11 +165,7 @@ export default function DesktopLoginPage() { {/* ---- Auth Section (second) ---- */} {isGoogleAuth ? ( - @@ -230,11 +210,7 @@ export default function DesktopLoginPage() { className="absolute inset-y-0 right-0 flex items-center pr-3 text-muted-foreground hover:text-foreground" tabIndex={-1} > - {showPassword ? ( - - ) : ( - - )} + {showPassword ? : }
diff --git a/surfsense_web/app/desktop/permissions/page.tsx b/surfsense_web/app/desktop/permissions/page.tsx index b636fcd7c..a2fadc8ff 100644 --- a/surfsense_web/app/desktop/permissions/page.tsx +++ b/surfsense_web/app/desktop/permissions/page.tsx @@ -80,7 +80,9 @@ export default function DesktopPermissionsPage() { poll(); interval = setInterval(poll, 2000); - return () => { if (interval) clearInterval(interval); }; + return () => { + if (interval) clearInterval(interval); + }; }, [api]); if (!api) { @@ -204,6 +206,7 @@ export default function DesktopPermissionsPage() { Grant permissions to continue

{label}

-

- {description} -

+

{description}

@@ -155,9 +147,7 @@ export function ShortcutRecorder({ )} > {recording ? ( - - Press keys... - + Press keys... ) : ( )} diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index 60f293005..c8dde97ee 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -1,21 +1,14 @@ "use client"; -import { AnimatePresence, motion } from "motion/react"; import { Monitor } from "lucide-react"; +import { AnimatePresence, motion } from "motion/react"; import Link from "next/link"; -import React, { useCallback, useEffect, useRef, useState, memo } from "react"; +import React, { memo, useCallback, useEffect, useRef, useState } from "react"; import Balancer from "react-wrap-balancer"; +import { ExpandedMediaOverlay, useExpandedMedia } from "@/components/ui/expanded-gif-overlay"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { AUTH_TYPE, BACKEND_URL } from "@/lib/env-config"; import { trackLoginAttempt } from "@/lib/posthog/events"; import { cn } from "@/lib/utils"; -import { - ExpandedMediaOverlay, - useExpandedMedia, -} from "@/components/ui/expanded-gif-overlay"; -import { - Tooltip, - TooltipContent, - TooltipTrigger, -} from "@/components/ui/tooltip"; const GoogleLogo = ({ className }: { className?: string }) => (

NotebookLM for Teams @@ -128,10 +117,11 @@ export function HeroSection() {

- An open source, privacy focused alternative to NotebookLM for teams with no data limits. + An open source, privacy focused alternative to NotebookLM for teams with no data + limits.

@@ -194,33 +184,34 @@ const BrowserWindow = () => {
{TAB_ITEMS.map((item, index) => ( - + {item.featured && ( + + + + + + + Desktop app only + + )} + {index !== TAB_ITEMS.length - 1 && (
)} @@ -263,13 +254,13 @@ const BrowserWindow = () => {

- {/* biome-ignore lint/a11y/useKeyWithClickEvents: wrapper for video expand */} -
-
+
@@ -277,11 +268,7 @@ const BrowserWindow = () => { {expanded && ( - + )} @@ -297,7 +284,7 @@ const TabVideo = memo(function TabVideo({ src }: { src: string }) { const video = videoRef.current; if (!video) return; video.currentTime = 0; - video.play().catch(() => { }); + video.play().catch(() => {}); }, [src]); const handleCanPlay = useCallback(() => { @@ -324,8 +311,7 @@ const TabVideo = memo(function TabVideo({ src }: { src: string }) { ); }); -const GITHUB_RELEASES_URL = - "https://github.com/MODSetter/SurfSense/releases/latest"; +const GITHUB_RELEASES_URL = "https://github.com/MODSetter/SurfSense/releases/latest"; const DownloadApp = memo(function DownloadApp() { return ( @@ -340,7 +326,16 @@ const DownloadApp = memo(function DownloadApp() { rel="noopener noreferrer" className="flex items-center gap-2 rounded-lg border border-neutral-200 bg-white px-4 py-2.5 text-sm font-medium text-neutral-700 shadow-sm transition hover:bg-neutral-50 dark:border-neutral-700 dark:bg-neutral-900 dark:text-neutral-200 dark:hover:bg-neutral-800" > - + @@ -353,7 +348,16 @@ const DownloadApp = memo(function DownloadApp() { rel="noopener noreferrer" className="flex items-center gap-2 rounded-lg border border-neutral-200 bg-white px-4 py-2.5 text-sm font-medium text-neutral-700 shadow-sm transition hover:bg-neutral-50 dark:border-neutral-700 dark:bg-neutral-900 dark:text-neutral-200 dark:hover:bg-neutral-800" > - + @@ -366,7 +370,16 @@ const DownloadApp = memo(function DownloadApp() { rel="noopener noreferrer" className="flex items-center gap-2 rounded-lg border border-neutral-200 bg-white px-4 py-2.5 text-sm font-medium text-neutral-700 shadow-sm transition hover:bg-neutral-50 dark:border-neutral-700 dark:bg-neutral-900 dark:text-neutral-200 dark:hover:bg-neutral-800" > - + diff --git a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx index 35489fe32..6de235d17 100644 --- a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx @@ -302,24 +302,27 @@ export function DocumentsSidebar({ [searchSpaceId, electronAPI] ); - const handleStopWatching = useCallback(async (folder: FolderDisplay) => { - if (!electronAPI) return; + const handleStopWatching = useCallback( + async (folder: FolderDisplay) => { + if (!electronAPI) return; - const watchedFolders = await electronAPI.getWatchedFolders(); - const matched = watchedFolders.find((wf) => wf.rootFolderId === folder.id); - if (!matched) { - toast.error("This folder is not being watched"); - return; - } + const watchedFolders = await electronAPI.getWatchedFolders(); + const matched = watchedFolders.find((wf) => wf.rootFolderId === folder.id); + if (!matched) { + toast.error("This folder is not being watched"); + return; + } - await electronAPI.removeWatchedFolder(matched.path); - try { - await foldersApiService.stopWatching(folder.id); - } catch (err) { - console.error("[DocumentsSidebar] Failed to clear watched metadata:", err); - } - toast.success(`Stopped watching: ${matched.name}`); - }, [electronAPI]); + await electronAPI.removeWatchedFolder(matched.path); + try { + await foldersApiService.stopWatching(folder.id); + } catch (err) { + console.error("[DocumentsSidebar] Failed to clear watched metadata:", err); + } + toast.success(`Stopped watching: ${matched.name}`); + }, + [electronAPI] + ); const handleRenameFolder = useCallback(async (folder: FolderDisplay, newName: string) => { try { @@ -330,22 +333,25 @@ export function DocumentsSidebar({ } }, []); - const handleDeleteFolder = useCallback(async (folder: FolderDisplay) => { - if (!confirm(`Delete folder "${folder.name}" and all its contents?`)) return; - try { - if (electronAPI) { - const watchedFolders = await electronAPI.getWatchedFolders(); - const matched = watchedFolders.find((wf) => wf.rootFolderId === folder.id); - if (matched) { - await electronAPI.removeWatchedFolder(matched.path); + const handleDeleteFolder = useCallback( + async (folder: FolderDisplay) => { + if (!confirm(`Delete folder "${folder.name}" and all its contents?`)) return; + try { + if (electronAPI) { + const watchedFolders = await electronAPI.getWatchedFolders(); + const matched = watchedFolders.find((wf) => wf.rootFolderId === folder.id); + if (matched) { + await electronAPI.removeWatchedFolder(matched.path); + } } + await foldersApiService.deleteFolder(folder.id); + toast.success("Folder deleted"); + } catch (e: unknown) { + toast.error((e as Error)?.message || "Failed to delete folder"); } - await foldersApiService.deleteFolder(folder.id); - toast.success("Folder deleted"); - } catch (e: unknown) { - toast.error((e as Error)?.message || "Failed to delete folder"); - } - }, [electronAPI]); + }, + [electronAPI] + ); const handleMoveFolder = useCallback( (folder: FolderDisplay) => { diff --git a/surfsense_web/components/sources/DocumentUploadTab.tsx b/surfsense_web/components/sources/DocumentUploadTab.tsx index 28e160261..76af48c45 100644 --- a/surfsense_web/components/sources/DocumentUploadTab.tsx +++ b/surfsense_web/components/sources/DocumentUploadTab.tsx @@ -25,8 +25,8 @@ import { import { Progress } from "@/components/ui/progress"; import { Spinner } from "@/components/ui/spinner"; import { Switch } from "@/components/ui/switch"; -import { documentsApiService } from "@/lib/apis/documents-api.service"; import { useElectronAPI } from "@/hooks/use-platform"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; import { trackDocumentUploadFailure, trackDocumentUploadStarted, diff --git a/surfsense_web/contexts/platform-context.tsx b/surfsense_web/contexts/platform-context.tsx index bb3e3800d..578901214 100644 --- a/surfsense_web/contexts/platform-context.tsx +++ b/surfsense_web/contexts/platform-context.tsx @@ -1,6 +1,6 @@ "use client"; -import { createContext, useEffect, useState, type ReactNode } from "react"; +import { createContext, type ReactNode, useEffect, useState } from "react"; export interface PlatformContextValue { isDesktop: boolean; @@ -25,7 +25,5 @@ export function PlatformProvider({ children }: { children: ReactNode }) { setValue({ isDesktop, isWeb: !isDesktop, electronAPI: api }); }, []); - return ( - {children} - ); + return {children}; } diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index 961ad9066..3f228066a 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -90,7 +90,9 @@ interface ElectronAPI { setAuthTokens: (bearer: string, refresh: string) => Promise; // Keyboard shortcut configuration getShortcuts: () => Promise<{ quickAsk: string; autocomplete: string }>; - setShortcuts: (config: Partial<{ quickAsk: string; autocomplete: string }>) => Promise<{ quickAsk: string; autocomplete: string }>; + setShortcuts: ( + config: Partial<{ quickAsk: string; autocomplete: string }> + ) => Promise<{ quickAsk: string; autocomplete: string }>; } declare global { From e574b5ec4a5fcc46b4ef6104138fe078ab027b3b Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 7 Apr 2026 03:17:10 -0700 Subject: [PATCH 08/47] refactor: remove prompt picker display on quick ask text retrieval - Eliminated the automatic display of the prompt picker when quick ask text is retrieved from the Electron API, streamlining the user experience. --- surfsense_web/components/assistant-ui/thread.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index 6c8c619b2..e0086cd66 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -364,7 +364,6 @@ const Composer: FC = () => { electronAPI.getQuickAskText().then((text) => { if (text) { setClipboardInitialText(text); - setShowPromptPicker(true); } }); }, [electronAPI]); From 27e9e8d8736e0b71d4082010f125e4b477e81b1d Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 7 Apr 2026 03:42:46 -0700 Subject: [PATCH 09/47] feat: add general assist feature and enhance shortcut management - Introduced a new "General Assist" shortcut, allowing users to open SurfSense from anywhere. - Updated shortcut management to include the new general assist functionality in both the desktop and web applications. - Enhanced the UI to reflect changes in shortcut labels and descriptions for better clarity. - Improved the Electron API to support the new shortcut configuration. --- surfsense_desktop/electron-builder.yml | 3 + surfsense_desktop/src/ipc/handlers.ts | 2 + surfsense_desktop/src/main.ts | 34 ++++++-- surfsense_desktop/src/modules/quick-ask.ts | 2 +- surfsense_desktop/src/modules/shortcuts.ts | 2 + surfsense_desktop/src/modules/tray.ts | 77 +++++++++++++++++++ .../components/DesktopContent.tsx | 44 +++++++---- surfsense_web/app/desktop/login/page.tsx | 21 +++-- .../components/desktop/shortcut-recorder.tsx | 1 + surfsense_web/types/window.d.ts | 6 +- 10 files changed, 159 insertions(+), 33 deletions(-) create mode 100644 surfsense_desktop/src/modules/tray.ts diff --git a/surfsense_desktop/electron-builder.yml b/surfsense_desktop/electron-builder.yml index 4d6f0b283..2c46c827a 100644 --- a/surfsense_desktop/electron-builder.yml +++ b/surfsense_desktop/electron-builder.yml @@ -19,6 +19,9 @@ files: - "!scripts" - "!release" extraResources: + - from: assets/ + to: assets/ + filter: ["*.ico", "*.png", "*.icns"] - from: ../surfsense_web/.next/standalone/surfsense_web/ to: standalone/ filter: diff --git a/surfsense_desktop/src/ipc/handlers.ts b/surfsense_desktop/src/ipc/handlers.ts index 7872e7a42..a583e5afc 100644 --- a/surfsense_desktop/src/ipc/handlers.ts +++ b/surfsense_desktop/src/ipc/handlers.ts @@ -23,6 +23,7 @@ import { import { getShortcuts, setShortcuts, type ShortcutConfig } from '../modules/shortcuts'; import { reregisterQuickAsk } from '../modules/quick-ask'; import { reregisterAutocomplete } from '../modules/autocomplete'; +import { reregisterGeneralAssist } from '../modules/tray'; let authTokens: { bearer: string; refresh: string } | null = null; @@ -107,6 +108,7 @@ export function registerIpcHandlers(): void { ipcMain.handle(IPC_CHANNELS.SET_SHORTCUTS, async (_event, config: Partial) => { const updated = await setShortcuts(config); + if (config.generalAssist) await reregisterGeneralAssist(); if (config.quickAsk) await reregisterQuickAsk(); if (config.autocomplete) await reregisterAutocomplete(); return updated; diff --git a/surfsense_desktop/src/main.ts b/surfsense_desktop/src/main.ts index 9eae8a4db..95b0359c8 100644 --- a/surfsense_desktop/src/main.ts +++ b/surfsense_desktop/src/main.ts @@ -1,7 +1,9 @@ import { app, BrowserWindow } from 'electron'; + +let isQuitting = false; import { registerGlobalErrorHandlers, showErrorDialog } from './modules/errors'; import { startNextServer } from './modules/server'; -import { createMainWindow } from './modules/window'; +import { createMainWindow, getMainWindow } from './modules/window'; import { setupDeepLinks, handlePendingDeepLink } from './modules/deep-links'; import { setupAutoUpdater } from './modules/auto-updater'; import { setupMenu } from './modules/menu'; @@ -9,6 +11,7 @@ import { registerQuickAsk, unregisterQuickAsk } from './modules/quick-ask'; import { registerAutocomplete, unregisterAutocomplete } from './modules/autocomplete'; import { registerFolderWatcher, unregisterFolderWatcher } from './modules/folder-watcher'; import { registerIpcHandlers } from './ipc/handlers'; +import { createTray, destroyTray } from './modules/tray'; registerGlobalErrorHandlers(); @@ -28,7 +31,18 @@ app.whenReady().then(async () => { return; } - createMainWindow('/dashboard'); + await createTray(); + + const win = createMainWindow('/dashboard'); + + // Minimize to tray instead of closing the app + win.on('close', (e) => { + if (!isQuitting) { + e.preventDefault(); + win.hide(); + } + }); + await registerQuickAsk(); await registerAutocomplete(); registerFolderWatcher(); @@ -37,20 +51,28 @@ app.whenReady().then(async () => { handlePendingDeepLink(); app.on('activate', () => { - if (BrowserWindow.getAllWindows().length === 0) { + const mw = getMainWindow(); + if (!mw || mw.isDestroyed()) { createMainWindow('/dashboard'); + } else { + mw.show(); + mw.focus(); } }); }); +// Keep running in the background — the tray "Quit" calls app.exit() app.on('window-all-closed', () => { - if (process.platform !== 'darwin') { - app.quit(); - } + // Do nothing: the app stays alive in the tray +}); + +app.on('before-quit', () => { + isQuitting = true; }); app.on('will-quit', () => { unregisterQuickAsk(); unregisterAutocomplete(); unregisterFolderWatcher(); + destroyTray(); }); diff --git a/surfsense_desktop/src/modules/quick-ask.ts b/surfsense_desktop/src/modules/quick-ask.ts index a015bfabf..224444be6 100644 --- a/surfsense_desktop/src/modules/quick-ask.ts +++ b/surfsense_desktop/src/modules/quick-ask.ts @@ -114,7 +114,7 @@ async function quickAskHandler(): Promise { const text = selected || savedClipboard.trim(); sourceApp = getFrontmostApp(); - console.log('[quick-ask] Source app:', sourceApp, '| Opening Quick Ask with', text.length, 'chars', selected ? '(selected)' : text ? '(clipboard fallback)' : '(empty)'); + console.log('[quick-ask] Source app:', sourceApp, '| Opening Quick Assist with', text.length, 'chars', selected ? '(selected)' : text ? '(clipboard fallback)' : '(empty)'); openQuickAsk(text); } diff --git a/surfsense_desktop/src/modules/shortcuts.ts b/surfsense_desktop/src/modules/shortcuts.ts index 8173b96c1..6948a005e 100644 --- a/surfsense_desktop/src/modules/shortcuts.ts +++ b/surfsense_desktop/src/modules/shortcuts.ts @@ -1,9 +1,11 @@ export interface ShortcutConfig { + generalAssist: string; quickAsk: string; autocomplete: string; } const DEFAULTS: ShortcutConfig = { + generalAssist: 'CommandOrControl+Shift+S', quickAsk: 'CommandOrControl+Alt+S', autocomplete: 'CommandOrControl+Shift+Space', }; diff --git a/surfsense_desktop/src/modules/tray.ts b/surfsense_desktop/src/modules/tray.ts new file mode 100644 index 000000000..1749145a1 --- /dev/null +++ b/surfsense_desktop/src/modules/tray.ts @@ -0,0 +1,77 @@ +import { app, globalShortcut, Menu, nativeImage, Tray } from 'electron'; +import path from 'path'; +import { getMainWindow, createMainWindow } from './window'; +import { getShortcuts } from './shortcuts'; + +let tray: Tray | null = null; +let currentShortcut: string | null = null; + +function getTrayIcon(): nativeImage { + const iconName = process.platform === 'win32' ? 'icon.ico' : 'icon.png'; + const iconPath = app.isPackaged + ? path.join(process.resourcesPath, 'assets', iconName) + : path.join(__dirname, '..', 'assets', iconName); + const img = nativeImage.createFromPath(iconPath); + return img.resize({ width: 16, height: 16 }); +} + +function showMainWindow(): void { + let win = getMainWindow(); + if (!win || win.isDestroyed()) { + win = createMainWindow('/dashboard'); + } else { + win.show(); + win.focus(); + } +} + +function registerShortcut(accelerator: string): void { + if (currentShortcut) { + globalShortcut.unregister(currentShortcut); + currentShortcut = null; + } + if (!accelerator) return; + try { + const ok = globalShortcut.register(accelerator, showMainWindow); + if (ok) { + currentShortcut = accelerator; + } else { + console.warn(`[tray] Failed to register General Assist shortcut: ${accelerator}`); + } + } catch (err) { + console.error(`[tray] Error registering General Assist shortcut:`, err); + } +} + +export async function createTray(): Promise { + if (tray) return; + + tray = new Tray(getTrayIcon()); + tray.setToolTip('SurfSense'); + + const contextMenu = Menu.buildFromTemplate([ + { label: 'Open SurfSense', click: showMainWindow }, + { type: 'separator' }, + { label: 'Quit', click: () => { app.exit(0); } }, + ]); + + tray.setContextMenu(contextMenu); + tray.on('double-click', showMainWindow); + + const shortcuts = await getShortcuts(); + registerShortcut(shortcuts.generalAssist); +} + +export async function reregisterGeneralAssist(): Promise { + const shortcuts = await getShortcuts(); + registerShortcut(shortcuts.generalAssist); +} + +export function destroyTray(): void { + if (currentShortcut) { + globalShortcut.unregister(currentShortcut); + currentShortcut = null; + } + tray?.destroy(); + tray = null; +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx index a2f9da0f8..eaf015740 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx @@ -1,12 +1,13 @@ "use client"; -import { Clipboard, Sparkles } from "lucide-react"; +import { AppWindow, Clipboard, Sparkles } from "lucide-react"; import { useCallback, useEffect, useState } from "react"; import { toast } from "sonner"; import { DEFAULT_SHORTCUTS, ShortcutRecorder } from "@/components/desktop/shortcut-recorder"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Label } from "@/components/ui/label"; import { Spinner } from "@/components/ui/spinner"; +import { Switch } from "@/components/ui/switch"; import { useElectronAPI } from "@/hooks/use-platform"; export function DesktopContent() { @@ -68,7 +69,7 @@ export function DesktopContent() { await api.setAutocompleteEnabled(checked); }; - const updateShortcut = (key: "quickAsk" | "autocomplete", accelerator: string) => { + const updateShortcut = (key: "generalAssist" | "quickAsk" | "autocomplete", accelerator: string) => { setShortcuts((prev) => { const updated = { ...prev, [key]: accelerator }; api.setShortcuts?.({ [key]: accelerator }).catch(() => { @@ -79,7 +80,7 @@ export function DesktopContent() { toast.success("Shortcut updated"); }; - const resetShortcut = (key: "quickAsk" | "autocomplete") => { + const resetShortcut = (key: "generalAssist" | "quickAsk" | "autocomplete") => { updateShortcut(key, DEFAULT_SHORTCUTS[key]); }; @@ -95,23 +96,32 @@ export function DesktopContent() { {shortcutsLoaded ? ( -
- updateShortcut("quickAsk", accel)} - onReset={() => resetShortcut("quickAsk")} - defaultValue={DEFAULT_SHORTCUTS.quickAsk} - label="Quick Ask" +
+ updateShortcut("generalAssist", accel)} + onReset={() => resetShortcut("generalAssist")} + defaultValue={DEFAULT_SHORTCUTS.generalAssist} + label="General Assist" + description="Open SurfSense from anywhere" + icon={AppWindow} + /> + updateShortcut("quickAsk", accel)} + onReset={() => resetShortcut("quickAsk")} + defaultValue={DEFAULT_SHORTCUTS.quickAsk} + label="Quick Assist" description="Copy selected text and ask AI about it" - icon={Clipboard} - /> + icon={Clipboard} + /> updateShortcut("autocomplete", accel)} onReset={() => resetShortcut("autocomplete")} defaultValue={DEFAULT_SHORTCUTS.autocomplete} - label="Autocomplete" - description="Get AI writing suggestions from a screenshot" + label="Extreme Assist" + description="AI writing powered by your screen and knowledge base" icon={Sparkles} />

@@ -126,10 +136,10 @@ export function DesktopContent() { - {/* Autocomplete Toggle */} + {/* Extreme Assist Toggle */} - Autocomplete + Extreme Assist Get inline writing suggestions powered by your knowledge base as you type in any app. @@ -138,7 +148,7 @@ export function DesktopContent() {

Show suggestions while typing in other applications. diff --git a/surfsense_web/app/desktop/login/page.tsx b/surfsense_web/app/desktop/login/page.tsx index c81e284ba..f442b5d26 100644 --- a/surfsense_web/app/desktop/login/page.tsx +++ b/surfsense_web/app/desktop/login/page.tsx @@ -2,7 +2,7 @@ import { IconBrandGoogleFilled } from "@tabler/icons-react"; import { useAtom } from "jotai"; -import { Clipboard, Eye, EyeOff, Keyboard, Sparkles } from "lucide-react"; +import { AppWindow, Clipboard, Eye, EyeOff, Keyboard, Sparkles } from "lucide-react"; import Image from "next/image"; import { useRouter } from "next/navigation"; import { useCallback, useEffect, useState } from "react"; @@ -48,7 +48,7 @@ export default function DesktopLoginPage() { }, [api]); const updateShortcut = useCallback( - (key: "quickAsk" | "autocomplete", accelerator: string) => { + (key: "generalAssist" | "quickAsk" | "autocomplete", accelerator: string) => { setShortcuts((prev) => { const updated = { ...prev, [key]: accelerator }; api?.setShortcuts?.({ [key]: accelerator }).catch(() => { @@ -62,7 +62,7 @@ export default function DesktopLoginPage() { ); const resetShortcut = useCallback( - (key: "quickAsk" | "autocomplete") => { + (key: "generalAssist" | "quickAsk" | "autocomplete") => { updateShortcut(key, DEFAULT_SHORTCUTS[key]); }, [updateShortcut] @@ -132,12 +132,21 @@ export default function DesktopLoginPage() { Keyboard Shortcuts

+ updateShortcut("generalAssist", accel)} + onReset={() => resetShortcut("generalAssist")} + defaultValue={DEFAULT_SHORTCUTS.generalAssist} + label="General Assist" + description="Open SurfSense from anywhere" + icon={AppWindow} + /> updateShortcut("quickAsk", accel)} onReset={() => resetShortcut("quickAsk")} defaultValue={DEFAULT_SHORTCUTS.quickAsk} - label="Quick Ask" + label="Quick Assist" description="Copy selected text and ask AI about it" icon={Clipboard} /> @@ -146,8 +155,8 @@ export default function DesktopLoginPage() { onChange={(accel) => updateShortcut("autocomplete", accel)} onReset={() => resetShortcut("autocomplete")} defaultValue={DEFAULT_SHORTCUTS.autocomplete} - label="Autocomplete" - description="Get AI writing suggestions from a screenshot" + label="Extreme Assist" + description="AI writing powered by your screen and knowledge base" icon={Sparkles} />

diff --git a/surfsense_web/components/desktop/shortcut-recorder.tsx b/surfsense_web/components/desktop/shortcut-recorder.tsx index 6d5e93a65..751579e50 100644 --- a/surfsense_web/components/desktop/shortcut-recorder.tsx +++ b/surfsense_web/components/desktop/shortcut-recorder.tsx @@ -36,6 +36,7 @@ export function acceleratorToDisplay(accel: string): string[] { } export const DEFAULT_SHORTCUTS = { + generalAssist: "CommandOrControl+Shift+S", quickAsk: "CommandOrControl+Alt+S", autocomplete: "CommandOrControl+Shift+Space", }; diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index 3f228066a..25077d1da 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -89,10 +89,10 @@ interface ElectronAPI { getAuthTokens: () => Promise<{ bearer: string; refresh: string } | null>; setAuthTokens: (bearer: string, refresh: string) => Promise; // Keyboard shortcut configuration - getShortcuts: () => Promise<{ quickAsk: string; autocomplete: string }>; + getShortcuts: () => Promise<{ generalAssist: string; quickAsk: string; autocomplete: string }>; setShortcuts: ( - config: Partial<{ quickAsk: string; autocomplete: string }> - ) => Promise<{ quickAsk: string; autocomplete: string }>; + config: Partial<{ generalAssist: string; quickAsk: string; autocomplete: string }> + ) => Promise<{ generalAssist: string; quickAsk: string; autocomplete: string }>; } declare global { From b74ac8a608fe5b5ad1e0ac0e76fb2d98d71230aa Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 7 Apr 2026 04:22:22 -0700 Subject: [PATCH 10/47] feat: update shortcut icons and descriptions for improved clarity - Replaced icons for "General Assist," "Quick Assist," and "Extreme Assist" shortcuts to better represent their functionalities. - Updated descriptions for each shortcut to enhance user understanding of their actions. - Refactored the layout of the shortcut recorder for a more streamlined user experience. --- .../components/DesktopContent.tsx | 30 +- surfsense_web/app/desktop/login/page.tsx | 265 ++++++++++-------- .../components/desktop/shortcut-recorder.tsx | 44 +-- 3 files changed, 182 insertions(+), 157 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx index eaf015740..5ecea6708 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx @@ -1,6 +1,6 @@ "use client"; -import { AppWindow, Clipboard, Sparkles } from "lucide-react"; +import { BrainCog, Rocket, Zap } from "lucide-react"; import { useCallback, useEffect, useState } from "react"; import { toast } from "sonner"; import { DEFAULT_SHORTCUTS, ShortcutRecorder } from "@/components/desktop/shortcut-recorder"; @@ -103,27 +103,27 @@ export function DesktopContent() { onReset={() => resetShortcut("generalAssist")} defaultValue={DEFAULT_SHORTCUTS.generalAssist} label="General Assist" - description="Open SurfSense from anywhere" - icon={AppWindow} + description="Launch SurfSense instantly from any application" + icon={Rocket} /> updateShortcut("quickAsk", accel)} onReset={() => resetShortcut("quickAsk")} defaultValue={DEFAULT_SHORTCUTS.quickAsk} - label="Quick Assist" - description="Copy selected text and ask AI about it" - icon={Clipboard} + label="Quick Assist" + description="Select text anywhere, then ask AI to explain, rewrite, or act on it" + icon={Zap} + /> + updateShortcut("autocomplete", accel)} + onReset={() => resetShortcut("autocomplete")} + defaultValue={DEFAULT_SHORTCUTS.autocomplete} + label="Extreme Assist" + description="AI drafts text using your screen context and knowledge base" + icon={BrainCog} /> - updateShortcut("autocomplete", accel)} - onReset={() => resetShortcut("autocomplete")} - defaultValue={DEFAULT_SHORTCUTS.autocomplete} - label="Extreme Assist" - description="AI writing powered by your screen and knowledge base" - icon={Sparkles} - />

Click a shortcut and press a new key combination to change it.

diff --git a/surfsense_web/app/desktop/login/page.tsx b/surfsense_web/app/desktop/login/page.tsx index f442b5d26..5d931b5c2 100644 --- a/surfsense_web/app/desktop/login/page.tsx +++ b/surfsense_web/app/desktop/login/page.tsx @@ -2,7 +2,7 @@ import { IconBrandGoogleFilled } from "@tabler/icons-react"; import { useAtom } from "jotai"; -import { AppWindow, Clipboard, Eye, EyeOff, Keyboard, Sparkles } from "lucide-react"; +import { BrainCog, Eye, EyeOff, Rocket, Zap } from "lucide-react"; import Image from "next/image"; import { useRouter } from "next/navigation"; import { useCallback, useEffect, useState } from "react"; @@ -10,7 +10,6 @@ import { toast } from "sonner"; import { loginMutationAtom } from "@/atoms/auth/auth-mutation.atoms"; import { DEFAULT_SHORTCUTS, ShortcutRecorder } from "@/components/desktop/shortcut-recorder"; import { Button } from "@/components/ui/button"; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Separator } from "@/components/ui/separator"; @@ -100,8 +99,9 @@ export default function DesktopLoginPage() { }; return ( -
-
+
+ {/* Subtle radial glow */} +
- - +
+ {/* Header */} +
SurfSense - Welcome to SurfSense Desktop App - Configure your shortcuts, then sign in to get started. - - - - {/* ---- Shortcuts Section (first) ---- */} - {shortcutsLoaded ? ( -
-
- - Keyboard Shortcuts -
- updateShortcut("generalAssist", accel)} - onReset={() => resetShortcut("generalAssist")} - defaultValue={DEFAULT_SHORTCUTS.generalAssist} - label="General Assist" - description="Open SurfSense from anywhere" - icon={AppWindow} - /> - updateShortcut("quickAsk", accel)} - onReset={() => resetShortcut("quickAsk")} - defaultValue={DEFAULT_SHORTCUTS.quickAsk} - label="Quick Assist" - description="Copy selected text and ask AI about it" - icon={Clipboard} - /> - updateShortcut("autocomplete", accel)} - onReset={() => resetShortcut("autocomplete")} - defaultValue={DEFAULT_SHORTCUTS.autocomplete} - label="Extreme Assist" - description="AI writing powered by your screen and knowledge base" - icon={Sparkles} - /> -

- Click a shortcut and press a new key combination to change it. -

-
- ) : ( -
- -
- )} - - {/* ---- Divider ---- */} - - - {/* ---- Auth Section (second) ---- */} - {isGoogleAuth ? ( - - ) : ( -
- {loginError && ( -
- {loginError} -
- )} +

+ Welcome to SurfSense Desktop +

+

+ Configure shortcuts, then sign in to get started. +

+
+ {/* Scrollable content */} +
+
+ {/* ---- Shortcuts ---- */} + {shortcutsLoaded ? (
- - setEmail(e.target.value)} - disabled={isLoggingIn} - autoFocus - /> -
- -
- -
- setPassword(e.target.value)} - disabled={isLoggingIn} - className="pr-10" +

+ Keyboard Shortcuts +

+
+ updateShortcut("generalAssist", accel)} + onReset={() => resetShortcut("generalAssist")} + defaultValue={DEFAULT_SHORTCUTS.generalAssist} + label="General Assist" + description="Launch SurfSense instantly from any application" + icon={Rocket} + /> + updateShortcut("quickAsk", accel)} + onReset={() => resetShortcut("quickAsk")} + defaultValue={DEFAULT_SHORTCUTS.quickAsk} + label="Quick Assist" + description="Select text anywhere, then ask AI to explain, rewrite, or act on it" + icon={Zap} + /> + updateShortcut("autocomplete", accel)} + onReset={() => resetShortcut("autocomplete")} + defaultValue={DEFAULT_SHORTCUTS.autocomplete} + label="Extreme Assist" + description="AI drafts text using your screen context and knowledge base" + icon={BrainCog} /> -
+

+ Click a shortcut and press a new key combination to change it. +

+ ) : ( +
+ +
+ )} - - - )} - - + + + {/* ---- Auth ---- */} +
+

+ Sign In +

+ + {isGoogleAuth ? ( + + ) : ( +
+ {loginError && ( +
+ {loginError} +
+ )} + +
+ + setEmail(e.target.value)} + disabled={isLoggingIn} + autoFocus + className="h-9" + /> +
+ +
+ +
+ setPassword(e.target.value)} + disabled={isLoggingIn} + className="h-9 pr-9" + /> + +
+
+ + +
+ )} +
+
+
+
); } diff --git a/surfsense_web/components/desktop/shortcut-recorder.tsx b/surfsense_web/components/desktop/shortcut-recorder.tsx index 751579e50..ec4e5a528 100644 --- a/surfsense_web/components/desktop/shortcut-recorder.tsx +++ b/surfsense_web/components/desktop/shortcut-recorder.tsx @@ -6,7 +6,7 @@ import { Button } from "@/components/ui/button"; import { cn } from "@/lib/utils"; // --------------------------------------------------------------------------- -// Accelerator ↔ display helpers +// Accelerator <-> display helpers // --------------------------------------------------------------------------- export function keyEventToAccelerator(e: React.KeyboardEvent): string | null { @@ -47,13 +47,13 @@ export const DEFAULT_SHORTCUTS = { export function Kbd({ keys, className }: { keys: string[]; className?: string }) { return ( - - {keys.map((key) => ( + + {keys.map((key, i) => ( 3 && "px-2" + "inline-flex h-6 min-w-6 items-center justify-center rounded border bg-muted px-1 font-mono text-[11px] font-medium text-muted-foreground", + key.length > 3 && "px-1.5" )} > {key} @@ -111,27 +111,29 @@ export function ShortcutRecorder({ const isDefault = value === defaultValue; return ( -
-
-
- -
-
-

{label}

-

{description}

-
+
+ {/* Icon */} +
+
-
+ {/* Label + description */} +
+

{label}

+

{description}

+
+ + {/* Actions */} +
{!isDefault && ( )} + + )} ); From 80f775581bd44dd980c1d75cdbc125dcc7b41f56 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 7 Apr 2026 05:11:41 -0700 Subject: [PATCH 13/47] feat: implement quick assist mode detection in AssistantActionBar - Added state management for quick assist mode using the Electron API. - Introduced a useEffect hook to asynchronously check and set the quick assist mode based on the API response, enhancing the component's interactivity. --- .../components/assistant-ui/assistant-message.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/surfsense_web/components/assistant-ui/assistant-message.tsx b/surfsense_web/components/assistant-ui/assistant-message.tsx index 5567cfca8..49853b0b5 100644 --- a/surfsense_web/components/assistant-ui/assistant-message.tsx +++ b/surfsense_web/components/assistant-ui/assistant-message.tsx @@ -465,8 +465,14 @@ const AssistantActionBar: FC = () => { const isLast = useAuiState((s) => s.message.isLast); const aui = useAui(); const api = useElectronAPI(); + const [isQuickAssist, setIsQuickAssist] = useState(false); - const isQuickAssist = !!api?.replaceText && !!api?.getQuickAskMode; + useEffect(() => { + if (!api?.getQuickAskMode) return; + api.getQuickAskMode().then((mode) => { + if (mode) setIsQuickAssist(true); + }); + }, [api]); return ( Date: Tue, 7 Apr 2026 17:38:39 +0200 Subject: [PATCH 14/47] feat: return 3 suggestion options from vision autocomplete agent --- .../agents/autocomplete/autocomplete_agent.py | 91 ++++++++++++++----- 1 file changed, 67 insertions(+), 24 deletions(-) diff --git a/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py b/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py index c6a071b0f..36b5bc086 100644 --- a/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py +++ b/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py @@ -14,7 +14,9 @@ LLM call — the window title is used directly as the KB search query. from __future__ import annotations import asyncio +import json import logging +import re import uuid from collections.abc import AsyncGenerator from typing import Any @@ -61,13 +63,21 @@ Key behavior: - If the text area already has text, continue it naturally — typically just a sentence or two. Rules: -- Output ONLY the text to be inserted. No quotes, no explanations, no meta-commentary. - Be CONCISE. Prefer a single paragraph or a few sentences. Autocomplete is a quick assist, not a full draft. - Match the tone and formality of the surrounding context. - If the screen shows code, write code. If it shows a casual chat, be casual. If it shows a formal email, be formal. - Do NOT describe the screenshot or explain your reasoning. - Do NOT cite or reference documents explicitly — just let the knowledge inform your writing naturally. -- If you cannot determine what to write, output nothing. +- If you cannot determine what to write, output an empty JSON array: [] + +## Output Format + +You MUST provide exactly 3 different suggestion options. Each should be a distinct, plausible completion — vary the tone, detail level, or angle. + +Return your suggestions as a JSON array of exactly 3 strings. Output ONLY the JSON array, nothing else — no markdown fences, no explanation, no commentary. + +Example format: +["First suggestion text here.", "Second suggestion — a different take.", "Third option with another approach."] ## Filesystem Tools `ls`, `read_file`, `write_file`, `edit_file`, `glob`, `grep` @@ -264,6 +274,50 @@ async def create_autocomplete_agent( return agent, kb +# --------------------------------------------------------------------------- +# JSON suggestion parsing (robust fallback) +# --------------------------------------------------------------------------- + + +def _parse_suggestions(raw: str) -> list[str]: + """Extract a list of suggestion strings from the agent's output. + + Tries, in order: + 1. Direct ``json.loads`` + 2. Extract content between ```json ... ``` fences + 3. Find the first ``[`` … ``]`` span + Falls back to wrapping the raw text as a single suggestion. + """ + text = raw.strip() + if not text: + return [] + + for candidate in _json_candidates(text): + try: + parsed = json.loads(candidate) + if isinstance(parsed, list) and all(isinstance(s, str) for s in parsed): + return [s for s in parsed if s.strip()] + except (json.JSONDecodeError, ValueError): + continue + + return [text] + + +def _json_candidates(text: str) -> list[str]: + """Yield candidate JSON strings from raw text.""" + candidates = [text] + + fence = re.search(r"```(?:json)?\s*\n?(.*?)```", text, re.DOTALL) + if fence: + candidates.append(fence.group(1).strip()) + + bracket = re.search(r"\[.*]", text, re.DOTALL) + if bracket: + candidates.append(bracket.group(0)) + + return candidates + + # --------------------------------------------------------------------------- # Streaming helper # --------------------------------------------------------------------------- @@ -285,7 +339,7 @@ async def stream_autocomplete_agent( thread_id = uuid.uuid4().hex config = {"configurable": {"thread_id": thread_id}} - current_text_id: str | None = None + text_buffer: list[str] = [] active_tool_depth = 0 thinking_step_counter = 0 tool_step_ids: dict[str, str] = {} @@ -315,14 +369,12 @@ async def stream_autocomplete_agent( if emit_message_start: yield streaming_service.format_message_start() - # Emit an initial "Generating completion" step so the UI immediately - # shows activity once the agent starts its first LLM call. gen_step_id = next_thinking_step_id() last_active_step_id = gen_step_id - step_titles[gen_step_id] = "Generating completion" + step_titles[gen_step_id] = "Generating suggestions" yield streaming_service.format_thinking_step( step_id=gen_step_id, - title="Generating completion", + title="Generating suggestions", status="in_progress", ) @@ -341,15 +393,7 @@ async def stream_autocomplete_agent( if chunk and hasattr(chunk, "content"): content = chunk.content if content and isinstance(content, str): - if current_text_id is None: - step_event = complete_current_step() - if step_event: - yield step_event - current_text_id = streaming_service.generate_text_id() - yield streaming_service.format_text_start(current_text_id) - yield streaming_service.format_text_delta( - current_text_id, content - ) + text_buffer.append(content) elif event_type == "on_tool_start": active_tool_depth += 1 @@ -357,10 +401,6 @@ async def stream_autocomplete_agent( run_id = event.get("run_id", "") tool_input = event.get("data", {}).get("input", {}) - if current_text_id is not None: - yield streaming_service.format_text_end(current_text_id) - current_text_id = None - step_event = complete_current_step() if step_event: yield step_event @@ -393,19 +433,22 @@ async def stream_autocomplete_agent( if last_active_step_id == step_id: last_active_step_id = None - if current_text_id is not None: - yield streaming_service.format_text_end(current_text_id) step_event = complete_current_step() if step_event: yield step_event + raw_text = "".join(text_buffer) + suggestions = _parse_suggestions(raw_text) + + yield streaming_service.format_data( + "suggestions", {"options": suggestions} + ) + yield streaming_service.format_finish() yield streaming_service.format_done() except Exception as e: logger.error(f"Autocomplete agent streaming error: {e}", exc_info=True) - if current_text_id is not None: - yield streaming_service.format_text_end(current_text_id) yield streaming_service.format_error("Autocomplete failed. Please try again.") yield streaming_service.format_done() From 2602248e7a552a19be0ac4e005f2c8183e18995a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 17:43:40 +0200 Subject: [PATCH 15/47] feat: handle multi-option suggestions in suggestion page UI --- surfsense_web/app/desktop/suggestion/page.tsx | 68 ++++++++++++++----- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index 8d9095320..e98da9a1c 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -14,6 +14,10 @@ type SSEEvent = | { type: "data-thinking-step"; data: { id: string; title: string; status: string; items: string[] }; + } + | { + type: "data-suggestions"; + data: { options: string[] }; }; interface AgentStep { @@ -70,10 +74,11 @@ function StepIcon({ status }: { status: string }) { export default function SuggestionPage() { const api = useElectronAPI(); - const [suggestion, setSuggestion] = useState(""); + const [options, setOptions] = useState([]); const [isLoading, setIsLoading] = useState(true); const [error, setError] = useState(null); const [steps, setSteps] = useState([]); + const [expandedOption, setExpandedOption] = useState(null); const abortRef = useRef(null); const isDesktop = !!api?.onAutocompleteContext; @@ -99,9 +104,10 @@ export default function SuggestionPage() { abortRef.current = controller; setIsLoading(true); - setSuggestion(""); + setOptions([]); setError(null); setSteps([]); + setExpandedOption(null); let token = getBearerToken(); if (!token) { @@ -165,8 +171,8 @@ export default function SuggestionPage() { try { const parsed: SSEEvent = JSON.parse(data); - if (parsed.type === "text-delta") { - setSuggestion((prev) => prev + parsed.delta); + if (parsed.type === "data-suggestions") { + setOptions(parsed.data.options); } else if (parsed.type === "error") { setError(friendlyError(parsed.errorText)); } else if (parsed.type === "data-thinking-step") { @@ -226,7 +232,7 @@ export default function SuggestionPage() { ); } - const showLoading = isLoading && !suggestion; + const showLoading = isLoading && options.length === 0; if (showLoading) { return ( @@ -258,29 +264,55 @@ export default function SuggestionPage() { ); } - const handleAccept = () => { - if (suggestion) { - api?.acceptSuggestion?.(suggestion); - } + const handleSelect = (text: string) => { + api?.acceptSuggestion?.(text); }; const handleDismiss = () => { api?.dismissSuggestion?.(); }; - if (!suggestion) return null; + const TRUNCATE_LENGTH = 120; + + if (options.length === 0) return null; return (
-

{suggestion}

+
+ {options.map((option, index) => { + const isExpanded = expandedOption === index; + const needsTruncation = option.length > TRUNCATE_LENGTH; + const displayText = + needsTruncation && !isExpanded + ? option.slice(0, TRUNCATE_LENGTH) + "…" + : option; + + return ( + + )} + + ); + })} +
- )} - +
); })}
From 70807cccd1cb55f47ea136c1c5cf8399fd5eec63 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 18:22:16 +0200 Subject: [PATCH 19/47] fix: hide scrollbar during streaming to prevent UI flicker --- surfsense_web/app/desktop/suggestion/suggestion.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/surfsense_web/app/desktop/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css index e0c56857d..f5471cf37 100644 --- a/surfsense_web/app/desktop/suggestion/suggestion.css +++ b/surfsense_web/app/desktop/suggestion/suggestion.css @@ -127,6 +127,10 @@ body:has(.suggestion-body) { max-height: 340px; } +.agent-activity::-webkit-scrollbar { + display: none; +} + .activity-initial { display: flex; align-items: center; From 5439b3991b6f61fd547ae8ef1d2020e04f6ec1a3 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 18:25:06 +0200 Subject: [PATCH 20/47] fix: auto-dismiss overlay when no suggestions are available --- surfsense_web/app/desktop/suggestion/page.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index af1c50be0..e458f6615 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -97,6 +97,14 @@ export default function SuggestionPage() { return () => clearTimeout(timer); }, [error, api]); + useEffect(() => { + if (isLoading || error || options.length > 0) return; + const timer = setTimeout(() => { + api?.dismissSuggestion?.(); + }, AUTO_DISMISS_MS); + return () => clearTimeout(timer); + }, [isLoading, error, options, api]); + const fetchSuggestion = useCallback( async (screenshot: string, searchSpaceId: string, appName?: string, windowTitle?: string) => { abortRef.current?.abort(); From 879945eeae05d22845a2dd682a6e3eb1cbeb578d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 18:49:04 +0200 Subject: [PATCH 21/47] Add VisionProvider enum, VisionLLMConfig table, and vision RBAC permissions --- surfsense_backend/app/db.py | 77 ++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 6e9553307..4689313f7 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -260,6 +260,24 @@ class ImageGenProvider(StrEnum): NSCALE = "NSCALE" +class VisionProvider(StrEnum): + OPENAI = "OPENAI" + ANTHROPIC = "ANTHROPIC" + GOOGLE = "GOOGLE" + AZURE_OPENAI = "AZURE_OPENAI" + VERTEX_AI = "VERTEX_AI" + BEDROCK = "BEDROCK" + XAI = "XAI" + OPENROUTER = "OPENROUTER" + OLLAMA = "OLLAMA" + GROQ = "GROQ" + TOGETHER_AI = "TOGETHER_AI" + FIREWORKS_AI = "FIREWORKS_AI" + DEEPSEEK = "DEEPSEEK" + MISTRAL = "MISTRAL" + CUSTOM = "CUSTOM" + + class LogLevel(StrEnum): DEBUG = "DEBUG" INFO = "INFO" @@ -377,6 +395,11 @@ class Permission(StrEnum): IMAGE_GENERATIONS_READ = "image_generations:read" IMAGE_GENERATIONS_DELETE = "image_generations:delete" + # Vision LLM Configs + VISION_CONFIGS_CREATE = "vision_configs:create" + VISION_CONFIGS_READ = "vision_configs:read" + VISION_CONFIGS_DELETE = "vision_configs:delete" + # Connectors CONNECTORS_CREATE = "connectors:create" CONNECTORS_READ = "connectors:read" @@ -445,6 +468,9 @@ DEFAULT_ROLE_PERMISSIONS = { # Image Generations (create and read, no delete) Permission.IMAGE_GENERATIONS_CREATE.value, Permission.IMAGE_GENERATIONS_READ.value, + # Vision Configs (create and read, no delete) + Permission.VISION_CONFIGS_CREATE.value, + Permission.VISION_CONFIGS_READ.value, # Connectors (no delete) Permission.CONNECTORS_CREATE.value, Permission.CONNECTORS_READ.value, @@ -478,6 +504,8 @@ DEFAULT_ROLE_PERMISSIONS = { Permission.VIDEO_PRESENTATIONS_READ.value, # Image Generations (read only) Permission.IMAGE_GENERATIONS_READ.value, + # Vision Configs (read only) + Permission.VISION_CONFIGS_READ.value, # Connectors (read only) Permission.CONNECTORS_READ.value, # Logs (read only) @@ -1263,6 +1291,35 @@ class ImageGenerationConfig(BaseModel, TimestampMixin): user = relationship("User", back_populates="image_generation_configs") +class VisionLLMConfig(BaseModel, TimestampMixin): + __tablename__ = "vision_llm_configs" + + name = Column(String(100), nullable=False, index=True) + description = Column(String(500), nullable=True) + + provider = Column(SQLAlchemyEnum(VisionProvider), nullable=False) + custom_provider = Column(String(100), nullable=True) + model_name = Column(String(100), nullable=False) + + api_key = Column(String, nullable=False) + api_base = Column(String(500), nullable=True) + api_version = Column(String(50), nullable=True) + + litellm_params = Column(JSON, nullable=True, default={}) + + search_space_id = Column( + Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False + ) + search_space = relationship( + "SearchSpace", back_populates="vision_llm_configs" + ) + + user_id = Column( + UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False + ) + user = relationship("User", back_populates="vision_llm_configs") + + class ImageGeneration(BaseModel, TimestampMixin): """ Stores image generation requests and results using litellm.aimage_generation(). @@ -1351,7 +1408,7 @@ class SearchSpace(BaseModel, TimestampMixin): image_generation_config_id = Column( Integer, nullable=True, default=0 ) # For image generation, defaults to Auto mode - vision_llm_id = Column( + vision_llm_config_id = Column( Integer, nullable=True, default=0 ) # For vision/screenshot analysis, defaults to Auto mode @@ -1432,6 +1489,12 @@ class SearchSpace(BaseModel, TimestampMixin): order_by="ImageGenerationConfig.id", cascade="all, delete-orphan", ) + vision_llm_configs = relationship( + "VisionLLMConfig", + back_populates="search_space", + order_by="VisionLLMConfig.id", + cascade="all, delete-orphan", + ) # RBAC relationships roles = relationship( @@ -1961,6 +2024,12 @@ if config.AUTH_TYPE == "GOOGLE": passive_deletes=True, ) + vision_llm_configs = relationship( + "VisionLLMConfig", + back_populates="user", + passive_deletes=True, + ) + # User memories for personalized AI responses memories = relationship( "UserMemory", @@ -2075,6 +2144,12 @@ else: passive_deletes=True, ) + vision_llm_configs = relationship( + "VisionLLMConfig", + back_populates="user", + passive_deletes=True, + ) + # User memories for personalized AI responses memories = relationship( "UserMemory", From 32a3356f55cc3c9003df88e1d529eca5cf8babbc Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 18:50:51 +0200 Subject: [PATCH 22/47] Add migration 120: vision_llm_configs table and column rename --- .../120_add_vision_llm_configs_table.py | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 surfsense_backend/alembic/versions/120_add_vision_llm_configs_table.py diff --git a/surfsense_backend/alembic/versions/120_add_vision_llm_configs_table.py b/surfsense_backend/alembic/versions/120_add_vision_llm_configs_table.py new file mode 100644 index 000000000..c0c915388 --- /dev/null +++ b/surfsense_backend/alembic/versions/120_add_vision_llm_configs_table.py @@ -0,0 +1,190 @@ +"""Add vision LLM configs table and rename preference column + +Revision ID: 120 +Revises: 119 + +Changes: +1. Create visionprovider enum type +2. Create vision_llm_configs table +3. Rename vision_llm_id -> vision_llm_config_id on searchspaces +4. Add vision config permissions to existing system roles +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import ENUM as PG_ENUM, UUID + +from alembic import op + +revision: str = "120" +down_revision: str | None = "119" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +VISION_PROVIDER_VALUES = ( + "OPENAI", + "ANTHROPIC", + "GOOGLE", + "AZURE_OPENAI", + "VERTEX_AI", + "BEDROCK", + "XAI", + "OPENROUTER", + "OLLAMA", + "GROQ", + "TOGETHER_AI", + "FIREWORKS_AI", + "DEEPSEEK", + "MISTRAL", + "CUSTOM", +) + + +def upgrade() -> None: + connection = op.get_bind() + + # 1. Create visionprovider enum + connection.execute( + sa.text( + """ + DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'visionprovider') THEN + CREATE TYPE visionprovider AS ENUM ( + 'OPENAI', 'ANTHROPIC', 'GOOGLE', 'AZURE_OPENAI', 'VERTEX_AI', + 'BEDROCK', 'XAI', 'OPENROUTER', 'OLLAMA', 'GROQ', + 'TOGETHER_AI', 'FIREWORKS_AI', 'DEEPSEEK', 'MISTRAL', 'CUSTOM' + ); + END IF; + END + $$; + """ + ) + ) + + # 2. Create vision_llm_configs table + result = connection.execute( + sa.text( + "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'vision_llm_configs')" + ) + ) + if not result.scalar(): + op.create_table( + "vision_llm_configs", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("name", sa.String(100), nullable=False), + sa.Column("description", sa.String(500), nullable=True), + sa.Column( + "provider", + PG_ENUM(*VISION_PROVIDER_VALUES, name="visionprovider", create_type=False), + nullable=False, + ), + sa.Column("custom_provider", sa.String(100), nullable=True), + sa.Column("model_name", sa.String(100), nullable=False), + sa.Column("api_key", sa.String(), nullable=False), + sa.Column("api_base", sa.String(500), nullable=True), + sa.Column("api_version", sa.String(50), nullable=True), + sa.Column("litellm_params", sa.JSON(), nullable=True), + sa.Column("search_space_id", sa.Integer(), nullable=False), + sa.Column("user_id", UUID(as_uuid=True), nullable=False), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint( + ["search_space_id"], ["searchspaces.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["user_id"], ["user.id"], ondelete="CASCADE" + ), + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_vision_llm_configs_name " + "ON vision_llm_configs (name)" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_vision_llm_configs_search_space_id " + "ON vision_llm_configs (search_space_id)" + ) + + # 3. Rename vision_llm_id -> vision_llm_config_id on searchspaces + existing_columns = [ + col["name"] for col in sa.inspect(connection).get_columns("searchspaces") + ] + if "vision_llm_id" in existing_columns and "vision_llm_config_id" not in existing_columns: + op.alter_column("searchspaces", "vision_llm_id", new_column_name="vision_llm_config_id") + elif "vision_llm_config_id" not in existing_columns: + op.add_column( + "searchspaces", + sa.Column("vision_llm_config_id", sa.Integer(), nullable=True, server_default="0"), + ) + + # 4. Add vision config permissions to existing system roles + connection.execute( + sa.text( + """ + UPDATE search_space_roles + SET permissions = array_cat( + permissions, + ARRAY['vision_configs:create', 'vision_configs:read'] + ) + WHERE is_system_role = true + AND name = 'Editor' + AND NOT ('vision_configs:create' = ANY(permissions)) + """ + ) + ) + connection.execute( + sa.text( + """ + UPDATE search_space_roles + SET permissions = array_cat( + permissions, + ARRAY['vision_configs:read'] + ) + WHERE is_system_role = true + AND name = 'Viewer' + AND NOT ('vision_configs:read' = ANY(permissions)) + """ + ) + ) + + +def downgrade() -> None: + connection = op.get_bind() + + # Remove permissions + connection.execute( + sa.text( + """ + UPDATE search_space_roles + SET permissions = array_remove( + array_remove( + array_remove(permissions, 'vision_configs:create'), + 'vision_configs:read' + ), + 'vision_configs:delete' + ) + WHERE is_system_role = true + """ + ) + ) + + # Rename column back + existing_columns = [ + col["name"] for col in sa.inspect(connection).get_columns("searchspaces") + ] + if "vision_llm_config_id" in existing_columns: + op.alter_column("searchspaces", "vision_llm_config_id", new_column_name="vision_llm_id") + + # Drop table and enum + op.execute("DROP INDEX IF EXISTS ix_vision_llm_configs_search_space_id") + op.execute("DROP INDEX IF EXISTS ix_vision_llm_configs_name") + op.execute("DROP TABLE IF EXISTS vision_llm_configs") + op.execute("DROP TYPE IF EXISTS visionprovider") From ecfcc6101112bcb59d95469785d9acb0b656319a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 18:52:37 +0200 Subject: [PATCH 23/47] Add VisionLLMConfig Pydantic schemas --- surfsense_backend/app/schemas/__init__.py | 13 ++++ surfsense_backend/app/schemas/vision_llm.py | 75 +++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 surfsense_backend/app/schemas/vision_llm.py diff --git a/surfsense_backend/app/schemas/__init__.py b/surfsense_backend/app/schemas/__init__.py index b94a30c19..fdf34672b 100644 --- a/surfsense_backend/app/schemas/__init__.py +++ b/surfsense_backend/app/schemas/__init__.py @@ -125,6 +125,13 @@ from .video_presentations import ( VideoPresentationRead, VideoPresentationUpdate, ) +from .vision_llm import ( + GlobalVisionLLMConfigRead, + VisionLLMConfigCreate, + VisionLLMConfigPublic, + VisionLLMConfigRead, + VisionLLMConfigUpdate, +) __all__ = [ # Folder schemas @@ -163,6 +170,8 @@ __all__ = [ "FolderUpdate", "GlobalImageGenConfigRead", "GlobalNewLLMConfigRead", + # Vision LLM Config schemas + "GlobalVisionLLMConfigRead", "GoogleDriveIndexRequest", "GoogleDriveIndexingOptions", # Base schemas @@ -264,4 +273,8 @@ __all__ = [ "VideoPresentationCreate", "VideoPresentationRead", "VideoPresentationUpdate", + "VisionLLMConfigCreate", + "VisionLLMConfigPublic", + "VisionLLMConfigRead", + "VisionLLMConfigUpdate", ] diff --git a/surfsense_backend/app/schemas/vision_llm.py b/surfsense_backend/app/schemas/vision_llm.py new file mode 100644 index 000000000..ab2e609dc --- /dev/null +++ b/surfsense_backend/app/schemas/vision_llm.py @@ -0,0 +1,75 @@ +import uuid +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from app.db import VisionProvider + + +class VisionLLMConfigBase(BaseModel): + name: str = Field(..., max_length=100) + description: str | None = Field(None, max_length=500) + provider: VisionProvider = Field(...) + custom_provider: str | None = Field(None, max_length=100) + model_name: str = Field(..., max_length=100) + api_key: str = Field(...) + api_base: str | None = Field(None, max_length=500) + api_version: str | None = Field(None, max_length=50) + litellm_params: dict[str, Any] | None = Field(default=None) + + +class VisionLLMConfigCreate(VisionLLMConfigBase): + search_space_id: int = Field(...) + + +class VisionLLMConfigUpdate(BaseModel): + name: str | None = Field(None, max_length=100) + description: str | None = Field(None, max_length=500) + provider: VisionProvider | None = None + custom_provider: str | None = Field(None, max_length=100) + model_name: str | None = Field(None, max_length=100) + api_key: str | None = None + api_base: str | None = Field(None, max_length=500) + api_version: str | None = Field(None, max_length=50) + litellm_params: dict[str, Any] | None = None + + +class VisionLLMConfigRead(VisionLLMConfigBase): + id: int + created_at: datetime + search_space_id: int + user_id: uuid.UUID + + model_config = ConfigDict(from_attributes=True) + + +class VisionLLMConfigPublic(BaseModel): + id: int + name: str + description: str | None = None + provider: VisionProvider + custom_provider: str | None = None + model_name: str + api_base: str | None = None + api_version: str | None = None + litellm_params: dict[str, Any] | None = None + created_at: datetime + search_space_id: int + user_id: uuid.UUID + + model_config = ConfigDict(from_attributes=True) + + +class GlobalVisionLLMConfigRead(BaseModel): + id: int = Field(...) + name: str + description: str | None = None + provider: str + custom_provider: str | None = None + model_name: str + api_base: str | None = None + api_version: str | None = None + litellm_params: dict[str, Any] | None = None + is_global: bool = True + is_auto_mode: bool = False From 362cd3590c2c68b9bed4a461f00af839e6130c73 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 18:59:33 +0200 Subject: [PATCH 24/47] Add VisionLLMRouterService for Auto mode routing --- .../app/services/vision_llm_router_service.py | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 surfsense_backend/app/services/vision_llm_router_service.py diff --git a/surfsense_backend/app/services/vision_llm_router_service.py b/surfsense_backend/app/services/vision_llm_router_service.py new file mode 100644 index 000000000..0d782ab2b --- /dev/null +++ b/surfsense_backend/app/services/vision_llm_router_service.py @@ -0,0 +1,193 @@ +import logging +from typing import Any + +from litellm import Router + +logger = logging.getLogger(__name__) + +VISION_AUTO_MODE_ID = 0 + +VISION_PROVIDER_MAP = { + "OPENAI": "openai", + "ANTHROPIC": "anthropic", + "GOOGLE": "gemini", + "AZURE_OPENAI": "azure", + "VERTEX_AI": "vertex_ai", + "BEDROCK": "bedrock", + "XAI": "xai", + "OPENROUTER": "openrouter", + "OLLAMA": "ollama_chat", + "GROQ": "groq", + "TOGETHER_AI": "together_ai", + "FIREWORKS_AI": "fireworks_ai", + "DEEPSEEK": "openai", + "MISTRAL": "mistral", + "CUSTOM": "custom", +} + + +class VisionLLMRouterService: + _instance = None + _router: Router | None = None + _model_list: list[dict] = [] + _router_settings: dict = {} + _initialized: bool = False + + def __new__(cls): + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + @classmethod + def get_instance(cls) -> "VisionLLMRouterService": + if cls._instance is None: + cls._instance = cls() + return cls._instance + + @classmethod + def initialize( + cls, + global_configs: list[dict], + router_settings: dict | None = None, + ) -> None: + instance = cls.get_instance() + + if instance._initialized: + logger.debug("Vision LLM Router already initialized, skipping") + return + + model_list = [] + for config in global_configs: + deployment = cls._config_to_deployment(config) + if deployment: + model_list.append(deployment) + + if not model_list: + logger.warning( + "No valid vision LLM configs found for router initialization" + ) + return + + instance._model_list = model_list + instance._router_settings = router_settings or {} + + default_settings = { + "routing_strategy": "usage-based-routing", + "num_retries": 3, + "allowed_fails": 3, + "cooldown_time": 60, + "retry_after": 5, + } + + final_settings = {**default_settings, **instance._router_settings} + + try: + instance._router = Router( + model_list=model_list, + routing_strategy=final_settings.get( + "routing_strategy", "usage-based-routing" + ), + num_retries=final_settings.get("num_retries", 3), + allowed_fails=final_settings.get("allowed_fails", 3), + cooldown_time=final_settings.get("cooldown_time", 60), + set_verbose=False, + ) + instance._initialized = True + logger.info( + "Vision LLM Router initialized with %d deployments, strategy: %s", + len(model_list), + final_settings.get("routing_strategy"), + ) + except Exception as e: + logger.error(f"Failed to initialize Vision LLM Router: {e}") + instance._router = None + + @classmethod + def _config_to_deployment(cls, config: dict) -> dict | None: + try: + if not config.get("model_name") or not config.get("api_key"): + return None + + if config.get("custom_provider"): + model_string = f"{config['custom_provider']}/{config['model_name']}" + else: + provider = config.get("provider", "").upper() + provider_prefix = VISION_PROVIDER_MAP.get(provider, provider.lower()) + model_string = f"{provider_prefix}/{config['model_name']}" + + litellm_params: dict[str, Any] = { + "model": model_string, + "api_key": config.get("api_key"), + } + + if config.get("api_base"): + litellm_params["api_base"] = config["api_base"] + + if config.get("api_version"): + litellm_params["api_version"] = config["api_version"] + + if config.get("litellm_params"): + litellm_params.update(config["litellm_params"]) + + deployment: dict[str, Any] = { + "model_name": "auto", + "litellm_params": litellm_params, + } + + if config.get("rpm"): + deployment["rpm"] = config["rpm"] + if config.get("tpm"): + deployment["tpm"] = config["tpm"] + + return deployment + + except Exception as e: + logger.warning(f"Failed to convert vision config to deployment: {e}") + return None + + @classmethod + def get_router(cls) -> Router | None: + instance = cls.get_instance() + return instance._router + + @classmethod + def is_initialized(cls) -> bool: + instance = cls.get_instance() + return instance._initialized and instance._router is not None + + @classmethod + def get_model_count(cls) -> int: + instance = cls.get_instance() + return len(instance._model_list) + + +def is_vision_auto_mode(config_id: int | None) -> bool: + return config_id == VISION_AUTO_MODE_ID + + +def build_vision_model_string( + provider: str, model_name: str, custom_provider: str | None +) -> str: + if custom_provider: + return f"{custom_provider}/{model_name}" + prefix = VISION_PROVIDER_MAP.get(provider.upper(), provider.lower()) + return f"{prefix}/{model_name}" + + +def get_global_vision_llm_config(config_id: int) -> dict | None: + from app.config import config + + if config_id == VISION_AUTO_MODE_ID: + return { + "id": VISION_AUTO_MODE_ID, + "name": "Auto (Fastest)", + "provider": "AUTO", + "model_name": "auto", + "is_auto_mode": True, + } + if config_id > 0: + return None + for cfg in config.GLOBAL_VISION_LLM_CONFIGS: + if cfg.get("id") == config_id: + return cfg + return None From 7448f27ee01ce1619ff02f61ab3b12b6f085e60e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:02:18 +0200 Subject: [PATCH 25/47] Add vision LLM config loading and router initialization to Config --- surfsense_backend/app/config/__init__.py | 67 ++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 7c4baf923..4c49a4f8b 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -102,6 +102,44 @@ def load_global_image_gen_configs(): return [] +def load_global_vision_llm_configs(): + global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml" + + if not global_config_file.exists(): + return [] + + try: + with open(global_config_file, encoding="utf-8") as f: + data = yaml.safe_load(f) + return data.get("global_vision_llm_configs", []) + except Exception as e: + print(f"Warning: Failed to load global vision LLM configs: {e}") + return [] + + +def load_vision_llm_router_settings(): + default_settings = { + "routing_strategy": "usage-based-routing", + "num_retries": 3, + "allowed_fails": 3, + "cooldown_time": 60, + } + + global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml" + + if not global_config_file.exists(): + return default_settings + + try: + with open(global_config_file, encoding="utf-8") as f: + data = yaml.safe_load(f) + settings = data.get("vision_llm_router_settings", {}) + return {**default_settings, **settings} + except Exception as e: + print(f"Warning: Failed to load vision LLM router settings: {e}") + return default_settings + + def load_image_gen_router_settings(): """ Load router settings for image generation Auto mode from YAML file. @@ -182,6 +220,29 @@ def initialize_image_gen_router(): print(f"Warning: Failed to initialize Image Generation Router: {e}") +def initialize_vision_llm_router(): + vision_configs = load_global_vision_llm_configs() + router_settings = load_vision_llm_router_settings() + + if not vision_configs: + print( + "Info: No global vision LLM configs found, " + "Vision LLM Auto mode will not be available" + ) + return + + try: + from app.services.vision_llm_router_service import VisionLLMRouterService + + VisionLLMRouterService.initialize(vision_configs, router_settings) + print( + f"Info: Vision LLM Router initialized with {len(vision_configs)} models " + f"(strategy: {router_settings.get('routing_strategy', 'usage-based-routing')})" + ) + except Exception as e: + print(f"Warning: Failed to initialize Vision LLM Router: {e}") + + class Config: # Check if ffmpeg is installed if not is_ffmpeg_installed(): @@ -335,6 +396,12 @@ class Config: # Router settings for Image Generation Auto mode IMAGE_GEN_ROUTER_SETTINGS = load_image_gen_router_settings() + # Global Vision LLM Configurations (optional) + GLOBAL_VISION_LLM_CONFIGS = load_global_vision_llm_configs() + + # Router settings for Vision LLM Auto mode + VISION_LLM_ROUTER_SETTINGS = load_vision_llm_router_settings() + # Chonkie Configuration | Edit this to your needs EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL") # Azure OpenAI credentials from environment variables From bdbc4ce4a1cc4b6d4426e7537a7818ec19a5bfb3 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:04:03 +0200 Subject: [PATCH 26/47] Add vision LLM config CRUD and global configs routes --- .../app/routes/vision_llm_routes.py | 267 ++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 surfsense_backend/app/routes/vision_llm_routes.py diff --git a/surfsense_backend/app/routes/vision_llm_routes.py b/surfsense_backend/app/routes/vision_llm_routes.py new file mode 100644 index 000000000..29d1a2757 --- /dev/null +++ b/surfsense_backend/app/routes/vision_llm_routes.py @@ -0,0 +1,267 @@ +import logging + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import config +from app.db import ( + Permission, + User, + VisionLLMConfig, + get_async_session, +) +from app.schemas import ( + GlobalVisionLLMConfigRead, + VisionLLMConfigCreate, + VisionLLMConfigRead, + VisionLLMConfigUpdate, +) +from app.users import current_active_user +from app.utils.rbac import check_permission + +router = APIRouter() +logger = logging.getLogger(__name__) + + +# ============================================================================= +# Global Vision LLM Configs (from YAML) +# ============================================================================= + + +@router.get( + "/global-vision-llm-configs", + response_model=list[GlobalVisionLLMConfigRead], +) +async def get_global_vision_llm_configs( + user: User = Depends(current_active_user), +): + try: + global_configs = config.GLOBAL_VISION_LLM_CONFIGS + safe_configs = [] + + if global_configs and len(global_configs) > 0: + safe_configs.append( + { + "id": 0, + "name": "Auto (Fastest)", + "description": "Automatically routes across available vision LLM providers.", + "provider": "AUTO", + "custom_provider": None, + "model_name": "auto", + "api_base": None, + "api_version": None, + "litellm_params": {}, + "is_global": True, + "is_auto_mode": True, + } + ) + + for cfg in global_configs: + safe_configs.append( + { + "id": cfg.get("id"), + "name": cfg.get("name"), + "description": cfg.get("description"), + "provider": cfg.get("provider"), + "custom_provider": cfg.get("custom_provider"), + "model_name": cfg.get("model_name"), + "api_base": cfg.get("api_base") or None, + "api_version": cfg.get("api_version") or None, + "litellm_params": cfg.get("litellm_params", {}), + "is_global": True, + } + ) + + return safe_configs + except Exception as e: + logger.exception("Failed to fetch global vision LLM configs") + raise HTTPException( + status_code=500, detail=f"Failed to fetch configs: {e!s}" + ) from e + + +# ============================================================================= +# VisionLLMConfig CRUD +# ============================================================================= + + +@router.post("/vision-llm-configs", response_model=VisionLLMConfigRead) +async def create_vision_llm_config( + config_data: VisionLLMConfigCreate, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + try: + await check_permission( + session, + user, + config_data.search_space_id, + Permission.VISION_CONFIGS_CREATE.value, + "You don't have permission to create vision LLM configs in this search space", + ) + + db_config = VisionLLMConfig(**config_data.model_dump(), user_id=user.id) + session.add(db_config) + await session.commit() + await session.refresh(db_config) + return db_config + + except HTTPException: + raise + except Exception as e: + await session.rollback() + logger.exception("Failed to create VisionLLMConfig") + raise HTTPException( + status_code=500, detail=f"Failed to create config: {e!s}" + ) from e + + +@router.get("/vision-llm-configs", response_model=list[VisionLLMConfigRead]) +async def list_vision_llm_configs( + search_space_id: int, + skip: int = 0, + limit: int = 100, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + try: + await check_permission( + session, + user, + search_space_id, + Permission.VISION_CONFIGS_READ.value, + "You don't have permission to view vision LLM configs in this search space", + ) + + result = await session.execute( + select(VisionLLMConfig) + .filter(VisionLLMConfig.search_space_id == search_space_id) + .order_by(VisionLLMConfig.created_at.desc()) + .offset(skip) + .limit(limit) + ) + return result.scalars().all() + + except HTTPException: + raise + except Exception as e: + logger.exception("Failed to list VisionLLMConfigs") + raise HTTPException( + status_code=500, detail=f"Failed to fetch configs: {e!s}" + ) from e + + +@router.get( + "/vision-llm-configs/{config_id}", response_model=VisionLLMConfigRead +) +async def get_vision_llm_config( + config_id: int, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + try: + result = await session.execute( + select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id) + ) + db_config = result.scalars().first() + if not db_config: + raise HTTPException(status_code=404, detail="Config not found") + + await check_permission( + session, + user, + db_config.search_space_id, + Permission.VISION_CONFIGS_READ.value, + "You don't have permission to view vision LLM configs in this search space", + ) + return db_config + + except HTTPException: + raise + except Exception as e: + logger.exception("Failed to get VisionLLMConfig") + raise HTTPException( + status_code=500, detail=f"Failed to fetch config: {e!s}" + ) from e + + +@router.put( + "/vision-llm-configs/{config_id}", response_model=VisionLLMConfigRead +) +async def update_vision_llm_config( + config_id: int, + update_data: VisionLLMConfigUpdate, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + try: + result = await session.execute( + select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id) + ) + db_config = result.scalars().first() + if not db_config: + raise HTTPException(status_code=404, detail="Config not found") + + await check_permission( + session, + user, + db_config.search_space_id, + Permission.VISION_CONFIGS_CREATE.value, + "You don't have permission to update vision LLM configs in this search space", + ) + + for key, value in update_data.model_dump(exclude_unset=True).items(): + setattr(db_config, key, value) + + await session.commit() + await session.refresh(db_config) + return db_config + + except HTTPException: + raise + except Exception as e: + await session.rollback() + logger.exception("Failed to update VisionLLMConfig") + raise HTTPException( + status_code=500, detail=f"Failed to update config: {e!s}" + ) from e + + +@router.delete("/vision-llm-configs/{config_id}", response_model=dict) +async def delete_vision_llm_config( + config_id: int, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + try: + result = await session.execute( + select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id) + ) + db_config = result.scalars().first() + if not db_config: + raise HTTPException(status_code=404, detail="Config not found") + + await check_permission( + session, + user, + db_config.search_space_id, + Permission.VISION_CONFIGS_DELETE.value, + "You don't have permission to delete vision LLM configs in this search space", + ) + + await session.delete(db_config) + await session.commit() + return { + "message": "Vision LLM config deleted successfully", + "id": config_id, + } + + except HTTPException: + raise + except Exception as e: + await session.rollback() + logger.exception("Failed to delete VisionLLMConfig") + raise HTTPException( + status_code=500, detail=f"Failed to delete config: {e!s}" + ) from e From 43b8862ac77b3351cd068d762563ff985568447e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:16:51 +0200 Subject: [PATCH 27/47] Update get_vision_llm to use dedicated VisionLLMConfig system --- surfsense_backend/app/services/llm_service.py | 119 +++++++++++++++++- 1 file changed, 113 insertions(+), 6 deletions(-) diff --git a/surfsense_backend/app/services/llm_service.py b/surfsense_backend/app/services/llm_service.py index 7c0f9e7e3..e531aeabb 100644 --- a/surfsense_backend/app/services/llm_service.py +++ b/surfsense_backend/app/services/llm_service.py @@ -32,7 +32,6 @@ logger = logging.getLogger(__name__) class LLMRole: AGENT = "agent" # For agent/chat operations DOCUMENT_SUMMARY = "document_summary" # For document summarization - VISION = "vision" # For vision/screenshot analysis def get_global_llm_config(llm_config_id: int) -> dict | None: @@ -188,7 +187,7 @@ async def get_search_space_llm_instance( Args: session: Database session search_space_id: Search Space ID - role: LLM role ('agent', 'document_summary', or 'vision') + role: LLM role ('agent' or 'document_summary') Returns: ChatLiteLLM or ChatLiteLLMRouter instance, or None if not found @@ -210,8 +209,6 @@ async def get_search_space_llm_instance( llm_config_id = search_space.agent_llm_id elif role == LLMRole.DOCUMENT_SUMMARY: llm_config_id = search_space.document_summary_llm_id - elif role == LLMRole.VISION: - llm_config_id = search_space.vision_llm_id else: logger.error(f"Invalid LLM role: {role}") return None @@ -411,8 +408,118 @@ async def get_document_summary_llm( async def get_vision_llm( session: AsyncSession, search_space_id: int ) -> ChatLiteLLM | ChatLiteLLMRouter | None: - """Get the search space's vision LLM instance for screenshot analysis.""" - return await get_search_space_llm_instance(session, search_space_id, LLMRole.VISION) + """Get the search space's vision LLM instance for screenshot analysis. + + Resolves from the dedicated VisionLLMConfig system: + - Auto mode (ID 0): VisionLLMRouterService + - Global (negative ID): YAML configs + - DB (positive ID): VisionLLMConfig table + """ + from app.db import VisionLLMConfig + from app.services.vision_llm_router_service import ( + VISION_PROVIDER_MAP, + VisionLLMRouterService, + get_global_vision_llm_config, + is_vision_auto_mode, + ) + + try: + result = await session.execute( + select(SearchSpace).where(SearchSpace.id == search_space_id) + ) + search_space = result.scalars().first() + if not search_space: + logger.error(f"Search space {search_space_id} not found") + return None + + config_id = search_space.vision_llm_config_id + if config_id is None: + logger.error( + f"No vision LLM configured for search space {search_space_id}" + ) + return None + + if is_vision_auto_mode(config_id): + if not VisionLLMRouterService.is_initialized(): + logger.error( + "Vision Auto mode requested but Vision LLM Router not initialized" + ) + return None + try: + return ChatLiteLLMRouter( + router=VisionLLMRouterService.get_router(), + streaming=True, + ) + except Exception as e: + logger.error(f"Failed to create vision ChatLiteLLMRouter: {e}") + return None + + if config_id < 0: + global_cfg = get_global_vision_llm_config(config_id) + if not global_cfg: + logger.error(f"Global vision LLM config {config_id} not found") + return None + + if global_cfg.get("custom_provider"): + model_string = ( + f"{global_cfg['custom_provider']}/{global_cfg['model_name']}" + ) + else: + prefix = VISION_PROVIDER_MAP.get( + global_cfg["provider"].upper(), + global_cfg["provider"].lower(), + ) + model_string = f"{prefix}/{global_cfg['model_name']}" + + litellm_kwargs = { + "model": model_string, + "api_key": global_cfg["api_key"], + } + if global_cfg.get("api_base"): + litellm_kwargs["api_base"] = global_cfg["api_base"] + if global_cfg.get("litellm_params"): + litellm_kwargs.update(global_cfg["litellm_params"]) + + return ChatLiteLLM(**litellm_kwargs) + + result = await session.execute( + select(VisionLLMConfig).where( + VisionLLMConfig.id == config_id, + VisionLLMConfig.search_space_id == search_space_id, + ) + ) + vision_cfg = result.scalars().first() + if not vision_cfg: + logger.error( + f"Vision LLM config {config_id} not found in search space {search_space_id}" + ) + return None + + if vision_cfg.custom_provider: + model_string = f"{vision_cfg.custom_provider}/{vision_cfg.model_name}" + else: + prefix = VISION_PROVIDER_MAP.get( + vision_cfg.provider.value.upper(), + vision_cfg.provider.value.lower(), + ) + model_string = f"{prefix}/{vision_cfg.model_name}" + + litellm_kwargs = { + "model": model_string, + "api_key": vision_cfg.api_key, + } + if vision_cfg.api_base: + litellm_kwargs["api_base"] = vision_cfg.api_base + if vision_cfg.litellm_params: + litellm_kwargs.update(vision_cfg.litellm_params) + + return ChatLiteLLM(**litellm_kwargs) + + except Exception as e: + logger.error( + f"Error getting vision LLM for search space {search_space_id}: {e!s}" + ) + return None # Backward-compatible alias (LLM preferences are now per-search-space, not per-user) From 6d85821ae95836978f63ccf18ddbfd693e2dd956 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:20:28 +0200 Subject: [PATCH 28/47] Wire vision_llm_config_id in preferences, register vision LLM router --- surfsense_backend/app/routes/__init__.py | 2 + .../app/routes/search_spaces_routes.py | 74 +++++++++++++++++-- .../app/schemas/new_llm_config.py | 10 +-- 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 22631bc1d..02367606b 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -49,6 +49,7 @@ from .stripe_routes import router as stripe_router from .surfsense_docs_routes import router as surfsense_docs_router from .teams_add_connector_route import router as teams_add_connector_router from .video_presentations_routes import router as video_presentations_router +from .vision_llm_routes import router as vision_llm_router from .youtube_routes import router as youtube_router router = APIRouter() @@ -68,6 +69,7 @@ router.include_router( ) # Video presentation status and streaming router.include_router(reports_router) # Report CRUD and multi-format export router.include_router(image_generation_router) # Image generation via litellm +router.include_router(vision_llm_router) # Vision LLM configs for screenshot analysis router.include_router(search_source_connectors_router) router.include_router(google_calendar_add_connector_router) router.include_router(google_gmail_add_connector_router) diff --git a/surfsense_backend/app/routes/search_spaces_routes.py b/surfsense_backend/app/routes/search_spaces_routes.py index c4f1ab035..78be97aa1 100644 --- a/surfsense_backend/app/routes/search_spaces_routes.py +++ b/surfsense_backend/app/routes/search_spaces_routes.py @@ -14,6 +14,7 @@ from app.db import ( SearchSpaceMembership, SearchSpaceRole, User, + VisionLLMConfig, get_async_session, get_default_roles_config, ) @@ -483,6 +484,63 @@ async def _get_image_gen_config_by_id( return None +async def _get_vision_llm_config_by_id( + session: AsyncSession, config_id: int | None +) -> dict | None: + if config_id is None: + return None + + if config_id == 0: + return { + "id": 0, + "name": "Auto (Fastest)", + "description": "Automatically routes requests across available vision LLM providers", + "provider": "AUTO", + "model_name": "auto", + "is_global": True, + "is_auto_mode": True, + } + + if config_id < 0: + for cfg in config.GLOBAL_VISION_LLM_CONFIGS: + if cfg.get("id") == config_id: + return { + "id": cfg.get("id"), + "name": cfg.get("name"), + "description": cfg.get("description"), + "provider": cfg.get("provider"), + "custom_provider": cfg.get("custom_provider"), + "model_name": cfg.get("model_name"), + "api_base": cfg.get("api_base") or None, + "api_version": cfg.get("api_version") or None, + "litellm_params": cfg.get("litellm_params", {}), + "is_global": True, + } + return None + + result = await session.execute( + select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id) + ) + db_config = result.scalars().first() + if db_config: + return { + "id": db_config.id, + "name": db_config.name, + "description": db_config.description, + "provider": db_config.provider.value if db_config.provider else None, + "custom_provider": db_config.custom_provider, + "model_name": db_config.model_name, + "api_base": db_config.api_base, + "api_version": db_config.api_version, + "litellm_params": db_config.litellm_params or {}, + "created_at": db_config.created_at.isoformat() + if db_config.created_at + else None, + "search_space_id": db_config.search_space_id, + } + return None + + @router.get( "/search-spaces/{search_space_id}/llm-preferences", response_model=LLMPreferencesRead, @@ -522,17 +580,19 @@ async def get_llm_preferences( image_generation_config = await _get_image_gen_config_by_id( session, search_space.image_generation_config_id ) - vision_llm = await _get_llm_config_by_id(session, search_space.vision_llm_id) + vision_llm_config = await _get_vision_llm_config_by_id( + session, search_space.vision_llm_config_id + ) return LLMPreferencesRead( agent_llm_id=search_space.agent_llm_id, document_summary_llm_id=search_space.document_summary_llm_id, image_generation_config_id=search_space.image_generation_config_id, - vision_llm_id=search_space.vision_llm_id, + vision_llm_config_id=search_space.vision_llm_config_id, agent_llm=agent_llm, document_summary_llm=document_summary_llm, image_generation_config=image_generation_config, - vision_llm=vision_llm, + vision_llm_config=vision_llm_config, ) except HTTPException: @@ -592,17 +652,19 @@ async def update_llm_preferences( image_generation_config = await _get_image_gen_config_by_id( session, search_space.image_generation_config_id ) - vision_llm = await _get_llm_config_by_id(session, search_space.vision_llm_id) + vision_llm_config = await _get_vision_llm_config_by_id( + session, search_space.vision_llm_config_id + ) return LLMPreferencesRead( agent_llm_id=search_space.agent_llm_id, document_summary_llm_id=search_space.document_summary_llm_id, image_generation_config_id=search_space.image_generation_config_id, - vision_llm_id=search_space.vision_llm_id, + vision_llm_config_id=search_space.vision_llm_config_id, agent_llm=agent_llm, document_summary_llm=document_summary_llm, image_generation_config=image_generation_config, - vision_llm=vision_llm, + vision_llm_config=vision_llm_config, ) except HTTPException: diff --git a/surfsense_backend/app/schemas/new_llm_config.py b/surfsense_backend/app/schemas/new_llm_config.py index 6c76ca512..a466f2c99 100644 --- a/surfsense_backend/app/schemas/new_llm_config.py +++ b/surfsense_backend/app/schemas/new_llm_config.py @@ -182,8 +182,8 @@ class LLMPreferencesRead(BaseModel): image_generation_config_id: int | None = Field( None, description="ID of the image generation config to use" ) - vision_llm_id: int | None = Field( - None, description="ID of the LLM config to use for vision/screenshot analysis" + vision_llm_config_id: int | None = Field( + None, description="ID of the vision LLM config to use for vision/screenshot analysis" ) agent_llm: dict[str, Any] | None = Field( None, description="Full config for agent LLM" @@ -194,7 +194,7 @@ class LLMPreferencesRead(BaseModel): image_generation_config: dict[str, Any] | None = Field( None, description="Full config for image generation" ) - vision_llm: dict[str, Any] | None = Field( + vision_llm_config: dict[str, Any] | None = Field( None, description="Full config for vision LLM" ) @@ -213,6 +213,6 @@ class LLMPreferencesUpdate(BaseModel): image_generation_config_id: int | None = Field( None, description="ID of the image generation config to use" ) - vision_llm_id: int | None = Field( - None, description="ID of the LLM config to use for vision/screenshot analysis" + vision_llm_config_id: int | None = Field( + None, description="ID of the vision LLM config to use for vision/screenshot analysis" ) From 4a675b64f4b066a6148126df0121ccfbdc9dbe0a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:21:10 +0200 Subject: [PATCH 29/47] Initialize vision LLM router at app and celery startup --- surfsense_backend/app/app.py | 8 +++++++- surfsense_backend/app/celery_app.py | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/app.py b/surfsense_backend/app/app.py index bba2f1f3a..7b2b421ac 100644 --- a/surfsense_backend/app/app.py +++ b/surfsense_backend/app/app.py @@ -25,7 +25,12 @@ from app.agents.new_chat.checkpointer import ( close_checkpointer, setup_checkpointer_tables, ) -from app.config import config, initialize_image_gen_router, initialize_llm_router +from app.config import ( + config, + initialize_image_gen_router, + initialize_llm_router, + initialize_vision_llm_router, +) from app.db import User, create_db_and_tables, get_async_session from app.routes import router as crud_router from app.routes.auth_routes import router as auth_router @@ -223,6 +228,7 @@ async def lifespan(app: FastAPI): await setup_checkpointer_tables() initialize_llm_router() initialize_image_gen_router() + initialize_vision_llm_router() try: await asyncio.wait_for(seed_surfsense_docs(), timeout=120) except TimeoutError: diff --git a/surfsense_backend/app/celery_app.py b/surfsense_backend/app/celery_app.py index 684da6a13..bf2fdcb39 100644 --- a/surfsense_backend/app/celery_app.py +++ b/surfsense_backend/app/celery_app.py @@ -18,10 +18,15 @@ def init_worker(**kwargs): This ensures the Auto mode (LiteLLM Router) is available for background tasks like document summarization and image generation. """ - from app.config import initialize_image_gen_router, initialize_llm_router + from app.config import ( + initialize_image_gen_router, + initialize_llm_router, + initialize_vision_llm_router, + ) initialize_llm_router() initialize_image_gen_router() + initialize_vision_llm_router() # Get Celery configuration from environment From 3369b8a83210e9afc62f9c0db91d87a13fb0c478 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:24:43 +0200 Subject: [PATCH 30/47] Add frontend vision LLM config types, API, atoms, and role manager wiring --- .../vision-llm-config-mutation.atoms.ts | 84 +++++++++++++++ .../vision-llm-config-query.atoms.ts | 27 +++++ .../components/settings/llm-role-manager.tsx | 66 +++++++++--- .../contracts/enums/vision-providers.ts | 102 ++++++++++++++++++ .../contracts/types/new-llm-config.types.ts | 99 +++++++++++++++-- .../lib/apis/vision-llm-config-api.service.ts | 58 ++++++++++ surfsense_web/lib/query-client/cache-keys.ts | 5 + 7 files changed, 421 insertions(+), 20 deletions(-) create mode 100644 surfsense_web/atoms/vision-llm-config/vision-llm-config-mutation.atoms.ts create mode 100644 surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts create mode 100644 surfsense_web/contracts/enums/vision-providers.ts create mode 100644 surfsense_web/lib/apis/vision-llm-config-api.service.ts diff --git a/surfsense_web/atoms/vision-llm-config/vision-llm-config-mutation.atoms.ts b/surfsense_web/atoms/vision-llm-config/vision-llm-config-mutation.atoms.ts new file mode 100644 index 000000000..b1aa01c6b --- /dev/null +++ b/surfsense_web/atoms/vision-llm-config/vision-llm-config-mutation.atoms.ts @@ -0,0 +1,84 @@ +import { atomWithMutation } from "jotai-tanstack-query"; +import { toast } from "sonner"; +import type { + CreateVisionLLMConfigRequest, + CreateVisionLLMConfigResponse, + DeleteVisionLLMConfigResponse, + GetVisionLLMConfigsResponse, + UpdateVisionLLMConfigRequest, + UpdateVisionLLMConfigResponse, +} from "@/contracts/types/new-llm-config.types"; +import { visionLLMConfigApiService } from "@/lib/apis/vision-llm-config-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { queryClient } from "@/lib/query-client/client"; +import { activeSearchSpaceIdAtom } from "../search-spaces/search-space-query.atoms"; + +export const createVisionLLMConfigMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + mutationKey: ["vision-llm-configs", "create"], + enabled: !!searchSpaceId, + mutationFn: async (request: CreateVisionLLMConfigRequest) => { + return visionLLMConfigApiService.createConfig(request); + }, + onSuccess: (_: CreateVisionLLMConfigResponse, request: CreateVisionLLMConfigRequest) => { + toast.success(`${request.name} created`); + queryClient.invalidateQueries({ + queryKey: cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)), + }); + }, + onError: (error: Error) => { + toast.error(error.message || "Failed to create vision model"); + }, + }; +}); + +export const updateVisionLLMConfigMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + mutationKey: ["vision-llm-configs", "update"], + enabled: !!searchSpaceId, + mutationFn: async (request: UpdateVisionLLMConfigRequest) => { + return visionLLMConfigApiService.updateConfig(request); + }, + onSuccess: (_: UpdateVisionLLMConfigResponse, request: UpdateVisionLLMConfigRequest) => { + toast.success(`${request.data.name ?? "Configuration"} updated`); + queryClient.invalidateQueries({ + queryKey: cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)), + }); + queryClient.invalidateQueries({ + queryKey: cacheKeys.visionLLMConfigs.byId(request.id), + }); + }, + onError: (error: Error) => { + toast.error(error.message || "Failed to update vision model"); + }, + }; +}); + +export const deleteVisionLLMConfigMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + mutationKey: ["vision-llm-configs", "delete"], + enabled: !!searchSpaceId, + mutationFn: async (request: { id: number; name: string }) => { + return visionLLMConfigApiService.deleteConfig(request.id); + }, + onSuccess: (_: DeleteVisionLLMConfigResponse, request: { id: number; name: string }) => { + toast.success(`${request.name} deleted`); + queryClient.setQueryData( + cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)), + (oldData: GetVisionLLMConfigsResponse | undefined) => { + if (!oldData) return oldData; + return oldData.filter((config) => config.id !== request.id); + } + ); + }, + onError: (error: Error) => { + toast.error(error.message || "Failed to delete vision model"); + }, + }; +}); diff --git a/surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts b/surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts new file mode 100644 index 000000000..53264fb24 --- /dev/null +++ b/surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts @@ -0,0 +1,27 @@ +import { atomWithQuery } from "jotai-tanstack-query"; +import { visionLLMConfigApiService } from "@/lib/apis/vision-llm-config-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { activeSearchSpaceIdAtom } from "../search-spaces/search-space-query.atoms"; + +export const visionLLMConfigsAtom = atomWithQuery((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + queryKey: cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)), + enabled: !!searchSpaceId, + staleTime: 5 * 60 * 1000, + queryFn: async () => { + return visionLLMConfigApiService.getConfigs(Number(searchSpaceId)); + }, + }; +}); + +export const globalVisionLLMConfigsAtom = atomWithQuery(() => { + return { + queryKey: cacheKeys.visionLLMConfigs.global(), + staleTime: 10 * 60 * 1000, + queryFn: async () => { + return visionLLMConfigApiService.getGlobalConfigs(); + }, + }; +}); diff --git a/surfsense_web/components/settings/llm-role-manager.tsx b/surfsense_web/components/settings/llm-role-manager.tsx index 386845d7d..995159d58 100644 --- a/surfsense_web/components/settings/llm-role-manager.tsx +++ b/surfsense_web/components/settings/llm-role-manager.tsx @@ -24,6 +24,10 @@ import { llmPreferencesAtom, newLLMConfigsAtom, } from "@/atoms/new-llm-config/new-llm-config-query.atoms"; +import { + globalVisionLLMConfigsAtom, + visionLLMConfigsAtom, +} from "@/atoms/vision-llm-config/vision-llm-config-query.atoms"; import { Alert, AlertDescription } from "@/components/ui/alert"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -77,8 +81,8 @@ const ROLE_DESCRIPTIONS = { description: "Vision-capable model for screenshot analysis and context extraction", color: "text-amber-600 dark:text-amber-400", bgColor: "bg-amber-500/10", - prefKey: "vision_llm_id" as const, - configType: "llm" as const, + prefKey: "vision_llm_config_id" as const, + configType: "vision" as const, }, }; @@ -112,6 +116,18 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { error: globalImageConfigsError, } = useAtomValue(globalImageGenConfigsAtom); + // Vision LLM configs + const { + data: userVisionConfigs = [], + isFetching: visionConfigsLoading, + error: visionConfigsError, + } = useAtomValue(visionLLMConfigsAtom); + const { + data: globalVisionConfigs = [], + isFetching: globalVisionConfigsLoading, + error: globalVisionConfigsError, + } = useAtomValue(globalVisionLLMConfigsAtom); + // Preferences const { data: preferences = {}, @@ -125,7 +141,7 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { agent_llm_id: preferences.agent_llm_id ?? "", document_summary_llm_id: preferences.document_summary_llm_id ?? "", image_generation_config_id: preferences.image_generation_config_id ?? "", - vision_llm_id: preferences.vision_llm_id ?? "", + vision_llm_config_id: preferences.vision_llm_config_id ?? "", })); const [savingRole, setSavingRole] = useState(null); @@ -137,14 +153,14 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { agent_llm_id: preferences.agent_llm_id ?? "", document_summary_llm_id: preferences.document_summary_llm_id ?? "", image_generation_config_id: preferences.image_generation_config_id ?? "", - vision_llm_id: preferences.vision_llm_id ?? "", + vision_llm_config_id: preferences.vision_llm_config_id ?? "", }); } }, [ preferences?.agent_llm_id, preferences?.document_summary_llm_id, preferences?.image_generation_config_id, - preferences?.vision_llm_id, + preferences?.vision_llm_config_id, ]); const handleRoleAssignment = useCallback( @@ -181,6 +197,14 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { ...(userImageConfigs ?? []).filter((config) => config.id && config.id.toString().trim() !== ""), ]; + // Combine global and custom vision LLM configs + const allVisionConfigs = [ + ...globalVisionConfigs.map((config) => ({ ...config, is_global: true })), + ...(userVisionConfigs ?? []).filter( + (config) => config.id && config.id.toString().trim() !== "" + ), + ]; + const isAssignmentComplete = allLLMConfigs.some((c) => c.id === assignments.agent_llm_id) && allLLMConfigs.some((c) => c.id === assignments.document_summary_llm_id) && @@ -191,13 +215,17 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { preferencesLoading || globalConfigsLoading || imageConfigsLoading || - globalImageConfigsLoading; + globalImageConfigsLoading || + visionConfigsLoading || + globalVisionConfigsLoading; const hasError = configsError || preferencesError || globalConfigsError || imageConfigsError || - globalImageConfigsError; + globalImageConfigsError || + visionConfigsError || + globalVisionConfigsError; const hasAnyConfigs = allLLMConfigs.length > 0 || allImageConfigs.length > 0; return ( @@ -291,15 +319,27 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
{Object.entries(ROLE_DESCRIPTIONS).map(([key, role]) => { const IconComponent = role.icon; - const isImageRole = role.configType === "image"; const currentAssignment = assignments[role.prefKey as keyof typeof assignments]; // Pick the right config lists based on role type - const roleGlobalConfigs = isImageRole ? globalImageConfigs : globalConfigs; - const roleUserConfigs = isImageRole - ? (userImageConfigs ?? []).filter((c) => c.id && c.id.toString().trim() !== "") - : newLLMConfigs.filter((c) => c.id && c.id.toString().trim() !== ""); - const roleAllConfigs = isImageRole ? allImageConfigs : allLLMConfigs; + const roleGlobalConfigs = + role.configType === "image" + ? globalImageConfigs + : role.configType === "vision" + ? globalVisionConfigs + : globalConfigs; + const roleUserConfigs = + role.configType === "image" + ? (userImageConfigs ?? []).filter((c) => c.id && c.id.toString().trim() !== "") + : role.configType === "vision" + ? (userVisionConfigs ?? []).filter((c) => c.id && c.id.toString().trim() !== "") + : newLLMConfigs.filter((c) => c.id && c.id.toString().trim() !== ""); + const roleAllConfigs = + role.configType === "image" + ? allImageConfigs + : role.configType === "vision" + ? allVisionConfigs + : allLLMConfigs; const assignedConfig = roleAllConfigs.find((config) => config.id === currentAssignment); const isAssigned = !!assignedConfig; diff --git a/surfsense_web/contracts/enums/vision-providers.ts b/surfsense_web/contracts/enums/vision-providers.ts new file mode 100644 index 000000000..260b03585 --- /dev/null +++ b/surfsense_web/contracts/enums/vision-providers.ts @@ -0,0 +1,102 @@ +export interface VisionProviderInfo { + value: string; + label: string; + example: string; + description: string; + apiBase?: string; +} + +export const VISION_PROVIDERS: VisionProviderInfo[] = [ + { + value: "OPENAI", + label: "OpenAI", + example: "gpt-4o, gpt-4o-mini", + description: "GPT-4o vision models", + }, + { + value: "ANTHROPIC", + label: "Anthropic", + example: "claude-sonnet-4-20250514", + description: "Claude vision models", + }, + { + value: "GOOGLE", + label: "Google AI Studio", + example: "gemini-2.5-flash, gemini-2.0-flash", + description: "Gemini vision models", + }, + { + value: "AZURE_OPENAI", + label: "Azure OpenAI", + example: "azure/gpt-4o", + description: "OpenAI vision models on Azure", + }, + { + value: "VERTEX_AI", + label: "Google Vertex AI", + example: "vertex_ai/gemini-2.5-flash", + description: "Gemini vision models on Vertex AI", + }, + { + value: "BEDROCK", + label: "AWS Bedrock", + example: "bedrock/anthropic.claude-sonnet-4-20250514-v1:0", + description: "Vision models on AWS Bedrock", + }, + { + value: "XAI", + label: "xAI", + example: "grok-2-vision", + description: "Grok vision models", + }, + { + value: "OPENROUTER", + label: "OpenRouter", + example: "openrouter/openai/gpt-4o", + description: "Vision models via OpenRouter", + }, + { + value: "OLLAMA", + label: "Ollama", + example: "llava, bakllava", + description: "Local vision models via Ollama", + apiBase: "http://localhost:11434", + }, + { + value: "GROQ", + label: "Groq", + example: "llama-4-scout-17b-16e-instruct", + description: "Vision models on Groq", + }, + { + value: "TOGETHER_AI", + label: "Together AI", + example: "meta-llama/Llama-4-Scout-17B-16E-Instruct", + description: "Vision models on Together AI", + }, + { + value: "FIREWORKS_AI", + label: "Fireworks AI", + example: "fireworks_ai/phi-3-vision-128k-instruct", + description: "Vision models on Fireworks AI", + }, + { + value: "DEEPSEEK", + label: "DeepSeek", + example: "deepseek-chat", + description: "DeepSeek vision models", + apiBase: "https://api.deepseek.com", + }, + { + value: "MISTRAL", + label: "Mistral", + example: "pixtral-large-latest", + description: "Pixtral vision models", + }, + { + value: "CUSTOM", + label: "Custom Provider", + example: "custom/my-vision-model", + description: "Custom OpenAI-compatible vision endpoint", + }, +]; diff --git a/surfsense_web/contracts/types/new-llm-config.types.ts b/surfsense_web/contracts/types/new-llm-config.types.ts index 02837cc73..6bef94bac 100644 --- a/surfsense_web/contracts/types/new-llm-config.types.ts +++ b/surfsense_web/contracts/types/new-llm-config.types.ts @@ -252,23 +252,99 @@ export const globalImageGenConfig = z.object({ export const getGlobalImageGenConfigsResponse = z.array(globalImageGenConfig); +// ============================================================================= +// Vision LLM Config (separate table for vision-capable models) +// ============================================================================= + +export const visionProviderEnum = z.enum([ + "OPENAI", + "ANTHROPIC", + "GOOGLE", + "AZURE_OPENAI", + "VERTEX_AI", + "BEDROCK", + "XAI", + "OPENROUTER", + "OLLAMA", + "GROQ", + "TOGETHER_AI", + "FIREWORKS_AI", + "DEEPSEEK", + "MISTRAL", + "CUSTOM", +]); + +export type VisionProvider = z.infer; + +export const visionLLMConfig = z.object({ + id: z.number(), + name: z.string().max(100), + description: z.string().max(500).nullable().optional(), + provider: visionProviderEnum, + custom_provider: z.string().max(100).nullable().optional(), + model_name: z.string().max(100), + api_key: z.string(), + api_base: z.string().max(500).nullable().optional(), + api_version: z.string().max(50).nullable().optional(), + litellm_params: z.record(z.string(), z.any()).nullable().optional(), + created_at: z.string(), + search_space_id: z.number(), + user_id: z.string(), +}); + +export const createVisionLLMConfigRequest = visionLLMConfig.omit({ + id: true, + created_at: true, + user_id: true, +}); + +export const createVisionLLMConfigResponse = visionLLMConfig; + +export const getVisionLLMConfigsResponse = z.array(visionLLMConfig); + +export const updateVisionLLMConfigRequest = z.object({ + id: z.number(), + data: visionLLMConfig + .omit({ id: true, created_at: true, search_space_id: true, user_id: true }) + .partial(), +}); + +export const updateVisionLLMConfigResponse = visionLLMConfig; + +export const deleteVisionLLMConfigResponse = z.object({ + message: z.string(), + id: z.number(), +}); + +export const globalVisionLLMConfig = z.object({ + id: z.number(), + name: z.string(), + description: z.string().nullable().optional(), + provider: z.string(), + custom_provider: z.string().nullable().optional(), + model_name: z.string(), + api_base: z.string().nullable().optional(), + api_version: z.string().nullable().optional(), + litellm_params: z.record(z.string(), z.any()).nullable().optional(), + is_global: z.literal(true), + is_auto_mode: z.boolean().optional().default(false), +}); + +export const getGlobalVisionLLMConfigsResponse = z.array(globalVisionLLMConfig); + // ============================================================================= // LLM Preferences (Role Assignments) // ============================================================================= -/** - * LLM Preferences schemas - for role assignments - * image_generation uses image_generation_config_id (not llm_id) - */ export const llmPreferences = z.object({ agent_llm_id: z.union([z.number(), z.null()]).optional(), document_summary_llm_id: z.union([z.number(), z.null()]).optional(), image_generation_config_id: z.union([z.number(), z.null()]).optional(), - vision_llm_id: z.union([z.number(), z.null()]).optional(), + vision_llm_config_id: z.union([z.number(), z.null()]).optional(), agent_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), document_summary_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), image_generation_config: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), - vision_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), + vision_llm_config: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), }); /** @@ -289,7 +365,7 @@ export const updateLLMPreferencesRequest = z.object({ agent_llm_id: true, document_summary_llm_id: true, image_generation_config_id: true, - vision_llm_id: true, + vision_llm_config_id: true, }), }); @@ -341,6 +417,15 @@ export type UpdateImageGenConfigResponse = z.infer; export type GlobalImageGenConfig = z.infer; export type GetGlobalImageGenConfigsResponse = z.infer; +export type VisionLLMConfig = z.infer; +export type CreateVisionLLMConfigRequest = z.infer; +export type CreateVisionLLMConfigResponse = z.infer; +export type GetVisionLLMConfigsResponse = z.infer; +export type UpdateVisionLLMConfigRequest = z.infer; +export type UpdateVisionLLMConfigResponse = z.infer; +export type DeleteVisionLLMConfigResponse = z.infer; +export type GlobalVisionLLMConfig = z.infer; +export type GetGlobalVisionLLMConfigsResponse = z.infer; export type LLMPreferences = z.infer; export type GetLLMPreferencesRequest = z.infer; export type GetLLMPreferencesResponse = z.infer; diff --git a/surfsense_web/lib/apis/vision-llm-config-api.service.ts b/surfsense_web/lib/apis/vision-llm-config-api.service.ts new file mode 100644 index 000000000..4099c6b39 --- /dev/null +++ b/surfsense_web/lib/apis/vision-llm-config-api.service.ts @@ -0,0 +1,58 @@ +import { + type CreateVisionLLMConfigRequest, + createVisionLLMConfigRequest, + createVisionLLMConfigResponse, + deleteVisionLLMConfigResponse, + getGlobalVisionLLMConfigsResponse, + getVisionLLMConfigsResponse, + type UpdateVisionLLMConfigRequest, + updateVisionLLMConfigRequest, + updateVisionLLMConfigResponse, +} from "@/contracts/types/new-llm-config.types"; +import { ValidationError } from "../error"; +import { baseApiService } from "./base-api.service"; + +class VisionLLMConfigApiService { + getGlobalConfigs = async () => { + return baseApiService.get( + `/api/v1/global-vision-llm-configs`, + getGlobalVisionLLMConfigsResponse + ); + }; + + createConfig = async (request: CreateVisionLLMConfigRequest) => { + const parsed = createVisionLLMConfigRequest.safeParse(request); + if (!parsed.success) { + const msg = parsed.error.issues.map((i) => i.message).join(", "); + throw new ValidationError(`Invalid request: ${msg}`); + } + return baseApiService.post(`/api/v1/vision-llm-configs`, createVisionLLMConfigResponse, { + body: parsed.data, + }); + }; + + getConfigs = async (searchSpaceId: number) => { + const params = new URLSearchParams({ + search_space_id: String(searchSpaceId), + }).toString(); + return baseApiService.get(`/api/v1/vision-llm-configs?${params}`, getVisionLLMConfigsResponse); + }; + + updateConfig = async (request: UpdateVisionLLMConfigRequest) => { + const parsed = updateVisionLLMConfigRequest.safeParse(request); + if (!parsed.success) { + const msg = parsed.error.issues.map((i) => i.message).join(", "); + throw new ValidationError(`Invalid request: ${msg}`); + } + const { id, data } = parsed.data; + return baseApiService.put(`/api/v1/vision-llm-configs/${id}`, updateVisionLLMConfigResponse, { + body: data, + }); + }; + + deleteConfig = async (id: number) => { + return baseApiService.delete(`/api/v1/vision-llm-configs/${id}`, deleteVisionLLMConfigResponse); + }; +} + +export const visionLLMConfigApiService = new VisionLLMConfigApiService(); diff --git a/surfsense_web/lib/query-client/cache-keys.ts b/surfsense_web/lib/query-client/cache-keys.ts index 754886618..04f348ff8 100644 --- a/surfsense_web/lib/query-client/cache-keys.ts +++ b/surfsense_web/lib/query-client/cache-keys.ts @@ -39,6 +39,11 @@ export const cacheKeys = { byId: (configId: number) => ["image-gen-configs", "detail", configId] as const, global: () => ["image-gen-configs", "global"] as const, }, + visionLLMConfigs: { + all: (searchSpaceId: number) => ["vision-llm-configs", searchSpaceId] as const, + byId: (configId: number) => ["vision-llm-configs", "detail", configId] as const, + global: () => ["vision-llm-configs", "global"] as const, + }, auth: { user: ["auth", "user"] as const, }, From 3bbe6c303737f63b089ddb5501d67f47d65a37b5 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:27:24 +0200 Subject: [PATCH 31/47] Add VisionModelManager and VisionConfigDialog components --- .../settings/vision-model-manager.tsx | 401 ++++++++++++++++++ .../shared/vision-config-dialog.tsx | 381 +++++++++++++++++ 2 files changed, 782 insertions(+) create mode 100644 surfsense_web/components/settings/vision-model-manager.tsx create mode 100644 surfsense_web/components/shared/vision-config-dialog.tsx diff --git a/surfsense_web/components/settings/vision-model-manager.tsx b/surfsense_web/components/settings/vision-model-manager.tsx new file mode 100644 index 000000000..31e6655cb --- /dev/null +++ b/surfsense_web/components/settings/vision-model-manager.tsx @@ -0,0 +1,401 @@ +"use client"; + +import { useAtomValue } from "jotai"; +import { AlertCircle, Dot, Edit3, Info, RefreshCw, Trash2 } from "lucide-react"; +import { useMemo, useState } from "react"; +import { membersAtom, myAccessAtom } from "@/atoms/members/members-query.atoms"; +import { deleteVisionLLMConfigMutationAtom } from "@/atoms/vision-llm-config/vision-llm-config-mutation.atoms"; +import { + globalVisionLLMConfigsAtom, + visionLLMConfigsAtom, +} from "@/atoms/vision-llm-config/vision-llm-config-query.atoms"; +import { VisionConfigDialog } from "@/components/shared/vision-config-dialog"; +import { Alert, AlertDescription } from "@/components/ui/alert"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Avatar, AvatarFallback, AvatarImage } from "@/components/ui/avatar"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Spinner } from "@/components/ui/spinner"; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; +import type { VisionLLMConfig } from "@/contracts/types/new-llm-config.types"; +import { useMediaQuery } from "@/hooks/use-media-query"; +import { getProviderIcon } from "@/lib/provider-icons"; +import { cn } from "@/lib/utils"; + +interface VisionModelManagerProps { + searchSpaceId: number; +} + +function getInitials(name: string): string { + const parts = name.trim().split(/\s+/); + if (parts.length >= 2) { + return (parts[0][0] + parts[1][0]).toUpperCase(); + } + return name.slice(0, 2).toUpperCase(); +} + +export function VisionModelManager({ searchSpaceId }: VisionModelManagerProps) { + const isDesktop = useMediaQuery("(min-width: 768px)"); + + const { + mutateAsync: deleteConfig, + isPending: isDeleting, + error: deleteError, + } = useAtomValue(deleteVisionLLMConfigMutationAtom); + + const { + data: userConfigs, + isFetching: configsLoading, + error: fetchError, + refetch: refreshConfigs, + } = useAtomValue(visionLLMConfigsAtom); + const { data: globalConfigs = [], isFetching: globalLoading } = useAtomValue( + globalVisionLLMConfigsAtom + ); + + const { data: members } = useAtomValue(membersAtom); + const memberMap = useMemo(() => { + const map = new Map(); + if (members) { + for (const m of members) { + map.set(m.user_id, { + name: m.user_display_name || m.user_email || "Unknown", + email: m.user_email || undefined, + avatarUrl: m.user_avatar_url || undefined, + }); + } + } + return map; + }, [members]); + + const { data: access } = useAtomValue(myAccessAtom); + const canCreate = useMemo(() => { + if (!access) return false; + if (access.is_owner) return true; + return access.permissions?.includes("vision_configs:create") ?? false; + }, [access]); + const canDelete = useMemo(() => { + if (!access) return false; + if (access.is_owner) return true; + return access.permissions?.includes("vision_configs:delete") ?? false; + }, [access]); + const canUpdate = canCreate; + const isReadOnly = !canCreate && !canDelete; + + const [isDialogOpen, setIsDialogOpen] = useState(false); + const [editingConfig, setEditingConfig] = useState(null); + const [configToDelete, setConfigToDelete] = useState(null); + + const isLoading = configsLoading || globalLoading; + const errors = [deleteError, fetchError].filter(Boolean) as Error[]; + + const openEditDialog = (config: VisionLLMConfig) => { + setEditingConfig(config); + setIsDialogOpen(true); + }; + + const openNewDialog = () => { + setEditingConfig(null); + setIsDialogOpen(true); + }; + + const handleDelete = async () => { + if (!configToDelete) return; + try { + await deleteConfig({ id: configToDelete.id, name: configToDelete.name }); + setConfigToDelete(null); + } catch { + // Error handled by mutation + } + }; + + return ( +
+
+ + {canCreate && ( + + )} +
+ + {errors.map((err) => ( +
+ + + {err?.message} + +
+ ))} + + {access && !isLoading && isReadOnly && ( +
+ + + + You have read-only access to vision model + configurations. Contact a space owner to request additional permissions. + + +
+ )} + {access && !isLoading && !isReadOnly && (!canCreate || !canDelete) && ( +
+ + + + You can{" "} + {[canCreate && "create and edit", canDelete && "delete"] + .filter(Boolean) + .join(" and ")}{" "} + vision model configurations + {!canDelete && ", but cannot delete them"}. + + +
+ )} + + {globalConfigs.filter((g) => !("is_auto_mode" in g && g.is_auto_mode)).length > 0 && ( + + + +

+ + {globalConfigs.filter((g) => !("is_auto_mode" in g && g.is_auto_mode)).length}{" "} + global vision{" "} + {globalConfigs.filter((g) => !("is_auto_mode" in g && g.is_auto_mode)).length === 1 + ? "model" + : "models"} + {" "} + available from your administrator. Use the model selector to view and select them. +

+ + + )} + + {isLoading && ( +
+
+
+ + +
+
+ {["skeleton-a", "skeleton-b", "skeleton-c"].map((key) => ( + + +
+
+ + +
+
+
+ + +
+
+ + + +
+ + + ))} +
+
+
+ )} + + {!isLoading && ( +
+ {(userConfigs?.length ?? 0) === 0 ? ( + + +

No Vision Models Yet

+

+ {canCreate + ? "Add your own vision-capable model (GPT-4o, Claude, Gemini, etc.)" + : "No vision models have been added to this space yet. Contact a space owner to add one."} +

+ + + ) : ( +
+ {userConfigs?.map((config) => { + const member = config.user_id ? memberMap.get(config.user_id) : null; + + return ( +
+ + +
+
+

+ {config.name} +

+ {config.description && ( +

+ {config.description} +

+ )} +
+ {(canUpdate || canDelete) && ( +
+ {canUpdate && ( + + + + + + Edit + + + )} + {canDelete && ( + + + + + + Delete + + + )} +
+ )} +
+ +
+ {getProviderIcon(config.provider, { + className: "size-3.5 shrink-0", + })} + + {config.model_name} + +
+ +
+ + {new Date(config.created_at).toLocaleDateString(undefined, { + year: "numeric", + month: "short", + day: "numeric", + })} + + {member && ( + <> + + + + +
+ + {member.avatarUrl && ( + + )} + + {getInitials(member.name)} + + + + {member.name} + +
+ + + {member.email || member.name} + + + + + )} +
+ + +
+ ); + })} +
+ )} +
+ )} + + { + setIsDialogOpen(open); + if (!open) setEditingConfig(null); + }} + config={editingConfig} + isGlobal={false} + searchSpaceId={searchSpaceId} + mode={editingConfig ? "edit" : "create"} + /> + + !open && setConfigToDelete(null)} + > + + + Delete Vision Model + + Are you sure you want to delete{" "} + {configToDelete?.name}? + + + + Cancel + + Delete + {isDeleting && } + + + + +
+ ); +} diff --git a/surfsense_web/components/shared/vision-config-dialog.tsx b/surfsense_web/components/shared/vision-config-dialog.tsx new file mode 100644 index 000000000..d69750316 --- /dev/null +++ b/surfsense_web/components/shared/vision-config-dialog.tsx @@ -0,0 +1,381 @@ +"use client"; + +import { useAtomValue } from "jotai"; +import { AlertCircle } from "lucide-react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { toast } from "sonner"; +import { updateLLMPreferencesMutationAtom } from "@/atoms/new-llm-config/new-llm-config-mutation.atoms"; +import { + createVisionLLMConfigMutationAtom, + updateVisionLLMConfigMutationAtom, +} from "@/atoms/vision-llm-config/vision-llm-config-mutation.atoms"; +import { Alert, AlertDescription } from "@/components/ui/alert"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Separator } from "@/components/ui/separator"; +import { Spinner } from "@/components/ui/spinner"; +import { VISION_PROVIDERS } from "@/contracts/enums/vision-providers"; +import type { + GlobalVisionLLMConfig, + VisionLLMConfig, + VisionProvider, +} from "@/contracts/types/new-llm-config.types"; + +interface VisionConfigDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + config: VisionLLMConfig | GlobalVisionLLMConfig | null; + isGlobal: boolean; + searchSpaceId: number; + mode: "create" | "edit" | "view"; +} + +const INITIAL_FORM = { + name: "", + description: "", + provider: "", + model_name: "", + api_key: "", + api_base: "", + api_version: "", +}; + +export function VisionConfigDialog({ + open, + onOpenChange, + config, + isGlobal, + searchSpaceId, + mode, +}: VisionConfigDialogProps) { + const [isSubmitting, setIsSubmitting] = useState(false); + const [formData, setFormData] = useState(INITIAL_FORM); + const [scrollPos, setScrollPos] = useState<"top" | "middle" | "bottom">("top"); + const scrollRef = useRef(null); + + useEffect(() => { + if (open) { + if (mode === "edit" && config && !isGlobal) { + setFormData({ + name: config.name || "", + description: config.description || "", + provider: config.provider || "", + model_name: config.model_name || "", + api_key: (config as VisionLLMConfig).api_key || "", + api_base: config.api_base || "", + api_version: (config as VisionLLMConfig).api_version || "", + }); + } else if (mode === "create") { + setFormData(INITIAL_FORM); + } + setScrollPos("top"); + } + }, [open, mode, config, isGlobal]); + + const { mutateAsync: createConfig } = useAtomValue(createVisionLLMConfigMutationAtom); + const { mutateAsync: updateConfig } = useAtomValue(updateVisionLLMConfigMutationAtom); + const { mutateAsync: updatePreferences } = useAtomValue(updateLLMPreferencesMutationAtom); + + const handleScroll = useCallback((e: React.UIEvent) => { + const el = e.currentTarget; + const atTop = el.scrollTop <= 2; + const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight <= 2; + setScrollPos(atTop ? "top" : atBottom ? "bottom" : "middle"); + }, []); + + const getTitle = () => { + if (mode === "create") return "Add Vision Model"; + if (isGlobal) return "View Global Vision Model"; + return "Edit Vision Model"; + }; + + const getSubtitle = () => { + if (mode === "create") return "Set up a new vision-capable LLM provider"; + if (isGlobal) return "Read-only global configuration"; + return "Update your vision model settings"; + }; + + const handleSubmit = useCallback(async () => { + setIsSubmitting(true); + try { + if (mode === "create") { + const result = await createConfig({ + name: formData.name, + provider: formData.provider as VisionProvider, + model_name: formData.model_name, + api_key: formData.api_key, + api_base: formData.api_base || undefined, + api_version: formData.api_version || undefined, + description: formData.description || undefined, + search_space_id: searchSpaceId, + }); + if (result?.id) { + await updatePreferences({ + search_space_id: searchSpaceId, + data: { vision_llm_config_id: result.id }, + }); + } + onOpenChange(false); + } else if (!isGlobal && config) { + await updateConfig({ + id: config.id, + data: { + name: formData.name, + description: formData.description || undefined, + provider: formData.provider as VisionProvider, + model_name: formData.model_name, + api_key: formData.api_key, + api_base: formData.api_base || undefined, + api_version: formData.api_version || undefined, + }, + }); + onOpenChange(false); + } + } catch (error) { + console.error("Failed to save vision config:", error); + toast.error("Failed to save vision model"); + } finally { + setIsSubmitting(false); + } + }, [ + mode, + isGlobal, + config, + formData, + searchSpaceId, + createConfig, + updateConfig, + updatePreferences, + onOpenChange, + ]); + + const handleUseGlobalConfig = useCallback(async () => { + if (!config || !isGlobal) return; + setIsSubmitting(true); + try { + await updatePreferences({ + search_space_id: searchSpaceId, + data: { vision_llm_config_id: config.id }, + }); + toast.success(`Now using ${config.name}`); + onOpenChange(false); + } catch (error) { + console.error("Failed to set vision model:", error); + toast.error("Failed to set vision model"); + } finally { + setIsSubmitting(false); + } + }, [config, isGlobal, searchSpaceId, updatePreferences, onOpenChange]); + + const isFormValid = formData.name && formData.provider && formData.model_name && formData.api_key; + const selectedProvider = VISION_PROVIDERS.find((p) => p.value === formData.provider); + + return ( + + e.preventDefault()} + > + {getTitle()} + +
+
+
+

{getTitle()}

+ {isGlobal && mode !== "create" && ( + + Global + + )} +
+

{getSubtitle()}

+ {config && mode !== "create" && ( +

{config.model_name}

+ )} +
+
+ +
+ {isGlobal && config && ( + <> + + + + Global configurations are read-only. To customize, create a new model. + + +
+
+
+
+ Name +
+

{config.name}

+
+ {config.description && ( +
+
+ Description +
+

{config.description}

+
+ )} +
+ +
+
+
+ Provider +
+

{config.provider}

+
+
+
+ Model +
+

{config.model_name}

+
+
+
+ + )} + + {(mode === "create" || (mode === "edit" && !isGlobal)) && ( +
+
+ + setFormData((p) => ({ ...p, name: e.target.value }))} + /> +
+ +
+ + setFormData((p) => ({ ...p, description: e.target.value }))} + /> +
+ + + +
+ + +
+ +
+ + setFormData((p) => ({ ...p, model_name: e.target.value }))} + /> +
+ +
+ + setFormData((p) => ({ ...p, api_key: e.target.value }))} + /> +
+ +
+ + setFormData((p) => ({ ...p, api_base: e.target.value }))} + /> +
+ + {formData.provider === "AZURE_OPENAI" && ( +
+ + setFormData((p) => ({ ...p, api_version: e.target.value }))} + /> +
+ )} +
+ )} +
+ +
+ + {mode === "create" || (mode === "edit" && !isGlobal) ? ( + + ) : isGlobal && config ? ( + + ) : null} +
+ + + ); +} From 035a4862f9c78fa75c3e209534e0b553148f12a5 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:29:24 +0200 Subject: [PATCH 32/47] Add Vision Models tab to settings dialog with i18n --- .../components/settings/search-space-settings-dialog.tsx | 9 ++++++++- surfsense_web/messages/en.json | 2 ++ surfsense_web/messages/es.json | 2 ++ surfsense_web/messages/hi.json | 2 ++ surfsense_web/messages/pt.json | 2 ++ surfsense_web/messages/zh.json | 2 ++ 6 files changed, 18 insertions(+), 1 deletion(-) diff --git a/surfsense_web/components/settings/search-space-settings-dialog.tsx b/surfsense_web/components/settings/search-space-settings-dialog.tsx index 47094d0c9..6573bc271 100644 --- a/surfsense_web/components/settings/search-space-settings-dialog.tsx +++ b/surfsense_web/components/settings/search-space-settings-dialog.tsx @@ -1,7 +1,7 @@ "use client"; import { useAtom } from "jotai"; -import { Bot, Brain, FileText, Globe, ImageIcon, MessageSquare, Shield } from "lucide-react"; +import { Bot, Brain, Eye, FileText, Globe, ImageIcon, MessageSquare, Shield } from "lucide-react"; import { useTranslations } from "next-intl"; import type React from "react"; import { searchSpaceSettingsDialogAtom } from "@/atoms/settings/settings-dialog.atoms"; @@ -13,6 +13,7 @@ import { ModelConfigManager } from "@/components/settings/model-config-manager"; import { PromptConfigManager } from "@/components/settings/prompt-config-manager"; import { RolesManager } from "@/components/settings/roles-manager"; import { SettingsDialog } from "@/components/settings/settings-dialog"; +import { VisionModelManager } from "@/components/settings/vision-model-manager"; interface SearchSpaceSettingsDialogProps { searchSpaceId: number; @@ -31,6 +32,11 @@ export function SearchSpaceSettingsDialog({ searchSpaceId }: SearchSpaceSettings label: t("nav_image_models"), icon: , }, + { + value: "vision-models", + label: t("nav_vision_models"), + icon: , + }, { value: "team-roles", label: t("nav_team_roles"), icon: }, { value: "prompts", @@ -45,6 +51,7 @@ export function SearchSpaceSettingsDialog({ searchSpaceId }: SearchSpaceSettings models: , roles: , "image-models": , + "vision-models": , "team-roles": , prompts: , "public-links": , diff --git a/surfsense_web/messages/en.json b/surfsense_web/messages/en.json index b67f9db22..a3a4e8853 100644 --- a/surfsense_web/messages/en.json +++ b/surfsense_web/messages/en.json @@ -738,6 +738,8 @@ "nav_role_assignments_desc": "Assign configs to agent roles", "nav_image_models": "Image Models", "nav_image_models_desc": "Configure image generation models", + "nav_vision_models": "Vision Models", + "nav_vision_models_desc": "Configure vision-capable LLM models", "nav_system_instructions": "System Instructions", "nav_system_instructions_desc": "SearchSpace-wide AI instructions", "nav_public_links": "Public Chat Links", diff --git a/surfsense_web/messages/es.json b/surfsense_web/messages/es.json index 5cf248a3a..fa620e271 100644 --- a/surfsense_web/messages/es.json +++ b/surfsense_web/messages/es.json @@ -738,6 +738,8 @@ "nav_role_assignments_desc": "Asignar configuraciones a roles de agente", "nav_image_models": "Modelos de imagen", "nav_image_models_desc": "Configurar modelos de generación de imágenes", + "nav_vision_models": "Modelos de visión", + "nav_vision_models_desc": "Configurar modelos LLM con capacidad de visión", "nav_system_instructions": "Instrucciones del sistema", "nav_system_instructions_desc": "Instrucciones de IA a nivel del espacio de búsqueda", "nav_public_links": "Enlaces de chat públicos", diff --git a/surfsense_web/messages/hi.json b/surfsense_web/messages/hi.json index 0e7194832..faeb4cb94 100644 --- a/surfsense_web/messages/hi.json +++ b/surfsense_web/messages/hi.json @@ -738,6 +738,8 @@ "nav_role_assignments_desc": "एजेंट भूमिकाओं को कॉन्फ़िगरेशन असाइन करें", "nav_image_models": "इमेज मॉडल", "nav_image_models_desc": "इमेज जनरेशन मॉडल कॉन्फ़िगर करें", + "nav_vision_models": "विज़न मॉडल", + "nav_vision_models_desc": "विज़न-सक्षम LLM मॉडल कॉन्फ़िगर करें", "nav_system_instructions": "सिस्टम निर्देश", "nav_system_instructions_desc": "सर्च स्पेस-व्यापी AI निर्देश", "nav_public_links": "सार्वजनिक चैट लिंक", diff --git a/surfsense_web/messages/pt.json b/surfsense_web/messages/pt.json index 00ae18eae..0bed7c6cc 100644 --- a/surfsense_web/messages/pt.json +++ b/surfsense_web/messages/pt.json @@ -738,6 +738,8 @@ "nav_role_assignments_desc": "Atribuir configurações a funções do agente", "nav_image_models": "Modelos de imagem", "nav_image_models_desc": "Configurar modelos de geração de imagens", + "nav_vision_models": "Modelos de visão", + "nav_vision_models_desc": "Configurar modelos LLM com capacidade de visão", "nav_system_instructions": "Instruções do sistema", "nav_system_instructions_desc": "Instruções de IA em nível do espaço de pesquisa", "nav_public_links": "Links de chat públicos", diff --git a/surfsense_web/messages/zh.json b/surfsense_web/messages/zh.json index a6f3b5b84..0d4f7e1c9 100644 --- a/surfsense_web/messages/zh.json +++ b/surfsense_web/messages/zh.json @@ -722,6 +722,8 @@ "nav_role_assignments_desc": "为代理角色分配配置", "nav_image_models": "图像模型", "nav_image_models_desc": "配置图像生成模型", + "nav_vision_models": "视觉模型", + "nav_vision_models_desc": "配置具有视觉能力的LLM模型", "nav_system_instructions": "系统指令", "nav_system_instructions_desc": "搜索空间级别的 AI 指令", "nav_public_links": "公开聊天链接", From e85c355592676d19bd47820981bf62c6ad8bdc1b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:45:30 +0200 Subject: [PATCH 33/47] Add NEXT_PUBLIC_POSTHOG_KEY to desktop release CI and .env.example --- .github/workflows/desktop-release.yml | 1 + surfsense_web/.env.example | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index 491df0992..4d217562a 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -60,6 +60,7 @@ jobs: NEXT_PUBLIC_ZERO_CACHE_URL: ${{ vars.NEXT_PUBLIC_ZERO_CACHE_URL }} NEXT_PUBLIC_DEPLOYMENT_MODE: ${{ vars.NEXT_PUBLIC_DEPLOYMENT_MODE }} NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${{ vars.NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE }} + NEXT_PUBLIC_POSTHOG_KEY: ${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }} - name: Install desktop dependencies run: pnpm install diff --git a/surfsense_web/.env.example b/surfsense_web/.env.example index b674d8e9b..b448c1f71 100644 --- a/surfsense_web/.env.example +++ b/surfsense_web/.env.example @@ -7,4 +7,7 @@ NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848 DATABASE_URL=postgresql://postgres:[YOUR-PASSWORD]@db.sdsf.supabase.co:5432/postgres # Deployment mode (optional) -NEXT_PUBLIC_DEPLOYMENT_MODE="self-hosted" or "cloud" \ No newline at end of file +NEXT_PUBLIC_DEPLOYMENT_MODE="self-hosted" or "cloud" + +# PostHog analytics (optional, leave empty to disable) +NEXT_PUBLIC_POSTHOG_KEY= \ No newline at end of file From 8566b03c91b5a69efd8359c52889b6bb0118a657 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 20:18:42 +0200 Subject: [PATCH 34/47] Add PostHog analytics to desktop main process --- .github/workflows/desktop-release.yml | 2 + surfsense_desktop/.env | 4 ++ surfsense_desktop/scripts/build-electron.mjs | 6 +++ surfsense_desktop/src/main.ts | 11 ++++- surfsense_desktop/src/modules/analytics.ts | 46 +++++++++++++++++++ .../src/modules/autocomplete/index.ts | 4 ++ surfsense_desktop/src/modules/quick-ask.ts | 3 ++ 7 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 surfsense_desktop/src/modules/analytics.ts diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index 4d217562a..62ba5d445 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -71,6 +71,8 @@ jobs: working-directory: surfsense_desktop env: HOSTED_FRONTEND_URL: ${{ vars.HOSTED_FRONTEND_URL }} + POSTHOG_KEY: ${{ secrets.POSTHOG_KEY }} + POSTHOG_HOST: ${{ vars.POSTHOG_HOST }} - name: Package & Publish run: pnpm exec electron-builder ${{ matrix.platform }} --config electron-builder.yml --publish always -c.extraMetadata.version=${{ steps.version.outputs.VERSION }} diff --git a/surfsense_desktop/.env b/surfsense_desktop/.env index d053aac97..a0463a39d 100644 --- a/surfsense_desktop/.env +++ b/surfsense_desktop/.env @@ -4,3 +4,7 @@ # The hosted web frontend URL. Used to intercept OAuth redirects and keep them # inside the desktop app. Set to your production frontend domain. HOSTED_FRONTEND_URL=https://surfsense.net + +# PostHog analytics (leave empty to disable) +POSTHOG_KEY= +POSTHOG_HOST=https://us.i.posthog.com diff --git a/surfsense_desktop/scripts/build-electron.mjs b/surfsense_desktop/scripts/build-electron.mjs index 9f507ea37..bfce6a9ad 100644 --- a/surfsense_desktop/scripts/build-electron.mjs +++ b/surfsense_desktop/scripts/build-electron.mjs @@ -111,6 +111,12 @@ async function buildElectron() { 'process.env.HOSTED_FRONTEND_URL': JSON.stringify( process.env.HOSTED_FRONTEND_URL || desktopEnv.HOSTED_FRONTEND_URL || 'https://surfsense.net' ), + 'process.env.POSTHOG_KEY': JSON.stringify( + process.env.POSTHOG_KEY || desktopEnv.POSTHOG_KEY || '' + ), + 'process.env.POSTHOG_HOST': JSON.stringify( + process.env.POSTHOG_HOST || desktopEnv.POSTHOG_HOST || 'https://us.i.posthog.com' + ), }, }; diff --git a/surfsense_desktop/src/main.ts b/surfsense_desktop/src/main.ts index 95b0359c8..231553f9a 100644 --- a/surfsense_desktop/src/main.ts +++ b/surfsense_desktop/src/main.ts @@ -12,6 +12,7 @@ import { registerAutocomplete, unregisterAutocomplete } from './modules/autocomp import { registerFolderWatcher, unregisterFolderWatcher } from './modules/folder-watcher'; import { registerIpcHandlers } from './ipc/handlers'; import { createTray, destroyTray } from './modules/tray'; +import { initAnalytics, shutdownAnalytics, trackEvent } from './modules/analytics'; registerGlobalErrorHandlers(); @@ -22,6 +23,8 @@ if (!setupDeepLinks()) { registerIpcHandlers(); app.whenReady().then(async () => { + initAnalytics(); + trackEvent('desktop_app_launched'); setupMenu(); try { await startNextServer(); @@ -70,9 +73,15 @@ app.on('before-quit', () => { isQuitting = true; }); -app.on('will-quit', () => { +let didCleanup = false; +app.on('will-quit', async (e) => { + if (didCleanup) return; + didCleanup = true; + e.preventDefault(); unregisterQuickAsk(); unregisterAutocomplete(); unregisterFolderWatcher(); destroyTray(); + await shutdownAnalytics(); + app.exit(); }); diff --git a/surfsense_desktop/src/modules/analytics.ts b/surfsense_desktop/src/modules/analytics.ts new file mode 100644 index 000000000..8f64c1bd8 --- /dev/null +++ b/surfsense_desktop/src/modules/analytics.ts @@ -0,0 +1,46 @@ +import { PostHog } from 'posthog-node'; +import { machineIdSync } from 'node-machine-id'; +import { app } from 'electron'; + +let client: PostHog | null = null; +let distinctId = ''; + +export function initAnalytics(): void { + const key = process.env.POSTHOG_KEY; + if (!key) return; + + try { + distinctId = machineIdSync(true); + } catch { + return; + } + + client = new PostHog(key, { + host: process.env.POSTHOG_HOST || 'https://us.i.posthog.com', + flushAt: 20, + flushInterval: 10000, + }); +} + +export function trackEvent(event: string, properties?: Record): void { + if (!client) return; + + client.capture({ + distinctId, + event, + properties: { + platform: 'desktop', + app_version: app.getVersion(), + os: process.platform, + ...properties, + }, + }); +} + +export async function shutdownAnalytics(): Promise { + if (!client) return; + + const timeout = new Promise((resolve) => setTimeout(resolve, 3000)); + await Promise.race([client.shutdown(), timeout]); + client = null; +} diff --git a/surfsense_desktop/src/modules/autocomplete/index.ts b/surfsense_desktop/src/modules/autocomplete/index.ts index cb09a42e1..d4eb727fd 100644 --- a/surfsense_desktop/src/modules/autocomplete/index.ts +++ b/surfsense_desktop/src/modules/autocomplete/index.ts @@ -6,6 +6,7 @@ import { captureScreen } from './screenshot'; import { createSuggestionWindow, destroySuggestion, getSuggestionWindow } from './suggestion-window'; import { getShortcuts } from '../shortcuts'; import { getActiveSearchSpaceId } from '../active-search-space'; +import { trackEvent } from '../analytics'; let currentShortcut = ''; let autocompleteEnabled = true; @@ -41,6 +42,7 @@ async function triggerAutocomplete(): Promise { console.warn('[autocomplete] No active search space. Select a search space first.'); return; } + trackEvent('desktop_autocomplete_triggered', { search_space_id: searchSpaceId }); const cursor = screen.getCursorScreenPoint(); const win = createSuggestionWindow(cursor.x, cursor.y); @@ -87,9 +89,11 @@ function registerIpcHandlers(): void { ipcRegistered = true; ipcMain.handle(IPC_CHANNELS.ACCEPT_SUGGESTION, async (_event, text: string) => { + trackEvent('desktop_autocomplete_accepted'); await acceptAndInject(text); }); ipcMain.handle(IPC_CHANNELS.DISMISS_SUGGESTION, () => { + trackEvent('desktop_autocomplete_dismissed'); destroySuggestion(); }); ipcMain.handle(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, (_event, enabled: boolean) => { diff --git a/surfsense_desktop/src/modules/quick-ask.ts b/surfsense_desktop/src/modules/quick-ask.ts index d5a2a9c2e..d700b421a 100644 --- a/surfsense_desktop/src/modules/quick-ask.ts +++ b/surfsense_desktop/src/modules/quick-ask.ts @@ -5,6 +5,7 @@ import { checkAccessibilityPermission, getFrontmostApp, simulateCopy, simulatePa import { getServerPort } from './server'; import { getShortcuts } from './shortcuts'; import { getActiveSearchSpaceId } from './active-search-space'; +import { trackEvent } from './analytics'; let currentShortcut = ''; let quickAskWindow: BrowserWindow | null = null; @@ -120,6 +121,7 @@ async function quickAskHandler(): Promise { sourceApp = getFrontmostApp(); console.log('[quick-ask] Source app:', sourceApp, '| Opening Quick Assist with', text.length, 'chars', selected ? '(selected)' : text ? '(clipboard fallback)' : '(empty)'); + trackEvent('desktop_quick_ask_opened', { has_selected_text: !!selected }); openQuickAsk(text); } @@ -151,6 +153,7 @@ function registerIpcHandlers(): void { if (!checkAccessibilityPermission()) return; + trackEvent('desktop_quick_ask_replaced'); clipboard.writeText(text); destroyQuickAsk(); From 556646fe9740c3e8f05aae9722f51b44dc5d55f0 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 20:20:56 +0200 Subject: [PATCH 35/47] Use assets.surfsense.com as PostHog host --- surfsense_desktop/.env | 2 +- surfsense_desktop/scripts/build-electron.mjs | 2 +- surfsense_desktop/src/modules/analytics.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/surfsense_desktop/.env b/surfsense_desktop/.env index a0463a39d..e127b99e0 100644 --- a/surfsense_desktop/.env +++ b/surfsense_desktop/.env @@ -7,4 +7,4 @@ HOSTED_FRONTEND_URL=https://surfsense.net # PostHog analytics (leave empty to disable) POSTHOG_KEY= -POSTHOG_HOST=https://us.i.posthog.com +POSTHOG_HOST=https://assets.surfsense.com diff --git a/surfsense_desktop/scripts/build-electron.mjs b/surfsense_desktop/scripts/build-electron.mjs index bfce6a9ad..90d76ef7a 100644 --- a/surfsense_desktop/scripts/build-electron.mjs +++ b/surfsense_desktop/scripts/build-electron.mjs @@ -115,7 +115,7 @@ async function buildElectron() { process.env.POSTHOG_KEY || desktopEnv.POSTHOG_KEY || '' ), 'process.env.POSTHOG_HOST': JSON.stringify( - process.env.POSTHOG_HOST || desktopEnv.POSTHOG_HOST || 'https://us.i.posthog.com' + process.env.POSTHOG_HOST || desktopEnv.POSTHOG_HOST || 'https://assets.surfsense.com' ), }, }; diff --git a/surfsense_desktop/src/modules/analytics.ts b/surfsense_desktop/src/modules/analytics.ts index 8f64c1bd8..ee6ae8722 100644 --- a/surfsense_desktop/src/modules/analytics.ts +++ b/surfsense_desktop/src/modules/analytics.ts @@ -16,7 +16,7 @@ export function initAnalytics(): void { } client = new PostHog(key, { - host: process.env.POSTHOG_HOST || 'https://us.i.posthog.com', + host: process.env.POSTHOG_HOST || 'https://assets.surfsense.com', flushAt: 20, flushInterval: 10000, }); From 0be3c796354347e8a1e570233a7f560cbd22181c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 20:22:00 +0200 Subject: [PATCH 36/47] Guard trackEvent with try-catch --- surfsense_desktop/src/modules/analytics.ts | 24 +++++++++++++--------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/surfsense_desktop/src/modules/analytics.ts b/surfsense_desktop/src/modules/analytics.ts index ee6ae8722..0bbcb3026 100644 --- a/surfsense_desktop/src/modules/analytics.ts +++ b/surfsense_desktop/src/modules/analytics.ts @@ -25,16 +25,20 @@ export function initAnalytics(): void { export function trackEvent(event: string, properties?: Record): void { if (!client) return; - client.capture({ - distinctId, - event, - properties: { - platform: 'desktop', - app_version: app.getVersion(), - os: process.platform, - ...properties, - }, - }); + try { + client.capture({ + distinctId, + event, + properties: { + platform: 'desktop', + app_version: app.getVersion(), + os: process.platform, + ...properties, + }, + }); + } catch { + // Analytics should never break the app + } } export async function shutdownAnalytics(): Promise { From c5646eef6620ef673b8dc7bf136068112a619b03 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 20:28:07 +0200 Subject: [PATCH 37/47] Formatting --- .../[search_space_id]/client-layout.tsx | 13 ++-- .../components/DesktopContent.tsx | 78 +++++++++++-------- surfsense_web/app/desktop/login/page.tsx | 4 +- surfsense_web/app/desktop/suggestion/page.tsx | 4 +- surfsense_web/components/TokenHandler.tsx | 2 +- .../components/desktop/shortcut-recorder.tsx | 4 +- 6 files changed, 59 insertions(+), 46 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx index 16af9ac6b..eceb46231 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx @@ -154,11 +154,14 @@ export function DashboardClientLayout({ // Sync to Electron store if stored value is null (first navigation) if (electronAPI?.setActiveSearchSpace) { - electronAPI.getActiveSearchSpace?.().then((stored) => { - if (!stored) { - electronAPI.setActiveSearchSpace!(activeSeacrhSpaceId); - } - }).catch(() => {}); + electronAPI + .getActiveSearchSpace?.() + .then((stored) => { + if (!stored) { + electronAPI.setActiveSearchSpace!(activeSeacrhSpaceId); + } + }) + .catch(() => {}); } }, [search_space_id, setActiveSearchSpaceIdState, electronAPI]); diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx index 596ed3e8b..c3f457f96 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx @@ -6,12 +6,18 @@ import { toast } from "sonner"; import { DEFAULT_SHORTCUTS, ShortcutRecorder } from "@/components/desktop/shortcut-recorder"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Label } from "@/components/ui/label"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { Spinner } from "@/components/ui/spinner"; import { Switch } from "@/components/ui/switch"; +import type { SearchSpace } from "@/contracts/types/search-space.types"; import { useElectronAPI } from "@/hooks/use-platform"; import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service"; -import type { SearchSpace } from "@/contracts/types/search-space.types"; export function DesktopContent() { const api = useElectronAPI(); @@ -82,7 +88,10 @@ export function DesktopContent() { await api.setAutocompleteEnabled(checked); }; - const updateShortcut = (key: "generalAssist" | "quickAsk" | "autocomplete", accelerator: string) => { + const updateShortcut = ( + key: "generalAssist" | "quickAsk" | "autocomplete", + accelerator: string + ) => { setShortcuts((prev) => { const updated = { ...prev, [key]: accelerator }; api.setShortcuts?.({ [key]: accelerator }).catch(() => { @@ -110,7 +119,8 @@ export function DesktopContent() { Default Search Space - Choose which search space General Assist, Quick Assist, and Extreme Assist operate against. + Choose which search space General Assist, Quick Assist, and Extreme Assist operate + against. @@ -128,7 +138,9 @@ export function DesktopContent() { ) : ( -

No search spaces found. Create one first.

+

+ No search spaces found. Create one first. +

)} @@ -143,34 +155,34 @@ export function DesktopContent() { {shortcutsLoaded ? ( -
- updateShortcut("generalAssist", accel)} - onReset={() => resetShortcut("generalAssist")} - defaultValue={DEFAULT_SHORTCUTS.generalAssist} - label="General Assist" - description="Launch SurfSense instantly from any application" - icon={Rocket} - /> - updateShortcut("quickAsk", accel)} - onReset={() => resetShortcut("quickAsk")} - defaultValue={DEFAULT_SHORTCUTS.quickAsk} - label="Quick Assist" - description="Select text anywhere, then ask AI to explain, rewrite, or act on it" - icon={Zap} - /> - updateShortcut("autocomplete", accel)} - onReset={() => resetShortcut("autocomplete")} - defaultValue={DEFAULT_SHORTCUTS.autocomplete} - label="Extreme Assist" - description="AI drafts text using your screen context and knowledge base" - icon={BrainCog} - /> +
+ updateShortcut("generalAssist", accel)} + onReset={() => resetShortcut("generalAssist")} + defaultValue={DEFAULT_SHORTCUTS.generalAssist} + label="General Assist" + description="Launch SurfSense instantly from any application" + icon={Rocket} + /> + updateShortcut("quickAsk", accel)} + onReset={() => resetShortcut("quickAsk")} + defaultValue={DEFAULT_SHORTCUTS.quickAsk} + label="Quick Assist" + description="Select text anywhere, then ask AI to explain, rewrite, or act on it" + icon={Zap} + /> + updateShortcut("autocomplete", accel)} + onReset={() => resetShortcut("autocomplete")} + defaultValue={DEFAULT_SHORTCUTS.autocomplete} + label="Extreme Assist" + description="AI drafts text using your screen context and knowledge base" + icon={BrainCog} + />

Click a shortcut and press a new key combination to change it.

diff --git a/surfsense_web/app/desktop/login/page.tsx b/surfsense_web/app/desktop/login/page.tsx index 744680010..8f68d20c1 100644 --- a/surfsense_web/app/desktop/login/page.tsx +++ b/surfsense_web/app/desktop/login/page.tsx @@ -139,9 +139,7 @@ export default function DesktopLoginPage() { height={48} priority /> -

- Welcome to SurfSense Desktop -

+

Welcome to SurfSense Desktop

Configure shortcuts, then sign in to get started.

diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index e458f6615..0815ba622 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -297,9 +297,7 @@ export default function SuggestionPage() { const isExpanded = expandedOption === index; const needsTruncation = option.length > TRUNCATE_LENGTH; const displayText = - needsTruncation && !isExpanded - ? option.slice(0, TRUNCATE_LENGTH) + "…" - : option; + needsTruncation && !isExpanded ? option.slice(0, TRUNCATE_LENGTH) + "…" : option; return (
{recording ? ( - Press keys… + + Press keys… + ) : ( )} From 13625acdd5ceddeb66a03d7aaeaf1cc9e1dc16b1 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 20:47:17 +0200 Subject: [PATCH 38/47] Add vision model tab to chat page model selector --- .../components/new-chat/chat-header.tsx | 44 +++ .../components/new-chat/model-selector.tsx | 304 +++++++++++++++++- 2 files changed, 339 insertions(+), 9 deletions(-) diff --git a/surfsense_web/components/new-chat/chat-header.tsx b/surfsense_web/components/new-chat/chat-header.tsx index 3263a2b07..0c5253c6c 100644 --- a/surfsense_web/components/new-chat/chat-header.tsx +++ b/surfsense_web/components/new-chat/chat-header.tsx @@ -3,11 +3,14 @@ import { useCallback, useState } from "react"; import { ImageConfigDialog } from "@/components/shared/image-config-dialog"; import { ModelConfigDialog } from "@/components/shared/model-config-dialog"; +import { VisionConfigDialog } from "@/components/shared/vision-config-dialog"; import type { GlobalImageGenConfig, GlobalNewLLMConfig, + GlobalVisionLLMConfig, ImageGenerationConfig, NewLLMConfigPublic, + VisionLLMConfig, } from "@/contracts/types/new-llm-config.types"; import { ModelSelector } from "./model-selector"; @@ -33,6 +36,14 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) { const [isImageGlobal, setIsImageGlobal] = useState(false); const [imageDialogMode, setImageDialogMode] = useState<"create" | "edit" | "view">("view"); + // Vision config dialog state + const [visionDialogOpen, setVisionDialogOpen] = useState(false); + const [selectedVisionConfig, setSelectedVisionConfig] = useState< + VisionLLMConfig | GlobalVisionLLMConfig | null + >(null); + const [isVisionGlobal, setIsVisionGlobal] = useState(false); + const [visionDialogMode, setVisionDialogMode] = useState<"create" | "edit" | "view">("view"); + // LLM handlers const handleEditLLMConfig = useCallback( (config: NewLLMConfigPublic | GlobalNewLLMConfig, global: boolean) => { @@ -79,6 +90,29 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) { if (!open) setSelectedImageConfig(null); }, []); + // Vision model handlers + const handleAddVisionModel = useCallback(() => { + setSelectedVisionConfig(null); + setIsVisionGlobal(false); + setVisionDialogMode("create"); + setVisionDialogOpen(true); + }, []); + + const handleEditVisionConfig = useCallback( + (config: VisionLLMConfig | GlobalVisionLLMConfig, global: boolean) => { + setSelectedVisionConfig(config); + setIsVisionGlobal(global); + setVisionDialogMode(global ? "view" : "edit"); + setVisionDialogOpen(true); + }, + [] + ); + + const handleVisionDialogClose = useCallback((open: boolean) => { + setVisionDialogOpen(open); + if (!open) setSelectedVisionConfig(null); + }, []); + return (
+
); } diff --git a/surfsense_web/components/new-chat/model-selector.tsx b/surfsense_web/components/new-chat/model-selector.tsx index 39f88f794..46b4a2c3a 100644 --- a/surfsense_web/components/new-chat/model-selector.tsx +++ b/surfsense_web/components/new-chat/model-selector.tsx @@ -1,7 +1,7 @@ "use client"; import { useAtomValue } from "jotai"; -import { Bot, Check, ChevronDown, Edit3, ImageIcon, Plus, Search, Zap } from "lucide-react"; +import { Bot, Check, ChevronDown, Edit3, Eye, ImageIcon, Plus, Search, Zap } from "lucide-react"; import { type UIEvent, useCallback, useMemo, useState } from "react"; import { toast } from "sonner"; import { @@ -15,6 +15,10 @@ import { newLLMConfigsAtom, } from "@/atoms/new-llm-config/new-llm-config-query.atoms"; import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; +import { + globalVisionLLMConfigsAtom, + visionLLMConfigsAtom, +} from "@/atoms/vision-llm-config/vision-llm-config-query.atoms"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { @@ -32,8 +36,10 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import type { GlobalImageGenConfig, GlobalNewLLMConfig, + GlobalVisionLLMConfig, ImageGenerationConfig, NewLLMConfigPublic, + VisionLLMConfig, } from "@/contracts/types/new-llm-config.types"; import { getProviderIcon } from "@/lib/provider-icons"; import { cn } from "@/lib/utils"; @@ -43,6 +49,8 @@ interface ModelSelectorProps { onAddNewLLM: () => void; onEditImage?: (config: ImageGenerationConfig | GlobalImageGenConfig, isGlobal: boolean) => void; onAddNewImage?: () => void; + onEditVision?: (config: VisionLLMConfig | GlobalVisionLLMConfig, isGlobal: boolean) => void; + onAddNewVision?: () => void; className?: string; } @@ -51,14 +59,18 @@ export function ModelSelector({ onAddNewLLM, onEditImage, onAddNewImage, + onEditVision, + onAddNewVision, className, }: ModelSelectorProps) { const [open, setOpen] = useState(false); - const [activeTab, setActiveTab] = useState<"llm" | "image">("llm"); + const [activeTab, setActiveTab] = useState<"llm" | "image" | "vision">("llm"); const [llmSearchQuery, setLlmSearchQuery] = useState(""); const [imageSearchQuery, setImageSearchQuery] = useState(""); + const [visionSearchQuery, setVisionSearchQuery] = useState(""); const [llmScrollPos, setLlmScrollPos] = useState<"top" | "middle" | "bottom">("top"); const [imageScrollPos, setImageScrollPos] = useState<"top" | "middle" | "bottom">("top"); + const [visionScrollPos, setVisionScrollPos] = useState<"top" | "middle" | "bottom">("top"); const handleListScroll = useCallback( (setter: typeof setLlmScrollPos) => (e: UIEvent) => { const el = e.currentTarget; @@ -82,8 +94,21 @@ export function ModelSelector({ useAtomValue(globalImageGenConfigsAtom); const { data: imageUserConfigs, isLoading: imageUserLoading } = useAtomValue(imageGenConfigsAtom); + // Vision data + const { data: visionGlobalConfigs, isLoading: visionGlobalLoading } = useAtomValue( + globalVisionLLMConfigsAtom + ); + const { data: visionUserConfigs, isLoading: visionUserLoading } = + useAtomValue(visionLLMConfigsAtom); + const isLoading = - llmUserLoading || llmGlobalLoading || prefsLoading || imageGlobalLoading || imageUserLoading; + llmUserLoading || + llmGlobalLoading || + prefsLoading || + imageGlobalLoading || + imageUserLoading || + visionGlobalLoading || + visionUserLoading; // ─── LLM current config ─── const currentLLMConfig = useMemo(() => { @@ -116,6 +141,24 @@ export function ModelSelector({ ); }, [currentImageConfig]); + // ─── Vision current config ─── + const currentVisionConfig = useMemo(() => { + if (!preferences) return null; + const id = preferences.vision_llm_config_id; + if (id === null || id === undefined) return null; + const globalMatch = visionGlobalConfigs?.find((c) => c.id === id); + if (globalMatch) return globalMatch; + return visionUserConfigs?.find((c) => c.id === id) ?? null; + }, [preferences, visionGlobalConfigs, visionUserConfigs]); + + const isVisionAutoMode = useMemo(() => { + return ( + currentVisionConfig && + "is_auto_mode" in currentVisionConfig && + currentVisionConfig.is_auto_mode + ); + }, [currentVisionConfig]); + // ─── LLM filtering ─── const filteredLLMGlobal = useMemo(() => { if (!llmGlobalConfigs) return []; @@ -170,6 +213,33 @@ export function ModelSelector({ const totalImageModels = (imageGlobalConfigs?.length ?? 0) + (imageUserConfigs?.length ?? 0); + // ─── Vision filtering ─── + const filteredVisionGlobal = useMemo(() => { + if (!visionGlobalConfigs) return []; + if (!visionSearchQuery) return visionGlobalConfigs; + const q = visionSearchQuery.toLowerCase(); + return visionGlobalConfigs.filter( + (c) => + c.name.toLowerCase().includes(q) || + c.model_name.toLowerCase().includes(q) || + c.provider.toLowerCase().includes(q) + ); + }, [visionGlobalConfigs, visionSearchQuery]); + + const filteredVisionUser = useMemo(() => { + if (!visionUserConfigs) return []; + if (!visionSearchQuery) return visionUserConfigs; + const q = visionSearchQuery.toLowerCase(); + return visionUserConfigs.filter( + (c) => + c.name.toLowerCase().includes(q) || + c.model_name.toLowerCase().includes(q) || + c.provider.toLowerCase().includes(q) + ); + }, [visionUserConfigs, visionSearchQuery]); + + const totalVisionModels = (visionGlobalConfigs?.length ?? 0) + (visionUserConfigs?.length ?? 0); + // ─── Handlers ─── const handleSelectLLM = useCallback( async (config: NewLLMConfigPublic | GlobalNewLLMConfig) => { @@ -229,6 +299,30 @@ export function ModelSelector({ [currentImageConfig, searchSpaceId, updatePreferences] ); + const handleSelectVision = useCallback( + async (configId: number) => { + if (currentVisionConfig?.id === configId) { + setOpen(false); + return; + } + if (!searchSpaceId) { + toast.error("No search space selected"); + return; + } + try { + await updatePreferences({ + search_space_id: Number(searchSpaceId), + data: { vision_llm_config_id: configId }, + }); + toast.success("Vision model updated"); + setOpen(false); + } catch { + toast.error("Failed to switch vision model"); + } + }, + [currentVisionConfig, searchSpaceId, updatePreferences] + ); + return ( @@ -282,6 +376,23 @@ export function ModelSelector({ ) : ( )} + + {/* Divider */} +
+ + {/* Vision section */} + {currentVisionConfig ? ( + <> + {getProviderIcon(currentVisionConfig.provider, { + isAutoMode: isVisionAutoMode ?? false, + })} + + {currentVisionConfig.name} + + + ) : ( + + )} )} @@ -295,25 +406,32 @@ export function ModelSelector({ > setActiveTab(v as "llm" | "image")} + onValueChange={(v) => setActiveTab(v as "llm" | "image" | "vision")} className="w-full" >
- + - + LLM - + Image + + + Vision +
@@ -676,6 +794,174 @@ export function ModelSelector({ + + {/* ─── Vision Tab ─── */} + + + {totalVisionModels > 3 && ( +
+ +
+ )} + + +
+ +

No vision models found

+

Try a different search term

+
+ + + {filteredVisionGlobal.length > 0 && ( + +
+ Global Vision Models +
+ {filteredVisionGlobal.map((config) => { + const isSelected = currentVisionConfig?.id === config.id; + const isAuto = "is_auto_mode" in config && config.is_auto_mode; + return ( + handleSelectVision(config.id)} + className={cn( + "mx-2 rounded-lg mb-1 cursor-pointer group transition-all hover:bg-accent/50 dark:hover:bg-white/[0.06]", + isSelected && "bg-accent/80 dark:bg-white/[0.06]" + )} + > +
+
+ {getProviderIcon(config.provider, { isAutoMode: isAuto })} +
+
+
+ {config.name} + {isAuto && ( + + Recommended + + )} + {isSelected && } +
+ + {isAuto ? "Auto Mode" : config.model_name} + +
+ {onEditVision && !isAuto && ( + + )} +
+ + ); + })} + + )} + + {filteredVisionUser.length > 0 && ( + <> + {filteredVisionGlobal.length > 0 && ( + + )} + +
+ Your Vision Models +
+ {filteredVisionUser.map((config) => { + const isSelected = currentVisionConfig?.id === config.id; + return ( + handleSelectVision(config.id)} + className={cn( + "mx-2 rounded-lg mb-1 cursor-pointer group transition-all hover:bg-accent/50 dark:hover:bg-white/[0.06]", + isSelected && "bg-accent/80 dark:bg-white/[0.06]" + )} + > +
+
{getProviderIcon(config.provider)}
+
+
+ {config.name} + {isSelected && ( + + )} +
+ + {config.model_name} + +
+ {onEditVision && ( + + )} +
+ + ); + })} + + + )} + + {onAddNewVision && ( +
+ +
+ )} + + + From 087b1498431ec60943a7d680af1f198876dc019b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 21:06:11 +0200 Subject: [PATCH 39/47] Add setup prompt in overlay when vision model not configured --- surfsense_web/app/desktop/suggestion/page.tsx | 74 +++++++++++++++---- .../app/desktop/suggestion/suggestion.css | 60 +++++++++++++++ 2 files changed, 120 insertions(+), 14 deletions(-) diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index 0815ba622..46e388568 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -27,24 +27,32 @@ interface AgentStep { items: string[]; } -function friendlyError(raw: string | number): string { +type FriendlyError = { message: string; isSetup?: boolean }; + +function friendlyError(raw: string | number): FriendlyError { if (typeof raw === "number") { - if (raw === 401) return "Please sign in to use suggestions."; - if (raw === 403) return "You don\u2019t have permission for this."; - if (raw === 404) return "Suggestion service not found. Is the backend running?"; - if (raw >= 500) return "Something went wrong on the server. Try again."; - return "Something went wrong. Try again."; + if (raw === 401) return { message: "Please sign in to use suggestions." }; + if (raw === 403) return { message: "You don\u2019t have permission for this." }; + if (raw === 404) return { message: "Suggestion service not found. Is the backend running?" }; + if (raw >= 500) return { message: "Something went wrong on the server. Try again." }; + return { message: "Something went wrong. Try again." }; } const lower = raw.toLowerCase(); if (lower.includes("not authenticated") || lower.includes("unauthorized")) - return "Please sign in to use suggestions."; + return { message: "Please sign in to use suggestions." }; if (lower.includes("no vision llm configured") || lower.includes("no llm configured")) - return "No Vision LLM configured. Set one in search space settings."; + return { + message: "Configure a vision-capable model (e.g. GPT-4o, Gemini) to enable autocomplete.", + isSetup: true, + }; if (lower.includes("does not support vision")) - return "Selected model doesn\u2019t support vision. Set a vision-capable model in settings."; + return { + message: "The selected model doesn\u2019t support vision. Choose a vision-capable model.", + isSetup: true, + }; if (lower.includes("fetch") || lower.includes("network") || lower.includes("econnrefused")) - return "Can\u2019t reach the server. Check your connection."; - return "Something went wrong. Try again."; + return { message: "Can\u2019t reach the server. Check your connection." }; + return { message: "Something went wrong. Try again." }; } const AUTO_DISMISS_MS = 3000; @@ -76,7 +84,7 @@ export default function SuggestionPage() { const api = useElectronAPI(); const [options, setOptions] = useState([]); const [isLoading, setIsLoading] = useState(true); - const [error, setError] = useState(null); + const [error, setError] = useState(null); const [steps, setSteps] = useState([]); const [expandedOption, setExpandedOption] = useState(null); const abortRef = useRef(null); @@ -90,7 +98,7 @@ export default function SuggestionPage() { }, [api]); useEffect(() => { - if (!error) return; + if (!error || error.isSetup) return; const timer = setTimeout(() => { api?.dismissSuggestion?.(); }, AUTO_DISMISS_MS); @@ -233,9 +241,47 @@ export default function SuggestionPage() { } if (error) { + if (error.isSetup) { + return ( +
+
+ +
+
+ Vision Model Required + {error.message} + Settings → Vision Models +
+ +
+ ); + } return (
- {error} + {error.message}
); } diff --git a/surfsense_web/app/desktop/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css index f5471cf37..fd5ec5d3b 100644 --- a/surfsense_web/app/desktop/suggestion/suggestion.css +++ b/surfsense_web/app/desktop/suggestion/suggestion.css @@ -117,6 +117,66 @@ body:has(.suggestion-body) { font-size: 12px; } +/* --- Setup prompt (vision model not configured) --- */ + +.suggestion-setup { + display: flex; + flex-direction: row; + align-items: flex-start; + gap: 10px; + border-color: #3b2d6b; + padding: 10px 14px; +} + +.setup-icon { + flex-shrink: 0; + margin-top: 1px; +} + +.setup-content { + display: flex; + flex-direction: column; + gap: 3px; + min-width: 0; +} + +.setup-title { + font-size: 13px; + font-weight: 600; + color: #c4b5fd; +} + +.setup-message { + font-size: 11.5px; + color: #a1a1aa; + line-height: 1.4; +} + +.setup-hint { + font-size: 10.5px; + color: #7c6dac; + margin-top: 2px; +} + +.setup-dismiss { + flex-shrink: 0; + align-self: flex-start; + background: none; + border: none; + color: #6b6b7b; + font-size: 14px; + cursor: pointer; + padding: 2px 4px; + line-height: 1; + border-radius: 4px; + transition: color 0.15s, background 0.15s; +} + +.setup-dismiss:hover { + color: #c4b5fd; + background: rgba(124, 109, 172, 0.15); +} + /* --- Agent activity indicator --- */ .agent-activity { From 36b8a84b0b6353e4bf608bd19fde2ff729faca2f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 21:55:58 +0200 Subject: [PATCH 40/47] Add vision LLM config examples to global_llm_config.example.yaml --- .../app/config/global_llm_config.example.yaml | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/surfsense_backend/app/config/global_llm_config.example.yaml b/surfsense_backend/app/config/global_llm_config.example.yaml index 49a8d0295..e382fdc74 100644 --- a/surfsense_backend/app/config/global_llm_config.example.yaml +++ b/surfsense_backend/app/config/global_llm_config.example.yaml @@ -263,6 +263,82 @@ global_image_generation_configs: # rpm: 30 # litellm_params: {} +# ============================================================================= +# Vision LLM Configuration +# ============================================================================= +# These configurations power the vision autocomplete feature (screenshot analysis). +# Only vision-capable models should be used here (e.g. GPT-4o, Gemini Pro, Claude 3). +# Supported providers: OpenAI, Anthropic, Google, Azure OpenAI, Vertex AI, Bedrock, +# xAI, OpenRouter, Ollama, Groq, Together AI, Fireworks AI, DeepSeek, Mistral, Custom +# +# Auto mode (ID 0) uses LiteLLM Router for load balancing across all vision configs. + +# Router Settings for Vision LLM Auto Mode +vision_llm_router_settings: + routing_strategy: "usage-based-routing" + num_retries: 3 + allowed_fails: 3 + cooldown_time: 60 + +global_vision_llm_configs: + # Example: OpenAI GPT-4o (recommended for vision) + - id: -1 + name: "Global GPT-4o Vision" + description: "OpenAI's GPT-4o with strong vision capabilities" + provider: "OPENAI" + model_name: "gpt-4o" + api_key: "sk-your-openai-api-key-here" + api_base: "" + rpm: 500 + tpm: 100000 + litellm_params: + temperature: 0.3 + max_tokens: 1000 + + # Example: Google Gemini 2.0 Flash + - id: -2 + name: "Global Gemini 2.0 Flash" + description: "Google's fast vision model with large context" + provider: "GOOGLE" + model_name: "gemini-2.0-flash" + api_key: "your-google-ai-api-key-here" + api_base: "" + rpm: 1000 + tpm: 200000 + litellm_params: + temperature: 0.3 + max_tokens: 1000 + + # Example: Anthropic Claude 3.5 Sonnet + - id: -3 + name: "Global Claude 3.5 Sonnet Vision" + description: "Anthropic's Claude 3.5 Sonnet with vision support" + provider: "ANTHROPIC" + model_name: "claude-3-5-sonnet-20241022" + api_key: "sk-ant-your-anthropic-api-key-here" + api_base: "" + rpm: 1000 + tpm: 100000 + litellm_params: + temperature: 0.3 + max_tokens: 1000 + + # Example: Azure OpenAI GPT-4o + # - id: -4 + # name: "Global Azure GPT-4o Vision" + # description: "Azure-hosted GPT-4o for vision analysis" + # provider: "AZURE_OPENAI" + # model_name: "azure/gpt-4o-deployment" + # api_key: "your-azure-api-key-here" + # api_base: "https://your-resource.openai.azure.com" + # api_version: "2024-02-15-preview" + # rpm: 500 + # tpm: 100000 + # litellm_params: + # temperature: 0.3 + # max_tokens: 1000 + # base_model: "gpt-4o" + # Notes: # - ID 0 is reserved for "Auto" mode - uses LiteLLM Router for load balancing # - Use negative IDs to distinguish global configs from user configs (NewLLMConfig in DB) @@ -283,3 +359,9 @@ global_image_generation_configs: # - The router uses litellm.aimage_generation() for async image generation # - Only RPM (requests per minute) is relevant for image generation rate limiting. # TPM (tokens per minute) does not apply since image APIs are billed/rate-limited per request, not per token. +# +# VISION LLM NOTES: +# - Vision configs use the same ID scheme (negative for global, positive for user DB) +# - Only use vision-capable models (GPT-4o, Gemini, Claude 3, etc.) +# - Lower temperature (0.3) is recommended for accurate screenshot analysis +# - Lower max_tokens (1000) is sufficient since autocomplete produces short suggestions From 00ee7974f6b4624050ca0065f303d1cca2d4da57 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 7 Apr 2026 13:13:16 -0700 Subject: [PATCH 41/47] fix(desktop): pasteback issues in quick ask - Updated the quick ask window URL to include a query parameter for quick assist mode. - Introduced a constant to detect quick assist mode based on the URL parameter in the assistant message component. - Simplified state management for quick assist detection, improving component performance and clarity. --- surfsense_desktop/src/modules/quick-ask.ts | 3 ++- .../components/assistant-ui/assistant-message.tsx | 13 ++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/surfsense_desktop/src/modules/quick-ask.ts b/surfsense_desktop/src/modules/quick-ask.ts index d5a2a9c2e..b738a864d 100644 --- a/surfsense_desktop/src/modules/quick-ask.ts +++ b/surfsense_desktop/src/modules/quick-ask.ts @@ -57,7 +57,7 @@ function createQuickAskWindow(x: number, y: number): BrowserWindow { const spaceId = pendingSearchSpaceId; const route = spaceId ? `/dashboard/${spaceId}/new-chat` : '/dashboard'; - quickAskWindow.loadURL(`http://localhost:${getServerPort()}${route}`); + quickAskWindow.loadURL(`http://localhost:${getServerPort()}${route}?quickAssist=true`); quickAskWindow.once('ready-to-show', () => { quickAskWindow?.show(); @@ -84,6 +84,7 @@ function createQuickAskWindow(x: number, y: number): BrowserWindow { async function openQuickAsk(text: string): Promise { pendingText = text; + pendingMode = 'quick-assist'; pendingSearchSpaceId = await getActiveSearchSpaceId(); const cursor = screen.getCursorScreenPoint(); const pos = clampToScreen(cursor.x, cursor.y, 450, 750); diff --git a/surfsense_web/components/assistant-ui/assistant-message.tsx b/surfsense_web/components/assistant-ui/assistant-message.tsx index 49853b0b5..605d9d518 100644 --- a/surfsense_web/components/assistant-ui/assistant-message.tsx +++ b/surfsense_web/components/assistant-ui/assistant-message.tsx @@ -90,6 +90,11 @@ import { useMediaQuery } from "@/hooks/use-media-query"; import { useElectronAPI } from "@/hooks/use-platform"; import { cn } from "@/lib/utils"; +// Captured once at module load — survives client-side navigations that strip the query param. +const IS_QUICK_ASSIST_WINDOW = + typeof window !== "undefined" && + new URLSearchParams(window.location.search).get("quickAssist") === "true"; + // Dynamically import video presentation tool to avoid loading Babel and Remotion in main bundle const GenerateVideoPresentationToolUI = dynamic( () => @@ -465,14 +470,8 @@ const AssistantActionBar: FC = () => { const isLast = useAuiState((s) => s.message.isLast); const aui = useAui(); const api = useElectronAPI(); - const [isQuickAssist, setIsQuickAssist] = useState(false); - useEffect(() => { - if (!api?.getQuickAskMode) return; - api.getQuickAskMode().then((mode) => { - if (mode) setIsQuickAssist(true); - }); - }, [api]); + const isQuickAssist = !!api?.replaceText && IS_QUICK_ASSIST_WINDOW; return ( Date: Tue, 7 Apr 2026 22:15:35 +0200 Subject: [PATCH 42/47] Add posthog-node and node-machine-id dependencies for desktop analytics --- surfsense_desktop/package.json | 4 +++- surfsense_desktop/pnpm-lock.yaml | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index 74f6274cb..7b91d70c9 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -34,6 +34,8 @@ "electron-store": "^11.0.2", "electron-updater": "^6.8.3", "get-port-please": "^3.2.0", - "node-mac-permissions": "^2.5.0" + "node-mac-permissions": "^2.5.0", + "node-machine-id": "^1.1.12", + "posthog-node": "^5.29.0" } } diff --git a/surfsense_desktop/pnpm-lock.yaml b/surfsense_desktop/pnpm-lock.yaml index e1df34fb2..e7b84cc01 100644 --- a/surfsense_desktop/pnpm-lock.yaml +++ b/surfsense_desktop/pnpm-lock.yaml @@ -26,6 +26,12 @@ importers: node-mac-permissions: specifier: ^2.5.0 version: 2.5.0 + node-machine-id: + specifier: ^1.1.12 + version: 1.1.12 + posthog-node: + specifier: ^5.29.0 + version: 5.29.0(rxjs@7.8.2) devDependencies: '@electron/rebuild': specifier: ^4.0.3 @@ -308,6 +314,9 @@ packages: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} + '@posthog/core@1.25.0': + resolution: {integrity: sha512-XKaHvRFIIN7Dw84r1eKimV1rl9DS+9XMCPPZ7P3+l8fE+rDsmumebiTFsY+q40bVXflcGW9wB+57LH0lvcGmhw==} + '@sindresorhus/is@4.6.0': resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} engines: {node: '>=10'} @@ -1194,6 +1203,9 @@ packages: resolution: {integrity: sha512-zR8SVCaN3WqV1xwWd04XVAdzm3UTdjbxciLrZtB0Cc7F2Kd34AJfhPD4hm1HU0YH3oGUZO4X9OBLY5ijSTHsGw==} os: [darwin] + node-machine-id@1.1.12: + resolution: {integrity: sha512-QNABxbrPa3qEIfrE6GOJ7BYIuignnJw7iQ2YPbc3Nla1HzRJjXzZOiikfF8m7eAMfichLt3M4VgLOetqgDmgGQ==} + nopt@8.1.0: resolution: {integrity: sha512-ieGu42u/Qsa4TFktmaKEwM6MQH0pOWnaB3htzh0JRtx84+Mebc0cbZYN5bC+6WTZ4+77xrL9Pn5m7CV6VIkV7A==} engines: {node: ^18.17.0 || >=20.5.0} @@ -1263,6 +1275,15 @@ packages: resolution: {integrity: sha512-uysumyrvkUX0rX/dEVqt8gC3sTBzd4zoWfLeS29nb53imdaXVvLINYXTI2GNqzaMuvacNx4uJQ8+b3zXR0pkgQ==} engines: {node: '>=10.4.0'} + posthog-node@5.29.0: + resolution: {integrity: sha512-po7N55haSKxV8VOulkBZJja938yILShl6+fFjoUV3iQgOBCg4Muu615/xRg8mpNiz+UASvL0EEiGvIxdhXfj6Q==} + engines: {node: ^20.20.0 || >=22.22.0} + peerDependencies: + rxjs: ^7.0.0 + peerDependenciesMeta: + rxjs: + optional: true + postject@1.0.0-alpha.6: resolution: {integrity: sha512-b9Eb8h2eVqNE8edvKdwqkrY6O7kAwmI8kcnBv1NScolYJbo59XUF0noFq+lxbC1yN20bmC0WBEbDC5H/7ASb0A==} engines: {node: '>=14.0.0'} @@ -1876,6 +1897,8 @@ snapshots: '@pkgjs/parseargs@0.11.0': optional: true + '@posthog/core@1.25.0': {} + '@sindresorhus/is@4.6.0': {} '@standard-schema/spec@1.1.0': {} @@ -2940,6 +2963,8 @@ snapshots: bindings: 1.5.0 node-addon-api: 7.1.1 + node-machine-id@1.1.12: {} + nopt@8.1.0: dependencies: abbrev: 3.0.1 @@ -3002,6 +3027,12 @@ snapshots: base64-js: 1.5.1 xmlbuilder: 15.1.1 + posthog-node@5.29.0(rxjs@7.8.2): + dependencies: + '@posthog/core': 1.25.0 + optionalDependencies: + rxjs: 7.8.2 + postject@1.0.0-alpha.6: dependencies: commander: 9.5.0 From 26bffbcc47d0d19148963815c83eefb7cf6cfeae Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 23:39:52 +0200 Subject: [PATCH 43/47] Add dynamic vision model list from OpenRouter with combobox selector --- .../config/vision_model_list_fallback.json | 23 +++ .../app/routes/vision_llm_routes.py | 28 ++++ .../app/services/vision_model_list_service.py | 132 ++++++++++++++++++ .../vision-llm-config-query.atoms.ts | 24 ++++ .../shared/vision-config-dialog.tsx | 114 ++++++++++++++- .../contracts/enums/vision-providers.ts | 26 ++++ .../lib/apis/vision-llm-config-api.service.ts | 5 + surfsense_web/lib/query-client/cache-keys.ts | 1 + 8 files changed, 346 insertions(+), 7 deletions(-) create mode 100644 surfsense_backend/app/config/vision_model_list_fallback.json create mode 100644 surfsense_backend/app/services/vision_model_list_service.py diff --git a/surfsense_backend/app/config/vision_model_list_fallback.json b/surfsense_backend/app/config/vision_model_list_fallback.json new file mode 100644 index 000000000..830eb6517 --- /dev/null +++ b/surfsense_backend/app/config/vision_model_list_fallback.json @@ -0,0 +1,23 @@ +[ + {"value": "gpt-4o", "label": "GPT-4o", "provider": "OPENAI", "context_window": "128K"}, + {"value": "gpt-4o-mini", "label": "GPT-4o Mini", "provider": "OPENAI", "context_window": "128K"}, + {"value": "gpt-4-turbo", "label": "GPT-4 Turbo", "provider": "OPENAI", "context_window": "128K"}, + {"value": "claude-sonnet-4-20250514", "label": "Claude Sonnet 4", "provider": "ANTHROPIC", "context_window": "200K"}, + {"value": "claude-3-7-sonnet-20250219", "label": "Claude 3.7 Sonnet", "provider": "ANTHROPIC", "context_window": "200K"}, + {"value": "claude-3-5-sonnet-20241022", "label": "Claude 3.5 Sonnet", "provider": "ANTHROPIC", "context_window": "200K"}, + {"value": "claude-3-opus-20240229", "label": "Claude 3 Opus", "provider": "ANTHROPIC", "context_window": "200K"}, + {"value": "claude-3-haiku-20240307", "label": "Claude 3 Haiku", "provider": "ANTHROPIC", "context_window": "200K"}, + {"value": "gemini-2.5-flash", "label": "Gemini 2.5 Flash", "provider": "GOOGLE", "context_window": "1M"}, + {"value": "gemini-2.5-pro", "label": "Gemini 2.5 Pro", "provider": "GOOGLE", "context_window": "1M"}, + {"value": "gemini-2.0-flash", "label": "Gemini 2.0 Flash", "provider": "GOOGLE", "context_window": "1M"}, + {"value": "gemini-1.5-pro", "label": "Gemini 1.5 Pro", "provider": "GOOGLE", "context_window": "1M"}, + {"value": "gemini-1.5-flash", "label": "Gemini 1.5 Flash", "provider": "GOOGLE", "context_window": "1M"}, + {"value": "pixtral-large-latest", "label": "Pixtral Large", "provider": "MISTRAL", "context_window": "128K"}, + {"value": "pixtral-12b-2409", "label": "Pixtral 12B", "provider": "MISTRAL", "context_window": "128K"}, + {"value": "grok-2-vision-1212", "label": "Grok 2 Vision", "provider": "XAI", "context_window": "32K"}, + {"value": "llava", "label": "LLaVA", "provider": "OLLAMA"}, + {"value": "bakllava", "label": "BakLLaVA", "provider": "OLLAMA"}, + {"value": "llava-llama3", "label": "LLaVA Llama 3", "provider": "OLLAMA"}, + {"value": "llama-4-scout-17b-16e-instruct", "label": "Llama 4 Scout 17B", "provider": "GROQ", "context_window": "128K"}, + {"value": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "label": "Llama 4 Scout 17B", "provider": "TOGETHER_AI", "context_window": "128K"} +] diff --git a/surfsense_backend/app/routes/vision_llm_routes.py b/surfsense_backend/app/routes/vision_llm_routes.py index 29d1a2757..eddd5e367 100644 --- a/surfsense_backend/app/routes/vision_llm_routes.py +++ b/surfsense_backend/app/routes/vision_llm_routes.py @@ -1,6 +1,7 @@ import logging from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -17,6 +18,7 @@ from app.schemas import ( VisionLLMConfigRead, VisionLLMConfigUpdate, ) +from app.services.vision_model_list_service import get_vision_model_list from app.users import current_active_user from app.utils.rbac import check_permission @@ -24,6 +26,32 @@ router = APIRouter() logger = logging.getLogger(__name__) +# ============================================================================= +# Vision Model Catalogue (from OpenRouter, filtered for image-input models) +# ============================================================================= + + +class VisionModelListItem(BaseModel): + value: str + label: str + provider: str + context_window: str | None = None + + +@router.get("/vision-models", response_model=list[VisionModelListItem]) +async def list_vision_models( + user: User = Depends(current_active_user), +): + """Return vision-capable models sourced from OpenRouter (filtered by image input).""" + try: + return await get_vision_model_list() + except Exception as e: + logger.exception("Failed to fetch vision model list") + raise HTTPException( + status_code=500, detail=f"Failed to fetch vision model list: {e!s}" + ) from e + + # ============================================================================= # Global Vision LLM Configs (from YAML) # ============================================================================= diff --git a/surfsense_backend/app/services/vision_model_list_service.py b/surfsense_backend/app/services/vision_model_list_service.py new file mode 100644 index 000000000..09893dd06 --- /dev/null +++ b/surfsense_backend/app/services/vision_model_list_service.py @@ -0,0 +1,132 @@ +""" +Service for fetching and caching the vision-capable model list. + +Reuses the same OpenRouter public API and local fallback as the LLM model +list service, but filters for models that accept image input. +""" + +import json +import logging +import time +from pathlib import Path + +import httpx + +logger = logging.getLogger(__name__) + +OPENROUTER_API_URL = "https://openrouter.ai/api/v1/models" +FALLBACK_FILE = Path(__file__).parent.parent / "config" / "vision_model_list_fallback.json" +CACHE_TTL_SECONDS = 86400 # 24 hours + +_cache: list[dict] | None = None +_cache_timestamp: float = 0 + +OPENROUTER_SLUG_TO_VISION_PROVIDER: dict[str, str] = { + "openai": "OPENAI", + "anthropic": "ANTHROPIC", + "google": "GOOGLE", + "mistralai": "MISTRAL", + "x-ai": "XAI", +} + + +def _format_context_length(length: int | None) -> str | None: + if not length: + return None + if length >= 1_000_000: + return f"{length / 1_000_000:g}M" + if length >= 1_000: + return f"{length / 1_000:g}K" + return str(length) + + +async def _fetch_from_openrouter() -> list[dict] | None: + try: + async with httpx.AsyncClient(timeout=15) as client: + response = await client.get(OPENROUTER_API_URL) + response.raise_for_status() + data = response.json() + return data.get("data", []) + except Exception as e: + logger.warning("Failed to fetch from OpenRouter API for vision models: %s", e) + return None + + +def _load_fallback() -> list[dict]: + try: + with open(FALLBACK_FILE, encoding="utf-8") as f: + return json.load(f) + except Exception as e: + logger.error("Failed to load vision model fallback list: %s", e) + return [] + + +def _is_vision_model(model: dict) -> bool: + """Return True if the model accepts image input and outputs text.""" + arch = model.get("architecture", {}) + input_mods = arch.get("input_modalities", []) + output_mods = arch.get("output_modalities", []) + return "image" in input_mods and "text" in output_mods + + +def _process_vision_models(raw_models: list[dict]) -> list[dict]: + processed: list[dict] = [] + + for model in raw_models: + model_id: str = model.get("id", "") + name: str = model.get("name", "") + context_length = model.get("context_length") + + if "/" not in model_id: + continue + + if not _is_vision_model(model): + continue + + provider_slug, model_name = model_id.split("/", 1) + context_window = _format_context_length(context_length) + + processed.append( + { + "value": model_id, + "label": name, + "provider": "OPENROUTER", + "context_window": context_window, + } + ) + + native_provider = OPENROUTER_SLUG_TO_VISION_PROVIDER.get(provider_slug) + if native_provider: + if native_provider == "GOOGLE" and not model_name.startswith("gemini-"): + continue + + processed.append( + { + "value": model_name, + "label": name, + "provider": native_provider, + "context_window": context_window, + } + ) + + return processed + + +async def get_vision_model_list() -> list[dict]: + global _cache, _cache_timestamp + + if _cache is not None and (time.time() - _cache_timestamp) < CACHE_TTL_SECONDS: + return _cache + + raw_models = await _fetch_from_openrouter() + + if raw_models is None: + logger.info("Using fallback vision model list") + return _load_fallback() + + processed = _process_vision_models(raw_models) + + _cache = processed + _cache_timestamp = time.time() + + return processed diff --git a/surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts b/surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts index 53264fb24..906ce638f 100644 --- a/surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts +++ b/surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts @@ -1,4 +1,6 @@ import { atomWithQuery } from "jotai-tanstack-query"; +import type { LLMModel } from "@/contracts/enums/llm-models"; +import { VISION_MODELS } from "@/contracts/enums/vision-providers"; import { visionLLMConfigApiService } from "@/lib/apis/vision-llm-config-api.service"; import { cacheKeys } from "@/lib/query-client/cache-keys"; import { activeSearchSpaceIdAtom } from "../search-spaces/search-space-query.atoms"; @@ -25,3 +27,25 @@ export const globalVisionLLMConfigsAtom = atomWithQuery(() => { }, }; }); + +export const visionModelListAtom = atomWithQuery(() => { + return { + queryKey: cacheKeys.visionLLMConfigs.modelList(), + staleTime: 60 * 60 * 1000, + placeholderData: VISION_MODELS, + queryFn: async (): Promise => { + const data = await visionLLMConfigApiService.getModels(); + const dynamicModels = data.map((m) => ({ + value: m.value, + label: m.label, + provider: m.provider, + contextWindow: m.context_window ?? undefined, + })); + + const coveredProviders = new Set(dynamicModels.map((m) => m.provider)); + const staticFallbacks = VISION_MODELS.filter((m) => !coveredProviders.has(m.provider)); + + return [...dynamicModels, ...staticFallbacks]; + }, + }; +}); diff --git a/surfsense_web/components/shared/vision-config-dialog.tsx b/surfsense_web/components/shared/vision-config-dialog.tsx index d69750316..6a494e0a6 100644 --- a/surfsense_web/components/shared/vision-config-dialog.tsx +++ b/surfsense_web/components/shared/vision-config-dialog.tsx @@ -1,20 +1,30 @@ "use client"; import { useAtomValue } from "jotai"; -import { AlertCircle } from "lucide-react"; -import { useCallback, useEffect, useRef, useState } from "react"; +import { AlertCircle, Check, ChevronsUpDown } from "lucide-react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; import { updateLLMPreferencesMutationAtom } from "@/atoms/new-llm-config/new-llm-config-mutation.atoms"; import { createVisionLLMConfigMutationAtom, updateVisionLLMConfigMutationAtom, } from "@/atoms/vision-llm-config/vision-llm-config-mutation.atoms"; +import { visionModelListAtom } from "@/atoms/vision-llm-config/vision-llm-config-query.atoms"; import { Alert, AlertDescription } from "@/components/ui/alert"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; +import { + Command, + CommandEmpty, + CommandGroup, + CommandInput, + CommandItem, + CommandList, +} from "@/components/ui/command"; import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { Select, SelectContent, @@ -30,6 +40,7 @@ import type { VisionLLMConfig, VisionProvider, } from "@/contracts/types/new-llm-config.types"; +import { cn } from "@/lib/utils"; interface VisionConfigDialogProps { open: boolean; @@ -177,6 +188,14 @@ export function VisionConfigDialog({ } }, [config, isGlobal, searchSpaceId, updatePreferences, onOpenChange]); + const { data: dynamicModels } = useAtomValue(visionModelListAtom); + const [modelComboboxOpen, setModelComboboxOpen] = useState(false); + + const availableModels = useMemo( + () => (dynamicModels ?? []).filter((m) => m.provider === formData.provider), + [dynamicModels, formData.provider] + ); + const isFormValid = formData.name && formData.provider && formData.model_name && formData.api_key; const selectedProvider = VISION_PROVIDERS.find((p) => p.value === formData.provider); @@ -303,11 +322,92 @@ export function VisionConfigDialog({
- setFormData((p) => ({ ...p, model_name: e.target.value }))} - /> + + + + + + + + setFormData((p) => ({ ...p, model_name: val })) + } + /> + + +
+ {formData.model_name + ? `Using: "${formData.model_name}"` + : "Type your model name"} +
+ + {availableModels.length > 0 && ( + + {availableModels + .filter( + (model) => + !formData.model_name || + model.value + .toLowerCase() + .includes(formData.model_name.toLowerCase()) || + model.label + .toLowerCase() + .includes(formData.model_name.toLowerCase()) + ) + .slice(0, 50) + .map((model) => ( + { + setFormData((p) => ({ + ...p, + model_name: value, + })); + setModelComboboxOpen(false); + }} + className="py-2" + > + +
+
{model.label}
+ {model.contextWindow && ( +
+ Context: {model.contextWindow} +
+ )} +
+ + ))} + + )} + + + +
diff --git a/surfsense_web/contracts/enums/vision-providers.ts b/surfsense_web/contracts/enums/vision-providers.ts index 260b03585..08be93b74 100644 --- a/surfsense_web/contracts/enums/vision-providers.ts +++ b/surfsense_web/contracts/enums/vision-providers.ts @@ -1,3 +1,5 @@ +import type { LLMModel } from "./llm-models"; + export interface VisionProviderInfo { value: string; label: string; @@ -100,3 +102,27 @@ export const VISION_PROVIDERS: VisionProviderInfo[] = [ description: "Custom OpenAI-compatible vision endpoint", }, ]; + +export const VISION_MODELS: LLMModel[] = [ + { value: "gpt-4o", label: "GPT-4o", provider: "OPENAI", contextWindow: "128K" }, + { value: "gpt-4o-mini", label: "GPT-4o Mini", provider: "OPENAI", contextWindow: "128K" }, + { value: "gpt-4-turbo", label: "GPT-4 Turbo", provider: "OPENAI", contextWindow: "128K" }, + { value: "claude-sonnet-4-20250514", label: "Claude Sonnet 4", provider: "ANTHROPIC", contextWindow: "200K" }, + { value: "claude-3-7-sonnet-20250219", label: "Claude 3.7 Sonnet", provider: "ANTHROPIC", contextWindow: "200K" }, + { value: "claude-3-5-sonnet-20241022", label: "Claude 3.5 Sonnet", provider: "ANTHROPIC", contextWindow: "200K" }, + { value: "claude-3-opus-20240229", label: "Claude 3 Opus", provider: "ANTHROPIC", contextWindow: "200K" }, + { value: "claude-3-haiku-20240307", label: "Claude 3 Haiku", provider: "ANTHROPIC", contextWindow: "200K" }, + { value: "gemini-2.5-flash", label: "Gemini 2.5 Flash", provider: "GOOGLE", contextWindow: "1M" }, + { value: "gemini-2.5-pro", label: "Gemini 2.5 Pro", provider: "GOOGLE", contextWindow: "1M" }, + { value: "gemini-2.0-flash", label: "Gemini 2.0 Flash", provider: "GOOGLE", contextWindow: "1M" }, + { value: "gemini-1.5-pro", label: "Gemini 1.5 Pro", provider: "GOOGLE", contextWindow: "1M" }, + { value: "gemini-1.5-flash", label: "Gemini 1.5 Flash", provider: "GOOGLE", contextWindow: "1M" }, + { value: "pixtral-large-latest", label: "Pixtral Large", provider: "MISTRAL", contextWindow: "128K" }, + { value: "pixtral-12b-2409", label: "Pixtral 12B", provider: "MISTRAL", contextWindow: "128K" }, + { value: "grok-2-vision-1212", label: "Grok 2 Vision", provider: "XAI", contextWindow: "32K" }, + { value: "llava", label: "LLaVA", provider: "OLLAMA" }, + { value: "bakllava", label: "BakLLaVA", provider: "OLLAMA" }, + { value: "llava-llama3", label: "LLaVA Llama 3", provider: "OLLAMA" }, + { value: "llama-4-scout-17b-16e-instruct", label: "Llama 4 Scout 17B", provider: "GROQ", contextWindow: "128K" }, + { value: "meta-llama/Llama-4-Scout-17B-16E-Instruct", label: "Llama 4 Scout 17B", provider: "TOGETHER_AI", contextWindow: "128K" }, +]; diff --git a/surfsense_web/lib/apis/vision-llm-config-api.service.ts b/surfsense_web/lib/apis/vision-llm-config-api.service.ts index 4099c6b39..537cecbd1 100644 --- a/surfsense_web/lib/apis/vision-llm-config-api.service.ts +++ b/surfsense_web/lib/apis/vision-llm-config-api.service.ts @@ -4,6 +4,7 @@ import { createVisionLLMConfigResponse, deleteVisionLLMConfigResponse, getGlobalVisionLLMConfigsResponse, + getModelListResponse, getVisionLLMConfigsResponse, type UpdateVisionLLMConfigRequest, updateVisionLLMConfigRequest, @@ -13,6 +14,10 @@ import { ValidationError } from "../error"; import { baseApiService } from "./base-api.service"; class VisionLLMConfigApiService { + getModels = async () => { + return baseApiService.get(`/api/v1/vision-models`, getModelListResponse); + }; + getGlobalConfigs = async () => { return baseApiService.get( `/api/v1/global-vision-llm-configs`, diff --git a/surfsense_web/lib/query-client/cache-keys.ts b/surfsense_web/lib/query-client/cache-keys.ts index 04f348ff8..10aba7ef4 100644 --- a/surfsense_web/lib/query-client/cache-keys.ts +++ b/surfsense_web/lib/query-client/cache-keys.ts @@ -43,6 +43,7 @@ export const cacheKeys = { all: (searchSpaceId: number) => ["vision-llm-configs", searchSpaceId] as const, byId: (configId: number) => ["vision-llm-configs", "detail", configId] as const, global: () => ["vision-llm-configs", "global"] as const, + modelList: () => ["vision-models", "catalogue"] as const, }, auth: { user: ["auth", "user"] as const, From a38ec3f5dc3ef20354a1acc4f4b4eb3e64c207d4 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 7 Apr 2026 14:59:40 -0700 Subject: [PATCH 44/47] fix: desktop release TODO: Move to monorepo here --- package.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 package.json diff --git a/package.json b/package.json new file mode 100644 index 000000000..8a1a6add8 --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "name": "surfsense", + "private": true, + "packageManager": "pnpm@10.24.0" +} From 9ac062ad7e18c524979c33cd3b2583905d825d5d Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 7 Apr 2026 15:03:33 -0700 Subject: [PATCH 45/47] ci: add workflow_dispatch to desktop release for manual testing --- .github/workflows/desktop-release.yml | 28 ++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index 62ba5d445..1f7e06a0c 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -5,6 +5,20 @@ on: tags: - 'v*' - 'beta-v*' + workflow_dispatch: + inputs: + version: + description: 'Version number (e.g. 0.0.15) — used for dry-run testing without a tag' + required: true + default: '0.0.0-test' + publish: + description: 'Publish to GitHub Releases' + required: true + type: choice + options: + - never + - always + default: 'never' permissions: contents: write @@ -27,13 +41,17 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Extract version from tag + - name: Extract version id: version shell: bash run: | - TAG=${GITHUB_REF#refs/tags/} - VERSION=${TAG#beta-} - VERSION=${VERSION#v} + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + VERSION="${{ inputs.version }}" + else + TAG=${GITHUB_REF#refs/tags/} + VERSION=${TAG#beta-} + VERSION=${VERSION#v} + fi echo "VERSION=$VERSION" >> "$GITHUB_OUTPUT" - name: Setup pnpm @@ -75,7 +93,7 @@ jobs: POSTHOG_HOST: ${{ vars.POSTHOG_HOST }} - name: Package & Publish - run: pnpm exec electron-builder ${{ matrix.platform }} --config electron-builder.yml --publish always -c.extraMetadata.version=${{ steps.version.outputs.VERSION }} + run: pnpm exec electron-builder ${{ matrix.platform }} --config electron-builder.yml --publish ${{ inputs.publish || 'always' }} -c.extraMetadata.version=${{ steps.version.outputs.VERSION }} working-directory: surfsense_desktop env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From b9b567fe303b933e4688306a1c2b860fc392902e Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 7 Apr 2026 15:13:26 -0700 Subject: [PATCH 46/47] chore: update desktop release workflow and configuration - Changed shell to bash in the desktop release workflow for consistency. - Updated the hosted frontend URL in the .env file to point to the new domain. - Enhanced package.json with homepage and author details for better project metadata. --- .github/workflows/desktop-release.yml | 1 + surfsense_desktop/.env | 2 +- surfsense_desktop/package.json | 6 +++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index 1f7e06a0c..f0fdee17d 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -93,6 +93,7 @@ jobs: POSTHOG_HOST: ${{ vars.POSTHOG_HOST }} - name: Package & Publish + shell: bash run: pnpm exec electron-builder ${{ matrix.platform }} --config electron-builder.yml --publish ${{ inputs.publish || 'always' }} -c.extraMetadata.version=${{ steps.version.outputs.VERSION }} working-directory: surfsense_desktop env: diff --git a/surfsense_desktop/.env b/surfsense_desktop/.env index e127b99e0..40e151c10 100644 --- a/surfsense_desktop/.env +++ b/surfsense_desktop/.env @@ -3,7 +3,7 @@ # The hosted web frontend URL. Used to intercept OAuth redirects and keep them # inside the desktop app. Set to your production frontend domain. -HOSTED_FRONTEND_URL=https://surfsense.net +HOSTED_FRONTEND_URL=https://surfsense.com # PostHog analytics (leave empty to disable) POSTHOG_KEY= diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index 7b91d70c9..634783e47 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -14,7 +14,11 @@ "typecheck": "tsc --noEmit", "postinstall": "electron-rebuild" }, - "author": "MODSetter", + "homepage": "https://github.com/MODSetter/SurfSense", + "author": { + "name": "MODSetter", + "email": "rohan@surfsense.com" + }, "license": "MIT", "packageManager": "pnpm@10.24.0", "devDependencies": { From 526057022d422908b3d7d6329c95f4488046c420 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 7 Apr 2026 15:25:48 -0700 Subject: [PATCH 47/47] chore: update dependencies in desktop release workflow - Upgraded actions/checkout from v4 to v5. - Upgraded pnpm/action-setup from v4 to v5. - Upgraded actions/setup-node from v4 to v5 and changed node version from 20 to 22. --- .github/workflows/desktop-release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index f0fdee17d..784dffb32 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Extract version id: version @@ -55,12 +55,12 @@ jobs: echo "VERSION=$VERSION" >> "$GITHUB_OUTPUT" - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@v5 - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v5 with: - node-version: 20 + node-version: 22 cache: 'pnpm' cache-dependency-path: | surfsense_web/pnpm-lock.yaml