From 2b5377846d5e91f960576505b9c91e852b40152f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 30 Dec 2025 16:38:57 +0200 Subject: [PATCH 1/6] refactor: modularize thread.tsx into focused component modules - Split 1,089-line thread.tsx into 10 smaller, focused modules - Created dedicated files for thinking-steps, welcome, composer, messages, etc. - No breaking changes - all logic preserved exactly as before - Improved code organization and maintainability --- .../assistant-ui/assistant-message.tsx | 118 ++ .../components/assistant-ui/branch-picker.tsx | 33 + .../assistant-ui/composer-action.tsx | 269 +++++ .../components/assistant-ui/composer.tsx | 240 ++++ .../components/assistant-ui/edit-composer.tsx | 27 + .../assistant-ui/thinking-steps.tsx | 207 ++++ .../assistant-ui/thread-scroll-to-bottom.tsx | 19 + .../assistant-ui/thread-welcome.tsx | 72 ++ .../components/assistant-ui/thread.tsx | 1045 +---------------- .../components/assistant-ui/user-message.tsx | 73 ++ 10 files changed, 1067 insertions(+), 1036 deletions(-) create mode 100644 surfsense_web/components/assistant-ui/assistant-message.tsx create mode 100644 surfsense_web/components/assistant-ui/branch-picker.tsx create mode 100644 surfsense_web/components/assistant-ui/composer-action.tsx create mode 100644 surfsense_web/components/assistant-ui/composer.tsx create mode 100644 surfsense_web/components/assistant-ui/edit-composer.tsx create mode 100644 surfsense_web/components/assistant-ui/thinking-steps.tsx create mode 100644 surfsense_web/components/assistant-ui/thread-scroll-to-bottom.tsx create mode 100644 surfsense_web/components/assistant-ui/thread-welcome.tsx create mode 100644 surfsense_web/components/assistant-ui/user-message.tsx diff --git a/surfsense_web/components/assistant-ui/assistant-message.tsx b/surfsense_web/components/assistant-ui/assistant-message.tsx new file mode 100644 index 000000000..62fbe0dd4 --- /dev/null +++ b/surfsense_web/components/assistant-ui/assistant-message.tsx @@ -0,0 +1,118 @@ +import { + ActionBarPrimitive, + AssistantIf, + ErrorPrimitive, + MessagePrimitive, + useAssistantState, +} from "@assistant-ui/react"; +import { CheckIcon, CopyIcon, DownloadIcon, RefreshCwIcon } from "lucide-react"; +import type { FC } from "react"; +import { useContext } from "react"; +import { MarkdownText } from "@/components/assistant-ui/markdown-text"; +import { ThinkingStepsContext, ThinkingStepsDisplay } from "@/components/assistant-ui/thinking-steps"; +import { ToolFallback } from "@/components/assistant-ui/tool-fallback"; +import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; +import { BranchPicker } from "@/components/assistant-ui/branch-picker"; + +export const MessageError: FC = () => { + return ( + + + + + + ); +}; + +/** + * Custom component to render thinking steps from Context + */ +const ThinkingStepsPart: FC = () => { + const thinkingStepsMap = useContext(ThinkingStepsContext); + + // Get the current message ID to look up thinking steps + const messageId = useAssistantState(({ message }) => message?.id); + const thinkingSteps = thinkingStepsMap.get(messageId) || []; + + // Check if this specific message is currently streaming + // A message is streaming if: thread is running AND this is the last assistant message + const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning); + const isLastMessage = useAssistantState(({ message }) => message?.isLast ?? false); + const isMessageStreaming = isThreadRunning && isLastMessage; + + if (thinkingSteps.length === 0) return null; + + return ( +
+ +
+ ); +}; + +const AssistantMessageInner: FC = () => { + return ( + <> + {/* Render thinking steps from message content - this ensures proper scroll tracking */} + + +
+ + +
+ +
+ + +
+ + ); +}; + +export const AssistantMessage: FC = () => { + return ( + + + + ); +}; + +const AssistantActionBar: FC = () => { + return ( + + + + message.isCopied}> + + + !message.isCopied}> + + + + + + + + + + + + + + + + ); +}; + diff --git a/surfsense_web/components/assistant-ui/branch-picker.tsx b/surfsense_web/components/assistant-ui/branch-picker.tsx new file mode 100644 index 000000000..1d9041309 --- /dev/null +++ b/surfsense_web/components/assistant-ui/branch-picker.tsx @@ -0,0 +1,33 @@ +import { BranchPickerPrimitive } from "@assistant-ui/react"; +import { ChevronLeftIcon, ChevronRightIcon } from "lucide-react"; +import type { FC } from "react"; +import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; +import { cn } from "@/lib/utils"; + +export const BranchPicker: FC = ({ className, ...rest }) => { + return ( + + + + + + + + / + + + + + + + + ); +}; + diff --git a/surfsense_web/components/assistant-ui/composer-action.tsx b/surfsense_web/components/assistant-ui/composer-action.tsx new file mode 100644 index 000000000..9c5a95d88 --- /dev/null +++ b/surfsense_web/components/assistant-ui/composer-action.tsx @@ -0,0 +1,269 @@ +import { AssistantIf, ComposerPrimitive, useAssistantState } from "@assistant-ui/react"; +import { useAtomValue } from "jotai"; +import { AlertCircle, ArrowUpIcon, Loader2, Plus, Plug2, SquareIcon } from "lucide-react"; +import Link from "next/link"; +import type { FC } from "react"; +import { useCallback, useMemo, useRef, useState } from "react"; +import { getDocumentTypeLabel } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon"; +import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; +import { + globalNewLLMConfigsAtom, + llmPreferencesAtom, + newLLMConfigsAtom, +} from "@/atoms/new-llm-config/new-llm-config-query.atoms"; +import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; +import { ComposerAddAttachment } from "@/components/assistant-ui/attachment"; +import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; +import { Button } from "@/components/ui/button"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; +import { cn } from "@/lib/utils"; +import { ChevronRightIcon } from "lucide-react"; + +const ConnectorIndicator: FC = () => { + const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); + const { connectors, isLoading: connectorsLoading } = useSearchSourceConnectors( + false, + searchSpaceId ? Number(searchSpaceId) : undefined + ); + const { data: documentTypeCounts, isLoading: documentTypesLoading } = + useAtomValue(documentTypeCountsAtom); + const [isOpen, setIsOpen] = useState(false); + const closeTimeoutRef = useRef(null); + + const isLoading = connectorsLoading || documentTypesLoading; + + // Get document types that have documents in the search space + const activeDocumentTypes = documentTypeCounts + ? Object.entries(documentTypeCounts).filter(([_, count]) => count > 0) + : []; + + const hasConnectors = connectors.length > 0; + const hasSources = hasConnectors || activeDocumentTypes.length > 0; + const totalSourceCount = connectors.length + activeDocumentTypes.length; + + const handleMouseEnter = useCallback(() => { + // Clear any pending close timeout + if (closeTimeoutRef.current) { + clearTimeout(closeTimeoutRef.current); + closeTimeoutRef.current = null; + } + setIsOpen(true); + }, []); + + const handleMouseLeave = useCallback(() => { + // Delay closing by 150ms for better UX + closeTimeoutRef.current = setTimeout(() => { + setIsOpen(false); + }, 150); + }, []); + + if (!searchSpaceId) return null; + + return ( + + + + + + {hasSources ? ( +
+
+

Connected Sources

+ + {totalSourceCount} + +
+
+ {/* Document types from the search space */} + {activeDocumentTypes.map(([docType]) => ( +
+ {getConnectorIcon(docType, "size-3.5")} + {getDocumentTypeLabel(docType)} +
+ ))} + {/* Search source connectors */} + {connectors.map((connector) => ( +
+ {getConnectorIcon(connector.connector_type, "size-3.5")} + {connector.name} +
+ ))} +
+
+ + + Add more sources + + +
+
+ ) : ( +
+

No sources yet

+

+ Add documents or connect data sources to enhance search results. +

+ + + Add Connector + +
+ )} + + + ); +}; + +export const ComposerAction: FC = () => { + // Check if any attachments are still being processed (running AND progress < 100) + // When progress is 100, processing is done but waiting for send() + const hasProcessingAttachments = useAssistantState(({ composer }) => + composer.attachments?.some((att) => { + const status = att.status; + if (status?.type !== "running") return false; + const progress = (status as { type: "running"; progress?: number }).progress; + return progress === undefined || progress < 100; + }) + ); + + // Check if composer text is empty + const isComposerEmpty = useAssistantState(({ composer }) => { + const text = composer.text?.trim() || ""; + return text.length === 0; + }); + + // Check if a model is configured + const { data: userConfigs } = useAtomValue(newLLMConfigsAtom); + const { data: globalConfigs } = useAtomValue(globalNewLLMConfigsAtom); + const { data: preferences } = useAtomValue(llmPreferencesAtom); + + const hasModelConfigured = useMemo(() => { + if (!preferences) return false; + const agentLlmId = preferences.agent_llm_id; + if (agentLlmId === null || agentLlmId === undefined) return false; + + // Check if the configured model actually exists + if (agentLlmId < 0) { + return globalConfigs?.some((c) => c.id === agentLlmId) ?? false; + } + return userConfigs?.some((c) => c.id === agentLlmId) ?? false; + }, [preferences, globalConfigs, userConfigs]); + + const isSendDisabled = hasProcessingAttachments || isComposerEmpty || !hasModelConfigured; + + return ( +
+
+ + +
+ + {/* Show processing indicator when attachments are being processed */} + {hasProcessingAttachments && ( +
+ + Processing... +
+ )} + + {/* Show warning when no model is configured */} + {!hasModelConfigured && !hasProcessingAttachments && ( +
+ + Select a model +
+ )} + + !thread.isRunning}> + + + + + + + + thread.isRunning}> + + + + +
+ ); +}; + diff --git a/surfsense_web/components/assistant-ui/composer.tsx b/surfsense_web/components/assistant-ui/composer.tsx new file mode 100644 index 000000000..1973726da --- /dev/null +++ b/surfsense_web/components/assistant-ui/composer.tsx @@ -0,0 +1,240 @@ +import { ComposerPrimitive, useAssistantState, useComposerRuntime } from "@assistant-ui/react"; +import { useAtom, useSetAtom } from "jotai"; +import { useParams } from "next/navigation"; +import type { FC } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { createPortal } from "react-dom"; +import { + mentionedDocumentIdsAtom, + mentionedDocumentsAtom, +} from "@/atoms/chat/mentioned-documents.atom"; +import { + ComposerAddAttachment, + ComposerAttachments, +} from "@/components/assistant-ui/attachment"; +import { ComposerAction } from "@/components/assistant-ui/composer-action"; +import { + InlineMentionEditor, + type InlineMentionEditorRef, +} from "@/components/assistant-ui/inline-mention-editor"; +import { + DocumentMentionPicker, + type DocumentMentionPickerRef, +} from "@/components/new-chat/document-mention-picker"; +import type { Document } from "@/contracts/types/document.types"; + +export const Composer: FC = () => { + // ---- State for document mentions (using atoms to persist across remounts) ---- + const [mentionedDocuments, setMentionedDocuments] = useAtom(mentionedDocumentsAtom); + const [showDocumentPopover, setShowDocumentPopover] = useState(false); + const [mentionQuery, setMentionQuery] = useState(""); + const editorRef = useRef(null); + const editorContainerRef = useRef(null); + const documentPickerRef = useRef(null); + const { search_space_id } = useParams(); + const setMentionedDocumentIds = useSetAtom(mentionedDocumentIdsAtom); + const composerRuntime = useComposerRuntime(); + const hasAutoFocusedRef = useRef(false); + + // Check if thread is empty (new chat) + const isThreadEmpty = useAssistantState(({ thread }) => thread.isEmpty); + + // Check if thread is currently running (streaming response) + const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning); + + // Auto-focus editor when on new chat page + useEffect(() => { + if (isThreadEmpty && !hasAutoFocusedRef.current && editorRef.current) { + // Small delay to ensure the editor is fully mounted + const timeoutId = setTimeout(() => { + editorRef.current?.focus(); + hasAutoFocusedRef.current = true; + }, 100); + return () => clearTimeout(timeoutId); + } + }, [isThreadEmpty]); + + // Sync mentioned document IDs to atom for use in chat request + useEffect(() => { + setMentionedDocumentIds(mentionedDocuments.map((doc) => doc.id)); + }, [mentionedDocuments, setMentionedDocumentIds]); + + // Handle text change from inline editor - sync with assistant-ui composer + const handleEditorChange = useCallback( + (text: string) => { + composerRuntime.setText(text); + }, + [composerRuntime] + ); + + // Handle @ mention trigger from inline editor + const handleMentionTrigger = useCallback((query: string) => { + setShowDocumentPopover(true); + setMentionQuery(query); + }, []); + + // Handle mention close + const handleMentionClose = useCallback(() => { + if (showDocumentPopover) { + setShowDocumentPopover(false); + setMentionQuery(""); + } + }, [showDocumentPopover]); + + // Handle keyboard navigation when popover is open + const handleKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (showDocumentPopover) { + if (e.key === "ArrowDown") { + e.preventDefault(); + documentPickerRef.current?.moveDown(); + return; + } + if (e.key === "ArrowUp") { + e.preventDefault(); + documentPickerRef.current?.moveUp(); + return; + } + if (e.key === "Enter") { + e.preventDefault(); + documentPickerRef.current?.selectHighlighted(); + return; + } + if (e.key === "Escape") { + e.preventDefault(); + setShowDocumentPopover(false); + setMentionQuery(""); + return; + } + } + }, + [showDocumentPopover] + ); + + // Handle submit from inline editor (Enter key) + const handleSubmit = useCallback(() => { + // Prevent sending while a response is still streaming + if (isThreadRunning) { + return; + } + if (!showDocumentPopover) { + composerRuntime.send(); + // Clear the editor after sending + editorRef.current?.clear(); + setMentionedDocuments([]); + setMentionedDocumentIds([]); + } + }, [ + showDocumentPopover, + isThreadRunning, + composerRuntime, + setMentionedDocuments, + setMentionedDocumentIds, + ]); + + // Handle document removal from inline editor + const handleDocumentRemove = useCallback( + (docId: number) => { + setMentionedDocuments((prev) => { + const updated = prev.filter((doc) => doc.id !== docId); + // Immediately sync document IDs to avoid race conditions + setMentionedDocumentIds(updated.map((doc) => doc.id)); + return updated; + }); + }, + [setMentionedDocuments, setMentionedDocumentIds] + ); + + // Handle document selection from picker + const handleDocumentsMention = useCallback( + (documents: Document[]) => { + // Insert chips into the inline editor for each new document + const existingIds = new Set(mentionedDocuments.map((d) => d.id)); + const newDocs = documents.filter((doc) => !existingIds.has(doc.id)); + + for (const doc of newDocs) { + editorRef.current?.insertDocumentChip(doc); + } + + // Update mentioned documents state + setMentionedDocuments((prev) => { + const existingIdSet = new Set(prev.map((d) => d.id)); + const uniqueNewDocs = documents.filter((doc) => !existingIdSet.has(doc.id)); + const updated = [...prev, ...uniqueNewDocs]; + // Immediately sync document IDs to avoid race conditions + setMentionedDocumentIds(updated.map((doc) => doc.id)); + return updated; + }); + + // Reset mention query but keep popover open for more selections + setMentionQuery(""); + }, + [mentionedDocuments, setMentionedDocuments, setMentionedDocumentIds] + ); + + return ( + + + + {/* -------- Inline Mention Editor -------- */} +
+ +
+ + {/* -------- Document mention popover (rendered via portal) -------- */} + {showDocumentPopover && + typeof document !== "undefined" && + createPortal( + <> + {/* Backdrop */} + + + + + + + + + ); +}; + diff --git a/surfsense_web/components/assistant-ui/thinking-steps.tsx b/surfsense_web/components/assistant-ui/thinking-steps.tsx new file mode 100644 index 000000000..f0cf4a7c1 --- /dev/null +++ b/surfsense_web/components/assistant-ui/thinking-steps.tsx @@ -0,0 +1,207 @@ +import { useAssistantState, useThreadViewport } from "@assistant-ui/react"; +import type { FC } from "react"; +import { createContext, useCallback, useContext, useEffect, useRef, useState } from "react"; +import { ChevronRightIcon } from "lucide-react"; +import { ChainOfThoughtItem } from "@/components/prompt-kit/chain-of-thought"; +import { TextShimmerLoader } from "@/components/prompt-kit/loader"; +import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking"; +import { cn } from "@/lib/utils"; + +// Context to pass thinking steps to AssistantMessage +export const ThinkingStepsContext = createContext>(new Map()); + +/** + * Chain of thought display component - single collapsible dropdown design + */ +export const ThinkingStepsDisplay: FC<{ steps: ThinkingStep[]; isThreadRunning?: boolean }> = ({ + steps, + isThreadRunning = true, +}) => { + const [isOpen, setIsOpen] = useState(true); + + // Derive effective status for each step + const getEffectiveStatus = useCallback( + (step: ThinkingStep): "pending" | "in_progress" | "completed" => { + if (step.status === "in_progress" && !isThreadRunning) { + return "completed"; + } + return step.status; + }, + [isThreadRunning] + ); + + // Calculate summary info + const completedSteps = steps.filter((s) => getEffectiveStatus(s) === "completed").length; + const inProgressStep = steps.find((s) => getEffectiveStatus(s) === "in_progress"); + const allCompleted = completedSteps === steps.length && steps.length > 0 && !isThreadRunning; + const isProcessing = isThreadRunning && !allCompleted; + + // Auto-collapse when all tasks are completed + useEffect(() => { + if (allCompleted) { + setIsOpen(false); + } + }, [allCompleted]); + + if (steps.length === 0) return null; + + // Generate header text + const getHeaderText = () => { + if (allCompleted) { + return `Reviewed ${completedSteps} ${completedSteps === 1 ? "step" : "steps"}`; + } + if (inProgressStep) { + return inProgressStep.title; + } + if (isProcessing) { + return `Processing ${completedSteps}/${steps.length} steps`; + } + return `Reviewed ${completedSteps} ${completedSteps === 1 ? "step" : "steps"}`; + }; + + return ( +
+
+ {/* Main collapsible header */} + + + {/* Collapsible content with CSS grid animation */} +
+
+
+ {steps.map((step, index) => { + const effectiveStatus = getEffectiveStatus(step); + const isLast = index === steps.length - 1; + + return ( +
+ {/* Dot and line column */} +
+ {/* Vertical connection line - extends to next dot */} + {!isLast && ( +
+ )} + {/* Step dot - on top of line */} +
+ {effectiveStatus === "in_progress" ? ( + + ) : ( + + )} +
+
+ + {/* Step content */} +
+ {/* Step title */} +
+ {effectiveStatus === "in_progress" ? ( + + ) : ( + step.title + )} +
+ + {/* Step items (sub-content) */} + {step.items && step.items.length > 0 && ( +
+ {step.items.map((item, idx) => ( + + {item} + + ))} +
+ )} +
+
+ ); + })} +
+
+
+
+
+ ); +}; + +/** + * Component that handles auto-scroll when thinking steps update. + * Uses useThreadViewport to scroll to bottom when thinking steps change, + * ensuring the user always sees the latest content during streaming. + */ +export const ThinkingStepsScrollHandler: FC = () => { + const thinkingStepsMap = useContext(ThinkingStepsContext); + const viewport = useThreadViewport(); + const isRunning = useAssistantState(({ thread }) => thread.isRunning); + // Track the serialized state to detect any changes + const prevStateRef = useRef(""); + + useEffect(() => { + // Only act during streaming + if (!isRunning) { + prevStateRef.current = ""; + return; + } + + // Serialize the thinking steps state to detect any changes + // This catches new steps, status changes, and item additions + let stateString = ""; + thinkingStepsMap.forEach((steps, msgId) => { + steps.forEach((step) => { + stateString += `${msgId}:${step.id}:${step.status}:${step.items?.length || 0};`; + }); + }); + + // If state changed at all during streaming, scroll + if (stateString !== prevStateRef.current && stateString !== "") { + prevStateRef.current = stateString; + + // Multiple attempts to ensure scroll happens after DOM updates + const scrollAttempt = () => { + try { + viewport.scrollToBottom(); + } catch { + // Ignore errors - viewport might not be ready + } + }; + + // Delayed attempts to handle async DOM updates + requestAnimationFrame(scrollAttempt); + setTimeout(scrollAttempt, 100); + } + }, [thinkingStepsMap, viewport, isRunning]); + + return null; // This component doesn't render anything +}; + diff --git a/surfsense_web/components/assistant-ui/thread-scroll-to-bottom.tsx b/surfsense_web/components/assistant-ui/thread-scroll-to-bottom.tsx new file mode 100644 index 000000000..6f641615e --- /dev/null +++ b/surfsense_web/components/assistant-ui/thread-scroll-to-bottom.tsx @@ -0,0 +1,19 @@ +import { ThreadPrimitive } from "@assistant-ui/react"; +import { ArrowDownIcon } from "lucide-react"; +import type { FC } from "react"; +import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; + +export const ThreadScrollToBottom: FC = () => { + return ( + + + + + + ); +}; + diff --git a/surfsense_web/components/assistant-ui/thread-welcome.tsx b/surfsense_web/components/assistant-ui/thread-welcome.tsx new file mode 100644 index 000000000..b5e4bbac0 --- /dev/null +++ b/surfsense_web/components/assistant-ui/thread-welcome.tsx @@ -0,0 +1,72 @@ +import { useAtomValue } from "jotai"; +import type { FC } from "react"; +import { useMemo } from "react"; +import { currentUserAtom } from "@/atoms/user/user-query.atoms"; +import { Composer } from "@/components/assistant-ui/composer"; + +const getTimeBasedGreeting = (userEmail?: string): string => { + const hour = new Date().getHours(); + + // Extract first name from email if available + const firstName = userEmail + ? userEmail.split("@")[0].split(".")[0].charAt(0).toUpperCase() + + userEmail.split("@")[0].split(".")[0].slice(1) + : null; + + // Array of greeting variations for each time period + const morningGreetings = ["Good morning", "Fresh start today", "Morning", "Hey there"]; + + const afternoonGreetings = ["Good afternoon", "Afternoon", "Hey there", "Hi there"]; + + const eveningGreetings = ["Good evening", "Evening", "Hey there", "Hi there"]; + + const nightGreetings = ["Good night", "Evening", "Hey there", "Winding down"]; + + const lateNightGreetings = ["Still up", "Night owl mode", "Up past bedtime", "Hi there"]; + + // Select a random greeting based on time + let greeting: string; + if (hour < 5) { + // Late night: midnight to 5 AM + greeting = lateNightGreetings[Math.floor(Math.random() * lateNightGreetings.length)]; + } else if (hour < 12) { + greeting = morningGreetings[Math.floor(Math.random() * morningGreetings.length)]; + } else if (hour < 18) { + greeting = afternoonGreetings[Math.floor(Math.random() * afternoonGreetings.length)]; + } else if (hour < 22) { + greeting = eveningGreetings[Math.floor(Math.random() * eveningGreetings.length)]; + } else { + // Night: 10 PM to midnight + greeting = nightGreetings[Math.floor(Math.random() * nightGreetings.length)]; + } + + // Add personalization with first name if available + if (firstName) { + return `${greeting}, ${firstName}!`; + } + + return `${greeting}!`; +}; + +export const ThreadWelcome: FC = () => { + const { data: user } = useAtomValue(currentUserAtom); + + // Memoize greeting so it doesn't change on re-renders (only on user change) + const greeting = useMemo(() => getTimeBasedGreeting(user?.email), [user?.email]); + + return ( +
+ {/* Greeting positioned above the composer - fixed position */} +
+

+ {greeting} +

+
+ {/* Composer - top edge fixed, expands downward only */} +
+ +
+
+ ); +}; + diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index cb01e7605..a354414f0 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -1,85 +1,13 @@ -import { - ActionBarPrimitive, - AssistantIf, - BranchPickerPrimitive, - ComposerPrimitive, - ErrorPrimitive, - MessagePrimitive, - ThreadPrimitive, - useAssistantState, - useComposerRuntime, - useThreadViewport, -} from "@assistant-ui/react"; -import { useAtom, useAtomValue, useSetAtom } from "jotai"; -import { - AlertCircle, - ArrowDownIcon, - ArrowUpIcon, - CheckIcon, - ChevronLeftIcon, - ChevronRightIcon, - CopyIcon, - DownloadIcon, - FileText, - Loader2, - PencilIcon, - Plug2, - Plus, - RefreshCwIcon, - SquareIcon, -} from "lucide-react"; -import Link from "next/link"; -import { useParams } from "next/navigation"; -import { - createContext, - type FC, - useCallback, - useContext, - useEffect, - useMemo, - useRef, - useState, -} from "react"; -import { createPortal } from "react-dom"; -import { getDocumentTypeLabel } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon"; -import { - mentionedDocumentIdsAtom, - mentionedDocumentsAtom, - messageDocumentsMapAtom, -} from "@/atoms/chat/mentioned-documents.atom"; -import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; -import { - globalNewLLMConfigsAtom, - llmPreferencesAtom, - newLLMConfigsAtom, -} from "@/atoms/new-llm-config/new-llm-config-query.atoms"; -import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; -import { currentUserAtom } from "@/atoms/user/user-query.atoms"; -import { - ComposerAddAttachment, - ComposerAttachments, - UserMessageAttachments, -} from "@/components/assistant-ui/attachment"; -import { - InlineMentionEditor, - type InlineMentionEditorRef, -} from "@/components/assistant-ui/inline-mention-editor"; -import { MarkdownText } from "@/components/assistant-ui/markdown-text"; -import { ToolFallback } from "@/components/assistant-ui/tool-fallback"; -import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; -import { - DocumentMentionPicker, - type DocumentMentionPickerRef, -} from "@/components/new-chat/document-mention-picker"; -import { ChainOfThoughtItem } from "@/components/prompt-kit/chain-of-thought"; -import { TextShimmerLoader } from "@/components/prompt-kit/loader"; +import { AssistantIf, ThreadPrimitive } from "@assistant-ui/react"; +import type { FC } from "react"; import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking"; -import { Button } from "@/components/ui/button"; -import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; -import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import type { Document } from "@/contracts/types/document.types"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; -import { cn } from "@/lib/utils"; +import { ThinkingStepsContext } from "@/components/assistant-ui/thinking-steps"; +import { ThreadWelcome } from "@/components/assistant-ui/thread-welcome"; +import { Composer } from "@/components/assistant-ui/composer"; +import { ThreadScrollToBottom } from "@/components/assistant-ui/thread-scroll-to-bottom"; +import { AssistantMessage } from "@/components/assistant-ui/assistant-message"; +import { UserMessage } from "@/components/assistant-ui/user-message"; +import { EditComposer } from "@/components/assistant-ui/edit-composer"; /** * Props for the Thread component @@ -90,204 +18,6 @@ interface ThreadProps { header?: React.ReactNode; } -// Context to pass thinking steps to AssistantMessage -const ThinkingStepsContext = createContext>(new Map()); - -/** - * Chain of thought display component - single collapsible dropdown design - */ -const ThinkingStepsDisplay: FC<{ steps: ThinkingStep[]; isThreadRunning?: boolean }> = ({ - steps, - isThreadRunning = true, -}) => { - const [isOpen, setIsOpen] = useState(true); - - // Derive effective status for each step - const getEffectiveStatus = useCallback( - (step: ThinkingStep): "pending" | "in_progress" | "completed" => { - if (step.status === "in_progress" && !isThreadRunning) { - return "completed"; - } - return step.status; - }, - [isThreadRunning] - ); - - // Calculate summary info - const completedSteps = steps.filter((s) => getEffectiveStatus(s) === "completed").length; - const inProgressStep = steps.find((s) => getEffectiveStatus(s) === "in_progress"); - const allCompleted = completedSteps === steps.length && steps.length > 0 && !isThreadRunning; - const isProcessing = isThreadRunning && !allCompleted; - - // Auto-collapse when all tasks are completed - useEffect(() => { - if (allCompleted) { - setIsOpen(false); - } - }, [allCompleted]); - - if (steps.length === 0) return null; - - // Generate header text - const getHeaderText = () => { - if (allCompleted) { - return `Reviewed ${completedSteps} ${completedSteps === 1 ? "step" : "steps"}`; - } - if (inProgressStep) { - return inProgressStep.title; - } - if (isProcessing) { - return `Processing ${completedSteps}/${steps.length} steps`; - } - return `Reviewed ${completedSteps} ${completedSteps === 1 ? "step" : "steps"}`; - }; - - return ( -
-
- {/* Main collapsible header */} - - - {/* Collapsible content with CSS grid animation */} -
-
-
- {steps.map((step, index) => { - const effectiveStatus = getEffectiveStatus(step); - const isLast = index === steps.length - 1; - - return ( -
- {/* Dot and line column */} -
- {/* Vertical connection line - extends to next dot */} - {!isLast && ( -
- )} - {/* Step dot - on top of line */} -
- {effectiveStatus === "in_progress" ? ( - - ) : ( - - )} -
-
- - {/* Step content */} -
- {/* Step title */} -
- {effectiveStatus === "in_progress" ? ( - - ) : ( - step.title - )} -
- - {/* Step items (sub-content) */} - {step.items && step.items.length > 0 && ( -
- {step.items.map((item, idx) => ( - - {item} - - ))} -
- )} -
-
- ); - })} -
-
-
-
-
- ); -}; - -/** - * Component that handles auto-scroll when thinking steps update. - * Uses useThreadViewport to scroll to bottom when thinking steps change, - * ensuring the user always sees the latest content during streaming. - */ -const _ThinkingStepsScrollHandler: FC = () => { - const thinkingStepsMap = useContext(ThinkingStepsContext); - const viewport = useThreadViewport(); - const isRunning = useAssistantState(({ thread }) => thread.isRunning); - // Track the serialized state to detect any changes - const prevStateRef = useRef(""); - - useEffect(() => { - // Only act during streaming - if (!isRunning) { - prevStateRef.current = ""; - return; - } - - // Serialize the thinking steps state to detect any changes - // This catches new steps, status changes, and item additions - let stateString = ""; - thinkingStepsMap.forEach((steps, msgId) => { - steps.forEach((step) => { - stateString += `${msgId}:${step.id}:${step.status}:${step.items?.length || 0};`; - }); - }); - - // If state changed at all during streaming, scroll - if (stateString !== prevStateRef.current && stateString !== "") { - prevStateRef.current = stateString; - - // Multiple attempts to ensure scroll happens after DOM updates - const scrollAttempt = () => { - try { - viewport.scrollToBottom(); - } catch { - // Ignore errors - viewport might not be ready - } - }; - - // Delayed attempts to handle async DOM updates - requestAnimationFrame(scrollAttempt); - setTimeout(scrollAttempt, 100); - } - }, [thinkingStepsMap, viewport, isRunning]); - - return null; // This component doesn't render anything -}; - export const Thread: FC = ({ messageThinkingSteps = new Map(), header }) => { return ( @@ -329,760 +59,3 @@ export const Thread: FC = ({ messageThinkingSteps = new Map(), head ); }; - -const ThreadScrollToBottom: FC = () => { - return ( - - - - - - ); -}; - -const getTimeBasedGreeting = (userEmail?: string): string => { - const hour = new Date().getHours(); - - // Extract first name from email if available - const firstName = userEmail - ? userEmail.split("@")[0].split(".")[0].charAt(0).toUpperCase() + - userEmail.split("@")[0].split(".")[0].slice(1) - : null; - - // Array of greeting variations for each time period - const morningGreetings = ["Good morning", "Fresh start today", "Morning", "Hey there"]; - - const afternoonGreetings = ["Good afternoon", "Afternoon", "Hey there", "Hi there"]; - - const eveningGreetings = ["Good evening", "Evening", "Hey there", "Hi there"]; - - const nightGreetings = ["Good night", "Evening", "Hey there", "Winding down"]; - - const lateNightGreetings = ["Still up", "Night owl mode", "Up past bedtime", "Hi there"]; - - // Select a random greeting based on time - let greeting: string; - if (hour < 5) { - // Late night: midnight to 5 AM - greeting = lateNightGreetings[Math.floor(Math.random() * lateNightGreetings.length)]; - } else if (hour < 12) { - greeting = morningGreetings[Math.floor(Math.random() * morningGreetings.length)]; - } else if (hour < 18) { - greeting = afternoonGreetings[Math.floor(Math.random() * afternoonGreetings.length)]; - } else if (hour < 22) { - greeting = eveningGreetings[Math.floor(Math.random() * eveningGreetings.length)]; - } else { - // Night: 10 PM to midnight - greeting = nightGreetings[Math.floor(Math.random() * nightGreetings.length)]; - } - - // Add personalization with first name if available - if (firstName) { - return `${greeting}, ${firstName}!`; - } - - return `${greeting}!`; -}; - -const ThreadWelcome: FC = () => { - const { data: user } = useAtomValue(currentUserAtom); - - // Memoize greeting so it doesn't change on re-renders (only on user change) - const greeting = useMemo(() => getTimeBasedGreeting(user?.email), [user?.email]); - - return ( -
- {/* Greeting positioned above the composer - fixed position */} -
-

- {greeting} -

-
- {/* Composer - top edge fixed, expands downward only */} -
- -
-
- ); -}; - -const Composer: FC = () => { - // ---- State for document mentions (using atoms to persist across remounts) ---- - const [mentionedDocuments, setMentionedDocuments] = useAtom(mentionedDocumentsAtom); - const [showDocumentPopover, setShowDocumentPopover] = useState(false); - const [mentionQuery, setMentionQuery] = useState(""); - const editorRef = useRef(null); - const editorContainerRef = useRef(null); - const documentPickerRef = useRef(null); - const { search_space_id } = useParams(); - const setMentionedDocumentIds = useSetAtom(mentionedDocumentIdsAtom); - const composerRuntime = useComposerRuntime(); - const hasAutoFocusedRef = useRef(false); - - // Check if thread is empty (new chat) - const isThreadEmpty = useAssistantState(({ thread }) => thread.isEmpty); - - // Check if thread is currently running (streaming response) - const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning); - - // Auto-focus editor when on new chat page - useEffect(() => { - if (isThreadEmpty && !hasAutoFocusedRef.current && editorRef.current) { - // Small delay to ensure the editor is fully mounted - const timeoutId = setTimeout(() => { - editorRef.current?.focus(); - hasAutoFocusedRef.current = true; - }, 100); - return () => clearTimeout(timeoutId); - } - }, [isThreadEmpty]); - - // Sync mentioned document IDs to atom for use in chat request - useEffect(() => { - setMentionedDocumentIds(mentionedDocuments.map((doc) => doc.id)); - }, [mentionedDocuments, setMentionedDocumentIds]); - - // Handle text change from inline editor - sync with assistant-ui composer - const handleEditorChange = useCallback( - (text: string) => { - composerRuntime.setText(text); - }, - [composerRuntime] - ); - - // Handle @ mention trigger from inline editor - const handleMentionTrigger = useCallback((query: string) => { - setShowDocumentPopover(true); - setMentionQuery(query); - }, []); - - // Handle mention close - const handleMentionClose = useCallback(() => { - if (showDocumentPopover) { - setShowDocumentPopover(false); - setMentionQuery(""); - } - }, [showDocumentPopover]); - - // Handle keyboard navigation when popover is open - const handleKeyDown = useCallback( - (e: React.KeyboardEvent) => { - if (showDocumentPopover) { - if (e.key === "ArrowDown") { - e.preventDefault(); - documentPickerRef.current?.moveDown(); - return; - } - if (e.key === "ArrowUp") { - e.preventDefault(); - documentPickerRef.current?.moveUp(); - return; - } - if (e.key === "Enter") { - e.preventDefault(); - documentPickerRef.current?.selectHighlighted(); - return; - } - if (e.key === "Escape") { - e.preventDefault(); - setShowDocumentPopover(false); - setMentionQuery(""); - return; - } - } - }, - [showDocumentPopover] - ); - - // Handle submit from inline editor (Enter key) - const handleSubmit = useCallback(() => { - // Prevent sending while a response is still streaming - if (isThreadRunning) { - return; - } - if (!showDocumentPopover) { - composerRuntime.send(); - // Clear the editor after sending - editorRef.current?.clear(); - setMentionedDocuments([]); - setMentionedDocumentIds([]); - } - }, [ - showDocumentPopover, - isThreadRunning, - composerRuntime, - setMentionedDocuments, - setMentionedDocumentIds, - ]); - - // Handle document removal from inline editor - const handleDocumentRemove = useCallback( - (docId: number) => { - setMentionedDocuments((prev) => { - const updated = prev.filter((doc) => doc.id !== docId); - // Immediately sync document IDs to avoid race conditions - setMentionedDocumentIds(updated.map((doc) => doc.id)); - return updated; - }); - }, - [setMentionedDocuments, setMentionedDocumentIds] - ); - - // Handle document selection from picker - const handleDocumentsMention = useCallback( - (documents: Document[]) => { - // Insert chips into the inline editor for each new document - const existingIds = new Set(mentionedDocuments.map((d) => d.id)); - const newDocs = documents.filter((doc) => !existingIds.has(doc.id)); - - for (const doc of newDocs) { - editorRef.current?.insertDocumentChip(doc); - } - - // Update mentioned documents state - setMentionedDocuments((prev) => { - const existingIdSet = new Set(prev.map((d) => d.id)); - const uniqueNewDocs = documents.filter((doc) => !existingIdSet.has(doc.id)); - const updated = [...prev, ...uniqueNewDocs]; - // Immediately sync document IDs to avoid race conditions - setMentionedDocumentIds(updated.map((doc) => doc.id)); - return updated; - }); - - // Reset mention query but keep popover open for more selections - setMentionQuery(""); - }, - [mentionedDocuments, setMentionedDocuments, setMentionedDocumentIds] - ); - - return ( - - - - {/* -------- Inline Mention Editor -------- */} -
- -
- - {/* -------- Document mention popover (rendered via portal) -------- */} - {showDocumentPopover && - typeof document !== "undefined" && - createPortal( - <> - {/* Backdrop */} - - - - {hasSources ? ( -
-
-

Connected Sources

- - {totalSourceCount} - -
-
- {/* Document types from the search space */} - {activeDocumentTypes.map(([docType]) => ( -
- {getConnectorIcon(docType, "size-3.5")} - {getDocumentTypeLabel(docType)} -
- ))} - {/* Search source connectors */} - {connectors.map((connector) => ( -
- {getConnectorIcon(connector.connector_type, "size-3.5")} - {connector.name} -
- ))} -
-
- - - Add more sources - - -
-
- ) : ( -
-

No sources yet

-

- Add documents or connect data sources to enhance search results. -

- - - Add Connector - -
- )} - - - ); -}; - -const ComposerAction: FC = () => { - // Check if any attachments are still being processed (running AND progress < 100) - // When progress is 100, processing is done but waiting for send() - const hasProcessingAttachments = useAssistantState(({ composer }) => - composer.attachments?.some((att) => { - const status = att.status; - if (status?.type !== "running") return false; - const progress = (status as { type: "running"; progress?: number }).progress; - return progress === undefined || progress < 100; - }) - ); - - // Check if composer text is empty - const isComposerEmpty = useAssistantState(({ composer }) => { - const text = composer.text?.trim() || ""; - return text.length === 0; - }); - - // Check if a model is configured - const { data: userConfigs } = useAtomValue(newLLMConfigsAtom); - const { data: globalConfigs } = useAtomValue(globalNewLLMConfigsAtom); - const { data: preferences } = useAtomValue(llmPreferencesAtom); - - const hasModelConfigured = useMemo(() => { - if (!preferences) return false; - const agentLlmId = preferences.agent_llm_id; - if (agentLlmId === null || agentLlmId === undefined) return false; - - // Check if the configured model actually exists - if (agentLlmId < 0) { - return globalConfigs?.some((c) => c.id === agentLlmId) ?? false; - } - return userConfigs?.some((c) => c.id === agentLlmId) ?? false; - }, [preferences, globalConfigs, userConfigs]); - - const isSendDisabled = hasProcessingAttachments || isComposerEmpty || !hasModelConfigured; - - return ( -
-
- - -
- - {/* Show processing indicator when attachments are being processed */} - {hasProcessingAttachments && ( -
- - Processing... -
- )} - - {/* Show warning when no model is configured */} - {!hasModelConfigured && !hasProcessingAttachments && ( -
- - Select a model -
- )} - - !thread.isRunning}> - - - - - - - - thread.isRunning}> - - - - -
- ); -}; - -const MessageError: FC = () => { - return ( - - - - - - ); -}; - -/** - * Custom component to render thinking steps from Context - */ -const ThinkingStepsPart: FC = () => { - const thinkingStepsMap = useContext(ThinkingStepsContext); - - // Get the current message ID to look up thinking steps - const messageId = useAssistantState(({ message }) => message?.id); - const thinkingSteps = thinkingStepsMap.get(messageId) || []; - - // Check if this specific message is currently streaming - // A message is streaming if: thread is running AND this is the last assistant message - const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning); - const isLastMessage = useAssistantState(({ message }) => message?.isLast ?? false); - const isMessageStreaming = isThreadRunning && isLastMessage; - - if (thinkingSteps.length === 0) return null; - - return ( -
- -
- ); -}; - -const AssistantMessageInner: FC = () => { - return ( - <> - {/* Render thinking steps from message content - this ensures proper scroll tracking */} - - -
- - -
- -
- - -
- - ); -}; - -const AssistantMessage: FC = () => { - return ( - - - - ); -}; - -const AssistantActionBar: FC = () => { - return ( - - - - message.isCopied}> - - - !message.isCopied}> - - - - - - - - - - - - - - - - ); -}; - -const UserMessage: FC = () => { - const messageId = useAssistantState(({ message }) => message?.id); - const messageDocumentsMap = useAtomValue(messageDocumentsMapAtom); - const mentionedDocs = messageId ? messageDocumentsMap[messageId] : undefined; - const hasAttachments = useAssistantState( - ({ message }) => message?.attachments && message.attachments.length > 0 - ); - - return ( - -
- {/* Display attachments and mentioned documents */} - {(hasAttachments || (mentionedDocs && mentionedDocs.length > 0)) && ( -
- {/* Attachments (images show as thumbnails, documents as chips) */} - - {/* Mentioned documents as chips */} - {mentionedDocs?.map((doc) => ( - - - {doc.title} - - ))} -
- )} - {/* Message bubble with action bar positioned relative to it */} -
-
- -
-
- -
-
-
- - - - ); -}; - -const UserActionBar: FC = () => { - return ( - - - - - - - - ); -}; - -const EditComposer: FC = () => { - return ( - - - -
- - - - - - -
- - - ); -}; - -const BranchPicker: FC = ({ className, ...rest }) => { - return ( - - - - - - - - / - - - - - - - - ); -}; diff --git a/surfsense_web/components/assistant-ui/user-message.tsx b/surfsense_web/components/assistant-ui/user-message.tsx new file mode 100644 index 000000000..fbbcf42bf --- /dev/null +++ b/surfsense_web/components/assistant-ui/user-message.tsx @@ -0,0 +1,73 @@ +import { ActionBarPrimitive, MessagePrimitive, useAssistantState } from "@assistant-ui/react"; +import { useAtomValue } from "jotai"; +import { FileText, PencilIcon } from "lucide-react"; +import type { FC } from "react"; +import { messageDocumentsMapAtom } from "@/atoms/chat/mentioned-documents.atom"; +import { UserMessageAttachments } from "@/components/assistant-ui/attachment"; +import { BranchPicker } from "@/components/assistant-ui/branch-picker"; +import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; + +export const UserMessage: FC = () => { + const messageId = useAssistantState(({ message }) => message?.id); + const messageDocumentsMap = useAtomValue(messageDocumentsMapAtom); + const mentionedDocs = messageId ? messageDocumentsMap[messageId] : undefined; + const hasAttachments = useAssistantState( + ({ message }) => message?.attachments && message.attachments.length > 0 + ); + + return ( + +
+ {/* Display attachments and mentioned documents */} + {(hasAttachments || (mentionedDocs && mentionedDocs.length > 0)) && ( +
+ {/* Attachments (images show as thumbnails, documents as chips) */} + + {/* Mentioned documents as chips */} + {mentionedDocs?.map((doc) => ( + + + {doc.title} + + ))} +
+ )} + {/* Message bubble with action bar positioned relative to it */} +
+
+ +
+
+ +
+
+
+ + + + ); +}; + +const UserActionBar: FC = () => { + return ( + + + + + + + + ); +}; + From 5b39b32ef6f5e8828e624db2c3782b1efb5291c0 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 30 Dec 2025 17:37:22 +0200 Subject: [PATCH 2/6] fix: improve connector popover filtering and display - Show document count badges for each document type - Filter to only show non-indexable connectors - Only display document types with at least 1 document - Update total source count to reflect filtered connectors --- .../components/assistant-ui/composer-action.tsx | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/surfsense_web/components/assistant-ui/composer-action.tsx b/surfsense_web/components/assistant-ui/composer-action.tsx index 9c5a95d88..ba27f40c2 100644 --- a/surfsense_web/components/assistant-ui/composer-action.tsx +++ b/surfsense_web/components/assistant-ui/composer-action.tsx @@ -34,14 +34,15 @@ const ConnectorIndicator: FC = () => { const isLoading = connectorsLoading || documentTypesLoading; - // Get document types that have documents in the search space const activeDocumentTypes = documentTypeCounts ? Object.entries(documentTypeCounts).filter(([_, count]) => count > 0) : []; - const hasConnectors = connectors.length > 0; + const nonIndexableConnectors = connectors.filter((connector) => !connector.is_indexable); + + const hasConnectors = nonIndexableConnectors.length > 0; const hasSources = hasConnectors || activeDocumentTypes.length > 0; - const totalSourceCount = connectors.length + activeDocumentTypes.length; + const totalSourceCount = nonIndexableConnectors.length + activeDocumentTypes.length; const handleMouseEnter = useCallback(() => { // Clear any pending close timeout @@ -110,18 +111,19 @@ const ConnectorIndicator: FC = () => {
- {/* Document types from the search space */} - {activeDocumentTypes.map(([docType]) => ( + {activeDocumentTypes.map(([docType, count]) => (
{getConnectorIcon(docType, "size-3.5")} {getDocumentTypeLabel(docType)} + + {count > 999 ? "999+" : count} +
))} - {/* Search source connectors */} - {connectors.map((connector) => ( + {nonIndexableConnectors.map((connector) => (
Date: Wed, 31 Dec 2025 14:15:07 +0200 Subject: [PATCH 3/6] feat: add file selection to Google Drive connector - Add structured request body with folders and files arrays - Support individual file indexing alongside folder indexing - Remove deprecated folder_ids/folder_names query params - Update UI to allow selecting both folders and files --- .../app/connectors/google_drive/__init__.py | 3 +- .../connectors/google_drive/folder_manager.py | 33 +++++ .../routes/search_source_connectors_routes.py | 74 ++++++---- surfsense_backend/app/schemas/__init__.py | 4 + surfsense_backend/app/schemas/google_drive.py | 42 ++++++ .../app/tasks/celery_tasks/connector_tasks.py | 16 +-- .../google_drive_indexer.py | 126 ++++++++++++++++++ .../connectors/(manage)/page.tsx | 80 +++++++---- .../connectors/google-drive-folder-tree.tsx | 46 +++++-- .../contracts/types/connector.types.ts | 22 ++- .../hooks/use-search-source-connectors.ts | 10 +- .../lib/apis/connectors-api.service.ts | 7 +- 12 files changed, 366 insertions(+), 97 deletions(-) create mode 100644 surfsense_backend/app/schemas/google_drive.py diff --git a/surfsense_backend/app/connectors/google_drive/__init__.py b/surfsense_backend/app/connectors/google_drive/__init__.py index 561072661..47cc8598e 100644 --- a/surfsense_backend/app/connectors/google_drive/__init__.py +++ b/surfsense_backend/app/connectors/google_drive/__init__.py @@ -4,13 +4,14 @@ from .change_tracker import categorize_change, fetch_all_changes, get_start_page from .client import GoogleDriveClient from .content_extractor import download_and_process_file from .credentials import get_valid_credentials, validate_credentials -from .folder_manager import get_files_in_folder, list_folder_contents +from .folder_manager import get_file_by_id, get_files_in_folder, list_folder_contents __all__ = [ "GoogleDriveClient", "categorize_change", "download_and_process_file", "fetch_all_changes", + "get_file_by_id", "get_files_in_folder", "get_start_page_token", "get_valid_credentials", diff --git a/surfsense_backend/app/connectors/google_drive/folder_manager.py b/surfsense_backend/app/connectors/google_drive/folder_manager.py index b0ed425ef..e28505f11 100644 --- a/surfsense_backend/app/connectors/google_drive/folder_manager.py +++ b/surfsense_backend/app/connectors/google_drive/folder_manager.py @@ -140,6 +140,39 @@ async def get_files_in_folder( return [], None, f"Error getting files in folder: {e!s}" +async def get_file_by_id( + client: GoogleDriveClient, + file_id: str, +) -> tuple[dict[str, Any] | None, str | None]: + """ + Get file metadata by ID. + + Args: + client: GoogleDriveClient instance + file_id: File ID to fetch + + Returns: + Tuple of (file metadata dict, error message) + """ + try: + file, error = await client.get_file_metadata( + file_id, + fields="id, name, mimeType, parents, createdTime, modifiedTime, size, webViewLink, iconLink", + ) + + if error: + return None, error + + if not file: + return None, f"File not found: {file_id}" + + return file, None + + except Exception as e: + logger.error(f"Error getting file by ID: {e!s}", exc_info=True) + return None, f"Error getting file by ID: {e!s}" + + def format_folder_path(hierarchy: list[dict[str, str]]) -> str: """ Format folder hierarchy as a path string. diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 8efbbfa5f..d6fdedd7c 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -14,7 +14,7 @@ import logging from datetime import UTC, datetime, timedelta from typing import Any -from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi import APIRouter, Body, Depends, HTTPException, Query from pydantic import BaseModel, Field, ValidationError from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession @@ -30,6 +30,7 @@ from app.db import ( get_async_session, ) from app.schemas import ( + GoogleDriveIndexRequest, SearchSourceConnectorBase, SearchSourceConnectorCreate, SearchSourceConnectorRead, @@ -542,13 +543,9 @@ async def index_connector_content( None, description="End date for indexing (YYYY-MM-DD format). If not provided, uses today's date", ), - folder_ids: str = Query( + drive_items: GoogleDriveIndexRequest | None = Body( None, - description="[Google Drive only] Comma-separated folder IDs to index", - ), - folder_names: str = Query( - None, - description="[Google Drive only] Comma-separated folder names for display purposes", + description="[Google Drive only] Structured request with folders and files to index", ), session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), @@ -762,22 +759,23 @@ async def index_connector_content( index_google_drive_files_task, ) - if not folder_ids or not folder_names: + if not drive_items or not drive_items.has_items(): raise HTTPException( status_code=400, - detail="Google Drive indexing requires folder_ids and folder_names parameters", + detail="Google Drive indexing requires drive_items body parameter with folders or files", ) logger.info( - f"Triggering Google Drive indexing for connector {connector_id} into search space {search_space_id}, folders: {folder_names}" + f"Triggering Google Drive indexing for connector {connector_id} into search space {search_space_id}, " + f"folders: {len(drive_items.folders)}, files: {len(drive_items.files)}" ) - # Pass comma-separated strings directly to Celery task + + # Pass structured data to Celery task index_google_drive_files_task.delay( connector_id, search_space_id, str(user.id), - folder_ids, # Pass as comma-separated string - folder_names, # Pass as comma-separated string + drive_items.model_dump(), # Convert to dict for JSON serialization ) response_message = "Google Drive indexing started in the background." @@ -1554,45 +1552,63 @@ async def run_google_drive_indexing( connector_id: int, search_space_id: int, user_id: str, - folder_ids: str, # Comma-separated folder IDs - folder_names: str, # Comma-separated folder names + items_dict: dict, # Dictionary with 'folders' and 'files' lists ): - """Runs the Google Drive indexing task for multiple folders and updates the timestamp.""" + """Runs the Google Drive indexing task for folders and files and updates the timestamp.""" try: from app.tasks.connector_indexers.google_drive_indexer import ( index_google_drive_files, + index_google_drive_single_file, ) - # Split comma-separated IDs and names into lists - folder_id_list = [fid.strip() for fid in folder_ids.split(",")] - folder_name_list = [fname.strip() for fname in folder_names.split(",")] - + # Parse the structured data + items = GoogleDriveIndexRequest(**items_dict) total_indexed = 0 errors = [] # Index each folder - for folder_id, folder_name in zip( - folder_id_list, folder_name_list, strict=False - ): + for folder in items.folders: try: indexed_count, error_message = await index_google_drive_files( session, connector_id, search_space_id, user_id, - folder_id, - folder_name, + folder_id=folder.id, + folder_name=folder.name, use_delta_sync=True, update_last_indexed=False, ) if error_message: - errors.append(f"{folder_name}: {error_message}") + errors.append(f"Folder '{folder.name}': {error_message}") else: total_indexed += indexed_count except Exception as e: - errors.append(f"{folder_name}: {e!s}") + errors.append(f"Folder '{folder.name}': {e!s}") logger.error( - f"Error indexing folder {folder_name} ({folder_id}): {e}", + f"Error indexing folder {folder.name} ({folder.id}): {e}", + exc_info=True, + ) + + # Index each individual file + for file in items.files: + try: + indexed_count, error_message = await index_google_drive_single_file( + session, + connector_id, + search_space_id, + user_id, + file_id=file.id, + file_name=file.name, + ) + if error_message: + errors.append(f"File '{file.name}': {error_message}") + else: + total_indexed += indexed_count + except Exception as e: + errors.append(f"File '{file.name}': {e!s}") + logger.error( + f"Error indexing file {file.name} ({file.id}): {e}", exc_info=True, ) @@ -1602,7 +1618,7 @@ async def run_google_drive_indexing( ) else: logger.info( - f"Google Drive indexing successful for connector {connector_id}. Indexed {total_indexed} documents from {len(folder_id_list)} folder(s)." + f"Google Drive indexing successful for connector {connector_id}. Indexed {total_indexed} documents from {len(items.folders)} folder(s) and {len(items.files)} file(s)." ) # Update the last indexed timestamp only on full success await update_connector_last_indexed(session, connector_id) diff --git a/surfsense_backend/app/schemas/__init__.py b/surfsense_backend/app/schemas/__init__.py index f5ae65e9d..751fd5af7 100644 --- a/surfsense_backend/app/schemas/__init__.py +++ b/surfsense_backend/app/schemas/__init__.py @@ -10,6 +10,7 @@ from .documents import ( ExtensionDocumentMetadata, PaginatedResponse, ) +from .google_drive import DriveItem, GoogleDriveIndexRequest from .logs import LogBase, LogCreate, LogFilter, LogRead, LogUpdate from .new_chat import ( ChatMessage, @@ -79,6 +80,8 @@ __all__ = [ "DefaultSystemInstructionsResponse", # Document schemas "DocumentBase", + # Google Drive schemas + "DriveItem", "DocumentRead", "DocumentUpdate", "DocumentWithChunksRead", @@ -86,6 +89,7 @@ __all__ = [ "ExtensionDocumentContent", "ExtensionDocumentMetadata", "GlobalNewLLMConfigRead", + "GoogleDriveIndexRequest", # Base schemas "IDModel", # RBAC schemas diff --git a/surfsense_backend/app/schemas/google_drive.py b/surfsense_backend/app/schemas/google_drive.py new file mode 100644 index 000000000..d8b79e388 --- /dev/null +++ b/surfsense_backend/app/schemas/google_drive.py @@ -0,0 +1,42 @@ +"""Schemas for Google Drive connector.""" + +from pydantic import BaseModel, Field + + +class DriveItem(BaseModel): + """Represents a Google Drive file or folder.""" + + id: str = Field(..., description="Google Drive item ID") + name: str = Field(..., description="Item display name") + + +class GoogleDriveIndexRequest(BaseModel): + """Request body for indexing Google Drive content.""" + + folders: list[DriveItem] = Field( + default_factory=list, description="List of folders to index" + ) + files: list[DriveItem] = Field( + default_factory=list, description="List of specific files to index" + ) + + def has_items(self) -> bool: + """Check if any items are selected.""" + return len(self.folders) > 0 or len(self.files) > 0 + + def get_folder_ids(self) -> list[str]: + """Get list of folder IDs.""" + return [folder.id for folder in self.folders] + + def get_folder_names(self) -> list[str]: + """Get list of folder names.""" + return [folder.name for folder in self.folders] + + def get_file_ids(self) -> list[str]: + """Get list of file IDs.""" + return [file.id for file in self.files] + + def get_file_names(self) -> list[str]: + """Get list of file names.""" + return [file.name for file in self.files] + diff --git a/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py b/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py index 44f57d464..3cae1bbdb 100644 --- a/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py @@ -479,10 +479,9 @@ def index_google_drive_files_task( connector_id: int, search_space_id: int, user_id: str, - folder_ids: str, # Comma-separated folder IDs - folder_names: str, # Comma-separated folder names + items_dict: dict, # Dictionary with 'folders' and 'files' lists ): - """Celery task to index Google Drive files from multiple folders.""" + """Celery task to index Google Drive folders and files.""" import asyncio loop = asyncio.new_event_loop() @@ -494,8 +493,7 @@ def index_google_drive_files_task( connector_id, search_space_id, user_id, - folder_ids, - folder_names, + items_dict, ) ) finally: @@ -506,10 +504,9 @@ async def _index_google_drive_files( connector_id: int, search_space_id: int, user_id: str, - folder_ids: str, # Comma-separated folder IDs - folder_names: str, # Comma-separated folder names + items_dict: dict, # Dictionary with 'folders' and 'files' lists ): - """Index Google Drive files from multiple folders with new session.""" + """Index Google Drive folders and files with new session.""" from app.routes.search_source_connectors_routes import ( run_google_drive_indexing, ) @@ -520,8 +517,7 @@ async def _index_google_drive_files( connector_id, search_space_id, user_id, - folder_ids, - folder_names, + items_dict, ) diff --git a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py index 10f4b672c..6ba9f31c3 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py @@ -10,6 +10,7 @@ from app.connectors.google_drive import ( categorize_change, download_and_process_file, fetch_all_changes, + get_file_by_id, get_files_in_folder, get_start_page_token, ) @@ -194,6 +195,131 @@ async def index_google_drive_files( return 0, f"Failed to index Google Drive files: {e!s}" +async def index_google_drive_single_file( + session: AsyncSession, + connector_id: int, + search_space_id: int, + user_id: str, + file_id: str, + file_name: str | None = None, +) -> tuple[int, str | None]: + """ + Index a single Google Drive file by its ID. + + Args: + session: Database session + connector_id: ID of the Drive connector + search_space_id: ID of the search space + user_id: ID of the user + file_id: Specific file ID to index + file_name: File name for display (optional) + + Returns: + Tuple of (number_of_indexed_files, error_message) + """ + task_logger = TaskLoggingService(session, search_space_id) + + log_entry = await task_logger.log_task_start( + task_name="google_drive_single_file_indexing", + source="connector_indexing_task", + message=f"Starting Google Drive single file indexing for file {file_id}", + metadata={ + "connector_id": connector_id, + "user_id": str(user_id), + "file_id": file_id, + "file_name": file_name, + }, + ) + + try: + connector = await get_connector_by_id( + session, connector_id, SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR + ) + + if not connector: + error_msg = f"Google Drive connector with ID {connector_id} not found" + await task_logger.log_task_failure( + log_entry, error_msg, {"error_type": "ConnectorNotFound"} + ) + return 0, error_msg + + await task_logger.log_task_progress( + log_entry, + f"Initializing Google Drive client for connector {connector_id}", + {"stage": "client_initialization"}, + ) + + drive_client = GoogleDriveClient(session, connector_id) + + # Fetch the file metadata + file, error = await get_file_by_id(drive_client, file_id) + + if error or not file: + error_msg = f"Failed to fetch file {file_id}: {error or 'File not found'}" + await task_logger.log_task_failure( + log_entry, error_msg, {"error_type": "FileNotFound"} + ) + return 0, error_msg + + display_name = file_name or file.get("name", "Unknown") + logger.info(f"Indexing Google Drive file: {display_name} ({file_id})") + + # Process the file + indexed, skipped = await _process_single_file( + drive_client=drive_client, + session=session, + file=file, + connector_id=connector_id, + search_space_id=search_space_id, + user_id=user_id, + task_logger=task_logger, + log_entry=log_entry, + ) + + await session.commit() + logger.info("Successfully committed Google Drive file indexing changes to database") + + if indexed > 0: + await task_logger.log_task_success( + log_entry, + f"Successfully indexed file {display_name}", + { + "file_name": display_name, + "file_id": file_id, + }, + ) + logger.info(f"Google Drive file indexing completed: {display_name}") + return 1, None + else: + await task_logger.log_task_progress( + log_entry, + f"File {display_name} was skipped", + {"status": "skipped"}, + ) + return 0, None + + except SQLAlchemyError as db_error: + await session.rollback() + await task_logger.log_task_failure( + log_entry, + f"Database error during file indexing", + str(db_error), + {"error_type": "SQLAlchemyError"}, + ) + logger.error(f"Database error: {db_error!s}", exc_info=True) + return 0, f"Database error: {db_error!s}" + except Exception as e: + await session.rollback() + await task_logger.log_task_failure( + log_entry, + f"Failed to index Google Drive file", + str(e), + {"error_type": type(e).__name__}, + ) + logger.error(f"Failed to index Google Drive file: {e!s}", exc_info=True) + return 0, f"Failed to index Google Drive file: {e!s}" + + async def _index_full_scan( drive_client: GoogleDriveClient, session: AsyncSession, diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx index a5d811c81..d12112110 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx @@ -119,9 +119,10 @@ export default function ConnectorsPage() { const [customFrequency, setCustomFrequency] = useState(""); const [isSavingPeriodic, setIsSavingPeriodic] = useState(false); - // Google Drive folder selection state + // Google Drive folder and file selection state const [driveFolderDialogOpen, setDriveFolderDialogOpen] = useState(false); const [selectedFolders, setSelectedFolders] = useState>([]); + const [selectedFiles, setSelectedFiles] = useState>([]); useEffect(() => { if (error) { @@ -162,10 +163,10 @@ export default function ConnectorsPage() { setDriveFolderDialogOpen(true); }; - // Handle Google Drive folder indexing - const handleIndexDriveFolder = async () => { - if (selectedConnectorForIndexing === null || selectedFolders.length === 0) { - toast.error("Please select at least one folder"); + // Handle Google Drive folder and file indexing + const handleIndexGoogleDrive = async () => { + if (selectedConnectorForIndexing === null || (selectedFolders.length === 0 && selectedFiles.length === 0)) { + toast.error("Please select at least one folder or file"); return; } @@ -174,15 +175,14 @@ export default function ConnectorsPage() { try { setIndexingConnectorId(selectedConnectorForIndexing); - const folderIds = selectedFolders.map((f) => f.id).join(","); - const folderNames = selectedFolders.map((f) => f.name).join(", "); - await indexConnector({ connector_id: selectedConnectorForIndexing, + body: { + folders: selectedFolders, + files: selectedFiles, + }, queryParams: { search_space_id: searchSpaceId, - folder_ids: folderIds, - folder_names: folderNames, }, }); toast.success(t("indexing_started")); @@ -190,10 +190,11 @@ export default function ConnectorsPage() { console.error("Error indexing connector content:", error); toast.error(error instanceof Error ? error.message : t("indexing_failed")); } finally { - setIndexingConnectorId(null); - setSelectedConnectorForIndexing(null); - setSelectedFolders([]); - } + setIndexingConnectorId(null); + setSelectedConnectorForIndexing(null); + setSelectedFolders([]); + setSelectedFiles([]); + } }; // Handle connector indexing with dates @@ -679,11 +680,11 @@ export default function ConnectorsPage() { - Select Google Drive Folders + Select Google Drive Folders & Files - Select folders to index. Only files directly in each folder will be + Select folders and/or individual files to index. For folders, only files directly in each folder will be processed—subfolders must be selected separately. @@ -698,23 +699,43 @@ export default function ConnectorsPage() { onSelectFolders={(folders) => { setSelectedFolders(folders); }} + selectedFiles={selectedFiles} + onSelectFiles={(files) => { + setSelectedFiles(files); + }} /> )}
- {selectedFolders.length > 0 && ( + {(selectedFolders.length > 0 || selectedFiles.length > 0) && (
-
-

- Selected {selectedFolders.length} folder{selectedFolders.length > 1 ? "s" : ""}: -

-
- {selectedFolders.map((folder) => ( -

- • {folder.name} -

- ))} + {selectedFolders.length > 0 && ( +
+

+ Selected {selectedFolders.length} folder{selectedFolders.length > 1 ? "s" : ""}: +

+
+ {selectedFolders.map((folder) => ( +

+ 📁 {folder.name} +

+ ))} +
-
+ )} + {selectedFiles.length > 0 && ( +
+

+ Selected {selectedFiles.length} file{selectedFiles.length > 1 ? "s" : ""}: +

+
+ {selectedFiles.map((file) => ( +

+ 📄 {file.name} +

+ ))} +
+
+ )}
)}
@@ -725,11 +746,12 @@ export default function ConnectorsPage() { setDriveFolderDialogOpen(false); setSelectedConnectorForIndexing(null); setSelectedFolders([]); + setSelectedFiles([]); }} > {tCommon("cancel")} - diff --git a/surfsense_web/components/connectors/google-drive-folder-tree.tsx b/surfsense_web/components/connectors/google-drive-folder-tree.tsx index cec207b2a..7cde39499 100644 --- a/surfsense_web/components/connectors/google-drive-folder-tree.tsx +++ b/surfsense_web/components/connectors/google-drive-folder-tree.tsx @@ -47,6 +47,8 @@ interface GoogleDriveFolderTreeProps { connectorId: number; selectedFolders: SelectedFolder[]; onSelectFolders: (folders: SelectedFolder[]) => void; + selectedFiles?: SelectedFolder[]; + onSelectFiles?: (files: SelectedFolder[]) => void; } // Helper to get appropriate icon for file type @@ -70,6 +72,8 @@ export function GoogleDriveFolderTree({ connectorId, selectedFolders, onSelectFolders, + selectedFiles = [], + onSelectFiles = () => {}, }: GoogleDriveFolderTreeProps) { const [itemStates, setItemStates] = useState>(new Map()); @@ -83,6 +87,10 @@ export function GoogleDriveFolderTree({ return selectedFolders.some((f) => f.id === folderId); }; + const isFileSelected = (fileId: string): boolean => { + return selectedFiles.some((f) => f.id === fileId); + }; + const toggleFolderSelection = (folderId: string, folderName: string) => { if (isFolderSelected(folderId)) { onSelectFolders(selectedFolders.filter((f) => f.id !== folderId)); @@ -91,6 +99,14 @@ export function GoogleDriveFolderTree({ } }; + const toggleFileSelection = (fileId: string, fileName: string) => { + if (isFileSelected(fileId)) { + onSelectFiles(selectedFiles.filter((f) => f.id !== fileId)); + } else { + onSelectFiles([...selectedFiles, { id: fileId, name: fileName }]); + } + }; + /** * Find an item by ID across all loaded items (root and nested). */ @@ -201,8 +217,8 @@ export function GoogleDriveFolderTree({ const isExpanded = state?.isExpanded || false; const isLoading = state?.isLoading || false; const children = state?.children; - const isSelected = isFolderSelected(item.id); const isFolder = item.isFolder; + const isSelected = isFolder ? isFolderSelected(item.id) : isFileSelected(item.id); const childFolders = children?.filter((c) => c.isFolder) || []; const childFiles = children?.filter((c) => !c.isFolder) || []; @@ -211,10 +227,8 @@ export function GoogleDriveFolderTree({
{isFolder ? ( @@ -237,16 +251,20 @@ export function GoogleDriveFolderTree({ )} - {isFolder && ( - toggleFolderSelection(item.id, item.name)} - className="shrink-0" - onClick={(e) => e.stopPropagation()} - /> - )} + { + if (isFolder) { + toggleFolderSelection(item.id, item.name); + } else { + toggleFileSelection(item.id, item.name); + } + }} + className="shrink-0 z-20 group-hover:border-white group-hover:border" + onClick={(e) => e.stopPropagation()} + /> -
+
{isFolder ? ( isExpanded ? ( diff --git a/surfsense_web/contracts/types/connector.types.ts b/surfsense_web/contracts/types/connector.types.ts index bc7664777..088584bb3 100644 --- a/surfsense_web/contracts/types/connector.types.ts +++ b/surfsense_web/contracts/types/connector.types.ts @@ -126,6 +126,24 @@ export const deleteConnectorResponse = z.object({ message: z.literal("Search source connector deleted successfully"), }); +/** + * Google Drive index request body + */ +export const googleDriveIndexBody = z.object({ + folders: z.array( + z.object({ + id: z.string(), + name: z.string(), + }) + ), + files: z.array( + z.object({ + id: z.string(), + name: z.string(), + }) + ), +}); + /** * Index connector */ @@ -135,10 +153,8 @@ export const indexConnectorRequest = z.object({ search_space_id: z.number().or(z.string()), start_date: z.string().optional(), end_date: z.string().optional(), - // Google Drive only - folder_ids: z.string().optional(), - folder_names: z.string().optional(), }), + body: googleDriveIndexBody.optional(), }); export const indexConnectorResponse = z.object({ diff --git a/surfsense_web/hooks/use-search-source-connectors.ts b/surfsense_web/hooks/use-search-source-connectors.ts index 14c21831b..2f77d7d82 100644 --- a/surfsense_web/hooks/use-search-source-connectors.ts +++ b/surfsense_web/hooks/use-search-source-connectors.ts @@ -267,9 +267,7 @@ export const useSearchSourceConnectors = (lazy: boolean = false, searchSpaceId?: connectorId: number, searchSpaceId: string | number, startDate?: string, - endDate?: string, - folderIds?: string, - folderNames?: string + endDate?: string ) => { try { // Build query parameters @@ -282,12 +280,6 @@ export const useSearchSourceConnectors = (lazy: boolean = false, searchSpaceId?: if (endDate) { params.append("end_date", endDate); } - if (folderIds) { - params.append("folder_ids", folderIds); - } - if (folderNames) { - params.append("folder_names", folderNames); - } const response = await authenticatedFetch( `${ diff --git a/surfsense_web/lib/apis/connectors-api.service.ts b/surfsense_web/lib/apis/connectors-api.service.ts index f6929391a..ca6a7f1ad 100644 --- a/surfsense_web/lib/apis/connectors-api.service.ts +++ b/surfsense_web/lib/apis/connectors-api.service.ts @@ -164,7 +164,7 @@ class ConnectorsApiService { throw new ValidationError(`Invalid request: ${errorMessage}`); } - const { connector_id, queryParams } = parsedRequest.data; + const { connector_id, queryParams, body } = parsedRequest.data; // Transform query params to be string values const transformedQueryParams = Object.fromEntries( @@ -177,7 +177,10 @@ class ConnectorsApiService { return baseApiService.post( `/api/v1/search-source-connectors/${connector_id}/index?${queryString}`, - indexConnectorResponse + indexConnectorResponse, + { + body: body || {}, + } ); }; From d3f83afb3db26c640f4d6b68c2330574c6c9b43e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 31 Dec 2025 21:53:30 +0200 Subject: [PATCH 4/6] feat: add colored connector icons - Add 18 branded SVG icons to public/connectors/ - Clean naming convention (e.g., slack.svg, google-drive.svg) - Includes: Slack, Notion, GitHub, Jira, Google (Drive/Gmail/Calendar), Linear, Discord, Confluence, BookStack, ClickUp, Elasticsearch, Baidu, Airtable, YouTube, Teams, Zoom --- surfsense_web/public/connectors/airtable.svg | 1 + .../public/connectors/baidu-search.svg | 6 + surfsense_web/public/connectors/bookstack.svg | 1 + surfsense_web/public/connectors/clickup.svg | 1 + .../public/connectors/confluence.svg | 15 ++ surfsense_web/public/connectors/discord.svg | 155 ++++++++++++++++++ .../public/connectors/elasticsearch.svg | 1 + surfsense_web/public/connectors/github.svg | 1 + .../public/connectors/google-calendar.svg | 44 +++++ .../public/connectors/google-drive.svg | 44 +++++ .../public/connectors/google-gmail.svg | 44 +++++ surfsense_web/public/connectors/jira.svg | 16 ++ surfsense_web/public/connectors/linear.svg | 1 + .../public/connectors/microsoft-teams.svg | 155 ++++++++++++++++++ surfsense_web/public/connectors/notion.svg | 4 + surfsense_web/public/connectors/slack.svg | 6 + surfsense_web/public/connectors/youtube.svg | 1 + surfsense_web/public/connectors/zoom.svg | 4 + 18 files changed, 500 insertions(+) create mode 100644 surfsense_web/public/connectors/airtable.svg create mode 100644 surfsense_web/public/connectors/baidu-search.svg create mode 100644 surfsense_web/public/connectors/bookstack.svg create mode 100644 surfsense_web/public/connectors/clickup.svg create mode 100644 surfsense_web/public/connectors/confluence.svg create mode 100644 surfsense_web/public/connectors/discord.svg create mode 100644 surfsense_web/public/connectors/elasticsearch.svg create mode 100644 surfsense_web/public/connectors/github.svg create mode 100644 surfsense_web/public/connectors/google-calendar.svg create mode 100644 surfsense_web/public/connectors/google-drive.svg create mode 100644 surfsense_web/public/connectors/google-gmail.svg create mode 100644 surfsense_web/public/connectors/jira.svg create mode 100644 surfsense_web/public/connectors/linear.svg create mode 100644 surfsense_web/public/connectors/microsoft-teams.svg create mode 100644 surfsense_web/public/connectors/notion.svg create mode 100644 surfsense_web/public/connectors/slack.svg create mode 100644 surfsense_web/public/connectors/youtube.svg create mode 100644 surfsense_web/public/connectors/zoom.svg diff --git a/surfsense_web/public/connectors/airtable.svg b/surfsense_web/public/connectors/airtable.svg new file mode 100644 index 000000000..d7bfc550a --- /dev/null +++ b/surfsense_web/public/connectors/airtable.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/surfsense_web/public/connectors/baidu-search.svg b/surfsense_web/public/connectors/baidu-search.svg new file mode 100644 index 000000000..5bf435123 --- /dev/null +++ b/surfsense_web/public/connectors/baidu-search.svg @@ -0,0 +1,6 @@ + \ No newline at end of file diff --git a/surfsense_web/public/connectors/bookstack.svg b/surfsense_web/public/connectors/bookstack.svg new file mode 100644 index 000000000..8b7829055 --- /dev/null +++ b/surfsense_web/public/connectors/bookstack.svg @@ -0,0 +1 @@ +BookStack \ No newline at end of file diff --git a/surfsense_web/public/connectors/clickup.svg b/surfsense_web/public/connectors/clickup.svg new file mode 100644 index 000000000..4bf99cfd8 --- /dev/null +++ b/surfsense_web/public/connectors/clickup.svg @@ -0,0 +1 @@ +ClickUp \ No newline at end of file diff --git a/surfsense_web/public/connectors/confluence.svg b/surfsense_web/public/connectors/confluence.svg new file mode 100644 index 000000000..f8c539608 --- /dev/null +++ b/surfsense_web/public/connectors/confluence.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/surfsense_web/public/connectors/discord.svg b/surfsense_web/public/connectors/discord.svg new file mode 100644 index 000000000..138d32844 --- /dev/null +++ b/surfsense_web/public/connectors/discord.svg @@ -0,0 +1,155 @@ + \ No newline at end of file diff --git a/surfsense_web/public/connectors/elasticsearch.svg b/surfsense_web/public/connectors/elasticsearch.svg new file mode 100644 index 000000000..5189b6751 --- /dev/null +++ b/surfsense_web/public/connectors/elasticsearch.svg @@ -0,0 +1 @@ +file_type_elastic \ No newline at end of file diff --git a/surfsense_web/public/connectors/github.svg b/surfsense_web/public/connectors/github.svg new file mode 100644 index 000000000..63c462cc3 --- /dev/null +++ b/surfsense_web/public/connectors/github.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/surfsense_web/public/connectors/google-calendar.svg b/surfsense_web/public/connectors/google-calendar.svg new file mode 100644 index 000000000..f1f6f96c3 --- /dev/null +++ b/surfsense_web/public/connectors/google-calendar.svg @@ -0,0 +1,44 @@ + \ No newline at end of file diff --git a/surfsense_web/public/connectors/google-drive.svg b/surfsense_web/public/connectors/google-drive.svg new file mode 100644 index 000000000..35f214efd --- /dev/null +++ b/surfsense_web/public/connectors/google-drive.svg @@ -0,0 +1,44 @@ + \ No newline at end of file diff --git a/surfsense_web/public/connectors/google-gmail.svg b/surfsense_web/public/connectors/google-gmail.svg new file mode 100644 index 000000000..47d9e973e --- /dev/null +++ b/surfsense_web/public/connectors/google-gmail.svg @@ -0,0 +1,44 @@ + \ No newline at end of file diff --git a/surfsense_web/public/connectors/jira.svg b/surfsense_web/public/connectors/jira.svg new file mode 100644 index 000000000..69c69f628 --- /dev/null +++ b/surfsense_web/public/connectors/jira.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/surfsense_web/public/connectors/linear.svg b/surfsense_web/public/connectors/linear.svg new file mode 100644 index 000000000..6252259bd --- /dev/null +++ b/surfsense_web/public/connectors/linear.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/surfsense_web/public/connectors/microsoft-teams.svg b/surfsense_web/public/connectors/microsoft-teams.svg new file mode 100644 index 000000000..caa352dff --- /dev/null +++ b/surfsense_web/public/connectors/microsoft-teams.svg @@ -0,0 +1,155 @@ + \ No newline at end of file diff --git a/surfsense_web/public/connectors/notion.svg b/surfsense_web/public/connectors/notion.svg new file mode 100644 index 000000000..38984c87b --- /dev/null +++ b/surfsense_web/public/connectors/notion.svg @@ -0,0 +1,4 @@ + + + + diff --git a/surfsense_web/public/connectors/slack.svg b/surfsense_web/public/connectors/slack.svg new file mode 100644 index 000000000..1832b4653 --- /dev/null +++ b/surfsense_web/public/connectors/slack.svg @@ -0,0 +1,6 @@ + \ No newline at end of file diff --git a/surfsense_web/public/connectors/youtube.svg b/surfsense_web/public/connectors/youtube.svg new file mode 100644 index 000000000..ba2395b5d --- /dev/null +++ b/surfsense_web/public/connectors/youtube.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/surfsense_web/public/connectors/zoom.svg b/surfsense_web/public/connectors/zoom.svg new file mode 100644 index 000000000..84dd78bcd --- /dev/null +++ b/surfsense_web/public/connectors/zoom.svg @@ -0,0 +1,4 @@ + + + + From 9548dd289c7761e791dc7236e44e6187b28e0a34 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 31 Dec 2025 21:53:47 +0200 Subject: [PATCH 5/6] feat: integrate colored SVG icons in getConnectorIcon - Use Next.js Image component for SVG icons (20x20px) - Replace 16 Tabler/Lucide icons with branded SVGs - Add support for Teams, Zoom, YouTube string cases - Clean up unused icon imports - Maintain fallback for connectors without SVG icons --- .../contracts/enums/connectorIcons.tsx | 55 ++++++++----------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/surfsense_web/contracts/enums/connectorIcons.tsx b/surfsense_web/contracts/enums/connectorIcons.tsx index 9281c00e9..e8c2b4ed0 100644 --- a/surfsense_web/contracts/enums/connectorIcons.tsx +++ b/surfsense_web/contracts/enums/connectorIcons.tsx @@ -1,20 +1,6 @@ import { - IconBook, - IconBooks, - IconBrandDiscord, - IconBrandElastic, - IconBrandGithub, - IconBrandNotion, - IconBrandSlack, - IconBrandYoutube, - IconCalendar, - IconChecklist, - IconLayoutKanban, IconLinkPlus, - IconMail, IconSparkles, - IconTable, - IconTicket, IconUsersGroup, IconWorldWww, } from "@tabler/icons-react"; @@ -27,52 +13,53 @@ import { Sparkles, Telescope, Webhook, - HardDrive, } from "lucide-react"; +import Image from "next/image"; import { EnumConnectorName } from "./connector"; export const getConnectorIcon = (connectorType: EnumConnectorName | string, className?: string) => { const iconProps = { className: className || "h-4 w-4" }; + const imgProps = { className: className || "h-5 w-5", width: 20, height: 20 }; switch (connectorType) { case EnumConnectorName.LINKUP_API: return ; case EnumConnectorName.LINEAR_CONNECTOR: - return ; + return Linear; case EnumConnectorName.GITHUB_CONNECTOR: - return ; + return GitHub; case EnumConnectorName.TAVILY_API: return ; case EnumConnectorName.SEARXNG_API: return ; case EnumConnectorName.BAIDU_SEARCH_API: - return ; + return Baidu; case EnumConnectorName.SLACK_CONNECTOR: - return ; + return Slack; case EnumConnectorName.NOTION_CONNECTOR: - return ; + return Notion; case EnumConnectorName.DISCORD_CONNECTOR: - return ; + return Discord; case EnumConnectorName.JIRA_CONNECTOR: - return ; + return Jira; case EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR: - return ; + return Google Calendar; case EnumConnectorName.GOOGLE_GMAIL_CONNECTOR: - return ; + return Gmail; case EnumConnectorName.GOOGLE_DRIVE_CONNECTOR: - return ; + return Google Drive; case EnumConnectorName.AIRTABLE_CONNECTOR: - return ; + return Airtable; case EnumConnectorName.CONFLUENCE_CONNECTOR: - return ; + return Confluence; case EnumConnectorName.BOOKSTACK_CONNECTOR: - return ; + return BookStack; case EnumConnectorName.CLICKUP_CONNECTOR: - return ; + return ClickUp; case EnumConnectorName.LUMA_CONNECTOR: return ; case EnumConnectorName.ELASTICSEARCH_CONNECTOR: - return ; + return Elasticsearch; case EnumConnectorName.WEBCRAWLER_CONNECTOR: return ; case EnumConnectorName.CIRCLEBACK_CONNECTOR: @@ -83,7 +70,13 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas case "CRAWLED_URL": return ; case "YOUTUBE_VIDEO": - return ; + return YouTube; + case "MICROSOFT_TEAMS": + case "ms-teams": + return Microsoft Teams; + case "ZOOM": + case "zoom": + return Zoom; case "FILE": return ; case "NOTE": From e2c062c079f67e976cda2a04a52259add210f91c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 31 Dec 2025 21:53:58 +0200 Subject: [PATCH 6/6] feat: use colored icons for Teams and Zoom in connector list - Replace IconBrandWindows with Teams SVG icon - Replace IconBrandZoom with Zoom SVG icon - Remove unused Tabler icon imports - Use getConnectorIcon helper for consistency --- surfsense_web/components/sources/connector-data.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/surfsense_web/components/sources/connector-data.tsx b/surfsense_web/components/sources/connector-data.tsx index 0ab696ceb..a1c6084d2 100644 --- a/surfsense_web/components/sources/connector-data.tsx +++ b/surfsense_web/components/sources/connector-data.tsx @@ -1,4 +1,3 @@ -import { IconBrandWindows, IconBrandZoom } from "@tabler/icons-react"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import type { ConnectorCategory } from "./types"; @@ -73,7 +72,7 @@ export const connectorCategories: ConnectorCategory[] = [ id: "ms-teams", title: "Microsoft Teams", description: "teams_desc", - icon: , + icon: getConnectorIcon("ms-teams", "h-6 w-6"), status: "coming-soon", }, ], @@ -208,7 +207,7 @@ export const connectorCategories: ConnectorCategory[] = [ id: "zoom", title: "Zoom", description: "zoom_desc", - icon: , + icon: getConnectorIcon("zoom", "h-6 w-6"), status: "coming-soon", }, ],