diff --git a/surfsense_backend/app/agents/new_chat/tools/link_preview.py b/surfsense_backend/app/agents/new_chat/tools/link_preview.py index 90b5da1d7..17e89345e 100644 --- a/surfsense_backend/app/agents/new_chat/tools/link_preview.py +++ b/surfsense_backend/app/agents/new_chat/tools/link_preview.py @@ -172,63 +172,63 @@ async def fetch_with_chromium(url: str) -> dict[str, Any] | None: """ Fetch page content using headless Chromium browser. Used as a fallback when simple HTTP requests are blocked (403, etc.). - + Args: url: URL to fetch - + Returns: Dict with title, description, image, and raw_html, or None if failed """ try: logger.info(f"[link_preview] Falling back to Chromium for {url}") - + # Generate a realistic User-Agent to avoid bot detection ua = UserAgent() user_agent = ua.random - + # Use AsyncChromiumLoader to fetch the page crawl_loader = AsyncChromiumLoader( urls=[url], headless=True, user_agent=user_agent ) documents = await crawl_loader.aload() - + if not documents: logger.warning(f"[link_preview] Chromium returned no documents for {url}") return None - + doc = documents[0] raw_html = doc.page_content - + if not raw_html or len(raw_html.strip()) == 0: logger.warning(f"[link_preview] Chromium returned empty content for {url}") return None - + # Extract metadata using Trafilatura trafilatura_metadata = trafilatura.extract_metadata(raw_html) - + # Extract OG image from raw HTML (trafilatura doesn't extract this) image = extract_image(raw_html) - + result = { "title": None, "description": None, "image": image, "raw_html": raw_html, } - + if trafilatura_metadata: result["title"] = trafilatura_metadata.title result["description"] = trafilatura_metadata.description - + # If trafilatura didn't get the title/description, try OG tags if not result["title"]: result["title"] = extract_title(raw_html) if not result["description"]: result["description"] = extract_description(raw_html) - + logger.info(f"[link_preview] Successfully fetched {url} via Chromium") return result - + except Exception as e: logger.error(f"[link_preview] Chromium fallback failed for {url}: {e}") return None @@ -346,13 +346,15 @@ def create_link_preview_tool(): except httpx.TimeoutException: # Timeout - try Chromium fallback - logger.warning(f"[link_preview] Timeout for {url}, trying Chromium fallback") + logger.warning( + f"[link_preview] Timeout for {url}, trying Chromium fallback" + ) chromium_result = await fetch_with_chromium(url) if chromium_result: title = chromium_result.get("title") or domain description = chromium_result.get("description") image = chromium_result.get("image") - + # Clean up and truncate if title: title = _unescape_html(title) @@ -360,11 +362,11 @@ def create_link_preview_tool(): description = _unescape_html(description) if len(description) > 200: description = description[:197] + "..." - + # Make sure image URL is absolute if image: image = _make_absolute_url(image, url) - + return { "id": preview_id, "assetId": url, @@ -375,7 +377,7 @@ def create_link_preview_tool(): "thumb": image, "domain": domain, } - + return { "id": preview_id, "assetId": url, @@ -387,7 +389,7 @@ def create_link_preview_tool(): } except httpx.HTTPStatusError as e: status_code = e.response.status_code - + # For 403 (Forbidden) and similar bot-detection errors, try Chromium fallback if status_code in (403, 401, 406, 429): logger.warning( @@ -398,7 +400,7 @@ def create_link_preview_tool(): title = chromium_result.get("title") or domain description = chromium_result.get("description") image = chromium_result.get("image") - + # Clean up and truncate if title: title = _unescape_html(title) @@ -406,11 +408,11 @@ def create_link_preview_tool(): description = _unescape_html(description) if len(description) > 200: description = description[:197] + "..." - + # Make sure image URL is absolute if image: image = _make_absolute_url(image, url) - + return { "id": preview_id, "assetId": url, @@ -421,7 +423,7 @@ def create_link_preview_tool(): "thumb": image, "domain": domain, } - + return { "id": preview_id, "assetId": url, diff --git a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx index 524d4470a..97e4b4373 100644 --- a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx +++ b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx @@ -45,6 +45,27 @@ interface InlineMentionEditorProps { const CHIP_DATA_ATTR = "data-mention-chip"; const CHIP_ID_ATTR = "data-mention-id"; +/** + * Type guard to check if a node is a chip element + */ +function isChipElement(node: Node | null): node is HTMLSpanElement { + return ( + node !== null && + node.nodeType === Node.ELEMENT_NODE && + (node as Element).hasAttribute(CHIP_DATA_ATTR) + ); +} + +/** + * Safely parse chip ID from element attribute + */ +function getChipId(element: Element): number | null { + const idStr = element.getAttribute(CHIP_ID_ATTR); + if (!idStr) return null; + const id = parseInt(idStr, 10); + return Number.isNaN(id) ? null : id; +} + export const InlineMentionEditor = forwardRef( ( { @@ -177,6 +198,12 @@ export const InlineMentionEditor = forwardRef { if (!editorRef.current) return; + // Validate required fields for type safety + if (typeof doc.id !== "number" || typeof doc.title !== "string") { + console.warn("[InlineMentionEditor] Invalid document passed to insertDocumentChip:", doc); + return; + } + const mentionDoc: MentionedDocument = { id: doc.id, title: doc.title, @@ -381,19 +408,21 @@ export const InlineMentionEditor = forwardRef { - const next = new Map(prev); - next.delete(chipId); - return next; - }); - // Notify parent that a document was removed - onDocumentRemove?.(chipId); + const chipId = getChipId(prevSibling); + if (chipId !== null) { + prevSibling.remove(); + setMentionedDocs((prev) => { + const next = new Map(prev); + next.delete(chipId); + return next; + }); + // Notify parent that a document was removed + onDocumentRemove?.(chipId); + } return; } // Check if we're about to delete @ at the start @@ -414,19 +443,21 @@ export const InlineMentionEditor = forwardRef 0) { - // Check if previous child is a chip + // Check if previous child is a chip using type guard const prevChild = (node as Element).childNodes[offset - 1]; - if (prevChild && (prevChild as Element).hasAttribute?.(CHIP_DATA_ATTR)) { + if (isChipElement(prevChild)) { e.preventDefault(); - const chipId = Number((prevChild as Element).getAttribute(CHIP_ID_ATTR)); - prevChild.parentNode?.removeChild(prevChild); - setMentionedDocs((prev) => { - const next = new Map(prev); - next.delete(chipId); - return next; - }); - // Notify parent that a document was removed - onDocumentRemove?.(chipId); + const chipId = getChipId(prevChild); + if (chipId !== null) { + prevChild.remove(); + setMentionedDocs((prev) => { + const next = new Map(prev); + next.delete(chipId); + return next; + }); + // Notify parent that a document was removed + onDocumentRemove?.(chipId); + } } } } diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index e6e9be45d..4f8c3d2ee 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -72,9 +72,9 @@ import { MarkdownText } from "@/components/assistant-ui/markdown-text"; import { ToolFallback } from "@/components/assistant-ui/tool-fallback"; import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; import { - DocumentsDataTable, - type DocumentsDataTableRef, -} from "@/components/new-chat/DocumentsDataTable"; + DocumentMentionPicker, + type DocumentMentionPickerRef, +} from "@/components/new-chat/DocumentMentionPicker"; import { ChainOfThought, ChainOfThoughtContent, @@ -404,7 +404,7 @@ const Composer: FC = () => { const [mentionQuery, setMentionQuery] = useState(""); const editorRef = useRef(null); const editorContainerRef = useRef(null); - const documentPickerRef = useRef(null); + const documentPickerRef = useRef(null); const { search_space_id } = useParams(); const setMentionedDocumentIds = useSetAtom(mentionedDocumentIdsAtom); const composerRuntime = useComposerRuntime(); @@ -598,7 +598,7 @@ const Composer: FC = () => { : "50%", }} > - void; + moveUp: () => void; + moveDown: () => void; +} + +interface DocumentMentionPickerProps { + searchSpaceId: number; + onSelectionChange: (documents: Document[]) => void; + onDone: () => void; + initialSelectedDocuments?: Document[]; + externalSearch?: string; +} + +function useDebounced(value: T, delay = 300) { + const [debounced, setDebounced] = useState(value); + useEffect(() => { + const t = setTimeout(() => setDebounced(value), delay); + return () => clearTimeout(t); + }, [value, delay]); + return debounced; +} + +export const DocumentMentionPicker = forwardRef< + DocumentMentionPickerRef, + DocumentMentionPickerProps +>(function DocumentMentionPicker( + { searchSpaceId, onSelectionChange, onDone, initialSelectedDocuments = [], externalSearch = "" }, + ref +) { + // Use external search + const search = externalSearch; + const debouncedSearch = useDebounced(search, 150); + const [highlightedIndex, setHighlightedIndex] = useState(0); + const itemRefs = useRef>(new Map()); + + const fetchQueryParams = useMemo( + () => ({ + search_space_id: searchSpaceId, + page: 0, + page_size: 20, + }), + [searchSpaceId] + ); + + const searchQueryParams = useMemo(() => { + return { + search_space_id: searchSpaceId, + page: 0, + page_size: 20, + title: debouncedSearch, + }; + }, [debouncedSearch, searchSpaceId]); + + // Use query for fetching documents + const { data: documents, isLoading: isDocumentsLoading } = useQuery({ + queryKey: cacheKeys.documents.withQueryParams(fetchQueryParams), + queryFn: () => documentsApiService.getDocuments({ queryParams: fetchQueryParams }), + staleTime: 3 * 60 * 1000, + enabled: !!searchSpaceId && !debouncedSearch.trim(), + }); + + // Searching + const { data: searchedDocuments, isLoading: isSearchedDocumentsLoading } = useQuery({ + queryKey: cacheKeys.documents.withQueryParams(searchQueryParams), + queryFn: () => documentsApiService.searchDocuments({ queryParams: searchQueryParams }), + staleTime: 3 * 60 * 1000, + enabled: !!searchSpaceId && !!debouncedSearch.trim(), + }); + + const actualDocuments = debouncedSearch.trim() + ? searchedDocuments?.items || [] + : documents?.items || []; + const actualLoading = debouncedSearch.trim() ? isSearchedDocumentsLoading : isDocumentsLoading; + + // Track already selected document IDs + const selectedIds = useMemo( + () => new Set(initialSelectedDocuments.map((d) => d.id)), + [initialSelectedDocuments] + ); + + // Filter out already selected documents for navigation + const selectableDocuments = useMemo( + () => actualDocuments.filter((doc) => !selectedIds.has(doc.id)), + [actualDocuments, selectedIds] + ); + + const handleSelectDocument = useCallback( + (doc: Document) => { + onSelectionChange([...initialSelectedDocuments, doc]); + onDone(); + }, + [initialSelectedDocuments, onSelectionChange, onDone] + ); + + // Scroll highlighted item into view + useEffect(() => { + const item = itemRefs.current.get(highlightedIndex); + if (item) { + item.scrollIntoView({ block: "nearest", behavior: "smooth" }); + } + }, [highlightedIndex]); + + // Reset highlighted index when external search changes + const prevSearchRef = useRef(search); + if (prevSearchRef.current !== search) { + prevSearchRef.current = search; + if (highlightedIndex !== 0) { + setHighlightedIndex(0); + } + } + + // Expose methods to parent via ref + useImperativeHandle( + ref, + () => ({ + selectHighlighted: () => { + if (selectableDocuments[highlightedIndex]) { + handleSelectDocument(selectableDocuments[highlightedIndex]); + } + }, + moveUp: () => { + setHighlightedIndex((prev) => (prev > 0 ? prev - 1 : selectableDocuments.length - 1)); + }, + moveDown: () => { + setHighlightedIndex((prev) => (prev < selectableDocuments.length - 1 ? prev + 1 : 0)); + }, + }), + [selectableDocuments, highlightedIndex, handleSelectDocument] + ); + + // Handle keyboard navigation + const handleKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (selectableDocuments.length === 0) return; + + switch (e.key) { + case "ArrowDown": + e.preventDefault(); + setHighlightedIndex((prev) => (prev < selectableDocuments.length - 1 ? prev + 1 : 0)); + break; + case "ArrowUp": + e.preventDefault(); + setHighlightedIndex((prev) => (prev > 0 ? prev - 1 : selectableDocuments.length - 1)); + break; + case "Enter": + e.preventDefault(); + if (selectableDocuments[highlightedIndex]) { + handleSelectDocument(selectableDocuments[highlightedIndex]); + } + break; + case "Escape": + e.preventDefault(); + onDone(); + break; + } + }, + [selectableDocuments, highlightedIndex, handleSelectDocument, onDone] + ); + + return ( +
+ {/* Document List */} +
+ {actualLoading ? ( +
+
+
+ ) : actualDocuments.length === 0 ? ( +
+ +

No documents found

+
+ ) : ( +
+ {actualDocuments.map((doc) => { + const isAlreadySelected = selectedIds.has(doc.id); + const selectableIndex = selectableDocuments.findIndex((d) => d.id === doc.id); + const isHighlighted = !isAlreadySelected && selectableIndex === highlightedIndex; + + return ( + + ); + })} +
+ )} +
+
+ ); +}); diff --git a/surfsense_web/components/new-chat/DocumentsDataTable.tsx b/surfsense_web/components/new-chat/DocumentsDataTable.tsx deleted file mode 100644 index 50b026dfc..000000000 --- a/surfsense_web/components/new-chat/DocumentsDataTable.tsx +++ /dev/null @@ -1,248 +0,0 @@ -"use client"; - -import { useQuery } from "@tanstack/react-query"; -import { FileText } from "lucide-react"; -import { - forwardRef, - useCallback, - useEffect, - useImperativeHandle, - useMemo, - useRef, - useState, -} from "react"; -import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import type { Document } from "@/contracts/types/document.types"; -import { documentsApiService } from "@/lib/apis/documents-api.service"; -import { cacheKeys } from "@/lib/query-client/cache-keys"; -import { cn } from "@/lib/utils"; - -export interface DocumentsDataTableRef { - selectHighlighted: () => void; - moveUp: () => void; - moveDown: () => void; -} - -interface DocumentsDataTableProps { - searchSpaceId: number; - onSelectionChange: (documents: Document[]) => void; - onDone: () => void; - initialSelectedDocuments?: Document[]; - externalSearch?: string; -} - -function useDebounced(value: T, delay = 300) { - const [debounced, setDebounced] = useState(value); - useEffect(() => { - const t = setTimeout(() => setDebounced(value), delay); - return () => clearTimeout(t); - }, [value, delay]); - return debounced; -} - -export const DocumentsDataTable = forwardRef( - function DocumentsDataTable( - { - searchSpaceId, - onSelectionChange, - onDone, - initialSelectedDocuments = [], - externalSearch = "", - }, - ref - ) { - // Use external search - const search = externalSearch; - const debouncedSearch = useDebounced(search, 150); - const [highlightedIndex, setHighlightedIndex] = useState(0); - const itemRefs = useRef>(new Map()); - - const fetchQueryParams = useMemo( - () => ({ - search_space_id: searchSpaceId, - page: 0, - page_size: 20, - }), - [searchSpaceId] - ); - - const searchQueryParams = useMemo(() => { - return { - search_space_id: searchSpaceId, - page: 0, - page_size: 20, - title: debouncedSearch, - }; - }, [debouncedSearch, searchSpaceId]); - - // Use query for fetching documents - const { data: documents, isLoading: isDocumentsLoading } = useQuery({ - queryKey: cacheKeys.documents.withQueryParams(fetchQueryParams), - queryFn: () => documentsApiService.getDocuments({ queryParams: fetchQueryParams }), - staleTime: 3 * 60 * 1000, - enabled: !!searchSpaceId && !debouncedSearch.trim(), - }); - - // Searching - const { data: searchedDocuments, isLoading: isSearchedDocumentsLoading } = useQuery({ - queryKey: cacheKeys.documents.withQueryParams(searchQueryParams), - queryFn: () => documentsApiService.searchDocuments({ queryParams: searchQueryParams }), - staleTime: 3 * 60 * 1000, - enabled: !!searchSpaceId && !!debouncedSearch.trim(), - }); - - const actualDocuments = debouncedSearch.trim() - ? searchedDocuments?.items || [] - : documents?.items || []; - const actualLoading = debouncedSearch.trim() ? isSearchedDocumentsLoading : isDocumentsLoading; - - // Track already selected document IDs - const selectedIds = useMemo( - () => new Set(initialSelectedDocuments.map((d) => d.id)), - [initialSelectedDocuments] - ); - - // Filter out already selected documents for navigation - const selectableDocuments = useMemo( - () => actualDocuments.filter((doc) => !selectedIds.has(doc.id)), - [actualDocuments, selectedIds] - ); - - const handleSelectDocument = useCallback( - (doc: Document) => { - onSelectionChange([...initialSelectedDocuments, doc]); - onDone(); - }, - [initialSelectedDocuments, onSelectionChange, onDone] - ); - - // Scroll highlighted item into view - useEffect(() => { - const item = itemRefs.current.get(highlightedIndex); - if (item) { - item.scrollIntoView({ block: "nearest", behavior: "smooth" }); - } - }, [highlightedIndex]); - - // Reset highlighted index when external search changes - const prevSearchRef = useRef(search); - if (prevSearchRef.current !== search) { - prevSearchRef.current = search; - if (highlightedIndex !== 0) { - setHighlightedIndex(0); - } - } - - // Expose methods to parent via ref - useImperativeHandle( - ref, - () => ({ - selectHighlighted: () => { - if (selectableDocuments[highlightedIndex]) { - handleSelectDocument(selectableDocuments[highlightedIndex]); - } - }, - moveUp: () => { - setHighlightedIndex((prev) => (prev > 0 ? prev - 1 : selectableDocuments.length - 1)); - }, - moveDown: () => { - setHighlightedIndex((prev) => (prev < selectableDocuments.length - 1 ? prev + 1 : 0)); - }, - }), - [selectableDocuments, highlightedIndex, handleSelectDocument] - ); - - // Handle keyboard navigation - const handleKeyDown = useCallback( - (e: React.KeyboardEvent) => { - if (selectableDocuments.length === 0) return; - - switch (e.key) { - case "ArrowDown": - e.preventDefault(); - setHighlightedIndex((prev) => (prev < selectableDocuments.length - 1 ? prev + 1 : 0)); - break; - case "ArrowUp": - e.preventDefault(); - setHighlightedIndex((prev) => (prev > 0 ? prev - 1 : selectableDocuments.length - 1)); - break; - case "Enter": - e.preventDefault(); - if (selectableDocuments[highlightedIndex]) { - handleSelectDocument(selectableDocuments[highlightedIndex]); - } - break; - case "Escape": - e.preventDefault(); - onDone(); - break; - } - }, - [selectableDocuments, highlightedIndex, handleSelectDocument, onDone] - ); - - return ( -
- {/* Document List */} -
- {actualLoading ? ( -
-
-
- ) : actualDocuments.length === 0 ? ( -
- -

No documents found

-
- ) : ( -
- {actualDocuments.map((doc) => { - const isAlreadySelected = selectedIds.has(doc.id); - const selectableIndex = selectableDocuments.findIndex((d) => d.id === doc.id); - const isHighlighted = !isAlreadySelected && selectableIndex === highlightedIndex; - - return ( - - ); - })} -
- )} -
-
- ); - } -);