Merge pull request #707 from AnishSarkar22/fix/mentions

feat: Revamped search in document mentions
2026-05-25 19:15:18 +02:00 · 2026-01-18 22:15:28 -08:00 · 2026-01-18 22:15:28 -08:00 · 26ef83fbaa
commit 26ef83fbaa
parent 87a174a1fd b158ddd083
11 changed files with 560 additions and 168 deletions
--- a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx
+++ b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx
@ -12,6 +12,7 @@ import {
 } from "react";
 import ReactDOMServer from "react-dom/server";
 import type { Document } from "@/contracts/types/document.types";
+import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
 import { cn } from "@/lib/utils";

 export interface MentionedDocument {
@ -166,12 +167,19 @@ export const InlineMentionEditor = forwardRef<InlineMentionEditorRef, InlineMent
 				chip.setAttribute(CHIP_DOCTYPE_ATTR, doc.document_type ?? "UNKNOWN");
 				chip.contentEditable = "false";
 				chip.className =
-					"inline-flex items-center gap-0.5 mx-0.5 pl-1 pr-0.5 py-0.5 rounded bg-primary/10 text-xs font-bold text-primary border border-primary/10 select-none";
+					"inline-flex items-center gap-1 mx-0.5 pl-1 pr-0.5 py-0.5 rounded bg-primary/10 text-xs font-bold text-primary/60 select-none";
 				chip.style.userSelect = "none";
 				chip.style.verticalAlign = "baseline";

+				// Add document type icon
+				const iconSpan = document.createElement("span");
+				iconSpan.className = "shrink-0 flex items-center text-muted-foreground";
+				iconSpan.innerHTML = ReactDOMServer.renderToString(
+					getConnectorIcon(doc.document_type ?? "UNKNOWN", "h-3 w-3")
+				);
+
 				const titleSpan = document.createElement("span");
-				titleSpan.className = "max-w-[80px] truncate";
+				titleSpan.className = "max-w-[120px] truncate";
 				titleSpan.textContent = doc.title;
 				titleSpan.title = doc.title;

@ -197,6 +205,7 @@ export const InlineMentionEditor = forwardRef<InlineMentionEditorRef, InlineMent
 					focusAtEnd();
 				};

+				chip.appendChild(iconSpan);
 				chip.appendChild(titleSpan);
 				chip.appendChild(removeBtn);

--- a/surfsense_web/components/assistant-ui/thread.tsx
+++ b/surfsense_web/components/assistant-ui/thread.tsx
@ -200,7 +200,7 @@ const ThreadWelcome: FC = () => {
 };

 const Composer: FC = () => {
-	// ---- State for document mentions (using atoms to persist across remounts) ----
+	// Document mention state (atoms persist across component remounts)
 	const [mentionedDocuments, setMentionedDocuments] = useAtom(mentionedDocumentsAtom);
 	const [showDocumentPopover, setShowDocumentPopover] = useState(false);
 	const [mentionQuery, setMentionQuery] = useState("");
@ -212,16 +212,12 @@ const Composer: FC = () => {
 	const composerRuntime = useComposerRuntime();
 	const hasAutoFocusedRef = useRef(false);

-	// Check if thread is empty (new chat)
 	const isThreadEmpty = useAssistantState(({ thread }) => thread.isEmpty);
-
-	// Check if thread is currently running (streaming response)
 	const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning);

-	// Auto-focus editor when on new chat page
+	// Auto-focus editor on new chat page after mount
 	useEffect(() => {
 		if (isThreadEmpty && !hasAutoFocusedRef.current && editorRef.current) {
-			// Small delay to ensure the editor is fully mounted
 			const timeoutId = setTimeout(() => {
 				editorRef.current?.focus();
 				hasAutoFocusedRef.current = true;
@ -230,7 +226,7 @@ const Composer: FC = () => {
 		}
 	}, [isThreadEmpty]);

-	// Sync mentioned document IDs to atom for use in chat request
+	// Sync mentioned document IDs to atom for inclusion in chat request payload
 	useEffect(() => {
 		setMentionedDocumentIds({
 			surfsense_doc_ids: mentionedDocuments
@ -242,7 +238,7 @@ const Composer: FC = () => {
 		});
 	}, [mentionedDocuments, setMentionedDocumentIds]);

-	// Handle text change from inline editor - sync with assistant-ui composer
+	// Sync editor text with assistant-ui composer runtime
 	const handleEditorChange = useCallback(
 		(text: string) => {
 			composerRuntime.setText(text);
@ -250,13 +246,13 @@ const Composer: FC = () => {
 		[composerRuntime]
 	);

-	// Handle @ mention trigger from inline editor
+	// Open document picker when @ mention is triggered
 	const handleMentionTrigger = useCallback((query: string) => {
 		setShowDocumentPopover(true);
 		setMentionQuery(query);
 	}, []);

-	// Handle mention close
+	// Close document picker and reset query
 	const handleMentionClose = useCallback(() => {
 		if (showDocumentPopover) {
 			setShowDocumentPopover(false);
@ -264,7 +260,7 @@ const Composer: FC = () => {
 		}
 	}, [showDocumentPopover]);

-	// Handle keyboard navigation when popover is open
+	// Keyboard navigation for document picker (arrow keys, Enter, Escape)
 	const handleKeyDown = useCallback(
 		(e: React.KeyboardEvent) => {
 			if (showDocumentPopover) {
@ -294,15 +290,13 @@ const Composer: FC = () => {
 		[showDocumentPopover]
 	);

-	// Handle submit from inline editor (Enter key)
+	// Submit message (blocked during streaming or when document picker is open)
 	const handleSubmit = useCallback(() => {
-		// Prevent sending while a response is still streaming
 		if (isThreadRunning) {
 			return;
 		}
 		if (!showDocumentPopover) {
 			composerRuntime.send();
-			// Clear the editor after sending
 			editorRef.current?.clear();
 			setMentionedDocuments([]);
 			setMentionedDocumentIds({
@ -318,6 +312,7 @@ const Composer: FC = () => {
 		setMentionedDocumentIds,
 	]);

+	// Remove document from mentions and sync IDs to atom
 	const handleDocumentRemove = useCallback(
 		(docId: number, docType?: string) => {
 			setMentionedDocuments((prev) => {
@ -336,6 +331,7 @@ const Composer: FC = () => {
 		[setMentionedDocuments, setMentionedDocumentIds]
 	);

+	// Add selected documents from picker, insert chips, and sync IDs to atom
 	const handleDocumentsMention = useCallback(
 		(documents: Pick<Document, "id" | "title" | "document_type">[]) => {
 			const existingKeys = new Set(mentionedDocuments.map((d) => `${d.document_type}:${d.id}`));
@ -373,7 +369,7 @@ const Composer: FC = () => {
 		<ComposerPrimitive.Root className="aui-composer-root relative flex w-full flex-col">
 			<ComposerPrimitive.AttachmentDropzone className="aui-composer-attachment-dropzone flex w-full flex-col rounded-2xl border-input bg-muted px-1 pt-2 outline-none transition-shadow data-[dragging=true]:border-ring data-[dragging=true]:border-dashed data-[dragging=true]:bg-accent/50">
 				<ComposerAttachments />
-				{/* -------- Inline Mention Editor -------- */}
+				{/* Inline editor with @mention support */}
 				<div ref={editorContainerRef} className="aui-composer-input-wrapper px-3 pt-3 pb-6">
 					<InlineMentionEditor
 						ref={editorRef}
@ -388,45 +384,29 @@ const Composer: FC = () => {
 					/>
 				</div>

-				{/* -------- Document mention popover (rendered via portal) -------- */}
+				{/* Document picker popover (portal to body for proper z-index stacking) */}
 				{showDocumentPopover &&
 					typeof document !== "undefined" &&
 					createPortal(
-						<>
-							{/* Backdrop */}
-							<button
-								type="button"
-								className="fixed inset-0 cursor-default"
-								style={{ zIndex: 9998 }}
-								onClick={() => setShowDocumentPopover(false)}
-								aria-label="Close document picker"
-							/>
-							{/* Popover positioned above input */}
-							<div
-								className="fixed shadow-2xl rounded-lg border border-border overflow-hidden bg-popover"
-								style={{
-									zIndex: 9999,
-									bottom: editorContainerRef.current
-										? `${window.innerHeight - editorContainerRef.current.getBoundingClientRect().top + 8}px`
-										: "200px",
-									left: editorContainerRef.current
-										? `${editorContainerRef.current.getBoundingClientRect().left}px`
-										: "50%",
-								}}
-							>
-								<DocumentMentionPicker
-									ref={documentPickerRef}
-									searchSpaceId={Number(search_space_id)}
-									onSelectionChange={handleDocumentsMention}
-									onDone={() => {
-										setShowDocumentPopover(false);
-										setMentionQuery("");
-									}}
-									initialSelectedDocuments={mentionedDocuments}
-									externalSearch={mentionQuery}
-								/>
-							</div>
-						</>,
+						<DocumentMentionPicker
+							ref={documentPickerRef}
+							searchSpaceId={Number(search_space_id)}
+							onSelectionChange={handleDocumentsMention}
+							onDone={() => {
+								setShowDocumentPopover(false);
+								setMentionQuery("");
+							}}
+							initialSelectedDocuments={mentionedDocuments}
+							externalSearch={mentionQuery}
+							containerStyle={{
+								bottom: editorContainerRef.current
+									? `${window.innerHeight - editorContainerRef.current.getBoundingClientRect().top + 8}px`
+									: "200px",
+								left: editorContainerRef.current
+									? `${editorContainerRef.current.getBoundingClientRect().left}px`
+									: "50%",
+							}}
+						/>,
 						document.body
 					)}
 				<ComposerAction />
--- a/surfsense_web/components/new-chat/chat-share-button.tsx
+++ b/surfsense_web/components/new-chat/chat-share-button.tsx
@ -1,7 +1,7 @@
 "use client";

 import { useQueryClient } from "@tanstack/react-query";
-import { Loader2, Lock, Share2, Users } from "lucide-react";
+import { Loader2, Lock, Users } from "lucide-react";
 import { useCallback, useState } from "react";
 import { toast } from "sonner";
 import { Button } from "@/components/ui/button";
--- a/surfsense_web/components/new-chat/document-mention-picker.tsx
+++ b/surfsense_web/components/new-chat/document-mention-picker.tsx
@ -1,7 +1,6 @@
 "use client";

-import { useQuery } from "@tanstack/react-query";
-import { FileText } from "lucide-react";
+import { keepPreviousData, useQuery, useQueryClient } from "@tanstack/react-query";
 import {
 	forwardRef,
 	useCallback,
@ -12,9 +11,8 @@ import {
 	useState,
 } from "react";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import type { Document, GetDocumentsResponse } from "@/contracts/types/document.types";
+import type { Document, SearchDocumentTitlesResponse } from "@/contracts/types/document.types";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
-import { cacheKeys } from "@/lib/query-client/cache-keys";
 import { cn } from "@/lib/utils";

 export interface DocumentMentionPickerRef {
@ -29,16 +27,39 @@ interface DocumentMentionPickerProps {
 	onDone: () => void;
 	initialSelectedDocuments?: Pick<Document, "id" | "title" | "document_type">[];
 	externalSearch?: string;
+	/** Positioning styles for the container */
+	containerStyle?: React.CSSProperties;
 }

 const PAGE_SIZE = 20;
+const MIN_SEARCH_LENGTH = 2;
+const DEBOUNCE_MS = 100;

-function useDebounced<T>(value: T, delay = 300) {
+/**
+ * Custom debounce hook that delays value updates until user input stabilizes.
+ * Preferred over throttling for search inputs as it reduces API request frequency
+ * and prevents race conditions from stale responses overtaking recent ones.
+ */
+function useDebounced<T>(value: T, delay = DEBOUNCE_MS) {
 	const [debounced, setDebounced] = useState(value);
+	const timeoutRef = useRef<ReturnType<typeof setTimeout> | undefined>(undefined);
+
 	useEffect(() => {
-		const t = setTimeout(() => setDebounced(value), delay);
-		return () => clearTimeout(t);
+		if (timeoutRef.current) {
+			clearTimeout(timeoutRef.current);
+		}
+
+		timeoutRef.current = setTimeout(() => {
+			setDebounced(value);
+		}, delay);
+
+		return () => {
+			if (timeoutRef.current) {
+				clearTimeout(timeoutRef.current);
+			}
+		};
 	}, [value, delay]);
+
 	return debounced;
 }

@ -46,17 +67,27 @@ export const DocumentMentionPicker = forwardRef<
 	DocumentMentionPickerRef,
 	DocumentMentionPickerProps
 >(function DocumentMentionPicker(
-	{ searchSpaceId, onSelectionChange, onDone, initialSelectedDocuments = [], externalSearch = "" },
+	{
+		searchSpaceId,
+		onSelectionChange,
+		onDone,
+		initialSelectedDocuments = [],
+		externalSearch = "",
+		containerStyle,
+	},
 	ref
 ) {
-	// Use external search
+	const queryClient = useQueryClient();
+
+	// Debounced search value to minimize API calls and prevent race conditions
 	const search = externalSearch;
-	const debouncedSearch = useDebounced(search, 150);
+	const debouncedSearch = useDebounced(search, DEBOUNCE_MS);
 	const [highlightedIndex, setHighlightedIndex] = useState(0);
 	const itemRefs = useRef<Map<number, HTMLButtonElement>>(new Map());
 	const scrollContainerRef = useRef<HTMLDivElement>(null);
+	const shouldScrollRef = useRef(false); // Keyboard navigation scroll flag

-	// State for pagination
+	// Pagination state for infinite scroll
 	const [accumulatedDocuments, setAccumulatedDocuments] = useState<
 		Pick<Document, "id" | "title" | "document_type">[]
 	>([]);
@ -64,74 +95,119 @@ export const DocumentMentionPicker = forwardRef<
 	const [hasMore, setHasMore] = useState(false);
 	const [isLoadingMore, setIsLoadingMore] = useState(false);

-	// Reset pagination when search or search space changes
-	// biome-ignore lint/correctness/useExhaustiveDependencies: intentionally reset pagination when search/space changes
+	/**
+	 * Search Strategy:
+	 * - Single character (length === 1): Client-side filtering for instant results
+	 * - Two or more characters (length >= 2): Server-side search with pg_trgm index
+	 * This hybrid approach optimizes UX by providing immediate feedback for short queries
+	 * while leveraging efficient database indexing for longer, more specific searches.
+	 */
+	const isSearchValid = debouncedSearch.trim().length >= MIN_SEARCH_LENGTH;
+	const shouldSearch = debouncedSearch.trim().length > 0;
+	const isSingleCharSearch = debouncedSearch.trim().length === 1;
+
+	// Prefetch initial data on mount for instant display when picker opens
+	useEffect(() => {
+		if (!searchSpaceId) return;
+
+		const prefetchParams = {
+			search_space_id: searchSpaceId,
+			page: 0,
+			page_size: PAGE_SIZE,
+		};
+
+		queryClient.prefetchQuery({
+			queryKey: ["document-titles", prefetchParams],
+			queryFn: () => documentsApiService.searchDocumentTitles({ queryParams: prefetchParams }),
+			staleTime: 60 * 1000,
+		});
+
+		queryClient.prefetchQuery({
+			queryKey: ["surfsense-docs-mention", "", false],
+			queryFn: () =>
+				documentsApiService.getSurfsenseDocs({
+					queryParams: { page: 0, page_size: PAGE_SIZE },
+				}),
+			staleTime: 3 * 60 * 1000,
+		});
+	}, [searchSpaceId, queryClient]);
+
+	// Reset pagination state when search query or search space changes.
+	// Documents are not cleared to maintain visual continuity during fetches.
+	// biome-ignore lint/correctness/useExhaustiveDependencies: Intentional reset on search/space change
 	useEffect(() => {
-		setAccumulatedDocuments([]);
 		setCurrentPage(0);
 		setHasMore(false);
 		setHighlightedIndex(0);
 	}, [debouncedSearch, searchSpaceId]);

-	// Query params for initial fetch (page 0)
-	const fetchQueryParams = useMemo(
+	// Query parameters for lightweight title search endpoint
+	const titleSearchParams = useMemo(
 		() => ({
 			search_space_id: searchSpaceId,
 			page: 0,
 			page_size: PAGE_SIZE,
+			...(isSearchValid ? { title: debouncedSearch.trim() } : {}),
 		}),
-		[searchSpaceId]
+		[searchSpaceId, debouncedSearch, isSearchValid]
 	);

-	const searchQueryParams = useMemo(() => {
-		return {
-			search_space_id: searchSpaceId,
-			page: 0,
-			page_size: PAGE_SIZE,
-			title: debouncedSearch,
-		};
-	}, [debouncedSearch, searchSpaceId]);
-
 	const surfsenseDocsQueryParams = useMemo(() => {
 		const params: { page: number; page_size: number; title?: string } = {
 			page: 0,
 			page_size: PAGE_SIZE,
 		};
-		if (debouncedSearch.trim()) {
-			params.title = debouncedSearch;
+		if (isSearchValid) {
+			params.title = debouncedSearch.trim();
 		}
 		return params;
-	}, [debouncedSearch]);
+	}, [debouncedSearch, isSearchValid]);

-	// Use query for fetching first page of documents
-	const { data: documents, isLoading: isDocumentsLoading } = useQuery({
-		queryKey: cacheKeys.documents.withQueryParams(fetchQueryParams),
-		queryFn: () => documentsApiService.getDocuments({ queryParams: fetchQueryParams }),
-		staleTime: 3 * 60 * 1000,
-		enabled: !!searchSpaceId && !debouncedSearch.trim() && currentPage === 0,
+	/**
+	 * TanStack Query for document title search.
+	 * - Uses AbortSignal for automatic request cancellation on query key changes
+	 * - placeholderData: keepPreviousData maintains UI stability during fetches
+	 * - Only triggers server-side search when isSearchValid (2+ characters)
+	 */
+	const { data: titleSearchResults, isLoading: isTitleSearchLoading } = useQuery({
+		queryKey: ["document-titles", titleSearchParams],
+		queryFn: ({ signal }) =>
+			documentsApiService.searchDocumentTitles({ queryParams: titleSearchParams }, signal),
+		staleTime: 60 * 1000,
+		enabled: !!searchSpaceId && currentPage === 0 && (!shouldSearch || isSearchValid),
+		placeholderData: keepPreviousData,
 	});

-	// Searching - first page
-	const { data: searchedDocuments, isLoading: isSearchedDocumentsLoading } = useQuery({
-		queryKey: cacheKeys.documents.withQueryParams(searchQueryParams),
-		queryFn: () => documentsApiService.searchDocuments({ queryParams: searchQueryParams }),
-		staleTime: 3 * 60 * 1000,
-		enabled: !!searchSpaceId && !!debouncedSearch.trim() && currentPage === 0,
-	});
-
-	// Use query for fetching first page of SurfSense docs
+	/**
+	 * TanStack Query for SurfSense documentation.
+	 * - Uses AbortSignal for automatic request cancellation
+	 * - placeholderData: keepPreviousData prevents UI flicker during refetches
+	 */
 	const { data: surfsenseDocs, isLoading: isSurfsenseDocsLoading } = useQuery({
-		queryKey: ["surfsense-docs-mention", debouncedSearch],
-		queryFn: () => documentsApiService.getSurfsenseDocs({ queryParams: surfsenseDocsQueryParams }),
+		queryKey: ["surfsense-docs-mention", debouncedSearch, isSearchValid],
+		queryFn: ({ signal }) =>
+			documentsApiService.getSurfsenseDocs({ queryParams: surfsenseDocsQueryParams }, signal),
 		staleTime: 3 * 60 * 1000,
+		enabled: !shouldSearch || isSearchValid,
+		placeholderData: keepPreviousData,
 	});

-	// Update accumulated documents when first page loads - combine both sources
+	// Post-fetch filter to eliminate false positives from backend fuzzy matching
+	const filterBySearchTerm = useCallback(
+		(docs: Pick<Document, "id" | "title" | "document_type">[]) => {
+			if (!isSearchValid) return docs; // No filtering when not searching
+			const searchLower = debouncedSearch.trim().toLowerCase();
+			return docs.filter((doc) => doc.title.toLowerCase().includes(searchLower));
+		},
+		[debouncedSearch, isSearchValid]
+	);
+
+	// Combine and update document list when first page data arrives
 	useEffect(() => {
 		if (currentPage === 0) {
 			const combinedDocs: Pick<Document, "id" | "title" | "document_type">[] = [];

-			// Add SurfSense docs first (they appear at top)
+			// SurfSense docs displayed first in the list
 			if (surfsenseDocs?.items) {
 				for (const doc of surfsenseDocs.items) {
 					combinedDocs.push({
@ -142,24 +218,16 @@ export const DocumentMentionPicker = forwardRef<
 				}
 			}

-			// Add regular documents
-			if (debouncedSearch.trim()) {
-				if (searchedDocuments?.items) {
-					combinedDocs.push(...searchedDocuments.items);
-					setHasMore(searchedDocuments.has_more);
-				}
-			} else {
-				if (documents?.items) {
-					combinedDocs.push(...documents.items);
-					setHasMore(documents.has_more);
-				}
+			if (titleSearchResults?.items) {
+				combinedDocs.push(...titleSearchResults.items);
+				setHasMore(titleSearchResults.has_more);
 			}

-			setAccumulatedDocuments(combinedDocs);
+			setAccumulatedDocuments(filterBySearchTerm(combinedDocs));
 		}
-	}, [documents, searchedDocuments, surfsenseDocs, debouncedSearch, currentPage]);
+	}, [titleSearchResults, surfsenseDocs, currentPage, filterBySearchTerm]);

-	// Function to load next page
+	// Load next page for infinite scroll pagination
 	const loadNextPage = useCallback(async () => {
 		if (isLoadingMore || !hasMore) return;

@ -167,23 +235,15 @@ export const DocumentMentionPicker = forwardRef<
 		setIsLoadingMore(true);

 		try {
-			let response: GetDocumentsResponse;
-			if (debouncedSearch.trim()) {
-				const queryParams = {
-					search_space_id: searchSpaceId,
-					page: nextPage,
-					page_size: PAGE_SIZE,
-					title: debouncedSearch,
-				};
-				response = await documentsApiService.searchDocuments({ queryParams });
-			} else {
-				const queryParams = {
-					search_space_id: searchSpaceId,
-					page: nextPage,
-					page_size: PAGE_SIZE,
-				};
-				response = await documentsApiService.getDocuments({ queryParams });
-			}
+			const queryParams = {
+				search_space_id: searchSpaceId,
+				page: nextPage,
+				page_size: PAGE_SIZE,
+				...(isSearchValid ? { title: debouncedSearch.trim() } : {}),
+			};
+			const response: SearchDocumentTitlesResponse = await documentsApiService.searchDocumentTitles(
+				{ queryParams }
+			);

 			setAccumulatedDocuments((prev) => [...prev, ...response.items]);
 			setHasMore(response.has_more);
@ -193,15 +253,14 @@ export const DocumentMentionPicker = forwardRef<
 		} finally {
 			setIsLoadingMore(false);
 		}
-	}, [currentPage, hasMore, isLoadingMore, debouncedSearch, searchSpaceId]);
+	}, [currentPage, hasMore, isLoadingMore, debouncedSearch, searchSpaceId, isSearchValid]);

-	// Infinite scroll handler
+	// Trigger pagination when user scrolls near the bottom (50px threshold)
 	const handleScroll = useCallback(
 		(e: React.UIEvent<HTMLDivElement>) => {
 			const target = e.currentTarget;
 			const scrollBottom = target.scrollHeight - target.scrollTop - target.clientHeight;

-			// Load more when within 50px of bottom
 			if (scrollBottom < 50 && hasMore && !isLoadingMore) {
 				loadNextPage();
 			}
@ -209,13 +268,26 @@ export const DocumentMentionPicker = forwardRef<
 		[hasMore, isLoadingMore, loadNextPage]
 	);

-	const actualDocuments = accumulatedDocuments;
-	const actualLoading =
-		((debouncedSearch.trim() ? isSearchedDocumentsLoading : isDocumentsLoading) ||
-			isSurfsenseDocsLoading) &&
-		currentPage === 0;
+	/**
+	 * Client-side filtering for single character searches.
+	 * Filters cached documents locally for instant feedback without additional API calls.
+	 * Server-side search is reserved for 2+ character queries to leverage database indexing.
+	 */
+	const clientFilteredDocs = useMemo(() => {
+		if (!isSingleCharSearch) return null;
+		const searchLower = debouncedSearch.trim().toLowerCase();
+		return accumulatedDocuments.filter((doc) => doc.title.toLowerCase().includes(searchLower));
+	}, [isSingleCharSearch, debouncedSearch, accumulatedDocuments]);

-	// Split documents into SurfSense docs and user docs for grouped rendering
+	// Select data source based on search length: client-filtered for single char, server results for 2+
+	const actualDocuments = isSingleCharSearch ? (clientFilteredDocs ?? []) : accumulatedDocuments;
+	// Only show loading spinner on initial load (no documents yet), not during subsequent searches
+	const actualLoading =
+		(isTitleSearchLoading || isSurfsenseDocsLoading) &&
+		currentPage === 0 &&
+		!isSingleCharSearch &&
+		accumulatedDocuments.length === 0;
+	// Partition documents by type for grouped UI rendering
 	const surfsenseDocsList = useMemo(
 		() => actualDocuments.filter((doc) => doc.document_type === "SURFSENSE_DOCS"),
 		[actualDocuments]
@ -225,13 +297,13 @@ export const DocumentMentionPicker = forwardRef<
 		[actualDocuments]
 	);

-	// Track already selected documents using unique key (document_type:id) to avoid ID collisions
+	// Track selected documents with composite key (document_type:id) to prevent cross-type ID collisions
 	const selectedKeys = useMemo(
 		() => new Set(initialSelectedDocuments.map((d) => `${d.document_type}:${d.id}`)),
 		[initialSelectedDocuments]
 	);

-	// Filter out already selected documents for navigation
+	// Exclude already-selected documents from keyboard navigation
 	const selectableDocuments = useMemo(
 		() => actualDocuments.filter((doc) => !selectedKeys.has(`${doc.document_type}:${doc.id}`)),
 		[actualDocuments, selectedKeys]
@ -245,15 +317,44 @@ export const DocumentMentionPicker = forwardRef<
 		[initialSelectedDocuments, onSelectionChange, onDone]
 	);

-	// Scroll highlighted item into view
+	// Auto-scroll highlighted item into view (keyboard navigation only, not mouse hover)
 	useEffect(() => {
-		const item = itemRefs.current.get(highlightedIndex);
-		if (item) {
-			item.scrollIntoView({ block: "nearest", behavior: "smooth" });
+		if (!shouldScrollRef.current) {
+			return;
 		}
+		shouldScrollRef.current = false;
+
+		const rafId = requestAnimationFrame(() => {
+			const item = itemRefs.current.get(highlightedIndex);
+			const container = scrollContainerRef.current;
+
+			if (item && container) {
+				const itemRect = item.getBoundingClientRect();
+				const containerRect = container.getBoundingClientRect();
+				const padding = 8;
+				const isAboveViewport = itemRect.top < containerRect.top + padding;
+				const isBelowViewport = itemRect.bottom > containerRect.bottom - padding;
+
+				if (isAboveViewport || isBelowViewport) {
+					const itemOffsetTop = item.offsetTop;
+					const containerHeight = container.clientHeight;
+					const itemHeight = item.offsetHeight;
+					const targetScrollTop = itemOffsetTop - containerHeight / 2 + itemHeight / 2;
+					const maxScrollTop = container.scrollHeight - containerHeight;
+					const clampedScrollTop = Math.max(0, Math.min(targetScrollTop, maxScrollTop));
+
+					container.scrollTo({
+						top: clampedScrollTop,
+						behavior: "smooth",
+					});
+				}
+			}
+		});
+
+		return () => cancelAnimationFrame(rafId);
 	}, [highlightedIndex]);

-	// Reset highlighted index when external search changes
+	// Reset highlight position when search query changes
 	const prevSearchRef = useRef(search);
 	if (prevSearchRef.current !== search) {
 		prevSearchRef.current = search;
@ -262,7 +363,7 @@ export const DocumentMentionPicker = forwardRef<
 		}
 	}

-	// Expose methods to parent via ref
+	// Expose navigation and selection methods to parent component via ref
 	useImperativeHandle(
 		ref,
 		() => ({
@ -272,16 +373,18 @@ export const DocumentMentionPicker = forwardRef<
 				}
 			},
 			moveUp: () => {
+				shouldScrollRef.current = true;
 				setHighlightedIndex((prev) => (prev > 0 ? prev - 1 : selectableDocuments.length - 1));
 			},
 			moveDown: () => {
+				shouldScrollRef.current = true;
 				setHighlightedIndex((prev) => (prev < selectableDocuments.length - 1 ? prev + 1 : 0));
 			},
 		}),
 		[selectableDocuments, highlightedIndex, handleSelectDocument]
 	);

-	// Handle keyboard navigation
+	// Keyboard navigation handler for arrow keys, Enter, and Escape
 	const handleKeyDown = useCallback(
 		(e: React.KeyboardEvent) => {
 			if (selectableDocuments.length === 0) return;
@ -289,10 +392,12 @@ export const DocumentMentionPicker = forwardRef<
 			switch (e.key) {
 				case "ArrowDown":
 					e.preventDefault();
+					shouldScrollRef.current = true;
 					setHighlightedIndex((prev) => (prev < selectableDocuments.length - 1 ? prev + 1 : 0));
 					break;
 				case "ArrowUp":
 					e.preventDefault();
+					shouldScrollRef.current = true;
 					setHighlightedIndex((prev) => (prev > 0 ? prev - 1 : selectableDocuments.length - 1));
 					break;
 				case "Enter":
@ -310,14 +415,24 @@ export const DocumentMentionPicker = forwardRef<
 		[selectableDocuments, highlightedIndex, handleSelectDocument, onDone]
 	);

+	// Hide popup when there are no documents to display (regardless of fetch state)
+	// Search continues in background; popup reappears when results arrive
+	if (!actualLoading && actualDocuments.length === 0) {
+		return null;
+	}
+
 	return (
 		<div
-			className="flex flex-col w-[280px] sm:w-[320px] bg-popover rounded-lg"
+			className="fixed shadow-2xl rounded-lg border border-border overflow-hidden bg-popover flex flex-col w-[280px] sm:w-[320px]"
+			style={{
+				zIndex: 9999,
+				...containerStyle,
+			}}
 			onKeyDown={handleKeyDown}
 			role="listbox"
 			tabIndex={-1}
 		>
-			{/* Document List - Shows max 5 items on mobile, 7-8 items on desktop */}
+			{/* Scrollable document list with responsive height */}
 			<div
 				ref={scrollContainerRef}
 				className="max-h-[180px] sm:max-h-[280px] overflow-y-auto"
@ -327,14 +442,9 @@ export const DocumentMentionPicker = forwardRef<
 					<div className="flex items-center justify-center py-4">
 						<div className="animate-spin h-5 w-5 border-2 border-primary border-t-transparent rounded-full" />
 					</div>
-				) : actualDocuments.length === 0 ? (
-					<div className="flex flex-col items-center justify-center py-4 text-center px-4">
-						<FileText className="h-5 w-5 text-muted-foreground/55 mb-1" />
-						<p className="text-sm text-muted-foreground">No documents found</p>
-					</div>
-				) : (
-					<div className="py-1">
-						{/* SurfSense Documentation Section */}
+				) : actualDocuments.length > 0 ? (
+					<div className="py-1 px-2">
+						{/* SurfSense Documentation */}
 						{surfsenseDocsList.length > 0 && (
 							<>
 								<div className="px-3 py-2 text-xs font-bold text-muted-foreground/55">
@ -365,7 +475,7 @@ export const DocumentMentionPicker = forwardRef<
 											}}
 											disabled={isAlreadySelected}
 											className={cn(
-												"w-full flex items-center gap-2 px-3 py-2 text-left transition-colors",
+												"w-full flex items-center gap-2 px-3 py-2 text-left transition-colors rounded-md",
 												isAlreadySelected ? "opacity-50 cursor-not-allowed" : "cursor-pointer",
 												isHighlighted && "bg-accent"
 											)}
@ -382,7 +492,7 @@ export const DocumentMentionPicker = forwardRef<
 							</>
 						)}

-						{/* User Documents Section */}
+						{/* User Documents */}
 						{userDocsList.length > 0 && (
 							<>
 								<div className="px-3 py-2 text-xs font-bold text-muted-foreground/55">
@ -413,7 +523,7 @@ export const DocumentMentionPicker = forwardRef<
 											}}
 											disabled={isAlreadySelected}
 											className={cn(
-												"w-full flex items-center gap-2 px-3 py-2 text-left transition-colors",
+												"w-full flex items-center gap-2 px-3 py-2 text-left transition-colors rounded-md",
 												isAlreadySelected ? "opacity-50 cursor-not-allowed" : "cursor-pointer",
 												isHighlighted && "bg-accent"
 											)}
@ -430,14 +540,14 @@ export const DocumentMentionPicker = forwardRef<
 							</>
 						)}

-						{/* Loading indicator for additional pages */}
+						{/* Pagination loading indicator */}
 						{isLoadingMore && (
 							<div className="flex items-center justify-center py-2">
 								<div className="animate-spin h-4 w-4 border-2 border-primary border-t-transparent rounded-full" />
 							</div>
 						)}
 					</div>
-				)}
+				) : null}
 			</div>
 		</div>
 	);