From b001b65067e00d5282202734146a12bf2ab15467 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 17 Jan 2026 20:45:10 +0530
Subject: [PATCH 01/10] feat: add pg_trgm indexes and lightweight document
 title search

- Introduced pg_trgm extension and GIN trigram indexes for efficient document title searches, enhancing performance for mention picker functionality.
- Implemented a new API endpoint for lightweight document title searches, returning only essential fields.
- Updated frontend components to utilize the new title search feature with throttling for improved user experience.
- Added necessary schemas and types for the new search functionality.
---
 ...pg_trgm_index_for_document_title_search.py |  77 +++++++
 surfsense_backend/app/db.py                   |  25 +++
 .../app/routes/documents_routes.py            |  95 +++++++++
 surfsense_backend/app/schemas/__init__.py     |   4 +
 surfsense_backend/app/schemas/documents.py    |  17 ++
 .../new-chat/document-mention-picker.tsx      | 194 +++++++++++-------
 .../contracts/types/document.types.ts         |  26 +++
 .../lib/apis/documents-api.service.ts         |  32 +++
 8 files changed, 393 insertions(+), 77 deletions(-)
 create mode 100644 surfsense_backend/alembic/versions/67_add_pg_trgm_index_for_document_title_search.py

diff --git a/surfsense_backend/alembic/versions/67_add_pg_trgm_index_for_document_title_search.py b/surfsense_backend/alembic/versions/67_add_pg_trgm_index_for_document_title_search.py
new file mode 100644
index 000000000..ba9cbdbac
--- /dev/null
+++ b/surfsense_backend/alembic/versions/67_add_pg_trgm_index_for_document_title_search.py
@@ -0,0 +1,77 @@
+"""Add pg_trgm indexes for efficient document title search
+
+Revision ID: 67
+Revises: 66
+
+Adds the pg_trgm extension and GIN trigram indexes on documents.title
+to enable efficient ILIKE searches with leading wildcards (e.g., '%search_term%').
+
+Indexes added:
+1. idx_documents_title_trgm - GIN trigram on title for ILIKE '%term%'
+2. idx_documents_search_space_id - B-tree on search_space_id for filtering
+3. idx_documents_search_space_updated - Composite for recent docs query (covering index)
+4. idx_surfsense_docs_title_trgm - GIN trigram on surfsense docs title
+
+This is critical for the document mention picker (@mentions) to scale to 10,000+ documents.
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "67"
+down_revision: str | None = "66"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Add pg_trgm extension and optimized indexes for document search."""
+
+    # Create pg_trgm extension if not exists
+    # This extension provides trigram-based text similarity functions and operators
+    op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
+
+    # 1. GIN trigram index on documents.title for ILIKE '%term%' searches
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS idx_documents_title_trgm 
+        ON documents USING gin (title gin_trgm_ops);
+        """
+    )
+
+    # 2. B-tree index on search_space_id for fast filtering
+    # (Every query filters by search_space_id first)
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS idx_documents_search_space_id 
+        ON documents (search_space_id);
+        """
+    )
+
+    # 3. Covering index for "recent documents" query (no search term)
+    # Includes id, title, document_type so PostgreSQL can do index-only scan
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS idx_documents_search_space_updated 
+        ON documents (search_space_id, updated_at DESC NULLS LAST)
+        INCLUDE (id, title, document_type);
+        """
+    )
+
+    # 4. GIN trigram index on surfsense_docs_documents.title
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS idx_surfsense_docs_title_trgm 
+        ON surfsense_docs_documents USING gin (title gin_trgm_ops);
+        """
+    )
+
+
+def downgrade() -> None:
+    """Remove all document search indexes (extension is left in place)."""
+    op.execute("DROP INDEX IF EXISTS idx_surfsense_docs_title_trgm;")
+    op.execute("DROP INDEX IF EXISTS idx_documents_search_space_updated;")
+    op.execute("DROP INDEX IF EXISTS idx_documents_search_space_id;")
+    op.execute("DROP INDEX IF EXISTS idx_documents_title_trgm;")
diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py
index 9b245ba44..ee0d0724d 100644
--- a/surfsense_backend/app/db.py
+++ b/surfsense_backend/app/db.py
@@ -1007,11 +1007,36 @@ async def setup_indexes():
                 "CREATE INDEX IF NOT EXISTS chucks_search_index ON chunks USING gin (to_tsvector('english', content))"
             )
         )
+        # pg_trgm indexes for efficient ILIKE '%term%' searches on titles
+        # Critical for document mention picker (@mentions) to scale
+        await conn.execute(
+            text(
+                "CREATE INDEX IF NOT EXISTS idx_documents_title_trgm ON documents USING gin (title gin_trgm_ops)"
+            )
+        )
+        # B-tree index on search_space_id for fast filtering
+        await conn.execute(
+            text(
+                "CREATE INDEX IF NOT EXISTS idx_documents_search_space_id ON documents (search_space_id)"
+            )
+        )
+        # Covering index for "recent documents" query - enables index-only scan
+        await conn.execute(
+            text(
+                "CREATE INDEX IF NOT EXISTS idx_documents_search_space_updated ON documents (search_space_id, updated_at DESC NULLS LAST) INCLUDE (id, title, document_type)"
+            )
+        )
+        await conn.execute(
+            text(
+                "CREATE INDEX IF NOT EXISTS idx_surfsense_docs_title_trgm ON surfsense_docs_documents USING gin (title gin_trgm_ops)"
+            )
+        )
 
 
 async def create_db_and_tables():
     async with engine.begin() as conn:
         await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
+        await conn.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm"))
         await conn.run_sync(Base.metadata.create_all)
     await setup_indexes()
 
diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py
index d06217fc4..a1a421b8a 100644
--- a/surfsense_backend/app/routes/documents_routes.py
+++ b/surfsense_backend/app/routes/documents_routes.py
@@ -19,6 +19,8 @@ from app.db import (
 from app.schemas import (
     DocumentRead,
     DocumentsCreate,
+    DocumentTitleRead,
+    DocumentTitleSearchResponse,
     DocumentUpdate,
     DocumentWithChunksRead,
     PaginatedResponse,
@@ -429,6 +431,99 @@ async def search_documents(
         ) from e
 
 
+@router.get("/documents/search/titles", response_model=DocumentTitleSearchResponse)
+async def search_document_titles(
+    search_space_id: int,
+    title: str = "",
+    page: int = 0,
+    page_size: int = 20,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    """
+    Lightweight document title search optimized for mention picker (@mentions).
+
+    Returns only id, title, and document_type - no content or metadata.
+    Results are ordered by relevance: prefix matches first, then contains matches.
+
+    Args:
+        search_space_id: The search space to search in. Required.
+        title: Search query (case-insensitive). If empty or < 2 chars, returns recent documents.
+        page: Zero-based page index. Default: 0.
+        page_size: Number of items per page. Default: 20.
+        session: Database session (injected).
+        user: Current authenticated user (injected).
+
+    Returns:
+        DocumentTitleSearchResponse: Lightweight list with has_more flag (no total count).
+    """
+    from sqlalchemy import case, literal
+
+    try:
+        # Check permission for the search space
+        await check_permission(
+            session,
+            user,
+            search_space_id,
+            Permission.DOCUMENTS_READ.value,
+            "You don't have permission to read documents in this search space",
+        )
+
+        # Base query - only select lightweight fields
+        query = select(
+            Document.id,
+            Document.title,
+            Document.document_type,
+        ).filter(Document.search_space_id == search_space_id)
+
+        # If query is too short, return recent documents ordered by updated_at
+        if len(title.strip()) < 2:
+            query = query.order_by(Document.updated_at.desc().nullslast())
+        else:
+            # Apply title filter with ILIKE (uses pg_trgm index)
+            search_term = title.strip()
+            query = query.filter(Document.title.ilike(f"%{search_term}%"))
+
+            # Order by relevance: prefix matches first, then alphabetical
+            # CASE WHEN title ILIKE 'term%' THEN 0 ELSE 1 END
+            prefix_priority = case(
+                (Document.title.ilike(f"{search_term}%"), literal(0)),
+                else_=literal(1),
+            )
+            query = query.order_by(prefix_priority, Document.title)
+
+        # Fetch page_size + 1 to determine has_more without COUNT query
+        offset = page * page_size
+        result = await session.execute(query.offset(offset).limit(page_size + 1))
+        rows = result.all()
+
+        # Check if there are more results
+        has_more = len(rows) > page_size
+        items = rows[:page_size]  # Only return requested page_size
+
+        # Convert to response format
+        api_documents = [
+            DocumentTitleRead(
+                id=row.id,
+                title=row.title,
+                document_type=row.document_type,
+            )
+            for row in items
+        ]
+
+        return DocumentTitleSearchResponse(
+            items=api_documents,
+            has_more=has_more,
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(
+            status_code=500, detail=f"Failed to search document titles: {e!s}"
+        ) from e
+
+
 @router.get("/documents/type-counts")
 async def get_document_type_counts(
     search_space_id: int | None = None,
diff --git a/surfsense_backend/app/schemas/__init__.py b/surfsense_backend/app/schemas/__init__.py
index 076ac5915..b9b371bc5 100644
--- a/surfsense_backend/app/schemas/__init__.py
+++ b/surfsense_backend/app/schemas/__init__.py
@@ -4,6 +4,8 @@ from .documents import (
     DocumentBase,
     DocumentRead,
     DocumentsCreate,
+    DocumentTitleRead,
+    DocumentTitleSearchResponse,
     DocumentUpdate,
     DocumentWithChunksRead,
     ExtensionDocumentContent,
@@ -85,6 +87,8 @@ __all__ = [
     # Document schemas
     "DocumentBase",
     "DocumentRead",
+    "DocumentTitleRead",
+    "DocumentTitleSearchResponse",
     "DocumentUpdate",
     "DocumentWithChunksRead",
     "DocumentsCreate",
diff --git a/surfsense_backend/app/schemas/documents.py b/surfsense_backend/app/schemas/documents.py
index e1e8b9248..2b4bda0ca 100644
--- a/surfsense_backend/app/schemas/documents.py
+++ b/surfsense_backend/app/schemas/documents.py
@@ -67,3 +67,20 @@ class PaginatedResponse[T](BaseModel):
     page: int
     page_size: int
     has_more: bool
+
+
+class DocumentTitleRead(BaseModel):
+    """Lightweight document response for mention picker - only essential fields."""
+
+    id: int
+    title: str
+    document_type: DocumentType
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class DocumentTitleSearchResponse(BaseModel):
+    """Response for document title search - optimized for typeahead."""
+
+    items: list[DocumentTitleRead]
+    has_more: bool
diff --git a/surfsense_web/components/new-chat/document-mention-picker.tsx b/surfsense_web/components/new-chat/document-mention-picker.tsx
index f7f948d41..66988adcc 100644
--- a/surfsense_web/components/new-chat/document-mention-picker.tsx
+++ b/surfsense_web/components/new-chat/document-mention-picker.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { useQuery } from "@tanstack/react-query";
+import { useQuery, useQueryClient } from "@tanstack/react-query";
 import { FileText } from "lucide-react";
 import {
 	forwardRef,
@@ -12,9 +12,8 @@ import {
 	useState,
 } from "react";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import type { Document, GetDocumentsResponse } from "@/contracts/types/document.types";
+import type { Document, SearchDocumentTitlesResponse } from "@/contracts/types/document.types";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
-import { cacheKeys } from "@/lib/query-client/cache-keys";
 import { cn } from "@/lib/utils";
 
 export interface DocumentMentionPickerRef {
@@ -32,14 +31,45 @@ interface DocumentMentionPickerProps {
 }
 
 const PAGE_SIZE = 20;
+const MIN_SEARCH_LENGTH = 2;
+const THROTTLE_MS = 200;
+
+/**
+ * Throttle hook - fires immediately, then at most once per interval
+ * Better than debounce for typeahead: user sees results updating as they type
+ */
+function useThrottled<T>(value: T, delay = THROTTLE_MS) {
+	const [throttled, setThrottled] = useState(value);
+	const lastExecuted = useRef(Date.now());
+	const timeoutRef = useRef<ReturnType<typeof setTimeout>>();
 
-function useDebounced<T>(value: T, delay = 300) {
-	const [debounced, setDebounced] = useState(value);
 	useEffect(() => {
-		const t = setTimeout(() => setDebounced(value), delay);
-		return () => clearTimeout(t);
+		const now = Date.now();
+		const elapsed = now - lastExecuted.current;
+
+		if (elapsed >= delay) {
+			// Enough time has passed, update immediately
+			lastExecuted.current = now;
+			setThrottled(value);
+		} else {
+			// Schedule update for remaining time
+			if (timeoutRef.current) {
+				clearTimeout(timeoutRef.current);
+			}
+			timeoutRef.current = setTimeout(() => {
+				lastExecuted.current = Date.now();
+				setThrottled(value);
+			}, delay - elapsed);
+		}
+
+		return () => {
+			if (timeoutRef.current) {
+				clearTimeout(timeoutRef.current);
+			}
+		};
 	}, [value, delay]);
-	return debounced;
+
+	return throttled;
 }
 
 export const DocumentMentionPicker = forwardRef<
@@ -49,9 +79,11 @@ export const DocumentMentionPicker = forwardRef<
 	{ searchSpaceId, onSelectionChange, onDone, initialSelectedDocuments = [], externalSearch = "" },
 	ref
 ) {
-	// Use external search
+	const queryClient = useQueryClient();
+
+	// Use external search with throttle (not debounce) for responsive feel
 	const search = externalSearch;
-	const debouncedSearch = useDebounced(search, 150);
+	const throttledSearch = useThrottled(search, THROTTLE_MS);
 	const [highlightedIndex, setHighlightedIndex] = useState(0);
 	const itemRefs = useRef<Map<number, HTMLButtonElement>>(new Map());
 	const scrollContainerRef = useRef<HTMLDivElement>(null);
@@ -64,6 +96,38 @@ export const DocumentMentionPicker = forwardRef<
 	const [hasMore, setHasMore] = useState(false);
 	const [isLoadingMore, setIsLoadingMore] = useState(false);
 
+	// Check if search is long enough
+	const isSearchValid = throttledSearch.trim().length >= MIN_SEARCH_LENGTH;
+	const shouldSearch = throttledSearch.trim().length > 0;
+
+	// Prefetch first page when picker mounts - results appear instantly
+	useEffect(() => {
+		if (!searchSpaceId) return;
+
+		const prefetchParams = {
+			search_space_id: searchSpaceId,
+			page: 0,
+			page_size: PAGE_SIZE,
+		};
+
+		// Prefetch document titles (user docs)
+		queryClient.prefetchQuery({
+			queryKey: ["document-titles", prefetchParams],
+			queryFn: () => documentsApiService.searchDocumentTitles({ queryParams: prefetchParams }),
+			staleTime: 60 * 1000,
+		});
+
+		// Prefetch SurfSense docs
+		queryClient.prefetchQuery({
+			queryKey: ["surfsense-docs-mention", "", false],
+			queryFn: () =>
+				documentsApiService.getSurfsenseDocs({
+					queryParams: { page: 0, page_size: PAGE_SIZE },
+				}),
+			staleTime: 3 * 60 * 1000,
+		});
+	}, [searchSpaceId, queryClient]);
+
 	// Reset pagination when search or search space changes
 	// biome-ignore lint/correctness/useExhaustiveDependencies: intentionally reset pagination when search/space changes
 	useEffect(() => {
@@ -71,59 +135,44 @@ export const DocumentMentionPicker = forwardRef<
 		setCurrentPage(0);
 		setHasMore(false);
 		setHighlightedIndex(0);
-	}, [debouncedSearch, searchSpaceId]);
+	}, [throttledSearch, searchSpaceId]);
 
-	// Query params for initial fetch (page 0)
-	const fetchQueryParams = useMemo(
+	// Query params for lightweight title search
+	const titleSearchParams = useMemo(
 		() => ({
 			search_space_id: searchSpaceId,
 			page: 0,
 			page_size: PAGE_SIZE,
+			...(isSearchValid ? { title: throttledSearch.trim() } : {}),
 		}),
-		[searchSpaceId]
+		[searchSpaceId, throttledSearch, isSearchValid]
 	);
 
-	const searchQueryParams = useMemo(() => {
-		return {
-			search_space_id: searchSpaceId,
-			page: 0,
-			page_size: PAGE_SIZE,
-			title: debouncedSearch,
-		};
-	}, [debouncedSearch, searchSpaceId]);
-
 	const surfsenseDocsQueryParams = useMemo(() => {
 		const params: { page: number; page_size: number; title?: string } = {
 			page: 0,
 			page_size: PAGE_SIZE,
 		};
-		if (debouncedSearch.trim()) {
-			params.title = debouncedSearch;
+		if (isSearchValid) {
+			params.title = throttledSearch.trim();
 		}
 		return params;
-	}, [debouncedSearch]);
+	}, [throttledSearch, isSearchValid]);
 
-	// Use query for fetching first page of documents
-	const { data: documents, isLoading: isDocumentsLoading } = useQuery({
-		queryKey: cacheKeys.documents.withQueryParams(fetchQueryParams),
-		queryFn: () => documentsApiService.getDocuments({ queryParams: fetchQueryParams }),
-		staleTime: 3 * 60 * 1000,
-		enabled: !!searchSpaceId && !debouncedSearch.trim() && currentPage === 0,
-	});
-
-	// Searching - first page
-	const { data: searchedDocuments, isLoading: isSearchedDocumentsLoading } = useQuery({
-		queryKey: cacheKeys.documents.withQueryParams(searchQueryParams),
-		queryFn: () => documentsApiService.searchDocuments({ queryParams: searchQueryParams }),
-		staleTime: 3 * 60 * 1000,
-		enabled: !!searchSpaceId && !!debouncedSearch.trim() && currentPage === 0,
+	// Use the new lightweight endpoint for document title search
+	const { data: titleSearchResults, isLoading: isTitleSearchLoading } = useQuery({
+		queryKey: ["document-titles", titleSearchParams],
+		queryFn: () => documentsApiService.searchDocumentTitles({ queryParams: titleSearchParams }),
+		staleTime: 60 * 1000, // 1 minute - shorter for fresher results
+		enabled: !!searchSpaceId && currentPage === 0 && (!shouldSearch || isSearchValid),
 	});
 
 	// Use query for fetching first page of SurfSense docs
 	const { data: surfsenseDocs, isLoading: isSurfsenseDocsLoading } = useQuery({
-		queryKey: ["surfsense-docs-mention", debouncedSearch],
+		queryKey: ["surfsense-docs-mention", throttledSearch, isSearchValid],
 		queryFn: () => documentsApiService.getSurfsenseDocs({ queryParams: surfsenseDocsQueryParams }),
 		staleTime: 3 * 60 * 1000,
+		enabled: !shouldSearch || isSearchValid,
 	});
 
 	// Update accumulated documents when first page loads - combine both sources
@@ -142,24 +191,17 @@ export const DocumentMentionPicker = forwardRef<
 				}
 			}
 
-			// Add regular documents
-			if (debouncedSearch.trim()) {
-				if (searchedDocuments?.items) {
-					combinedDocs.push(...searchedDocuments.items);
-					setHasMore(searchedDocuments.has_more);
-				}
-			} else {
-				if (documents?.items) {
-					combinedDocs.push(...documents.items);
-					setHasMore(documents.has_more);
-				}
+			// Add regular documents from lightweight endpoint
+			if (titleSearchResults?.items) {
+				combinedDocs.push(...titleSearchResults.items);
+				setHasMore(titleSearchResults.has_more);
 			}
 
 			setAccumulatedDocuments(combinedDocs);
 		}
-	}, [documents, searchedDocuments, surfsenseDocs, debouncedSearch, currentPage]);
+	}, [titleSearchResults, surfsenseDocs, currentPage]);
 
-	// Function to load next page
+	// Function to load next page using lightweight endpoint
 	const loadNextPage = useCallback(async () => {
 		if (isLoadingMore || !hasMore) return;
 
@@ -167,23 +209,14 @@ export const DocumentMentionPicker = forwardRef<
 		setIsLoadingMore(true);
 
 		try {
-			let response: GetDocumentsResponse;
-			if (debouncedSearch.trim()) {
-				const queryParams = {
-					search_space_id: searchSpaceId,
-					page: nextPage,
-					page_size: PAGE_SIZE,
-					title: debouncedSearch,
-				};
-				response = await documentsApiService.searchDocuments({ queryParams });
-			} else {
-				const queryParams = {
-					search_space_id: searchSpaceId,
-					page: nextPage,
-					page_size: PAGE_SIZE,
-				};
-				response = await documentsApiService.getDocuments({ queryParams });
-			}
+			const queryParams = {
+				search_space_id: searchSpaceId,
+				page: nextPage,
+				page_size: PAGE_SIZE,
+				...(isSearchValid ? { title: throttledSearch.trim() } : {}),
+			};
+			const response: SearchDocumentTitlesResponse =
+				await documentsApiService.searchDocumentTitles({ queryParams });
 
 			setAccumulatedDocuments((prev) => [...prev, ...response.items]);
 			setHasMore(response.has_more);
@@ -193,7 +226,7 @@ export const DocumentMentionPicker = forwardRef<
 		} finally {
 			setIsLoadingMore(false);
 		}
-	}, [currentPage, hasMore, isLoadingMore, debouncedSearch, searchSpaceId]);
+	}, [currentPage, hasMore, isLoadingMore, throttledSearch, searchSpaceId, isSearchValid]);
 
 	// Infinite scroll handler
 	const handleScroll = useCallback(
@@ -210,10 +243,10 @@ export const DocumentMentionPicker = forwardRef<
 	);
 
 	const actualDocuments = accumulatedDocuments;
-	const actualLoading =
-		((debouncedSearch.trim() ? isSearchedDocumentsLoading : isDocumentsLoading) ||
-			isSurfsenseDocsLoading) &&
-		currentPage === 0;
+	const actualLoading = (isTitleSearchLoading || isSurfsenseDocsLoading) && currentPage === 0;
+
+	// Show hint when search is too short
+	const showSearchHint = shouldSearch && !isSearchValid;
 
 	// Split documents into SurfSense docs and user docs for grouped rendering
 	const surfsenseDocsList = useMemo(
@@ -323,7 +356,14 @@ export const DocumentMentionPicker = forwardRef<
 				className="max-h-[180px] sm:max-h-[280px] overflow-y-auto"
 				onScroll={handleScroll}
 			>
-				{actualLoading ? (
+				{showSearchHint ? (
+					<div className="flex flex-col items-center justify-center py-4 text-center px-4">
+						<p className="text-sm text-muted-foreground">
+							Type {MIN_SEARCH_LENGTH - throttledSearch.trim().length} more character
+							{MIN_SEARCH_LENGTH - throttledSearch.trim().length > 1 ? "s" : ""} to search
+						</p>
+					</div>
+				) : actualLoading ? (
 					<div className="flex items-center justify-center py-4">
 						<div className="animate-spin h-5 w-5 border-2 border-primary border-t-transparent rounded-full" />
 					</div>
diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts
index 2b144bd68..ba81562b1 100644
--- a/surfsense_web/contracts/types/document.types.ts
+++ b/surfsense_web/contracts/types/document.types.ts
@@ -155,6 +155,29 @@ export const searchDocumentsResponse = z.object({
 	has_more: z.boolean(),
 });
 
+/**
+ * Search document titles (lightweight, for mention picker)
+ */
+export const documentTitleRead = z.object({
+	id: z.number(),
+	title: z.string(),
+	document_type: documentTypeEnum,
+});
+
+export const searchDocumentTitlesRequest = z.object({
+	queryParams: z.object({
+		search_space_id: z.number(),
+		title: z.string().optional(),
+		page: z.number().optional(),
+		page_size: z.number().optional(),
+	}),
+});
+
+export const searchDocumentTitlesResponse = z.object({
+	items: z.array(documentTitleRead),
+	has_more: z.boolean(),
+});
+
 /**
  * Get document type counts
  */
@@ -223,6 +246,7 @@ export const deleteDocumentResponse = z.object({
 });
 
 export type Document = z.infer<typeof document>;
+export type DocumentTitleRead = z.infer<typeof documentTitleRead>;
 export type GetDocumentsRequest = z.infer<typeof getDocumentsRequest>;
 export type GetDocumentsResponse = z.infer<typeof getDocumentsResponse>;
 export type GetDocumentRequest = z.infer<typeof getDocumentRequest>;
@@ -233,6 +257,8 @@ export type UploadDocumentRequest = z.infer<typeof uploadDocumentRequest>;
 export type UploadDocumentResponse = z.infer<typeof uploadDocumentResponse>;
 export type SearchDocumentsRequest = z.infer<typeof searchDocumentsRequest>;
 export type SearchDocumentsResponse = z.infer<typeof searchDocumentsResponse>;
+export type SearchDocumentTitlesRequest = z.infer<typeof searchDocumentTitlesRequest>;
+export type SearchDocumentTitlesResponse = z.infer<typeof searchDocumentTitlesResponse>;
 export type GetDocumentTypeCountsRequest = z.infer<typeof getDocumentTypeCountsRequest>;
 export type GetDocumentTypeCountsResponse = z.infer<typeof getDocumentTypeCountsResponse>;
 export type GetDocumentByChunkRequest = z.infer<typeof getDocumentByChunkRequest>;
diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts
index bea399f98..daa67f6d5 100644
--- a/surfsense_web/lib/apis/documents-api.service.ts
+++ b/surfsense_web/lib/apis/documents-api.service.ts
@@ -22,8 +22,11 @@ import {
 	getSurfsenseDocsRequest,
 	getSurfsenseDocsResponse,
 	type SearchDocumentsRequest,
+	type SearchDocumentTitlesRequest,
 	searchDocumentsRequest,
 	searchDocumentsResponse,
+	searchDocumentTitlesRequest,
+	searchDocumentTitlesResponse,
 	type UpdateDocumentRequest,
 	type UploadDocumentRequest,
 	updateDocumentRequest,
@@ -160,6 +163,35 @@ class DocumentsApiService {
 		return baseApiService.get(`/api/v1/documents/search?${queryParams}`, searchDocumentsResponse);
 	};
 
+	/**
+	 * Search document titles (lightweight, optimized for mention picker)
+	 * Returns only id, title, document_type - no content or metadata
+	 */
+	searchDocumentTitles = async (request: SearchDocumentTitlesRequest) => {
+		const parsedRequest = searchDocumentTitlesRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		// Transform query params to be string values
+		const transformedQueryParams = Object.fromEntries(
+			Object.entries(parsedRequest.data.queryParams)
+				.filter(([, v]) => v !== undefined)
+				.map(([k, v]) => [k, String(v)])
+		);
+
+		const queryParams = new URLSearchParams(transformedQueryParams).toString();
+
+		return baseApiService.get(
+			`/api/v1/documents/search/titles?${queryParams}`,
+			searchDocumentTitlesResponse
+		);
+	};
+
 	/**
 	 * Get document type counts
 	 */

From 293de6876ac92febb7f0f42495b44a68a9eb144e Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 17 Jan 2026 20:46:47 +0530
Subject: [PATCH 02/10] feat: implement fuzzy search in mention document

---
 .../app/routes/documents_routes.py            | 33 +++++++++++++------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py
index a1a421b8a..be90df459 100644
--- a/surfsense_backend/app/routes/documents_routes.py
+++ b/surfsense_backend/app/routes/documents_routes.py
@@ -444,7 +444,8 @@ async def search_document_titles(
     Lightweight document title search optimized for mention picker (@mentions).
 
     Returns only id, title, and document_type - no content or metadata.
-    Results are ordered by relevance: prefix matches first, then contains matches.
+    Uses pg_trgm fuzzy search with similarity scoring for typo tolerance.
+    Results are ordered by relevance using trigram similarity scores.
 
     Args:
         search_space_id: The search space to search in. Required.
@@ -457,7 +458,7 @@ async def search_document_titles(
     Returns:
         DocumentTitleSearchResponse: Lightweight list with has_more flag (no total count).
     """
-    from sqlalchemy import case, literal
+    from sqlalchemy import desc, func, or_
 
     try:
         # Check permission for the search space
@@ -480,17 +481,29 @@ async def search_document_titles(
         if len(title.strip()) < 2:
             query = query.order_by(Document.updated_at.desc().nullslast())
         else:
-            # Apply title filter with ILIKE (uses pg_trgm index)
+            # Fuzzy search using pg_trgm similarity + ILIKE fallback
             search_term = title.strip()
-            query = query.filter(Document.title.ilike(f"%{search_term}%"))
 
-            # Order by relevance: prefix matches first, then alphabetical
-            # CASE WHEN title ILIKE 'term%' THEN 0 ELSE 1 END
-            prefix_priority = case(
-                (Document.title.ilike(f"{search_term}%"), literal(0)),
-                else_=literal(1),
+            # Similarity threshold for fuzzy matching (0.3 = ~30% trigram overlap)
+            # Lower values = more fuzzy, higher values = stricter matching
+            similarity_threshold = 0.3
+
+            # Match documents that either:
+            # 1. Have high trigram similarity (fuzzy match - handles typos)
+            # 2. Contain the exact substring (ILIKE - handles partial matches)
+            query = query.filter(
+                or_(
+                    func.similarity(Document.title, search_term) > similarity_threshold,
+                    Document.title.ilike(f"%{search_term}%"),
+                )
+            )
+
+            # Order by similarity score (descending) for best relevance ranking
+            # Higher similarity = better match = appears first
+            query = query.order_by(
+                desc(func.similarity(Document.title, search_term)),
+                Document.title,  # Alphabetical tiebreaker
             )
-            query = query.order_by(prefix_priority, Document.title)
 
         # Fetch page_size + 1 to determine has_more without COUNT query
         offset = page * page_size

From 720c13667ea0a234054e7731a119a812f8161c93 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 17 Jan 2026 21:44:10 +0530
Subject: [PATCH 03/10] refactor: update document mention picker and chat share
 button for improved performance and UX

- Replaced throttling with debouncing in DocumentMentionPicker to reduce request spam and enhance user experience.
- Updated API service methods to support request cancellation using AbortSignal.
- Simplified imports in ChatShareButton by removing unused components.
---
 .../components/new-chat/chat-share-button.tsx |  2 +-
 .../new-chat/document-mention-picker.tsx      | 88 ++++++++++---------
 .../lib/apis/documents-api.service.ts         | 13 ++-
 3 files changed, 55 insertions(+), 48 deletions(-)

diff --git a/surfsense_web/components/new-chat/chat-share-button.tsx b/surfsense_web/components/new-chat/chat-share-button.tsx
index 5b49212dc..d9e269794 100644
--- a/surfsense_web/components/new-chat/chat-share-button.tsx
+++ b/surfsense_web/components/new-chat/chat-share-button.tsx
@@ -1,7 +1,7 @@
 "use client";
 
 import { useQueryClient } from "@tanstack/react-query";
-import { Loader2, Lock, Share2, Users } from "lucide-react";
+import { Loader2, Lock, Users } from "lucide-react";
 import { useCallback, useState } from "react";
 import { toast } from "sonner";
 import { Button } from "@/components/ui/button";
diff --git a/surfsense_web/components/new-chat/document-mention-picker.tsx b/surfsense_web/components/new-chat/document-mention-picker.tsx
index 66988adcc..81c7b9a33 100644
--- a/surfsense_web/components/new-chat/document-mention-picker.tsx
+++ b/surfsense_web/components/new-chat/document-mention-picker.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { useQuery, useQueryClient } from "@tanstack/react-query";
+import { keepPreviousData, useQuery, useQueryClient } from "@tanstack/react-query";
 import { FileText } from "lucide-react";
 import {
 	forwardRef,
@@ -32,36 +32,27 @@ interface DocumentMentionPickerProps {
 
 const PAGE_SIZE = 20;
 const MIN_SEARCH_LENGTH = 2;
-const THROTTLE_MS = 200;
+const DEBOUNCE_MS = 300;
 
 /**
- * Throttle hook - fires immediately, then at most once per interval
- * Better than debounce for typeahead: user sees results updating as they type
+ * Debounce hook - waits until user stops typing before firing
+ * Better than throttle for search: reduces request spam and prevents race conditions
  */
-function useThrottled<T>(value: T, delay = THROTTLE_MS) {
-	const [throttled, setThrottled] = useState(value);
-	const lastExecuted = useRef(Date.now());
-	const timeoutRef = useRef<ReturnType<typeof setTimeout>>();
+function useDebounced<T>(value: T, delay = DEBOUNCE_MS) {
+	const [debounced, setDebounced] = useState(value);
+	const timeoutRef = useRef<ReturnType<typeof setTimeout> | undefined>(undefined);
 
 	useEffect(() => {
-		const now = Date.now();
-		const elapsed = now - lastExecuted.current;
-
-		if (elapsed >= delay) {
-			// Enough time has passed, update immediately
-			lastExecuted.current = now;
-			setThrottled(value);
-		} else {
-			// Schedule update for remaining time
-			if (timeoutRef.current) {
-				clearTimeout(timeoutRef.current);
-			}
-			timeoutRef.current = setTimeout(() => {
-				lastExecuted.current = Date.now();
-				setThrottled(value);
-			}, delay - elapsed);
+		// Clear any existing timeout
+		if (timeoutRef.current) {
+			clearTimeout(timeoutRef.current);
 		}
 
+		// Set new timeout - only fires after user stops typing for `delay` ms
+		timeoutRef.current = setTimeout(() => {
+			setDebounced(value);
+		}, delay);
+
 		return () => {
 			if (timeoutRef.current) {
 				clearTimeout(timeoutRef.current);
@@ -69,7 +60,7 @@ function useThrottled<T>(value: T, delay = THROTTLE_MS) {
 		};
 	}, [value, delay]);
 
-	return throttled;
+	return debounced;
 }
 
 export const DocumentMentionPicker = forwardRef<
@@ -81,9 +72,10 @@ export const DocumentMentionPicker = forwardRef<
 ) {
 	const queryClient = useQueryClient();
 
-	// Use external search with throttle (not debounce) for responsive feel
+	// Use external search with debounce - waits until user stops typing
+	// Reduces request spam and prevents race conditions with stale results
 	const search = externalSearch;
-	const throttledSearch = useThrottled(search, THROTTLE_MS);
+	const debouncedSearch = useDebounced(search, DEBOUNCE_MS);
 	const [highlightedIndex, setHighlightedIndex] = useState(0);
 	const itemRefs = useRef<Map<number, HTMLButtonElement>>(new Map());
 	const scrollContainerRef = useRef<HTMLDivElement>(null);
@@ -97,8 +89,8 @@ export const DocumentMentionPicker = forwardRef<
 	const [isLoadingMore, setIsLoadingMore] = useState(false);
 
 	// Check if search is long enough
-	const isSearchValid = throttledSearch.trim().length >= MIN_SEARCH_LENGTH;
-	const shouldSearch = throttledSearch.trim().length > 0;
+	const isSearchValid = debouncedSearch.trim().length >= MIN_SEARCH_LENGTH;
+	const shouldSearch = debouncedSearch.trim().length > 0;
 
 	// Prefetch first page when picker mounts - results appear instantly
 	useEffect(() => {
@@ -129,13 +121,15 @@ export const DocumentMentionPicker = forwardRef<
 	}, [searchSpaceId, queryClient]);
 
 	// Reset pagination when search or search space changes
+	// Don't clear accumulatedDocuments - let new data replace it smoothly (prevents "No documents found" flash)
 	// biome-ignore lint/correctness/useExhaustiveDependencies: intentionally reset pagination when search/space changes
 	useEffect(() => {
-		setAccumulatedDocuments([]);
+		// Keep previous documents visible while new query is fetching (smooth UX)
+		// setAccumulatedDocuments([]); // Removed to prevent flash of "No documents found"
 		setCurrentPage(0);
 		setHasMore(false);
 		setHighlightedIndex(0);
-	}, [throttledSearch, searchSpaceId]);
+	}, [debouncedSearch, searchSpaceId]);
 
 	// Query params for lightweight title search
 	const titleSearchParams = useMemo(
@@ -143,9 +137,9 @@ export const DocumentMentionPicker = forwardRef<
 			search_space_id: searchSpaceId,
 			page: 0,
 			page_size: PAGE_SIZE,
-			...(isSearchValid ? { title: throttledSearch.trim() } : {}),
+			...(isSearchValid ? { title: debouncedSearch.trim() } : {}),
 		}),
-		[searchSpaceId, throttledSearch, isSearchValid]
+		[searchSpaceId, debouncedSearch, isSearchValid]
 	);
 
 	const surfsenseDocsQueryParams = useMemo(() => {
@@ -154,25 +148,33 @@ export const DocumentMentionPicker = forwardRef<
 			page_size: PAGE_SIZE,
 		};
 		if (isSearchValid) {
-			params.title = throttledSearch.trim();
+			params.title = debouncedSearch.trim();
 		}
 		return params;
-	}, [throttledSearch, isSearchValid]);
+	}, [debouncedSearch, isSearchValid]);
 
 	// Use the new lightweight endpoint for document title search
+	// TanStack Query provides signal for automatic request cancellation
+	// keepPreviousData: shows old results while fetching new ones (no spinner flicker)
 	const { data: titleSearchResults, isLoading: isTitleSearchLoading } = useQuery({
 		queryKey: ["document-titles", titleSearchParams],
-		queryFn: () => documentsApiService.searchDocumentTitles({ queryParams: titleSearchParams }),
+		queryFn: ({ signal }) =>
+			documentsApiService.searchDocumentTitles({ queryParams: titleSearchParams }, signal),
 		staleTime: 60 * 1000, // 1 minute - shorter for fresher results
 		enabled: !!searchSpaceId && currentPage === 0 && (!shouldSearch || isSearchValid),
+		placeholderData: keepPreviousData,
 	});
 
 	// Use query for fetching first page of SurfSense docs
+	// TanStack Query provides signal for automatic request cancellation
+	// keepPreviousData: shows old results while fetching new ones (no spinner flicker)
 	const { data: surfsenseDocs, isLoading: isSurfsenseDocsLoading } = useQuery({
-		queryKey: ["surfsense-docs-mention", throttledSearch, isSearchValid],
-		queryFn: () => documentsApiService.getSurfsenseDocs({ queryParams: surfsenseDocsQueryParams }),
+		queryKey: ["surfsense-docs-mention", debouncedSearch, isSearchValid],
+		queryFn: ({ signal }) =>
+			documentsApiService.getSurfsenseDocs({ queryParams: surfsenseDocsQueryParams }, signal),
 		staleTime: 3 * 60 * 1000,
 		enabled: !shouldSearch || isSearchValid,
+		placeholderData: keepPreviousData,
 	});
 
 	// Update accumulated documents when first page loads - combine both sources
@@ -213,7 +215,7 @@ export const DocumentMentionPicker = forwardRef<
 				search_space_id: searchSpaceId,
 				page: nextPage,
 				page_size: PAGE_SIZE,
-				...(isSearchValid ? { title: throttledSearch.trim() } : {}),
+				...(isSearchValid ? { title: debouncedSearch.trim() } : {}),
 			};
 			const response: SearchDocumentTitlesResponse =
 				await documentsApiService.searchDocumentTitles({ queryParams });
@@ -226,7 +228,7 @@ export const DocumentMentionPicker = forwardRef<
 		} finally {
 			setIsLoadingMore(false);
 		}
-	}, [currentPage, hasMore, isLoadingMore, throttledSearch, searchSpaceId, isSearchValid]);
+	}, [currentPage, hasMore, isLoadingMore, debouncedSearch, searchSpaceId, isSearchValid]);
 
 	// Infinite scroll handler
 	const handleScroll = useCallback(
@@ -359,8 +361,8 @@ export const DocumentMentionPicker = forwardRef<
 				{showSearchHint ? (
 					<div className="flex flex-col items-center justify-center py-4 text-center px-4">
 						<p className="text-sm text-muted-foreground">
-							Type {MIN_SEARCH_LENGTH - throttledSearch.trim().length} more character
-							{MIN_SEARCH_LENGTH - throttledSearch.trim().length > 1 ? "s" : ""} to search
+							Type {MIN_SEARCH_LENGTH - debouncedSearch.trim().length} more character
+							{MIN_SEARCH_LENGTH - debouncedSearch.trim().length > 1 ? "s" : ""} to search
 						</p>
 					</div>
 				) : actualLoading ? (
@@ -369,7 +371,7 @@ export const DocumentMentionPicker = forwardRef<
 					</div>
 				) : actualDocuments.length === 0 ? (
 					<div className="flex flex-col items-center justify-center py-4 text-center px-4">
-						<FileText className="h-5 w-5 text-muted-foreground/55 mb-1" />
+						<FileText className="h-5 w-5 text-muted-foreground mb-1" />
 						<p className="text-sm text-muted-foreground">No documents found</p>
 					</div>
 				) : (
diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts
index daa67f6d5..03d86a253 100644
--- a/surfsense_web/lib/apis/documents-api.service.ts
+++ b/surfsense_web/lib/apis/documents-api.service.ts
@@ -166,8 +166,10 @@ class DocumentsApiService {
 	/**
 	 * Search document titles (lightweight, optimized for mention picker)
 	 * Returns only id, title, document_type - no content or metadata
+	 * @param request - The search request with query params
+	 * @param signal - Optional AbortSignal for request cancellation
 	 */
-	searchDocumentTitles = async (request: SearchDocumentTitlesRequest) => {
+	searchDocumentTitles = async (request: SearchDocumentTitlesRequest, signal?: AbortSignal) => {
 		const parsedRequest = searchDocumentTitlesRequest.safeParse(request);
 
 		if (!parsedRequest.success) {
@@ -188,7 +190,8 @@ class DocumentsApiService {
 
 		return baseApiService.get(
 			`/api/v1/documents/search/titles?${queryParams}`,
-			searchDocumentTitlesResponse
+			searchDocumentTitlesResponse,
+			{ signal }
 		);
 	};
 
@@ -258,8 +261,10 @@ class DocumentsApiService {
 
 	/**
 	 * List all Surfsense documentation documents
+	 * @param request - The request with query params
+	 * @param signal - Optional AbortSignal for request cancellation
 	 */
-	getSurfsenseDocs = async (request: GetSurfsenseDocsRequest) => {
+	getSurfsenseDocs = async (request: GetSurfsenseDocsRequest, signal?: AbortSignal) => {
 		const parsedRequest = getSurfsenseDocsRequest.safeParse(request);
 
 		if (!parsedRequest.success) {
@@ -282,7 +287,7 @@ class DocumentsApiService {
 
 		const url = `/api/v1/surfsense-docs?${queryParams}`;
 
-		return baseApiService.get(url, getSurfsenseDocsResponse);
+		return baseApiService.get(url, getSurfsenseDocsResponse, { signal });
 	};
 
 	/**

From 0b5568d7ab9a2dd7630a484ef43edcc01fec0b96 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 17 Jan 2026 22:25:40 +0530
Subject: [PATCH 04/10] feat: enhance document mention editor and picker for
 better user experience

- Added document type icons in InlineMentionEditor for improved visual context.
- Updated DocumentMentionPicker to include client-side filtering, reducing false positives in search results.
- Enhanced loading state management by incorporating fetching indicators for better UX during data retrieval.
---
 .../assistant-ui/inline-mention-editor.tsx    | 13 +++++++--
 .../new-chat/document-mention-picker.tsx      | 28 ++++++++++++++++---
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx
index f35019216..917660cf6 100644
--- a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx
+++ b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx
@@ -12,6 +12,7 @@ import {
 } from "react";
 import ReactDOMServer from "react-dom/server";
 import type { Document } from "@/contracts/types/document.types";
+import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
 import { cn } from "@/lib/utils";
 
 export interface MentionedDocument {
@@ -166,12 +167,19 @@ export const InlineMentionEditor = forwardRef<InlineMentionEditorRef, InlineMent
 				chip.setAttribute(CHIP_DOCTYPE_ATTR, doc.document_type ?? "UNKNOWN");
 				chip.contentEditable = "false";
 				chip.className =
-					"inline-flex items-center gap-0.5 mx-0.5 pl-1 pr-0.5 py-0.5 rounded bg-primary/10 text-xs font-bold text-primary border border-primary/10 select-none";
+					"inline-flex items-center gap-1 mx-0.5 pl-1 pr-0.5 py-0.5 rounded bg-primary/10 text-xs font-bold text-primary border border-primary/10 select-none";
 				chip.style.userSelect = "none";
 				chip.style.verticalAlign = "baseline";
 
+				// Add document type icon
+				const iconSpan = document.createElement("span");
+				iconSpan.className = "shrink-0 flex items-center text-muted-foreground";
+				iconSpan.innerHTML = ReactDOMServer.renderToString(
+					getConnectorIcon(doc.document_type ?? "UNKNOWN", "h-3 w-3")
+				);
+
 				const titleSpan = document.createElement("span");
-				titleSpan.className = "max-w-[80px] truncate";
+				titleSpan.className = "max-w-[120px] truncate";
 				titleSpan.textContent = doc.title;
 				titleSpan.title = doc.title;
 
@@ -197,6 +205,7 @@ export const InlineMentionEditor = forwardRef<InlineMentionEditorRef, InlineMent
 					focusAtEnd();
 				};
 
+				chip.appendChild(iconSpan);
 				chip.appendChild(titleSpan);
 				chip.appendChild(removeBtn);
 
diff --git a/surfsense_web/components/new-chat/document-mention-picker.tsx b/surfsense_web/components/new-chat/document-mention-picker.tsx
index 81c7b9a33..5b23f4432 100644
--- a/surfsense_web/components/new-chat/document-mention-picker.tsx
+++ b/surfsense_web/components/new-chat/document-mention-picker.tsx
@@ -156,7 +156,7 @@ export const DocumentMentionPicker = forwardRef<
 	// Use the new lightweight endpoint for document title search
 	// TanStack Query provides signal for automatic request cancellation
 	// keepPreviousData: shows old results while fetching new ones (no spinner flicker)
-	const { data: titleSearchResults, isLoading: isTitleSearchLoading } = useQuery({
+	const { data: titleSearchResults, isLoading: isTitleSearchLoading, isFetching: isTitleSearchFetching } = useQuery({
 		queryKey: ["document-titles", titleSearchParams],
 		queryFn: ({ signal }) =>
 			documentsApiService.searchDocumentTitles({ queryParams: titleSearchParams }, signal),
@@ -168,7 +168,7 @@ export const DocumentMentionPicker = forwardRef<
 	// Use query for fetching first page of SurfSense docs
 	// TanStack Query provides signal for automatic request cancellation
 	// keepPreviousData: shows old results while fetching new ones (no spinner flicker)
-	const { data: surfsenseDocs, isLoading: isSurfsenseDocsLoading } = useQuery({
+	const { data: surfsenseDocs, isLoading: isSurfsenseDocsLoading, isFetching: isSurfsenseDocsFetching } = useQuery({
 		queryKey: ["surfsense-docs-mention", debouncedSearch, isSearchValid],
 		queryFn: ({ signal }) =>
 			documentsApiService.getSurfsenseDocs({ queryParams: surfsenseDocsQueryParams }, signal),
@@ -177,6 +177,16 @@ export const DocumentMentionPicker = forwardRef<
 		placeholderData: keepPreviousData,
 	});
 
+	// Client-side filter to verify search term is actually in the title (handles backend fuzzy false positives)
+	const filterBySearchTerm = useCallback(
+		(docs: Pick<Document, "id" | "title" | "document_type">[]) => {
+			if (!isSearchValid) return docs; // No filtering when not searching
+			const searchLower = debouncedSearch.trim().toLowerCase();
+			return docs.filter((doc) => doc.title.toLowerCase().includes(searchLower));
+		},
+		[debouncedSearch, isSearchValid]
+	);
+
 	// Update accumulated documents when first page loads - combine both sources
 	useEffect(() => {
 		if (currentPage === 0) {
@@ -199,9 +209,10 @@ export const DocumentMentionPicker = forwardRef<
 				setHasMore(titleSearchResults.has_more);
 			}
 
-			setAccumulatedDocuments(combinedDocs);
+			// Apply client-side filter to remove fuzzy false positives
+			setAccumulatedDocuments(filterBySearchTerm(combinedDocs));
 		}
-	}, [titleSearchResults, surfsenseDocs, currentPage]);
+	}, [titleSearchResults, surfsenseDocs, currentPage, filterBySearchTerm]);
 
 	// Function to load next page using lightweight endpoint
 	const loadNextPage = useCallback(async () => {
@@ -246,10 +257,14 @@ export const DocumentMentionPicker = forwardRef<
 
 	const actualDocuments = accumulatedDocuments;
 	const actualLoading = (isTitleSearchLoading || isSurfsenseDocsLoading) && currentPage === 0;
+	const isFetchingResults = isTitleSearchFetching || isSurfsenseDocsFetching;
 
 	// Show hint when search is too short
 	const showSearchHint = shouldSearch && !isSearchValid;
 
+	// Hide popup entirely when user is searching and no documents match (only after fetch completes)
+	const hasNoSearchResults = isSearchValid && !actualLoading && !isFetchingResults && actualDocuments.length === 0;
+
 	// Split documents into SurfSense docs and user docs for grouped rendering
 	const surfsenseDocsList = useMemo(
 		() => actualDocuments.filter((doc) => doc.document_type === "SURFSENSE_DOCS"),
@@ -345,6 +360,11 @@ export const DocumentMentionPicker = forwardRef<
 		[selectableDocuments, highlightedIndex, handleSelectDocument, onDone]
 	);
 
+	// Don't show popup when user is searching and no documents match
+	if (hasNoSearchResults) {
+		return null;
+	}
+
 	return (
 		<div
 			className="flex flex-col w-[280px] sm:w-[320px] bg-popover rounded-lg"

From 685774ecb7411933e3f53fb8058bed9f0f0dbca8 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 17 Jan 2026 23:36:37 +0530
Subject: [PATCH 05/10] refactor: improve document mention editor and picker
 functionality

- Updated InlineMentionEditor to enhance visual styling of mention chips.
- Refactored DocumentMentionPicker to streamline the rendering of the document selection popover and improve keyboard navigation with smooth scrolling.
- Enhanced user experience by ensuring the mention mode remains active during search without closing the popup when no results are found.
---
 .../assistant-ui/inline-mention-editor.tsx    |  2 +-
 .../components/assistant-ui/thread.tsx        | 57 ++++++--------
 .../new-chat/document-mention-picker.tsx      | 78 +++++++++++++++----
 3 files changed, 85 insertions(+), 52 deletions(-)

diff --git a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx
index 917660cf6..570440f6a 100644
--- a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx
+++ b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx
@@ -167,7 +167,7 @@ export const InlineMentionEditor = forwardRef<InlineMentionEditorRef, InlineMent
 				chip.setAttribute(CHIP_DOCTYPE_ATTR, doc.document_type ?? "UNKNOWN");
 				chip.contentEditable = "false";
 				chip.className =
-					"inline-flex items-center gap-1 mx-0.5 pl-1 pr-0.5 py-0.5 rounded bg-primary/10 text-xs font-bold text-primary border border-primary/10 select-none";
+					"inline-flex items-center gap-1 mx-0.5 pl-1 pr-0.5 py-0.5 rounded bg-primary/10 text-xs font-bold text-primary/60 select-none";
 				chip.style.userSelect = "none";
 				chip.style.verticalAlign = "baseline";
 
diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx
index d8e0d55b9..1bdc424cc 100644
--- a/surfsense_web/components/assistant-ui/thread.tsx
+++ b/surfsense_web/components/assistant-ui/thread.tsx
@@ -392,41 +392,30 @@ const Composer: FC = () => {
 				{showDocumentPopover &&
 					typeof document !== "undefined" &&
 					createPortal(
-						<>
-							{/* Backdrop */}
-							<button
-								type="button"
-								className="fixed inset-0 cursor-default"
-								style={{ zIndex: 9998 }}
-								onClick={() => setShowDocumentPopover(false)}
-								aria-label="Close document picker"
-							/>
-							{/* Popover positioned above input */}
-							<div
-								className="fixed shadow-2xl rounded-lg border border-border overflow-hidden bg-popover"
-								style={{
-									zIndex: 9999,
-									bottom: editorContainerRef.current
-										? `${window.innerHeight - editorContainerRef.current.getBoundingClientRect().top + 8}px`
-										: "200px",
-									left: editorContainerRef.current
-										? `${editorContainerRef.current.getBoundingClientRect().left}px`
-										: "50%",
+						<div
+							className="fixed shadow-2xl rounded-lg border border-border overflow-hidden bg-popover"
+							style={{
+								zIndex: 9999,
+								bottom: editorContainerRef.current
+									? `${window.innerHeight - editorContainerRef.current.getBoundingClientRect().top + 8}px`
+									: "200px",
+								left: editorContainerRef.current
+									? `${editorContainerRef.current.getBoundingClientRect().left}px`
+									: "50%",
+							}}
+						>
+							<DocumentMentionPicker
+								ref={documentPickerRef}
+								searchSpaceId={Number(search_space_id)}
+								onSelectionChange={handleDocumentsMention}
+								onDone={() => {
+									setShowDocumentPopover(false);
+									setMentionQuery("");
 								}}
-							>
-								<DocumentMentionPicker
-									ref={documentPickerRef}
-									searchSpaceId={Number(search_space_id)}
-									onSelectionChange={handleDocumentsMention}
-									onDone={() => {
-										setShowDocumentPopover(false);
-										setMentionQuery("");
-									}}
-									initialSelectedDocuments={mentionedDocuments}
-									externalSearch={mentionQuery}
-								/>
-							</div>
-						</>,
+								initialSelectedDocuments={mentionedDocuments}
+								externalSearch={mentionQuery}
+							/>
+						</div>,
 						document.body
 					)}
 				<ComposerAction />
diff --git a/surfsense_web/components/new-chat/document-mention-picker.tsx b/surfsense_web/components/new-chat/document-mention-picker.tsx
index 5b23f4432..c14e740e1 100644
--- a/surfsense_web/components/new-chat/document-mention-picker.tsx
+++ b/surfsense_web/components/new-chat/document-mention-picker.tsx
@@ -1,7 +1,6 @@
 "use client";
 
 import { keepPreviousData, useQuery, useQueryClient } from "@tanstack/react-query";
-import { FileText } from "lucide-react";
 import {
 	forwardRef,
 	useCallback,
@@ -79,6 +78,7 @@ export const DocumentMentionPicker = forwardRef<
 	const [highlightedIndex, setHighlightedIndex] = useState(0);
 	const itemRefs = useRef<Map<number, HTMLButtonElement>>(new Map());
 	const scrollContainerRef = useRef<HTMLDivElement>(null);
+	const shouldScrollRef = useRef(false); // Track if scroll should happen (only for keyboard navigation)
 
 	// State for pagination
 	const [accumulatedDocuments, setAccumulatedDocuments] = useState<
@@ -262,7 +262,9 @@ export const DocumentMentionPicker = forwardRef<
 	// Show hint when search is too short
 	const showSearchHint = shouldSearch && !isSearchValid;
 
-	// Hide popup entirely when user is searching and no documents match (only after fetch completes)
+	// Hide popup when user is searching and no documents match (only after fetch completes)
+	// We return null instead of calling onDone() so that mention mode stays active
+	// This allows the popup to reappear when user deletes characters and results come back
 	const hasNoSearchResults = isSearchValid && !actualLoading && !isFetchingResults && actualDocuments.length === 0;
 
 	// Split documents into SurfSense docs and user docs for grouped rendering
@@ -295,12 +297,54 @@ export const DocumentMentionPicker = forwardRef<
 		[initialSelectedDocuments, onSelectionChange, onDone]
 	);
 
-	// Scroll highlighted item into view
+	// Scroll highlighted item into view - only for keyboard navigation, not mouse hover
 	useEffect(() => {
-		const item = itemRefs.current.get(highlightedIndex);
-		if (item) {
-			item.scrollIntoView({ block: "nearest", behavior: "smooth" });
+		// Only scroll if this was triggered by keyboard navigation
+		if (!shouldScrollRef.current) {
+			return;
 		}
+
+		// Reset the flag after checking
+		shouldScrollRef.current = false;
+
+		// Use requestAnimationFrame to ensure DOM is updated
+		const rafId = requestAnimationFrame(() => {
+			const item = itemRefs.current.get(highlightedIndex);
+			const container = scrollContainerRef.current;
+			
+			if (item && container) {
+				// Get item and container positions
+				const itemRect = item.getBoundingClientRect();
+				const containerRect = container.getBoundingClientRect();
+				
+				// Calculate if item is outside viewport (with some padding)
+				const padding = 8; // Small padding to ensure item is fully visible
+				const isAboveViewport = itemRect.top < containerRect.top + padding;
+				const isBelowViewport = itemRect.bottom > containerRect.bottom - padding;
+				
+				if (isAboveViewport || isBelowViewport) {
+					// Calculate scroll position to center the item in viewport
+					const itemOffsetTop = item.offsetTop;
+					const containerHeight = container.clientHeight;
+					const itemHeight = item.offsetHeight;
+					
+					// Center the item in the viewport
+					const targetScrollTop = itemOffsetTop - containerHeight / 2 + itemHeight / 2;
+					
+					// Ensure we don't scroll beyond bounds
+					const maxScrollTop = container.scrollHeight - containerHeight;
+					const clampedScrollTop = Math.max(0, Math.min(targetScrollTop, maxScrollTop));
+					
+					// Smooth scroll to target position
+					container.scrollTo({
+						top: clampedScrollTop,
+						behavior: "smooth",
+					});
+				}
+			}
+		});
+		
+		return () => cancelAnimationFrame(rafId);
 	}, [highlightedIndex]);
 
 	// Reset highlighted index when external search changes
@@ -322,9 +366,11 @@ export const DocumentMentionPicker = forwardRef<
 				}
 			},
 			moveUp: () => {
+				shouldScrollRef.current = true; // Enable scrolling for keyboard navigation
 				setHighlightedIndex((prev) => (prev > 0 ? prev - 1 : selectableDocuments.length - 1));
 			},
 			moveDown: () => {
+				shouldScrollRef.current = true; // Enable scrolling for keyboard navigation
 				setHighlightedIndex((prev) => (prev < selectableDocuments.length - 1 ? prev + 1 : 0));
 			},
 		}),
@@ -339,10 +385,12 @@ export const DocumentMentionPicker = forwardRef<
 			switch (e.key) {
 				case "ArrowDown":
 					e.preventDefault();
+					shouldScrollRef.current = true; // Enable scrolling for keyboard navigation
 					setHighlightedIndex((prev) => (prev < selectableDocuments.length - 1 ? prev + 1 : 0));
 					break;
 				case "ArrowUp":
 					e.preventDefault();
+					shouldScrollRef.current = true; // Enable scrolling for keyboard navigation
 					setHighlightedIndex((prev) => (prev > 0 ? prev - 1 : selectableDocuments.length - 1));
 					break;
 				case "Enter":
@@ -360,7 +408,8 @@ export const DocumentMentionPicker = forwardRef<
 		[selectableDocuments, highlightedIndex, handleSelectDocument, onDone]
 	);
 
-	// Don't show popup when user is searching and no documents match
+	// Hide popup visually when searching returns no results
+	// Don't call onDone() - this keeps mention mode active so popup reappears when results come back
 	if (hasNoSearchResults) {
 		return null;
 	}
@@ -389,13 +438,8 @@ export const DocumentMentionPicker = forwardRef<
 					<div className="flex items-center justify-center py-4">
 						<div className="animate-spin h-5 w-5 border-2 border-primary border-t-transparent rounded-full" />
 					</div>
-				) : actualDocuments.length === 0 ? (
-					<div className="flex flex-col items-center justify-center py-4 text-center px-4">
-						<FileText className="h-5 w-5 text-muted-foreground mb-1" />
-						<p className="text-sm text-muted-foreground">No documents found</p>
-					</div>
-				) : (
-					<div className="py-1">
+				) : actualDocuments.length > 0 ? (
+					<div className="py-1 px-2">
 						{/* SurfSense Documentation Section */}
 						{surfsenseDocsList.length > 0 && (
 							<>
@@ -427,7 +471,7 @@ export const DocumentMentionPicker = forwardRef<
 											}}
 											disabled={isAlreadySelected}
 											className={cn(
-												"w-full flex items-center gap-2 px-3 py-2 text-left transition-colors",
+												"w-full flex items-center gap-2 px-3 py-2 text-left transition-colors rounded-md",
 												isAlreadySelected ? "opacity-50 cursor-not-allowed" : "cursor-pointer",
 												isHighlighted && "bg-accent"
 											)}
@@ -475,7 +519,7 @@ export const DocumentMentionPicker = forwardRef<
 											}}
 											disabled={isAlreadySelected}
 											className={cn(
-												"w-full flex items-center gap-2 px-3 py-2 text-left transition-colors",
+												"w-full flex items-center gap-2 px-3 py-2 text-left transition-colors rounded-md",
 												isAlreadySelected ? "opacity-50 cursor-not-allowed" : "cursor-pointer",
 												isHighlighted && "bg-accent"
 											)}
@@ -499,7 +543,7 @@ export const DocumentMentionPicker = forwardRef<
 							</div>
 						)}
 					</div>
-				)}
+				) : null}
 			</div>
 		</div>
 	);

From 8c29f21acb0e7c5e1367b67677dbbb54fb5c081e Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 17 Jan 2026 23:44:53 +0530
Subject: [PATCH 06/10] chore: ran frontend linting

---
 .../new-chat/document-mention-picker.tsx      | 34 ++++++++++++-------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/surfsense_web/components/new-chat/document-mention-picker.tsx b/surfsense_web/components/new-chat/document-mention-picker.tsx
index c14e740e1..c7d4d9e84 100644
--- a/surfsense_web/components/new-chat/document-mention-picker.tsx
+++ b/surfsense_web/components/new-chat/document-mention-picker.tsx
@@ -156,7 +156,11 @@ export const DocumentMentionPicker = forwardRef<
 	// Use the new lightweight endpoint for document title search
 	// TanStack Query provides signal for automatic request cancellation
 	// keepPreviousData: shows old results while fetching new ones (no spinner flicker)
-	const { data: titleSearchResults, isLoading: isTitleSearchLoading, isFetching: isTitleSearchFetching } = useQuery({
+	const {
+		data: titleSearchResults,
+		isLoading: isTitleSearchLoading,
+		isFetching: isTitleSearchFetching,
+	} = useQuery({
 		queryKey: ["document-titles", titleSearchParams],
 		queryFn: ({ signal }) =>
 			documentsApiService.searchDocumentTitles({ queryParams: titleSearchParams }, signal),
@@ -168,7 +172,11 @@ export const DocumentMentionPicker = forwardRef<
 	// Use query for fetching first page of SurfSense docs
 	// TanStack Query provides signal for automatic request cancellation
 	// keepPreviousData: shows old results while fetching new ones (no spinner flicker)
-	const { data: surfsenseDocs, isLoading: isSurfsenseDocsLoading, isFetching: isSurfsenseDocsFetching } = useQuery({
+	const {
+		data: surfsenseDocs,
+		isLoading: isSurfsenseDocsLoading,
+		isFetching: isSurfsenseDocsFetching,
+	} = useQuery({
 		queryKey: ["surfsense-docs-mention", debouncedSearch, isSearchValid],
 		queryFn: ({ signal }) =>
 			documentsApiService.getSurfsenseDocs({ queryParams: surfsenseDocsQueryParams }, signal),
@@ -228,8 +236,9 @@ export const DocumentMentionPicker = forwardRef<
 				page_size: PAGE_SIZE,
 				...(isSearchValid ? { title: debouncedSearch.trim() } : {}),
 			};
-			const response: SearchDocumentTitlesResponse =
-				await documentsApiService.searchDocumentTitles({ queryParams });
+			const response: SearchDocumentTitlesResponse = await documentsApiService.searchDocumentTitles(
+				{ queryParams }
+			);
 
 			setAccumulatedDocuments((prev) => [...prev, ...response.items]);
 			setHasMore(response.has_more);
@@ -265,7 +274,8 @@ export const DocumentMentionPicker = forwardRef<
 	// Hide popup when user is searching and no documents match (only after fetch completes)
 	// We return null instead of calling onDone() so that mention mode stays active
 	// This allows the popup to reappear when user deletes characters and results come back
-	const hasNoSearchResults = isSearchValid && !actualLoading && !isFetchingResults && actualDocuments.length === 0;
+	const hasNoSearchResults =
+		isSearchValid && !actualLoading && !isFetchingResults && actualDocuments.length === 0;
 
 	// Split documents into SurfSense docs and user docs for grouped rendering
 	const surfsenseDocsList = useMemo(
@@ -311,30 +321,30 @@ export const DocumentMentionPicker = forwardRef<
 		const rafId = requestAnimationFrame(() => {
 			const item = itemRefs.current.get(highlightedIndex);
 			const container = scrollContainerRef.current;
-			
+
 			if (item && container) {
 				// Get item and container positions
 				const itemRect = item.getBoundingClientRect();
 				const containerRect = container.getBoundingClientRect();
-				
+
 				// Calculate if item is outside viewport (with some padding)
 				const padding = 8; // Small padding to ensure item is fully visible
 				const isAboveViewport = itemRect.top < containerRect.top + padding;
 				const isBelowViewport = itemRect.bottom > containerRect.bottom - padding;
-				
+
 				if (isAboveViewport || isBelowViewport) {
 					// Calculate scroll position to center the item in viewport
 					const itemOffsetTop = item.offsetTop;
 					const containerHeight = container.clientHeight;
 					const itemHeight = item.offsetHeight;
-					
+
 					// Center the item in the viewport
 					const targetScrollTop = itemOffsetTop - containerHeight / 2 + itemHeight / 2;
-					
+
 					// Ensure we don't scroll beyond bounds
 					const maxScrollTop = container.scrollHeight - containerHeight;
 					const clampedScrollTop = Math.max(0, Math.min(targetScrollTop, maxScrollTop));
-					
+
 					// Smooth scroll to target position
 					container.scrollTo({
 						top: clampedScrollTop,
@@ -343,7 +353,7 @@ export const DocumentMentionPicker = forwardRef<
 				}
 			}
 		});
-		
+
 		return () => cancelAnimationFrame(rafId);
 	}, [highlightedIndex]);
 

From 2f84f1b547166c55782adbd99e12ee884b965a48 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sun, 18 Jan 2026 19:43:46 +0530
Subject: [PATCH 07/10] refactor: streamline DocumentMentionPicker integration
 in Composer

- Removed unnecessary wrapper div around DocumentMentionPicker for cleaner rendering.
- Added containerStyle prop to DocumentMentionPicker for improved positioning flexibility.
- Adjusted debounce timing in DocumentMentionPicker to enhance responsiveness during user input.
---
 .../components/assistant-ui/thread.tsx        | 29 ++++++++-----------
 .../new-chat/document-mention-picker.tsx      | 12 ++++++--
 2 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx
index 1bdc424cc..7a06f82eb 100644
--- a/surfsense_web/components/assistant-ui/thread.tsx
+++ b/surfsense_web/components/assistant-ui/thread.tsx
@@ -392,10 +392,17 @@ const Composer: FC = () => {
 				{showDocumentPopover &&
 					typeof document !== "undefined" &&
 					createPortal(
-						<div
-							className="fixed shadow-2xl rounded-lg border border-border overflow-hidden bg-popover"
-							style={{
-								zIndex: 9999,
+						<DocumentMentionPicker
+							ref={documentPickerRef}
+							searchSpaceId={Number(search_space_id)}
+							onSelectionChange={handleDocumentsMention}
+							onDone={() => {
+								setShowDocumentPopover(false);
+								setMentionQuery("");
+							}}
+							initialSelectedDocuments={mentionedDocuments}
+							externalSearch={mentionQuery}
+							containerStyle={{
 								bottom: editorContainerRef.current
 									? `${window.innerHeight - editorContainerRef.current.getBoundingClientRect().top + 8}px`
 									: "200px",
@@ -403,19 +410,7 @@ const Composer: FC = () => {
 									? `${editorContainerRef.current.getBoundingClientRect().left}px`
 									: "50%",
 							}}
-						>
-							<DocumentMentionPicker
-								ref={documentPickerRef}
-								searchSpaceId={Number(search_space_id)}
-								onSelectionChange={handleDocumentsMention}
-								onDone={() => {
-									setShowDocumentPopover(false);
-									setMentionQuery("");
-								}}
-								initialSelectedDocuments={mentionedDocuments}
-								externalSearch={mentionQuery}
-							/>
-						</div>,
+						/>,
 						document.body
 					)}
 				<ComposerAction />
diff --git a/surfsense_web/components/new-chat/document-mention-picker.tsx b/surfsense_web/components/new-chat/document-mention-picker.tsx
index c7d4d9e84..7e39c6100 100644
--- a/surfsense_web/components/new-chat/document-mention-picker.tsx
+++ b/surfsense_web/components/new-chat/document-mention-picker.tsx
@@ -27,11 +27,13 @@ interface DocumentMentionPickerProps {
 	onDone: () => void;
 	initialSelectedDocuments?: Pick<Document, "id" | "title" | "document_type">[];
 	externalSearch?: string;
+	/** Positioning styles for the container */
+	containerStyle?: React.CSSProperties;
 }
 
 const PAGE_SIZE = 20;
 const MIN_SEARCH_LENGTH = 2;
-const DEBOUNCE_MS = 300;
+const DEBOUNCE_MS = 100;
 
 /**
  * Debounce hook - waits until user stops typing before firing
@@ -66,7 +68,7 @@ export const DocumentMentionPicker = forwardRef<
 	DocumentMentionPickerRef,
 	DocumentMentionPickerProps
 >(function DocumentMentionPicker(
-	{ searchSpaceId, onSelectionChange, onDone, initialSelectedDocuments = [], externalSearch = "" },
+	{ searchSpaceId, onSelectionChange, onDone, initialSelectedDocuments = [], externalSearch = "", containerStyle },
 	ref
 ) {
 	const queryClient = useQueryClient();
@@ -426,7 +428,11 @@ export const DocumentMentionPicker = forwardRef<
 
 	return (
 		<div
-			className="flex flex-col w-[280px] sm:w-[320px] bg-popover rounded-lg"
+			className="fixed shadow-2xl rounded-lg border border-border overflow-hidden bg-popover flex flex-col w-[280px] sm:w-[320px]"
+			style={{
+				zIndex: 9999,
+				...containerStyle,
+			}}
 			onKeyDown={handleKeyDown}
 			role="listbox"
 			tabIndex={-1}

From 068699b0c62b2107a8fdeb12aad111b671f30de6 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sun, 18 Jan 2026 19:47:52 +0530
Subject: [PATCH 08/10] refactor: enhance DocumentMentionPicker with
 client-side filtering for single character searches

- Implemented client-side filtering for single character searches to improve responsiveness and user experience.
- Adjusted loading state management to differentiate between single character and multi-character searches.
- Removed unnecessary search hint display for single character searches, streamlining the user interface.
---
 .../new-chat/document-mention-picker.tsx      | 45 ++++++++++++-------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/surfsense_web/components/new-chat/document-mention-picker.tsx b/surfsense_web/components/new-chat/document-mention-picker.tsx
index 7e39c6100..af3eeabb6 100644
--- a/surfsense_web/components/new-chat/document-mention-picker.tsx
+++ b/surfsense_web/components/new-chat/document-mention-picker.tsx
@@ -68,7 +68,14 @@ export const DocumentMentionPicker = forwardRef<
 	DocumentMentionPickerRef,
 	DocumentMentionPickerProps
 >(function DocumentMentionPicker(
-	{ searchSpaceId, onSelectionChange, onDone, initialSelectedDocuments = [], externalSearch = "", containerStyle },
+	{
+		searchSpaceId,
+		onSelectionChange,
+		onDone,
+		initialSelectedDocuments = [],
+		externalSearch = "",
+		containerStyle,
+	},
 	ref
 ) {
 	const queryClient = useQueryClient();
@@ -90,9 +97,11 @@ export const DocumentMentionPicker = forwardRef<
 	const [hasMore, setHasMore] = useState(false);
 	const [isLoadingMore, setIsLoadingMore] = useState(false);
 
-	// Check if search is long enough
+	// Check if search is long enough for server-side search
 	const isSearchValid = debouncedSearch.trim().length >= MIN_SEARCH_LENGTH;
 	const shouldSearch = debouncedSearch.trim().length > 0;
+	// Single character search uses client-side filtering (no API call, instant)
+	const isSingleCharSearch = debouncedSearch.trim().length === 1;
 
 	// Prefetch first page when picker mounts - results appear instantly
 	useEffect(() => {
@@ -266,18 +275,29 @@ export const DocumentMentionPicker = forwardRef<
 		[hasMore, isLoadingMore, loadNextPage]
 	);
 
-	const actualDocuments = accumulatedDocuments;
-	const actualLoading = (isTitleSearchLoading || isSurfsenseDocsLoading) && currentPage === 0;
-	const isFetchingResults = isTitleSearchFetching || isSurfsenseDocsFetching;
+	// Client-side filtered results for single character search (instant, no API call)
+	// This filters the cached/accumulated documents instead of hitting the server
+	const clientFilteredDocs = useMemo(() => {
+		if (!isSingleCharSearch) return null;
+		const searchLower = debouncedSearch.trim().toLowerCase();
+		return accumulatedDocuments.filter((doc) => doc.title.toLowerCase().includes(searchLower));
+	}, [isSingleCharSearch, debouncedSearch, accumulatedDocuments]);
 
-	// Show hint when search is too short
-	const showSearchHint = shouldSearch && !isSearchValid;
+	// Use client-side filtering for single char, server results for 2+ chars
+	const actualDocuments = isSingleCharSearch ? (clientFilteredDocs ?? []) : accumulatedDocuments;
+	const actualLoading =
+		(isTitleSearchLoading || isSurfsenseDocsLoading) && currentPage === 0 && !isSingleCharSearch;
+	const isFetchingResults =
+		(isTitleSearchFetching || isSurfsenseDocsFetching) && !isSingleCharSearch;
 
 	// Hide popup when user is searching and no documents match (only after fetch completes)
 	// We return null instead of calling onDone() so that mention mode stays active
 	// This allows the popup to reappear when user deletes characters and results come back
 	const hasNoSearchResults =
-		isSearchValid && !actualLoading && !isFetchingResults && actualDocuments.length === 0;
+		(isSearchValid || isSingleCharSearch) &&
+		!actualLoading &&
+		!isFetchingResults &&
+		actualDocuments.length === 0;
 
 	// Split documents into SurfSense docs and user docs for grouped rendering
 	const surfsenseDocsList = useMemo(
@@ -443,14 +463,7 @@ export const DocumentMentionPicker = forwardRef<
 				className="max-h-[180px] sm:max-h-[280px] overflow-y-auto"
 				onScroll={handleScroll}
 			>
-				{showSearchHint ? (
-					<div className="flex flex-col items-center justify-center py-4 text-center px-4">
-						<p className="text-sm text-muted-foreground">
-							Type {MIN_SEARCH_LENGTH - debouncedSearch.trim().length} more character
-							{MIN_SEARCH_LENGTH - debouncedSearch.trim().length > 1 ? "s" : ""} to search
-						</p>
-					</div>
-				) : actualLoading ? (
+				{actualLoading ? (
 					<div className="flex items-center justify-center py-4">
 						<div className="animate-spin h-5 w-5 border-2 border-primary border-t-transparent rounded-full" />
 					</div>

From 8654c98afe60f1ff0636c3a65c775dd56f10b65a Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sun, 18 Jan 2026 20:13:51 +0530
Subject: [PATCH 09/10] chore: update redundant comments

---
 ...pg_trgm_index_for_document_title_search.py |   1 -
 .../components/assistant-ui/thread.tsx        |  28 ++--
 .../new-chat/document-mention-picker.tsx      | 130 ++++++++----------
 3 files changed, 71 insertions(+), 88 deletions(-)

diff --git a/surfsense_backend/alembic/versions/67_add_pg_trgm_index_for_document_title_search.py b/surfsense_backend/alembic/versions/67_add_pg_trgm_index_for_document_title_search.py
index ba9cbdbac..85d34c4f2 100644
--- a/surfsense_backend/alembic/versions/67_add_pg_trgm_index_for_document_title_search.py
+++ b/surfsense_backend/alembic/versions/67_add_pg_trgm_index_for_document_title_search.py
@@ -12,7 +12,6 @@ Indexes added:
 3. idx_documents_search_space_updated - Composite for recent docs query (covering index)
 4. idx_surfsense_docs_title_trgm - GIN trigram on surfsense docs title
 
-This is critical for the document mention picker (@mentions) to scale to 10,000+ documents.
 """
 
 from collections.abc import Sequence
diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx
index 7a06f82eb..9229e25be 100644
--- a/surfsense_web/components/assistant-ui/thread.tsx
+++ b/surfsense_web/components/assistant-ui/thread.tsx
@@ -200,7 +200,7 @@ const ThreadWelcome: FC = () => {
 };
 
 const Composer: FC = () => {
-	// ---- State for document mentions (using atoms to persist across remounts) ----
+	// Document mention state (atoms persist across component remounts)
 	const [mentionedDocuments, setMentionedDocuments] = useAtom(mentionedDocumentsAtom);
 	const [showDocumentPopover, setShowDocumentPopover] = useState(false);
 	const [mentionQuery, setMentionQuery] = useState("");
@@ -212,16 +212,12 @@ const Composer: FC = () => {
 	const composerRuntime = useComposerRuntime();
 	const hasAutoFocusedRef = useRef(false);
 
-	// Check if thread is empty (new chat)
 	const isThreadEmpty = useAssistantState(({ thread }) => thread.isEmpty);
-
-	// Check if thread is currently running (streaming response)
 	const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning);
 
-	// Auto-focus editor when on new chat page
+	// Auto-focus editor on new chat page after mount
 	useEffect(() => {
 		if (isThreadEmpty && !hasAutoFocusedRef.current && editorRef.current) {
-			// Small delay to ensure the editor is fully mounted
 			const timeoutId = setTimeout(() => {
 				editorRef.current?.focus();
 				hasAutoFocusedRef.current = true;
@@ -230,7 +226,7 @@ const Composer: FC = () => {
 		}
 	}, [isThreadEmpty]);
 
-	// Sync mentioned document IDs to atom for use in chat request
+	// Sync mentioned document IDs to atom for inclusion in chat request payload
 	useEffect(() => {
 		setMentionedDocumentIds({
 			surfsense_doc_ids: mentionedDocuments
@@ -242,7 +238,7 @@ const Composer: FC = () => {
 		});
 	}, [mentionedDocuments, setMentionedDocumentIds]);
 
-	// Handle text change from inline editor - sync with assistant-ui composer
+	// Sync editor text with assistant-ui composer runtime
 	const handleEditorChange = useCallback(
 		(text: string) => {
 			composerRuntime.setText(text);
@@ -250,13 +246,13 @@ const Composer: FC = () => {
 		[composerRuntime]
 	);
 
-	// Handle @ mention trigger from inline editor
+	// Open document picker when @ mention is triggered
 	const handleMentionTrigger = useCallback((query: string) => {
 		setShowDocumentPopover(true);
 		setMentionQuery(query);
 	}, []);
 
-	// Handle mention close
+	// Close document picker and reset query
 	const handleMentionClose = useCallback(() => {
 		if (showDocumentPopover) {
 			setShowDocumentPopover(false);
@@ -264,7 +260,7 @@ const Composer: FC = () => {
 		}
 	}, [showDocumentPopover]);
 
-	// Handle keyboard navigation when popover is open
+	// Keyboard navigation for document picker (arrow keys, Enter, Escape)
 	const handleKeyDown = useCallback(
 		(e: React.KeyboardEvent) => {
 			if (showDocumentPopover) {
@@ -294,15 +290,13 @@ const Composer: FC = () => {
 		[showDocumentPopover]
 	);
 
-	// Handle submit from inline editor (Enter key)
+	// Submit message (blocked during streaming or when document picker is open)
 	const handleSubmit = useCallback(() => {
-		// Prevent sending while a response is still streaming
 		if (isThreadRunning) {
 			return;
 		}
 		if (!showDocumentPopover) {
 			composerRuntime.send();
-			// Clear the editor after sending
 			editorRef.current?.clear();
 			setMentionedDocuments([]);
 			setMentionedDocumentIds({
@@ -318,6 +312,7 @@ const Composer: FC = () => {
 		setMentionedDocumentIds,
 	]);
 
+	// Remove document from mentions and sync IDs to atom
 	const handleDocumentRemove = useCallback(
 		(docId: number, docType?: string) => {
 			setMentionedDocuments((prev) => {
@@ -336,6 +331,7 @@ const Composer: FC = () => {
 		[setMentionedDocuments, setMentionedDocumentIds]
 	);
 
+	// Add selected documents from picker, insert chips, and sync IDs to atom
 	const handleDocumentsMention = useCallback(
 		(documents: Pick<Document, "id" | "title" | "document_type">[]) => {
 			const existingKeys = new Set(mentionedDocuments.map((d) => `${d.document_type}:${d.id}`));
@@ -373,7 +369,7 @@ const Composer: FC = () => {
 		<ComposerPrimitive.Root className="aui-composer-root relative flex w-full flex-col">
 			<ComposerPrimitive.AttachmentDropzone className="aui-composer-attachment-dropzone flex w-full flex-col rounded-2xl border-input bg-muted px-1 pt-2 outline-none transition-shadow data-[dragging=true]:border-ring data-[dragging=true]:border-dashed data-[dragging=true]:bg-accent/50">
 				<ComposerAttachments />
-				{/* -------- Inline Mention Editor -------- */}
+				{/* Inline editor with @mention support */}
 				<div ref={editorContainerRef} className="aui-composer-input-wrapper px-3 pt-3 pb-6">
 					<InlineMentionEditor
 						ref={editorRef}
@@ -388,7 +384,7 @@ const Composer: FC = () => {
 					/>
 				</div>
 
-				{/* -------- Document mention popover (rendered via portal) -------- */}
+				{/* Document picker popover (portal to body for proper z-index stacking) */}
 				{showDocumentPopover &&
 					typeof document !== "undefined" &&
 					createPortal(
diff --git a/surfsense_web/components/new-chat/document-mention-picker.tsx b/surfsense_web/components/new-chat/document-mention-picker.tsx
index af3eeabb6..98bf61520 100644
--- a/surfsense_web/components/new-chat/document-mention-picker.tsx
+++ b/surfsense_web/components/new-chat/document-mention-picker.tsx
@@ -36,20 +36,19 @@ const MIN_SEARCH_LENGTH = 2;
 const DEBOUNCE_MS = 100;
 
 /**
- * Debounce hook - waits until user stops typing before firing
- * Better than throttle for search: reduces request spam and prevents race conditions
+ * Custom debounce hook that delays value updates until user input stabilizes.
+ * Preferred over throttling for search inputs as it reduces API request frequency
+ * and prevents race conditions from stale responses overtaking recent ones.
  */
 function useDebounced<T>(value: T, delay = DEBOUNCE_MS) {
 	const [debounced, setDebounced] = useState(value);
 	const timeoutRef = useRef<ReturnType<typeof setTimeout> | undefined>(undefined);
 
 	useEffect(() => {
-		// Clear any existing timeout
 		if (timeoutRef.current) {
 			clearTimeout(timeoutRef.current);
 		}
 
-		// Set new timeout - only fires after user stops typing for `delay` ms
 		timeoutRef.current = setTimeout(() => {
 			setDebounced(value);
 		}, delay);
@@ -80,16 +79,15 @@ export const DocumentMentionPicker = forwardRef<
 ) {
 	const queryClient = useQueryClient();
 
-	// Use external search with debounce - waits until user stops typing
-	// Reduces request spam and prevents race conditions with stale results
+	// Debounced search value to minimize API calls and prevent race conditions
 	const search = externalSearch;
 	const debouncedSearch = useDebounced(search, DEBOUNCE_MS);
 	const [highlightedIndex, setHighlightedIndex] = useState(0);
 	const itemRefs = useRef<Map<number, HTMLButtonElement>>(new Map());
 	const scrollContainerRef = useRef<HTMLDivElement>(null);
-	const shouldScrollRef = useRef(false); // Track if scroll should happen (only for keyboard navigation)
+	const shouldScrollRef = useRef(false); // Keyboard navigation scroll flag
 
-	// State for pagination
+	// Pagination state for infinite scroll
 	const [accumulatedDocuments, setAccumulatedDocuments] = useState<
 		Pick<Document, "id" | "title" | "document_type">[]
 	>([]);
@@ -97,13 +95,18 @@ export const DocumentMentionPicker = forwardRef<
 	const [hasMore, setHasMore] = useState(false);
 	const [isLoadingMore, setIsLoadingMore] = useState(false);
 
-	// Check if search is long enough for server-side search
+	/**
+	 * Search Strategy:
+	 * - Single character (length === 1): Client-side filtering for instant results
+	 * - Two or more characters (length >= 2): Server-side search with pg_trgm index
+	 * This hybrid approach optimizes UX by providing immediate feedback for short queries
+	 * while leveraging efficient database indexing for longer, more specific searches.
+	 */
 	const isSearchValid = debouncedSearch.trim().length >= MIN_SEARCH_LENGTH;
 	const shouldSearch = debouncedSearch.trim().length > 0;
-	// Single character search uses client-side filtering (no API call, instant)
 	const isSingleCharSearch = debouncedSearch.trim().length === 1;
 
-	// Prefetch first page when picker mounts - results appear instantly
+	// Prefetch initial data on mount for instant display when picker opens
 	useEffect(() => {
 		if (!searchSpaceId) return;
 
@@ -113,14 +116,12 @@ export const DocumentMentionPicker = forwardRef<
 			page_size: PAGE_SIZE,
 		};
 
-		// Prefetch document titles (user docs)
 		queryClient.prefetchQuery({
 			queryKey: ["document-titles", prefetchParams],
 			queryFn: () => documentsApiService.searchDocumentTitles({ queryParams: prefetchParams }),
 			staleTime: 60 * 1000,
 		});
 
-		// Prefetch SurfSense docs
 		queryClient.prefetchQuery({
 			queryKey: ["surfsense-docs-mention", "", false],
 			queryFn: () =>
@@ -131,18 +132,16 @@ export const DocumentMentionPicker = forwardRef<
 		});
 	}, [searchSpaceId, queryClient]);
 
-	// Reset pagination when search or search space changes
-	// Don't clear accumulatedDocuments - let new data replace it smoothly (prevents "No documents found" flash)
-	// biome-ignore lint/correctness/useExhaustiveDependencies: intentionally reset pagination when search/space changes
+	// Reset pagination state when search query or search space changes.
+	// Documents are not cleared to maintain visual continuity during fetches.
+	// biome-ignore lint/correctness/useExhaustiveDependencies: Intentional reset on search/space change
 	useEffect(() => {
-		// Keep previous documents visible while new query is fetching (smooth UX)
-		// setAccumulatedDocuments([]); // Removed to prevent flash of "No documents found"
 		setCurrentPage(0);
 		setHasMore(false);
 		setHighlightedIndex(0);
 	}, [debouncedSearch, searchSpaceId]);
 
-	// Query params for lightweight title search
+	// Query parameters for lightweight title search endpoint
 	const titleSearchParams = useMemo(
 		() => ({
 			search_space_id: searchSpaceId,
@@ -164,9 +163,12 @@ export const DocumentMentionPicker = forwardRef<
 		return params;
 	}, [debouncedSearch, isSearchValid]);
 
-	// Use the new lightweight endpoint for document title search
-	// TanStack Query provides signal for automatic request cancellation
-	// keepPreviousData: shows old results while fetching new ones (no spinner flicker)
+	/**
+	 * TanStack Query for document title search.
+	 * - Uses AbortSignal for automatic request cancellation on query key changes
+	 * - placeholderData: keepPreviousData maintains UI stability during fetches
+	 * - Only triggers server-side search when isSearchValid (2+ characters)
+	 */
 	const {
 		data: titleSearchResults,
 		isLoading: isTitleSearchLoading,
@@ -175,14 +177,16 @@ export const DocumentMentionPicker = forwardRef<
 		queryKey: ["document-titles", titleSearchParams],
 		queryFn: ({ signal }) =>
 			documentsApiService.searchDocumentTitles({ queryParams: titleSearchParams }, signal),
-		staleTime: 60 * 1000, // 1 minute - shorter for fresher results
+		staleTime: 60 * 1000,
 		enabled: !!searchSpaceId && currentPage === 0 && (!shouldSearch || isSearchValid),
 		placeholderData: keepPreviousData,
 	});
 
-	// Use query for fetching first page of SurfSense docs
-	// TanStack Query provides signal for automatic request cancellation
-	// keepPreviousData: shows old results while fetching new ones (no spinner flicker)
+	/**
+	 * TanStack Query for SurfSense documentation.
+	 * - Uses AbortSignal for automatic request cancellation
+	 * - placeholderData: keepPreviousData prevents UI flicker during refetches
+	 */
 	const {
 		data: surfsenseDocs,
 		isLoading: isSurfsenseDocsLoading,
@@ -196,7 +200,7 @@ export const DocumentMentionPicker = forwardRef<
 		placeholderData: keepPreviousData,
 	});
 
-	// Client-side filter to verify search term is actually in the title (handles backend fuzzy false positives)
+	// Post-fetch filter to eliminate false positives from backend fuzzy matching
 	const filterBySearchTerm = useCallback(
 		(docs: Pick<Document, "id" | "title" | "document_type">[]) => {
 			if (!isSearchValid) return docs; // No filtering when not searching
@@ -206,12 +210,12 @@ export const DocumentMentionPicker = forwardRef<
 		[debouncedSearch, isSearchValid]
 	);
 
-	// Update accumulated documents when first page loads - combine both sources
+	// Combine and update document list when first page data arrives
 	useEffect(() => {
 		if (currentPage === 0) {
 			const combinedDocs: Pick<Document, "id" | "title" | "document_type">[] = [];
 
-			// Add SurfSense docs first (they appear at top)
+			// SurfSense docs displayed first in the list
 			if (surfsenseDocs?.items) {
 				for (const doc of surfsenseDocs.items) {
 					combinedDocs.push({
@@ -222,18 +226,16 @@ export const DocumentMentionPicker = forwardRef<
 				}
 			}
 
-			// Add regular documents from lightweight endpoint
 			if (titleSearchResults?.items) {
 				combinedDocs.push(...titleSearchResults.items);
 				setHasMore(titleSearchResults.has_more);
 			}
 
-			// Apply client-side filter to remove fuzzy false positives
 			setAccumulatedDocuments(filterBySearchTerm(combinedDocs));
 		}
 	}, [titleSearchResults, surfsenseDocs, currentPage, filterBySearchTerm]);
 
-	// Function to load next page using lightweight endpoint
+	// Load next page for infinite scroll pagination
 	const loadNextPage = useCallback(async () => {
 		if (isLoadingMore || !hasMore) return;
 
@@ -261,13 +263,12 @@ export const DocumentMentionPicker = forwardRef<
 		}
 	}, [currentPage, hasMore, isLoadingMore, debouncedSearch, searchSpaceId, isSearchValid]);
 
-	// Infinite scroll handler
+	// Trigger pagination when user scrolls near the bottom (50px threshold)
 	const handleScroll = useCallback(
 		(e: React.UIEvent<HTMLDivElement>) => {
 			const target = e.currentTarget;
 			const scrollBottom = target.scrollHeight - target.scrollTop - target.clientHeight;
 
-			// Load more when within 50px of bottom
 			if (scrollBottom < 50 && hasMore && !isLoadingMore) {
 				loadNextPage();
 			}
@@ -275,31 +276,32 @@ export const DocumentMentionPicker = forwardRef<
 		[hasMore, isLoadingMore, loadNextPage]
 	);
 
-	// Client-side filtered results for single character search (instant, no API call)
-	// This filters the cached/accumulated documents instead of hitting the server
+	/**
+	 * Client-side filtering for single character searches.
+	 * Filters cached documents locally for instant feedback without additional API calls.
+	 * Server-side search is reserved for 2+ character queries to leverage database indexing.
+	 */
 	const clientFilteredDocs = useMemo(() => {
 		if (!isSingleCharSearch) return null;
 		const searchLower = debouncedSearch.trim().toLowerCase();
 		return accumulatedDocuments.filter((doc) => doc.title.toLowerCase().includes(searchLower));
 	}, [isSingleCharSearch, debouncedSearch, accumulatedDocuments]);
 
-	// Use client-side filtering for single char, server results for 2+ chars
+	// Select data source based on search length: client-filtered for single char, server results for 2+
 	const actualDocuments = isSingleCharSearch ? (clientFilteredDocs ?? []) : accumulatedDocuments;
 	const actualLoading =
 		(isTitleSearchLoading || isSurfsenseDocsLoading) && currentPage === 0 && !isSingleCharSearch;
 	const isFetchingResults =
 		(isTitleSearchFetching || isSurfsenseDocsFetching) && !isSingleCharSearch;
 
-	// Hide popup when user is searching and no documents match (only after fetch completes)
-	// We return null instead of calling onDone() so that mention mode stays active
-	// This allows the popup to reappear when user deletes characters and results come back
+	// Determine if search yields no results (hide popup but keep mention mode active for recovery)
 	const hasNoSearchResults =
 		(isSearchValid || isSingleCharSearch) &&
 		!actualLoading &&
 		!isFetchingResults &&
 		actualDocuments.length === 0;
 
-	// Split documents into SurfSense docs and user docs for grouped rendering
+	// Partition documents by type for grouped UI rendering
 	const surfsenseDocsList = useMemo(
 		() => actualDocuments.filter((doc) => doc.document_type === "SURFSENSE_DOCS"),
 		[actualDocuments]
@@ -309,13 +311,13 @@ export const DocumentMentionPicker = forwardRef<
 		[actualDocuments]
 	);
 
-	// Track already selected documents using unique key (document_type:id) to avoid ID collisions
+	// Track selected documents with composite key (document_type:id) to prevent cross-type ID collisions
 	const selectedKeys = useMemo(
 		() => new Set(initialSelectedDocuments.map((d) => `${d.document_type}:${d.id}`)),
 		[initialSelectedDocuments]
 	);
 
-	// Filter out already selected documents for navigation
+	// Exclude already-selected documents from keyboard navigation
 	const selectableDocuments = useMemo(
 		() => actualDocuments.filter((doc) => !selectedKeys.has(`${doc.document_type}:${doc.id}`)),
 		[actualDocuments, selectedKeys]
@@ -329,45 +331,32 @@ export const DocumentMentionPicker = forwardRef<
 		[initialSelectedDocuments, onSelectionChange, onDone]
 	);
 
-	// Scroll highlighted item into view - only for keyboard navigation, not mouse hover
+	// Auto-scroll highlighted item into view (keyboard navigation only, not mouse hover)
 	useEffect(() => {
-		// Only scroll if this was triggered by keyboard navigation
 		if (!shouldScrollRef.current) {
 			return;
 		}
-
-		// Reset the flag after checking
 		shouldScrollRef.current = false;
 
-		// Use requestAnimationFrame to ensure DOM is updated
 		const rafId = requestAnimationFrame(() => {
 			const item = itemRefs.current.get(highlightedIndex);
 			const container = scrollContainerRef.current;
 
 			if (item && container) {
-				// Get item and container positions
 				const itemRect = item.getBoundingClientRect();
 				const containerRect = container.getBoundingClientRect();
-
-				// Calculate if item is outside viewport (with some padding)
-				const padding = 8; // Small padding to ensure item is fully visible
+				const padding = 8;
 				const isAboveViewport = itemRect.top < containerRect.top + padding;
 				const isBelowViewport = itemRect.bottom > containerRect.bottom - padding;
 
 				if (isAboveViewport || isBelowViewport) {
-					// Calculate scroll position to center the item in viewport
 					const itemOffsetTop = item.offsetTop;
 					const containerHeight = container.clientHeight;
 					const itemHeight = item.offsetHeight;
-
-					// Center the item in the viewport
 					const targetScrollTop = itemOffsetTop - containerHeight / 2 + itemHeight / 2;
-
-					// Ensure we don't scroll beyond bounds
 					const maxScrollTop = container.scrollHeight - containerHeight;
 					const clampedScrollTop = Math.max(0, Math.min(targetScrollTop, maxScrollTop));
 
-					// Smooth scroll to target position
 					container.scrollTo({
 						top: clampedScrollTop,
 						behavior: "smooth",
@@ -379,7 +368,7 @@ export const DocumentMentionPicker = forwardRef<
 		return () => cancelAnimationFrame(rafId);
 	}, [highlightedIndex]);
 
-	// Reset highlighted index when external search changes
+	// Reset highlight position when search query changes
 	const prevSearchRef = useRef(search);
 	if (prevSearchRef.current !== search) {
 		prevSearchRef.current = search;
@@ -388,7 +377,7 @@ export const DocumentMentionPicker = forwardRef<
 		}
 	}
 
-	// Expose methods to parent via ref
+	// Expose navigation and selection methods to parent component via ref
 	useImperativeHandle(
 		ref,
 		() => ({
@@ -398,18 +387,18 @@ export const DocumentMentionPicker = forwardRef<
 				}
 			},
 			moveUp: () => {
-				shouldScrollRef.current = true; // Enable scrolling for keyboard navigation
+				shouldScrollRef.current = true;
 				setHighlightedIndex((prev) => (prev > 0 ? prev - 1 : selectableDocuments.length - 1));
 			},
 			moveDown: () => {
-				shouldScrollRef.current = true; // Enable scrolling for keyboard navigation
+				shouldScrollRef.current = true;
 				setHighlightedIndex((prev) => (prev < selectableDocuments.length - 1 ? prev + 1 : 0));
 			},
 		}),
 		[selectableDocuments, highlightedIndex, handleSelectDocument]
 	);
 
-	// Handle keyboard navigation
+	// Keyboard navigation handler for arrow keys, Enter, and Escape
 	const handleKeyDown = useCallback(
 		(e: React.KeyboardEvent) => {
 			if (selectableDocuments.length === 0) return;
@@ -417,12 +406,12 @@ export const DocumentMentionPicker = forwardRef<
 			switch (e.key) {
 				case "ArrowDown":
 					e.preventDefault();
-					shouldScrollRef.current = true; // Enable scrolling for keyboard navigation
+					shouldScrollRef.current = true;
 					setHighlightedIndex((prev) => (prev < selectableDocuments.length - 1 ? prev + 1 : 0));
 					break;
 				case "ArrowUp":
 					e.preventDefault();
-					shouldScrollRef.current = true; // Enable scrolling for keyboard navigation
+					shouldScrollRef.current = true;
 					setHighlightedIndex((prev) => (prev > 0 ? prev - 1 : selectableDocuments.length - 1));
 					break;
 				case "Enter":
@@ -440,8 +429,7 @@ export const DocumentMentionPicker = forwardRef<
 		[selectableDocuments, highlightedIndex, handleSelectDocument, onDone]
 	);
 
-	// Hide popup visually when searching returns no results
-	// Don't call onDone() - this keeps mention mode active so popup reappears when results come back
+	// Return null when no results; mention mode remains active for result recovery on backspace
 	if (hasNoSearchResults) {
 		return null;
 	}
@@ -457,7 +445,7 @@ export const DocumentMentionPicker = forwardRef<
 			role="listbox"
 			tabIndex={-1}
 		>
-			{/* Document List - Shows max 5 items on mobile, 7-8 items on desktop */}
+			{/* Scrollable document list with responsive height */}
 			<div
 				ref={scrollContainerRef}
 				className="max-h-[180px] sm:max-h-[280px] overflow-y-auto"
@@ -469,7 +457,7 @@ export const DocumentMentionPicker = forwardRef<
 					</div>
 				) : actualDocuments.length > 0 ? (
 					<div className="py-1 px-2">
-						{/* SurfSense Documentation Section */}
+						{/* SurfSense Documentation */}
 						{surfsenseDocsList.length > 0 && (
 							<>
 								<div className="px-3 py-2 text-xs font-bold text-muted-foreground/55">
@@ -517,7 +505,7 @@ export const DocumentMentionPicker = forwardRef<
 							</>
 						)}
 
-						{/* User Documents Section */}
+						{/* User Documents */}
 						{userDocsList.length > 0 && (
 							<>
 								<div className="px-3 py-2 text-xs font-bold text-muted-foreground/55">
@@ -565,7 +553,7 @@ export const DocumentMentionPicker = forwardRef<
 							</>
 						)}
 
-						{/* Loading indicator for additional pages */}
+						{/* Pagination loading indicator */}
 						{isLoadingMore && (
 							<div className="flex items-center justify-center py-2">
 								<div className="animate-spin h-4 w-4 border-2 border-primary border-t-transparent rounded-full" />

From b158ddd083c22e0a7275c948bd67b99387110a8f Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sun, 18 Jan 2026 22:04:28 +0530
Subject: [PATCH 10/10] refactor: simplify loading state management in
 DocumentMentionPicker

- Consolidated loading state checks to improve clarity and performance.
- Updated logic to hide the results popup when no documents are available, regardless of fetch state.
- Removed redundant variables related to fetching states, streamlining the component's functionality.
---
 .../new-chat/document-mention-picker.tsx      | 33 ++++++-------------
 1 file changed, 10 insertions(+), 23 deletions(-)

diff --git a/surfsense_web/components/new-chat/document-mention-picker.tsx b/surfsense_web/components/new-chat/document-mention-picker.tsx
index 98bf61520..5c4c9bc61 100644
--- a/surfsense_web/components/new-chat/document-mention-picker.tsx
+++ b/surfsense_web/components/new-chat/document-mention-picker.tsx
@@ -169,11 +169,7 @@ export const DocumentMentionPicker = forwardRef<
 	 * - placeholderData: keepPreviousData maintains UI stability during fetches
 	 * - Only triggers server-side search when isSearchValid (2+ characters)
 	 */
-	const {
-		data: titleSearchResults,
-		isLoading: isTitleSearchLoading,
-		isFetching: isTitleSearchFetching,
-	} = useQuery({
+	const { data: titleSearchResults, isLoading: isTitleSearchLoading } = useQuery({
 		queryKey: ["document-titles", titleSearchParams],
 		queryFn: ({ signal }) =>
 			documentsApiService.searchDocumentTitles({ queryParams: titleSearchParams }, signal),
@@ -187,11 +183,7 @@ export const DocumentMentionPicker = forwardRef<
 	 * - Uses AbortSignal for automatic request cancellation
 	 * - placeholderData: keepPreviousData prevents UI flicker during refetches
 	 */
-	const {
-		data: surfsenseDocs,
-		isLoading: isSurfsenseDocsLoading,
-		isFetching: isSurfsenseDocsFetching,
-	} = useQuery({
+	const { data: surfsenseDocs, isLoading: isSurfsenseDocsLoading } = useQuery({
 		queryKey: ["surfsense-docs-mention", debouncedSearch, isSearchValid],
 		queryFn: ({ signal }) =>
 			documentsApiService.getSurfsenseDocs({ queryParams: surfsenseDocsQueryParams }, signal),
@@ -289,18 +281,12 @@ export const DocumentMentionPicker = forwardRef<
 
 	// Select data source based on search length: client-filtered for single char, server results for 2+
 	const actualDocuments = isSingleCharSearch ? (clientFilteredDocs ?? []) : accumulatedDocuments;
+	// Only show loading spinner on initial load (no documents yet), not during subsequent searches
 	const actualLoading =
-		(isTitleSearchLoading || isSurfsenseDocsLoading) && currentPage === 0 && !isSingleCharSearch;
-	const isFetchingResults =
-		(isTitleSearchFetching || isSurfsenseDocsFetching) && !isSingleCharSearch;
-
-	// Determine if search yields no results (hide popup but keep mention mode active for recovery)
-	const hasNoSearchResults =
-		(isSearchValid || isSingleCharSearch) &&
-		!actualLoading &&
-		!isFetchingResults &&
-		actualDocuments.length === 0;
-
+		(isTitleSearchLoading || isSurfsenseDocsLoading) &&
+		currentPage === 0 &&
+		!isSingleCharSearch &&
+		accumulatedDocuments.length === 0;
 	// Partition documents by type for grouped UI rendering
 	const surfsenseDocsList = useMemo(
 		() => actualDocuments.filter((doc) => doc.document_type === "SURFSENSE_DOCS"),
@@ -429,8 +415,9 @@ export const DocumentMentionPicker = forwardRef<
 		[selectableDocuments, highlightedIndex, handleSelectDocument, onDone]
 	);
 
-	// Return null when no results; mention mode remains active for result recovery on backspace
-	if (hasNoSearchResults) {
+	// Hide popup when there are no documents to display (regardless of fetch state)
+	// Search continues in background; popup reappears when results arrive
+	if (!actualLoading && actualDocuments.length === 0) {
 		return null;
 	}