diff --git a/surfsense_web/components/chat/ChatInputGroup.tsx b/surfsense_web/components/chat/ChatInputGroup.tsx index 866ac7b0f..dfce22227 100644 --- a/surfsense_web/components/chat/ChatInputGroup.tsx +++ b/surfsense_web/components/chat/ChatInputGroup.tsx @@ -25,7 +25,7 @@ import { SelectValue, } from "@/components/ui/select"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { type Document, useDocuments } from "@/hooks/use-documents"; +import type { Document } from "@/hooks/use-documents"; import { useLLMConfigs, useLLMPreferences } from "@/hooks/use-llm-configs"; import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; @@ -40,20 +40,9 @@ const DocumentSelector = React.memo( const { search_space_id } = useParams(); const [isOpen, setIsOpen] = useState(false); - const { documents, loading, isLoaded, fetchDocuments } = useDocuments(Number(search_space_id), { - lazy: true, - pageSize: -1, // Fetch all documents with large page size - }); - - const handleOpenChange = useCallback( - (open: boolean) => { - setIsOpen(open); - if (open && !isLoaded) { - fetchDocuments(); - } - }, - [fetchDocuments, isLoaded] - ); + const handleOpenChange = useCallback((open: boolean) => { + setIsOpen(open); + }, []); const handleSelectionChange = useCallback( (documents: Document[]) => { @@ -91,21 +80,12 @@ const DocumentSelector = React.memo(
- {loading ? ( -
-
-
-

Loading documents...

-
-
- ) : isLoaded ? ( - - ) : null} +
diff --git a/surfsense_web/components/chat/DocumentsDataTable.tsx b/surfsense_web/components/chat/DocumentsDataTable.tsx index 79b6216b3..ae647f341 100644 --- a/surfsense_web/components/chat/DocumentsDataTable.tsx +++ b/surfsense_web/components/chat/DocumentsDataTable.tsx @@ -2,21 +2,18 @@ import { type ColumnDef, - type ColumnFiltersState, flexRender, getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, type SortingState, useReactTable, - type VisibilityState, } from "@tanstack/react-table"; -import { ArrowUpDown, Calendar, FileText, Search } from "lucide-react"; -import { useEffect, useMemo, useState } from "react"; +import { ArrowUpDown, Calendar, FileText, Filter, Search } from "lucide-react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { Select, SelectContent, @@ -32,26 +29,24 @@ import { TableHeader, TableRow, } from "@/components/ui/table"; -import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import type { Document, DocumentType } from "@/hooks/use-documents"; +import { type Document, type DocumentType, useDocuments } from "@/hooks/use-documents"; interface DocumentsDataTableProps { - documents: Document[]; + searchSpaceId: number; onSelectionChange: (documents: Document[]) => void; onDone: () => void; initialSelectedDocuments?: Document[]; } -// Combine EnumConnectorName with additional document types -const DOCUMENT_TYPES: (string | "ALL")[] = [ - "ALL", - "FILE", - "EXTENSION", - "CRAWLED_URL", - "YOUTUBE_VIDEO", - ...Object.values(EnumConnectorName), -]; +function useDebounced(value: T, delay = 300) { + const [debounced, setDebounced] = useState(value); + useEffect(() => { + const t = setTimeout(() => setDebounced(value), delay); + return () => clearTimeout(t); + }, [value, delay]); + return debounced; +} const columns: ColumnDef[] = [ { @@ -177,93 +172,193 @@ const columns: ColumnDef[] = [ ]; export function DocumentsDataTable({ - documents, + searchSpaceId, onSelectionChange, onDone, initialSelectedDocuments = [], }: DocumentsDataTableProps) { const [sorting, setSorting] = useState([]); - const [columnFilters, setColumnFilters] = useState([]); - const [columnVisibility, setColumnVisibility] = useState({}); - const [documentTypeFilter, setDocumentTypeFilter] = useState("ALL"); + const [search, setSearch] = useState(""); + const debouncedSearch = useDebounced(search, 300); + const [documentTypeFilter, setDocumentTypeFilter] = useState([]); + const [pageIndex, setPageIndex] = useState(0); + const [pageSize, setPageSize] = useState(10); + const [typeCounts, setTypeCounts] = useState>({}); + + // Use server-side pagination, search, and filtering + const { documents, total, loading, fetchDocuments, searchDocuments, getDocumentTypeCounts } = + useDocuments(searchSpaceId, { + page: pageIndex, + pageSize: pageSize, + }); + + // Fetch document type counts on mount + useEffect(() => { + if (searchSpaceId && getDocumentTypeCounts) { + getDocumentTypeCounts().then(setTypeCounts); + } + }, [searchSpaceId, getDocumentTypeCounts]); + + // Refetch when pagination changes or when search/filters change + useEffect(() => { + if (searchSpaceId) { + if (debouncedSearch.trim()) { + searchDocuments?.( + debouncedSearch, + pageIndex, + pageSize, + documentTypeFilter.length > 0 ? documentTypeFilter : undefined + ); + } else { + fetchDocuments?.( + pageIndex, + pageSize, + documentTypeFilter.length > 0 ? documentTypeFilter : undefined + ); + } + } + }, [ + pageIndex, + pageSize, + debouncedSearch, + documentTypeFilter, + searchSpaceId, + fetchDocuments, + searchDocuments, + ]); // Memoize initial row selection to prevent infinite loops const initialRowSelection = useMemo(() => { - if (!documents.length || !initialSelectedDocuments.length) return {}; + if (!initialSelectedDocuments.length) return {}; const selection: Record = {}; initialSelectedDocuments.forEach((selectedDoc) => { selection[selectedDoc.id] = true; }); return selection; - }, [documents, initialSelectedDocuments]); + }, [initialSelectedDocuments]); - const [rowSelection, setRowSelection] = useState>({}); + const [rowSelection, setRowSelection] = useState>( + () => initialRowSelection + ); - // Only update row selection when initialRowSelection actually changes and is not empty + // Maintain a separate state for actually selected documents (across all pages) + const [selectedDocumentsMap, setSelectedDocumentsMap] = useState>(() => { + const map = new Map(); + initialSelectedDocuments.forEach((doc) => map.set(doc.id, doc)); + return map; + }); + + // Track the last notified selection to avoid redundant parent calls + const lastNotifiedSelection = useRef(""); + + // Update row selection only when initialSelectedDocuments changes (not rowSelection itself) useEffect(() => { - const hasChanges = JSON.stringify(rowSelection) !== JSON.stringify(initialRowSelection); - if (hasChanges && Object.keys(initialRowSelection).length > 0) { - setRowSelection(initialRowSelection); - } - }, [initialRowSelection]); + const initialKeys = Object.keys(initialRowSelection); + if (initialKeys.length === 0) return; - // Initialize row selection on mount + const currentKeys = Object.keys(rowSelection); + // Quick length check before expensive comparison + if (currentKeys.length === initialKeys.length) { + // Check if all keys match (order doesn't matter for Sets) + const hasAllKeys = initialKeys.every((key) => rowSelection[key]); + if (hasAllKeys) return; + } + + setRowSelection(initialRowSelection); + }, [initialRowSelection]); // Remove rowSelection from dependencies to prevent loop + + // Update the selected documents map when row selection changes useEffect(() => { - if (Object.keys(rowSelection).length === 0 && Object.keys(initialRowSelection).length > 0) { - setRowSelection(initialRowSelection); - } - }, []); + if (!documents || documents.length === 0) return; - const filteredDocuments = useMemo(() => { - if (documentTypeFilter === "ALL") return documents; - return documents.filter((doc) => doc.document_type === documentTypeFilter); - }, [documents, documentTypeFilter]); + setSelectedDocumentsMap((prev) => { + const newMap = new Map(prev); + let hasChanges = false; + + // Process only current page documents + for (const doc of documents) { + const docId = doc.id; + const isSelected = rowSelection[docId.toString()]; + const wasInMap = newMap.has(docId); + + if (isSelected && !wasInMap) { + newMap.set(docId, doc); + hasChanges = true; + } else if (!isSelected && wasInMap) { + newMap.delete(docId); + hasChanges = true; + } + } + + // Return same reference if no changes to avoid unnecessary re-renders + return hasChanges ? newMap : prev; + }); + }, [rowSelection, documents]); + + // Memoize selected documents array + const selectedDocumentsArray = useMemo(() => { + return Array.from(selectedDocumentsMap.values()); + }, [selectedDocumentsMap]); + + // Notify parent of selection changes only when content actually changes + useEffect(() => { + // Create a stable string representation for comparison + const selectionKey = selectedDocumentsArray + .map((d) => d.id) + .sort() + .join(","); + + // Skip if selection hasn't actually changed + if (selectionKey === lastNotifiedSelection.current) return; + + lastNotifiedSelection.current = selectionKey; + onSelectionChange(selectedDocumentsArray); + }, [selectedDocumentsArray, onSelectionChange]); const table = useReactTable({ - data: filteredDocuments, + data: documents || [], columns, getRowId: (row) => row.id.toString(), onSortingChange: setSorting, - onColumnFiltersChange: setColumnFilters, getCoreRowModel: getCoreRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - onColumnVisibilityChange: setColumnVisibility, onRowSelectionChange: setRowSelection, - initialState: { pagination: { pageSize: 10 } }, - state: { sorting, columnFilters, columnVisibility, rowSelection }, + manualPagination: true, + pageCount: Math.ceil(total / pageSize), + state: { sorting, rowSelection, pagination: { pageIndex, pageSize } }, }); - useEffect(() => { - const selectedRows = table.getFilteredSelectedRowModel().rows; - const selectedDocuments = selectedRows.map((row) => row.original); - onSelectionChange(selectedDocuments); - }, [rowSelection, onSelectionChange, table]); + const handleClearAll = useCallback(() => { + setRowSelection({}); + setSelectedDocumentsMap(new Map()); + }, []); - const handleClearAll = () => setRowSelection({}); - - const handleSelectPage = () => { + const handleSelectPage = useCallback(() => { const currentPageRows = table.getRowModel().rows; const newSelection = { ...rowSelection }; currentPageRows.forEach((row) => { newSelection[row.id] = true; }); setRowSelection(newSelection); - }; + }, [table, rowSelection]); - const handleSelectAllFiltered = () => { - const allFilteredRows = table.getFilteredRowModel().rows; - const newSelection: Record = {}; - allFilteredRows.forEach((row) => { - newSelection[row.id] = true; + const handleToggleType = useCallback((type: string, checked: boolean) => { + setDocumentTypeFilter((prev) => { + if (checked) { + return [...prev, type]; + } + return prev.filter((t) => t !== type); }); - setRowSelection(newSelection); - }; + setPageIndex(0); // Reset to first page when filter changes + }, []); - const selectedCount = table.getFilteredSelectedRowModel().rows.length; - const totalFiltered = table.getFilteredRowModel().rows.length; + const selectedCount = selectedDocumentsMap.size; + + // Get available document types from type counts (memoized) + const availableTypes = useMemo(() => { + const types = Object.keys(typeCounts); + return types.length > 0 ? types.sort() : []; + }, [typeCounts]); return (
@@ -275,33 +370,70 @@ export function DocumentsDataTable({ table.getColumn("title")?.setFilterValue(event.target.value)} + value={search} + onChange={(event) => { + setSearch(event.target.value); + setPageIndex(0); // Reset to first page on search + }} className="pl-10 text-sm" />
- + + + + + +
+
Filter by Type
+
+ {availableTypes.map((type) => ( +
+ handleToggleType(type, !!checked)} + /> + +
+ ))} +
+ {documentTypeFilter.length > 0 && ( + + )} +
+
+
{/* Action Controls Row */}
- {selectedCount} of {totalFiltered} selected + {selectedCount} selected {loading && "ยท Loading..."}
@@ -319,25 +451,28 @@ export function DocumentsDataTable({ size="sm" onClick={handleSelectPage} className="text-xs sm:text-sm" + disabled={loading} > Select Page - - + + {pageSize} per page + + + {[10, 25, 50, 100].map((size) => ( + + {size} per page + + ))} + +
{/* Footer Pagination */}
- Showing {table.getState().pagination.pageIndex * table.getState().pagination.pageSize + 1}{" "} - to{" "} - {Math.min( - (table.getState().pagination.pageIndex + 1) * table.getState().pagination.pageSize, - table.getFilteredRowModel().rows.length - )}{" "} - of {table.getFilteredRowModel().rows.length} documents + Showing {pageIndex * pageSize + 1} to {Math.min((pageIndex + 1) * pageSize, total)} of{" "} + {total} documents
Page - {table.getState().pagination.pageIndex + 1} + {pageIndex + 1} of - {table.getPageCount()} + {Math.ceil(total / pageSize)}