diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index 4a69a7533..8387ad9b4 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -26,10 +26,8 @@ export default function DocumentsTable() { const params = useParams(); const searchSpaceId = Number(params.search_space_id); - const { documents, loading, error, refreshDocuments, deleteDocument } = - useDocuments(searchSpaceId); - - const [data, setData] = useState([]); + const [pageIndex, setPageIndex] = useState(0); + const [pageSize, setPageSize] = useState(100); // Increased page size for better performance const [search, setSearch] = useState(""); const debouncedSearch = useDebounced(search, 250); const [activeTypes, setActiveTypes] = useState([]); @@ -39,16 +37,28 @@ export default function DocumentsTable() { content: true, created_at: true, }); - const [pageIndex, setPageIndex] = useState(0); - const [pageSize, setPageSize] = useState(10); const [sortKey, setSortKey] = useState("title"); const [sortDesc, setSortDesc] = useState(false); const [selectedIds, setSelectedIds] = useState>(new Set()); + // Calculate skip value for pagination + const skip = pageIndex * pageSize; + + const { documents, loading, error, refreshDocuments, deleteDocument } = useDocuments( + searchSpaceId, + false, + skip, + pageSize + ); + + const [data, setData] = useState([]); + useEffect(() => { if (documents) setData(documents as Document[]); }, [documents]); + // For server-side pagination, we apply client-side filtering to the current page data + // Note: This is a simplified approach. For production, you might want to implement server-side filtering const filtered = useMemo(() => { let result = data; if (debouncedSearch.trim()) { @@ -61,16 +71,21 @@ export default function DocumentsTable() { return result; }, [data, debouncedSearch, activeTypes]); - const total = filtered.length; - const pageStart = pageIndex * pageSize; - const pageEnd = Math.min(pageStart + pageSize, total); - const pageDocs = filtered.slice(pageStart, pageEnd); + // For server-side pagination, we show the filtered results from current page + // Total count is estimated based on current page size (this could be improved with a count endpoint) + const pageDocs = filtered; + const total = pageSize * (pageIndex + 1); // Estimated total, could be improved with actual count const onToggleType = (type: string, checked: boolean) => { setActiveTypes((prev) => (checked ? [...prev, type] : prev.filter((t) => t !== type))); setPageIndex(0); }; + // Reset to first page when search changes + useEffect(() => { + setPageIndex(0); + }, [debouncedSearch]); + const onToggleColumn = (id: keyof ColumnVisibility, checked: boolean) => { setColumnVisibility((prev) => ({ ...prev, [id]: checked })); }; @@ -157,10 +172,10 @@ export default function DocumentsTable() { }} onFirst={() => setPageIndex(0)} onPrev={() => setPageIndex((i) => Math.max(0, i - 1))} - onNext={() => setPageIndex((i) => (pageEnd < total ? i + 1 : i))} + onNext={() => setPageIndex((i) => i + 1)} onLast={() => setPageIndex(Math.max(0, Math.ceil(total / pageSize) - 1))} canPrev={pageIndex > 0} - canNext={pageEnd < total} + canNext={pageDocs.length === pageSize} // Show next if current page is full id={id} /> diff --git a/surfsense_web/components/chat/ChatInputGroup.tsx b/surfsense_web/components/chat/ChatInputGroup.tsx index 1f5cc6d31..d1a0c7538 100644 --- a/surfsense_web/components/chat/ChatInputGroup.tsx +++ b/surfsense_web/components/chat/ChatInputGroup.tsx @@ -39,11 +39,25 @@ const DocumentSelector = React.memo( }) => { const { search_space_id } = useParams(); const [isOpen, setIsOpen] = useState(false); + const [pageIndex, setPageIndex] = useState(0); + const [pageSize, setPageSize] = useState(100); // Larger page size for document selector - const { documents, loading, isLoaded, fetchDocuments } = useDocuments( - Number(search_space_id), - true - ); + // Calculate skip value for pagination + const skip = pageIndex * pageSize; + + const { documents, loading, isLoaded, fetchDocuments, error } = useDocuments( + Number(search_space_id), + true, + skip, + pageSize + ); + + // Refetch documents when pagination changes + useEffect(() => { + if (isOpen) { + fetchDocuments(); + } + }, [skip, pageSize, isOpen, fetchDocuments]); const handleOpenChange = useCallback( (open: boolean) => { @@ -98,12 +112,29 @@ const DocumentSelector = React.memo(

Loading documents...

+ ) : error ? ( +
+
+

Error loading documents: {error}

+ +
+
) : isLoaded ? ( { + setPageSize(newSize); + setPageIndex(0); + }} + canNext={documents.length === pageSize} /> ) : null} diff --git a/surfsense_web/components/chat/DocumentsDataTable.tsx b/surfsense_web/components/chat/DocumentsDataTable.tsx index 79b6216b3..f901c46e3 100644 --- a/surfsense_web/components/chat/DocumentsDataTable.tsx +++ b/surfsense_web/components/chat/DocumentsDataTable.tsx @@ -1,16 +1,16 @@ "use client"; import { - type ColumnDef, - type ColumnFiltersState, - flexRender, - getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, - type SortingState, - useReactTable, - type VisibilityState, + type ColumnDef, + type ColumnFiltersState, + flexRender, + getCoreRowModel, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + type SortingState, + useReactTable, + type VisibilityState, } from "@tanstack/react-table"; import { ArrowUpDown, Calendar, FileText, Search } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; @@ -18,423 +18,462 @@ import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; import { Input } from "@/components/ui/input"; import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, } from "@/components/ui/select"; import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, } from "@/components/ui/table"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import type { Document, DocumentType } from "@/hooks/use-documents"; interface DocumentsDataTableProps { - documents: Document[]; - onSelectionChange: (documents: Document[]) => void; - onDone: () => void; - initialSelectedDocuments?: Document[]; + documents: Document[]; + onSelectionChange: (documents: Document[]) => void; + onDone: () => void; + initialSelectedDocuments?: Document[]; + pageIndex?: number; + pageSize?: number; + onPageIndexChange?: (pageIndex: number) => void; + onPageSizeChange?: (pageSize: number) => void; + canNext?: boolean; } // Combine EnumConnectorName with additional document types const DOCUMENT_TYPES: (string | "ALL")[] = [ - "ALL", - "FILE", - "EXTENSION", - "CRAWLED_URL", - "YOUTUBE_VIDEO", - ...Object.values(EnumConnectorName), + "ALL", + "FILE", + "EXTENSION", + "CRAWLED_URL", + "YOUTUBE_VIDEO", + ...Object.values(EnumConnectorName), ]; const columns: ColumnDef[] = [ - { - id: "select", - header: ({ table }) => ( - table.toggleAllPageRowsSelected(!!value)} - aria-label="Select all" - /> - ), - cell: ({ row }) => ( - row.toggleSelected(!!value)} - aria-label="Select row" - /> - ), - enableSorting: false, - enableHiding: false, - size: 40, - }, - { - accessorKey: "title", - header: ({ column }) => ( - - ), - cell: ({ row }) => { - const title = row.getValue("title") as string; - return ( -
- {title} -
- ); - }, - }, - { - accessorKey: "document_type", - header: "Type", - cell: ({ row }) => { - const type = row.getValue("document_type") as DocumentType; - return ( -
- {getConnectorIcon(type)} -
- ); - }, - size: 80, - meta: { - className: "hidden sm:table-cell", - }, - }, - { - accessorKey: "content", - header: "Preview", - cell: ({ row }) => { - const content = row.getValue("content") as string; - return ( -
- {content.substring(0, 30)}... - {content.substring(0, 100)}... -
- ); - }, - enableSorting: false, - meta: { - className: "hidden md:table-cell", - }, - }, - { - accessorKey: "created_at", - header: ({ column }) => ( - - ), - cell: ({ row }) => { - const date = new Date(row.getValue("created_at")); - return ( -
- - {date.toLocaleDateString("en-US", { - month: "short", - day: "numeric", - year: "numeric", - })} - - - {date.toLocaleDateString("en-US", { - month: "numeric", - day: "numeric", - })} - -
- ); - }, - size: 80, - }, + { + id: "select", + header: ({ table }) => ( + { + table.toggleAllPageRowsSelected(!!value); + }} + aria-label="Select all" + /> + ), + cell: ({ row }) => ( + { + row.toggleSelected(!!value); + }} + aria-label="Select row" + /> + ), + enableSorting: false, + enableHiding: false, + size: 40, + }, + { + accessorKey: "title", + header: ({ column }) => ( + + ), + cell: ({ row }) => { + const title = row.getValue("title") as string; + return ( +
+ {title} +
+ ); + }, + }, + { + accessorKey: "document_type", + header: "Type", + cell: ({ row }) => { + const type = row.getValue("document_type") as DocumentType; + return ( +
+ {getConnectorIcon(type)} +
+ ); + }, + size: 80, + meta: { + className: "hidden sm:table-cell", + }, + }, + { + accessorKey: "content", + header: "Preview", + cell: ({ row }) => { + const content = row.getValue("content") as string; + return ( +
+ {content.substring(0, 30)}... + + {content.substring(0, 100)}... + +
+ ); + }, + enableSorting: false, + meta: { + className: "hidden md:table-cell", + }, + }, + { + accessorKey: "created_at", + header: ({ column }) => ( + + ), + cell: ({ row }) => { + const date = new Date(row.getValue("created_at")); + return ( +
+ + {date.toLocaleDateString("en-US", { + month: "short", + day: "numeric", + year: "numeric", + })} + + + {date.toLocaleDateString("en-US", { + month: "numeric", + day: "numeric", + })} + +
+ ); + }, + size: 80, + }, ]; export function DocumentsDataTable({ - documents, - onSelectionChange, - onDone, - initialSelectedDocuments = [], + documents, + onSelectionChange, + onDone, + initialSelectedDocuments = [], + pageIndex = 0, + pageSize = 100, + onPageIndexChange, + onPageSizeChange, + canNext = false, }: DocumentsDataTableProps) { - const [sorting, setSorting] = useState([]); - const [columnFilters, setColumnFilters] = useState([]); - const [columnVisibility, setColumnVisibility] = useState({}); - const [documentTypeFilter, setDocumentTypeFilter] = useState("ALL"); + const [sorting, setSorting] = useState([]); + const [columnFilters, setColumnFilters] = useState([]); + const [columnVisibility, setColumnVisibility] = useState({}); + const [documentTypeFilter, setDocumentTypeFilter] = useState( + "ALL" + ); - // Memoize initial row selection to prevent infinite loops - const initialRowSelection = useMemo(() => { - if (!documents.length || !initialSelectedDocuments.length) return {}; + // Memoize initial row selection to prevent infinite loops + const initialRowSelection = useMemo(() => { + if (!documents.length || !initialSelectedDocuments.length) return {}; - const selection: Record = {}; - initialSelectedDocuments.forEach((selectedDoc) => { - selection[selectedDoc.id] = true; - }); - return selection; - }, [documents, initialSelectedDocuments]); + const selection: Record = {}; + initialSelectedDocuments.forEach((selectedDoc) => { + // Find the document in the current documents array to get the correct row ID + const docInCurrentList = documents.find( + (doc) => doc.id === selectedDoc.id + ); + if (docInCurrentList) { + selection[docInCurrentList.id.toString()] = true; + } + }); + return selection; + }, [documents, initialSelectedDocuments]); - const [rowSelection, setRowSelection] = useState>({}); + const [rowSelection, setRowSelection] = useState>( + () => initialRowSelection + ); - // Only update row selection when initialRowSelection actually changes and is not empty - useEffect(() => { - const hasChanges = JSON.stringify(rowSelection) !== JSON.stringify(initialRowSelection); - if (hasChanges && Object.keys(initialRowSelection).length > 0) { - setRowSelection(initialRowSelection); - } - }, [initialRowSelection]); + // Update row selection when initial selection changes + useEffect(() => { + setRowSelection(initialRowSelection); + }, [initialRowSelection]); - // Initialize row selection on mount - useEffect(() => { - if (Object.keys(rowSelection).length === 0 && Object.keys(initialRowSelection).length > 0) { - setRowSelection(initialRowSelection); - } - }, []); + const filteredDocuments = useMemo(() => { + if (documentTypeFilter === "ALL") return documents; + return documents.filter((doc) => doc.document_type === documentTypeFilter); + }, [documents, documentTypeFilter]); - const filteredDocuments = useMemo(() => { - if (documentTypeFilter === "ALL") return documents; - return documents.filter((doc) => doc.document_type === documentTypeFilter); - }, [documents, documentTypeFilter]); + const table = useReactTable({ + data: filteredDocuments, + columns, + getRowId: (row) => row.id.toString(), + onSortingChange: setSorting, + onColumnFiltersChange: setColumnFilters, + getCoreRowModel: getCoreRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + onColumnVisibilityChange: setColumnVisibility, + onRowSelectionChange: setRowSelection, + // Disable internal pagination since we're handling it at the parent level + manualPagination: true, + pageCount: -1, + enableRowSelection: true, + state: { sorting, columnFilters, columnVisibility, rowSelection }, + }); - const table = useReactTable({ - data: filteredDocuments, - columns, - getRowId: (row) => row.id.toString(), - onSortingChange: setSorting, - onColumnFiltersChange: setColumnFilters, - getCoreRowModel: getCoreRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - onColumnVisibilityChange: setColumnVisibility, - onRowSelectionChange: setRowSelection, - initialState: { pagination: { pageSize: 10 } }, - state: { sorting, columnFilters, columnVisibility, rowSelection }, - }); + useEffect(() => { + const selectedRows = table.getFilteredSelectedRowModel().rows; + const selectedDocuments = selectedRows.map((row) => row.original); + onSelectionChange(selectedDocuments); + }, [rowSelection, table]); - useEffect(() => { - const selectedRows = table.getFilteredSelectedRowModel().rows; - const selectedDocuments = selectedRows.map((row) => row.original); - onSelectionChange(selectedDocuments); - }, [rowSelection, onSelectionChange, table]); + const handleClearAll = () => setRowSelection({}); - const handleClearAll = () => setRowSelection({}); + const handleSelectPage = () => { + const currentPageRows = table.getRowModel().rows; + const newSelection = { ...rowSelection }; + currentPageRows.forEach((row) => { + newSelection[row.id] = true; + }); + setRowSelection(newSelection); + }; - const handleSelectPage = () => { - const currentPageRows = table.getRowModel().rows; - const newSelection = { ...rowSelection }; - currentPageRows.forEach((row) => { - newSelection[row.id] = true; - }); - setRowSelection(newSelection); - }; + const handleSelectAllFiltered = () => { + const allFilteredRows = table.getFilteredRowModel().rows; + const newSelection: Record = {}; + allFilteredRows.forEach((row) => { + newSelection[row.id] = true; + }); + setRowSelection(newSelection); + }; - const handleSelectAllFiltered = () => { - const allFilteredRows = table.getFilteredRowModel().rows; - const newSelection: Record = {}; - allFilteredRows.forEach((row) => { - newSelection[row.id] = true; - }); - setRowSelection(newSelection); - }; + const selectedCount = table.getFilteredSelectedRowModel().rows.length; + const totalFiltered = table.getFilteredRowModel().rows.length; - const selectedCount = table.getFilteredSelectedRowModel().rows.length; - const totalFiltered = table.getFilteredRowModel().rows.length; + return ( +
+ {/* Header Controls */} +
+ {/* Search and Filter Row */} +
+
+ + + table.getColumn("title")?.setFilterValue(event.target.value) + } + className="pl-10 text-sm" + /> +
+ +
- return ( -
- {/* Header Controls */} -
- {/* Search and Filter Row */} -
-
- - table.getColumn("title")?.setFilterValue(event.target.value)} - className="pl-10 text-sm" - /> -
- -
+ {/* Action Controls Row */} +
+
+ + {selectedCount} of {totalFiltered} selected + +
+
+ + + + +
+
+ +
+
- {/* Action Controls Row */} -
-
- - {selectedCount} of {totalFiltered} selected - -
-
- - - - -
-
- -
-
+ {/* Table Container */} +
+
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext() + )} + + ))} + + ))} + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + {flexRender( + cell.column.columnDef.cell, + cell.getContext() + )} + + ))} + + )) + ) : ( + + + No documents found. + + + )} + +
+
+
- {/* Table Container */} -
-
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => ( - - {header.isPlaceholder - ? null - : flexRender(header.column.columnDef.header, header.getContext())} - - ))} - - ))} - - - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - - {flexRender(cell.column.columnDef.cell, cell.getContext())} - - ))} - - )) - ) : ( - - - No documents found. - - - )} - -
-
-
- - {/* Footer Pagination */} -
-
- Showing {table.getState().pagination.pageIndex * table.getState().pagination.pageSize + 1}{" "} - to{" "} - {Math.min( - (table.getState().pagination.pageIndex + 1) * table.getState().pagination.pageSize, - table.getFilteredRowModel().rows.length - )}{" "} - of {table.getFilteredRowModel().rows.length} documents -
-
- -
- Page - {table.getState().pagination.pageIndex + 1} - of - {table.getPageCount()} -
- -
-
-
- ); + {/* Footer Pagination */} +
+
+ Showing {pageIndex * pageSize + 1} to{" "} + {Math.min( + (pageIndex + 1) * pageSize, + pageIndex * pageSize + documents.length + )}{" "} + of{" "} + {canNext + ? `${(pageIndex + 1) * pageSize}+` + : `${pageIndex * pageSize + documents.length}`}{" "} + documents +
+
+ +
+ Page + {pageIndex + 1} + {canNext && +} +
+ +
+
+
+ ); } diff --git a/surfsense_web/hooks/use-documents.ts b/surfsense_web/hooks/use-documents.ts index 4c7536689..79c9a9766 100644 --- a/surfsense_web/hooks/use-documents.ts +++ b/surfsense_web/hooks/use-documents.ts @@ -29,19 +29,24 @@ export type DocumentType = | "GOOGLE_GMAIL_CONNECTOR" | "AIRTABLE_CONNECTOR"; -export function useDocuments(searchSpaceId: number, lazy: boolean = false) { +export function useDocuments( + searchSpaceId: number, + lazy: boolean = false, + skip: number = 0, + limit: number = 300 +) { const [documents, setDocuments] = useState([]); const [loading, setLoading] = useState(!lazy); // Don't show loading initially for lazy mode const [error, setError] = useState(null); const [isLoaded, setIsLoaded] = useState(false); // Memoization flag const fetchDocuments = useCallback(async () => { - if (isLoaded && lazy) return; // Avoid redundant calls in lazy mode - try { setLoading(true); + setError(null); + const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents?search_space_id=${searchSpaceId}`, + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents?search_space_id=${searchSpaceId}&skip=${skip}&limit=${limit}`, { headers: { Authorization: `Bearer ${localStorage.getItem("surfsense_bearer_token")}`, @@ -51,27 +56,28 @@ export function useDocuments(searchSpaceId: number, lazy: boolean = false) { ); if (!response.ok) { + const errorText = await response.text(); toast.error("Failed to fetch documents"); - throw new Error("Failed to fetch documents"); + throw new Error(`Failed to fetch documents: ${response.status} ${errorText}`); } const data = await response.json(); - setDocuments(data); - setError(null); + setDocuments(data || []); setIsLoaded(true); } catch (err: any) { setError(err.message || "Failed to fetch documents"); console.error("Error fetching documents:", err); + setDocuments([]); } finally { setLoading(false); } - }, [searchSpaceId, isLoaded, lazy]); + }, [searchSpaceId, skip, limit]); useEffect(() => { if (!lazy && searchSpaceId) { fetchDocuments(); } - }, [searchSpaceId, lazy, fetchDocuments]); + }, [searchSpaceId, skip, limit, lazy, fetchDocuments]); // Function to refresh the documents list const refreshDocuments = useCallback(async () => {