Merge 40732faba0 into 8bbc8dba4e

surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx

@@ -26,10 +26,8 @@ export default function DocumentsTable() {
 	const params = useParams();
 	const searchSpaceId = Number(params.search_space_id);
 
-	const { documents, loading, error, refreshDocuments, deleteDocument } =
+	const [pageIndex, setPageIndex] = useState(0);
-		useDocuments(searchSpaceId);
+	const [pageSize, setPageSize] = useState(100); // Increased page size for better performance
-
-	const [data, setData] = useState<Document[]>([]);
 	const [search, setSearch] = useState("");
 	const debouncedSearch = useDebounced(search, 250);
 	const [activeTypes, setActiveTypes] = useState<string[]>([]);
@@ -39,16 +37,28 @@ export default function DocumentsTable() {
 		content: true,
 		created_at: true,
 	});
-	const [pageIndex, setPageIndex] = useState(0);
-	const [pageSize, setPageSize] = useState(10);
 	const [sortKey, setSortKey] = useState<SortKey>("title");
 	const [sortDesc, setSortDesc] = useState(false);
 	const [selectedIds, setSelectedIds] = useState<Set<number>>(new Set());
 
+	// Calculate skip value for pagination
+	const skip = pageIndex * pageSize;
+
+	const { documents, loading, error, refreshDocuments, deleteDocument } = useDocuments(
+		searchSpaceId,
+		false,
+		skip,
+		pageSize
+	);
+
+	const [data, setData] = useState<Document[]>([]);
+
 	useEffect(() => {
 		if (documents) setData(documents as Document[]);
 	}, [documents]);
 
+	// For server-side pagination, we apply client-side filtering to the current page data
+	// Note: This is a simplified approach. For production, you might want to implement server-side filtering
 	const filtered = useMemo(() => {
 		let result = data;
 		if (debouncedSearch.trim()) {
@@ -61,16 +71,21 @@ export default function DocumentsTable() {
 		return result;
 	}, [data, debouncedSearch, activeTypes]);
 
-	const total = filtered.length;
+	// For server-side pagination, we show the filtered results from current page
-	const pageStart = pageIndex * pageSize;
+	// Total count is estimated based on current page size (this could be improved with a count endpoint)
-	const pageEnd = Math.min(pageStart + pageSize, total);
+	const pageDocs = filtered;
-	const pageDocs = filtered.slice(pageStart, pageEnd);
+	const total = pageSize * (pageIndex + 1); // Estimated total, could be improved with actual count
 
 	const onToggleType = (type: string, checked: boolean) => {
 		setActiveTypes((prev) => (checked ? [...prev, type] : prev.filter((t) => t !== type)));
 		setPageIndex(0);
 	};
 
+	// Reset to first page when search changes
+	useEffect(() => {
+		setPageIndex(0);
+	}, [debouncedSearch]);
+
 	const onToggleColumn = (id: keyof ColumnVisibility, checked: boolean) => {
 		setColumnVisibility((prev) => ({ ...prev, [id]: checked }));
 	};
@@ -157,10 +172,10 @@ export default function DocumentsTable() {
 				}}
 				onFirst={() => setPageIndex(0)}
 				onPrev={() => setPageIndex((i) => Math.max(0, i - 1))}
-				onNext={() => setPageIndex((i) => (pageEnd < total ? i + 1 : i))}
+				onNext={() => setPageIndex((i) => i + 1)}
 				onLast={() => setPageIndex(Math.max(0, Math.ceil(total / pageSize) - 1))}
 				canPrev={pageIndex > 0}
-				canNext={pageEnd < total}
+				canNext={pageDocs.length === pageSize} // Show next if current page is full
 				id={id}
 			/>
 		</motion.div>

surfsense_web/components/chat/ChatInputGroup.tsx

@@ -39,12 +39,26 @@ const DocumentSelector = React.memo(
 	}) => {
 		const { search_space_id } = useParams();
 		const [isOpen, setIsOpen] = useState(false);
+		const [pageIndex, setPageIndex] = useState(0);
+		const [pageSize, setPageSize] = useState(100); // Larger page size for document selector
 
-		const { documents, loading, isLoaded, fetchDocuments } = useDocuments(
+		// Calculate skip value for pagination
+		const skip = pageIndex * pageSize;
+
+	const { documents, loading, isLoaded, fetchDocuments, error } = useDocuments(
 		Number(search_space_id),
-			true
+		true,
+		skip,
+		pageSize
 	);
 
+	// Refetch documents when pagination changes
+	useEffect(() => {
+		if (isOpen) {
+			fetchDocuments();
+		}
+	}, [skip, pageSize, isOpen, fetchDocuments]);
+
 		const handleOpenChange = useCallback(
 			(open: boolean) => {
 				setIsOpen(open);
@@ -98,12 +112,29 @@ const DocumentSelector = React.memo(
 										<p className="text-sm text-muted-foreground">Loading documents...</p>
 									</div>
 								</div>
+							) : error ? (
+								<div className="flex items-center justify-center h-full">
+									<div className="text-center space-y-2">
+										<p className="text-sm text-destructive">Error loading documents: {error}</p>
+										<Button onClick={fetchDocuments} variant="outline" size="sm">
+											Retry
+										</Button>
+									</div>
+								</div>
 							) : isLoaded ? (
 								<DocumentsDataTable
 									documents={documents}
 									onSelectionChange={handleSelectionChange}
 									onDone={handleDone}
 									initialSelectedDocuments={selectedDocuments}
+									pageIndex={pageIndex}
+									pageSize={pageSize}
+									onPageIndexChange={setPageIndex}
+									onPageSizeChange={(newSize) => {
+										setPageSize(newSize);
+										setPageIndex(0);
+									}}
+									canNext={documents.length === pageSize}
 								/>
 							) : null}
 						</div>

surfsense_web/components/chat/DocumentsDataTable.tsx

@@ -41,6 +41,11 @@ interface DocumentsDataTableProps {
   onSelectionChange: (documents: Document[]) => void;
   onDone: () => void;
   initialSelectedDocuments?: Document[];
+  pageIndex?: number;
+  pageSize?: number;
+  onPageIndexChange?: (pageIndex: number) => void;
+  onPageSizeChange?: (pageSize: number) => void;
+  canNext?: boolean;
 }
 
 // Combine EnumConnectorName with additional document types
@@ -59,16 +64,21 @@ const columns: ColumnDef<Document>[] = [
     header: ({ table }) => (
       <Checkbox
         checked={
-					table.getIsAllPageRowsSelected() || (table.getIsSomePageRowsSelected() && "indeterminate")
+          table.getIsAllPageRowsSelected() ||
+          (table.getIsSomePageRowsSelected() && "indeterminate")
         }
-				onCheckedChange={(value) => table.toggleAllPageRowsSelected(!!value)}
+        onCheckedChange={(value) => {
+          table.toggleAllPageRowsSelected(!!value);
+        }}
         aria-label="Select all"
       />
     ),
     cell: ({ row }) => (
       <Checkbox
         checked={row.getIsSelected()}
-				onCheckedChange={(value) => row.toggleSelected(!!value)}
+        onCheckedChange={(value) => {
+          row.toggleSelected(!!value);
+        }}
         aria-label="Select row"
       />
     ),
@@ -129,7 +139,9 @@ const columns: ColumnDef<Document>[] = [
           title={content}
         >
           <span className="sm:hidden">{content.substring(0, 30)}...</span>
-					<span className="hidden sm:inline">{content.substring(0, 100)}...</span>
+          <span className="hidden sm:inline">
+            {content.substring(0, 100)}...
+          </span>
         </div>
       );
     },
@@ -181,11 +193,18 @@ export function DocumentsDataTable({
   onSelectionChange,
   onDone,
   initialSelectedDocuments = [],
+  pageIndex = 0,
+  pageSize = 100,
+  onPageIndexChange,
+  onPageSizeChange,
+  canNext = false,
 }: DocumentsDataTableProps) {
   const [sorting, setSorting] = useState<SortingState>([]);
   const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
   const [columnVisibility, setColumnVisibility] = useState<VisibilityState>({});
-	const [documentTypeFilter, setDocumentTypeFilter] = useState<string | "ALL">("ALL");
+  const [documentTypeFilter, setDocumentTypeFilter] = useState<string | "ALL">(
+    "ALL"
+  );
 
   // Memoize initial row selection to prevent infinite loops
   const initialRowSelection = useMemo(() => {
@@ -193,28 +212,26 @@ export function DocumentsDataTable({
 
     const selection: Record<string, boolean> = {};
     initialSelectedDocuments.forEach((selectedDoc) => {
-			selection[selectedDoc.id] = true;
+      // Find the document in the current documents array to get the correct row ID
+      const docInCurrentList = documents.find(
+        (doc) => doc.id === selectedDoc.id
+      );
+      if (docInCurrentList) {
+        selection[docInCurrentList.id.toString()] = true;
+      }
     });
     return selection;
   }, [documents, initialSelectedDocuments]);
 
-	const [rowSelection, setRowSelection] = useState<Record<string, boolean>>({});
+  const [rowSelection, setRowSelection] = useState<Record<string, boolean>>(
+    () => initialRowSelection
+  );
 
-	// Only update row selection when initialRowSelection actually changes and is not empty
+  // Update row selection when initial selection changes
   useEffect(() => {
-		const hasChanges = JSON.stringify(rowSelection) !== JSON.stringify(initialRowSelection);
-		if (hasChanges && Object.keys(initialRowSelection).length > 0) {
     setRowSelection(initialRowSelection);
-		}
   }, [initialRowSelection]);
 
-	// Initialize row selection on mount
-	useEffect(() => {
-		if (Object.keys(rowSelection).length === 0 && Object.keys(initialRowSelection).length > 0) {
-			setRowSelection(initialRowSelection);
-		}
-	}, []);
-
   const filteredDocuments = useMemo(() => {
     if (documentTypeFilter === "ALL") return documents;
     return documents.filter((doc) => doc.document_type === documentTypeFilter);
@@ -227,12 +244,14 @@ export function DocumentsDataTable({
     onSortingChange: setSorting,
     onColumnFiltersChange: setColumnFilters,
     getCoreRowModel: getCoreRowModel(),
-		getPaginationRowModel: getPaginationRowModel(),
     getSortedRowModel: getSortedRowModel(),
     getFilteredRowModel: getFilteredRowModel(),
     onColumnVisibilityChange: setColumnVisibility,
     onRowSelectionChange: setRowSelection,
-		initialState: { pagination: { pageSize: 10 } },
+    // Disable internal pagination since we're handling it at the parent level
+    manualPagination: true,
+    pageCount: -1,
+    enableRowSelection: true,
     state: { sorting, columnFilters, columnVisibility, rowSelection },
   });
 
@@ -240,7 +259,7 @@ export function DocumentsDataTable({
     const selectedRows = table.getFilteredSelectedRowModel().rows;
     const selectedDocuments = selectedRows.map((row) => row.original);
     onSelectionChange(selectedDocuments);
-	}, [rowSelection, onSelectionChange, table]);
+  }, [rowSelection, table]);
 
   const handleClearAll = () => setRowSelection({});
 
@@ -275,14 +294,20 @@ export function DocumentsDataTable({
             <Search className="absolute left-3 top-1/2 transform -translate-y-1/2 h-4 w-4 text-muted-foreground" />
             <Input
               placeholder="Search documents..."
-							value={(table.getColumn("title")?.getFilterValue() as string) ?? ""}
+              value={
-							onChange={(event) => table.getColumn("title")?.setFilterValue(event.target.value)}
+                (table.getColumn("title")?.getFilterValue() as string) ?? ""
+              }
+              onChange={(event) =>
+                table.getColumn("title")?.setFilterValue(event.target.value)
+              }
               className="pl-10 text-sm"
             />
           </div>
           <Select
             value={documentTypeFilter}
-						onValueChange={(value) => setDocumentTypeFilter(value as string | "ALL")}
+            onValueChange={(value) =>
+              setDocumentTypeFilter(value as string | "ALL")
+            }
           >
             <SelectTrigger className="w-full sm:w-[180px]">
               <SelectValue />
@@ -358,10 +383,16 @@ export function DocumentsDataTable({
               {table.getHeaderGroups().map((headerGroup) => (
                 <TableRow key={headerGroup.id} className="border-b">
                   {headerGroup.headers.map((header) => (
-										<TableHead key={header.id} className="h-12 text-xs sm:text-sm">
+                    <TableHead
+                      key={header.id}
+                      className="h-12 text-xs sm:text-sm"
+                    >
                       {header.isPlaceholder
                         ? null
-												: flexRender(header.column.columnDef.header, header.getContext())}
+                        : flexRender(
+                            header.column.columnDef.header,
+                            header.getContext()
+                          )}
                     </TableHead>
                   ))}
                 </TableRow>
@@ -376,8 +407,14 @@ export function DocumentsDataTable({
                     className="hover:bg-muted/30"
                   >
                     {row.getVisibleCells().map((cell) => (
-											<TableCell key={cell.id} className="py-3 text-xs sm:text-sm">
+                      <TableCell
-												{flexRender(cell.column.columnDef.cell, cell.getContext())}
+                        key={cell.id}
+                        className="py-3 text-xs sm:text-sm"
+                      >
+                        {flexRender(
+                          cell.column.columnDef.cell,
+                          cell.getContext()
+                        )}
                       </TableCell>
                     ))}
                   </TableRow>
@@ -400,35 +437,37 @@ export function DocumentsDataTable({
       {/* Footer Pagination */}
       <div className="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-3 text-xs sm:text-sm text-muted-foreground border-t pt-3 md:pt-4 flex-shrink-0">
         <div className="text-center sm:text-left">
-					Showing {table.getState().pagination.pageIndex * table.getState().pagination.pageSize + 1}{" "}
+          Showing {pageIndex * pageSize + 1} to{" "}
-					to{" "}
           {Math.min(
-						(table.getState().pagination.pageIndex + 1) * table.getState().pagination.pageSize,
+            (pageIndex + 1) * pageSize,
-						table.getFilteredRowModel().rows.length
+            pageIndex * pageSize + documents.length
           )}{" "}
-					of {table.getFilteredRowModel().rows.length} documents
+          of{" "}
+          {canNext
+            ? `${(pageIndex + 1) * pageSize}+`
+            : `${pageIndex * pageSize + documents.length}`}{" "}
+          documents
         </div>
         <div className="flex items-center justify-center sm:justify-end space-x-2">
           <Button
             variant="outline"
             size="sm"
-						onClick={() => table.previousPage()}
+            onClick={() => onPageIndexChange?.(pageIndex - 1)}
-						disabled={!table.getCanPreviousPage()}
+            disabled={pageIndex === 0}
             className="text-xs sm:text-sm"
           >
             Previous
           </Button>
           <div className="flex items-center space-x-1 text-xs sm:text-sm">
             <span>Page</span>
-						<strong>{table.getState().pagination.pageIndex + 1}</strong>
+            <strong>{pageIndex + 1}</strong>
-						<span>of</span>
+            {canNext && <span>+</span>}
-						<strong>{table.getPageCount()}</strong>
           </div>
           <Button
             variant="outline"
             size="sm"
-						onClick={() => table.nextPage()}
+            onClick={() => onPageIndexChange?.(pageIndex + 1)}
-						disabled={!table.getCanNextPage()}
+            disabled={!canNext}
             className="text-xs sm:text-sm"
           >
             Next

surfsense_web/hooks/use-documents.ts

@@ -29,19 +29,24 @@ export type DocumentType =
 	| "GOOGLE_GMAIL_CONNECTOR"
 	| "AIRTABLE_CONNECTOR";
 
-export function useDocuments(searchSpaceId: number, lazy: boolean = false) {
+export function useDocuments(
+	searchSpaceId: number,
+	lazy: boolean = false,
+	skip: number = 0,
+	limit: number = 300
+) {
 	const [documents, setDocuments] = useState<Document[]>([]);
 	const [loading, setLoading] = useState(!lazy); // Don't show loading initially for lazy mode
 	const [error, setError] = useState<string | null>(null);
 	const [isLoaded, setIsLoaded] = useState(false); // Memoization flag
 
 	const fetchDocuments = useCallback(async () => {
-		if (isLoaded && lazy) return; // Avoid redundant calls in lazy mode
-
 		try {
 			setLoading(true);
+			setError(null);
+			
 			const response = await fetch(
-				`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents?search_space_id=${searchSpaceId}`,
+				`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents?search_space_id=${searchSpaceId}&skip=${skip}&limit=${limit}`,
 				{
 					headers: {
 						Authorization: `Bearer ${localStorage.getItem("surfsense_bearer_token")}`,
@@ -51,27 +56,28 @@ export function useDocuments(searchSpaceId: number, lazy: boolean = false) {
 			);
 
 			if (!response.ok) {
+				const errorText = await response.text();
 				toast.error("Failed to fetch documents");
-				throw new Error("Failed to fetch documents");
+				throw new Error(`Failed to fetch documents: ${response.status} ${errorText}`);
 			}
 
 			const data = await response.json();
-			setDocuments(data);
+			setDocuments(data || []);
-			setError(null);
 			setIsLoaded(true);
 		} catch (err: any) {
 			setError(err.message || "Failed to fetch documents");
 			console.error("Error fetching documents:", err);
+			setDocuments([]);
 		} finally {
 			setLoading(false);
 		}
-	}, [searchSpaceId, isLoaded, lazy]);
+	}, [searchSpaceId, skip, limit]);
 
 	useEffect(() => {
 		if (!lazy && searchSpaceId) {
 			fetchDocuments();
 		}
-	}, [searchSpaceId, lazy, fetchDocuments]);
+	}, [searchSpaceId, skip, limit, lazy, fetchDocuments]);
 
 	// Function to refresh the documents list
 	const refreshDocuments = useCallback(async () => {