fix(ui): Fix pagination for documents #320 solved

surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx

@@ -26,10 +26,8 @@ export default function DocumentsTable() {
 	const params = useParams();
 	const searchSpaceId = Number(params.search_space_id);
 
-	const { documents, loading, error, refreshDocuments, deleteDocument } =
+	const [pageIndex, setPageIndex] = useState(0);
-		useDocuments(searchSpaceId);
+	const [pageSize, setPageSize] = useState(100); // Increased page size for better performance
-
-	const [data, setData] = useState<Document[]>([]);
 	const [search, setSearch] = useState("");
 	const debouncedSearch = useDebounced(search, 250);
 	const [activeTypes, setActiveTypes] = useState<string[]>([]);
@@ -39,16 +37,28 @@ export default function DocumentsTable() {
 		content: true,
 		created_at: true,
 	});
-	const [pageIndex, setPageIndex] = useState(0);
-	const [pageSize, setPageSize] = useState(10);
 	const [sortKey, setSortKey] = useState<SortKey>("title");
 	const [sortDesc, setSortDesc] = useState(false);
 	const [selectedIds, setSelectedIds] = useState<Set<number>>(new Set());
 
+	// Calculate skip value for pagination
+	const skip = pageIndex * pageSize;
+
+	const { documents, loading, error, refreshDocuments, deleteDocument } = useDocuments(
+		searchSpaceId,
+		false,
+		skip,
+		pageSize
+	);
+
+	const [data, setData] = useState<Document[]>([]);
+
 	useEffect(() => {
 		if (documents) setData(documents as Document[]);
 	}, [documents]);
 
+	// For server-side pagination, we apply client-side filtering to the current page data
+	// Note: This is a simplified approach. For production, you might want to implement server-side filtering
 	const filtered = useMemo(() => {
 		let result = data;
 		if (debouncedSearch.trim()) {
@@ -61,16 +71,21 @@ export default function DocumentsTable() {
 		return result;
 	}, [data, debouncedSearch, activeTypes]);
 
-	const total = filtered.length;
+	// For server-side pagination, we show the filtered results from current page
-	const pageStart = pageIndex * pageSize;
+	// Total count is estimated based on current page size (this could be improved with a count endpoint)
-	const pageEnd = Math.min(pageStart + pageSize, total);
+	const pageDocs = filtered;
-	const pageDocs = filtered.slice(pageStart, pageEnd);
+	const total = pageSize * (pageIndex + 1); // Estimated total, could be improved with actual count
 
 	const onToggleType = (type: string, checked: boolean) => {
 		setActiveTypes((prev) => (checked ? [...prev, type] : prev.filter((t) => t !== type)));
 		setPageIndex(0);
 	};
 
+	// Reset to first page when search changes
+	useEffect(() => {
+		setPageIndex(0);
+	}, [debouncedSearch]);
+
 	const onToggleColumn = (id: keyof ColumnVisibility, checked: boolean) => {
 		setColumnVisibility((prev) => ({ ...prev, [id]: checked }));
 	};
@@ -157,10 +172,10 @@ export default function DocumentsTable() {
 				}}
 				onFirst={() => setPageIndex(0)}
 				onPrev={() => setPageIndex((i) => Math.max(0, i - 1))}
-				onNext={() => setPageIndex((i) => (pageEnd < total ? i + 1 : i))}
+				onNext={() => setPageIndex((i) => i + 1)}
 				onLast={() => setPageIndex(Math.max(0, Math.ceil(total / pageSize) - 1))}
 				canPrev={pageIndex > 0}
-				canNext={pageEnd < total}
+				canNext={pageDocs.length === pageSize} // Show next if current page is full
 				id={id}
 			/>
 		</motion.div>

surfsense_web/components/chat/ChatInputGroup.tsx

@@ -39,10 +39,17 @@ const DocumentSelector = React.memo(
 	}) => {
 		const { search_space_id } = useParams();
 		const [isOpen, setIsOpen] = useState(false);
+		const [pageIndex, setPageIndex] = useState(0);
+		const [pageSize, setPageSize] = useState(100); // Larger page size for document selector
+
+		// Calculate skip value for pagination
+		const skip = pageIndex * pageSize;
 
 		const { documents, loading, isLoaded, fetchDocuments } = useDocuments(
 			Number(search_space_id),
-			true
+			true,
+			skip,
+			pageSize
 		);
 
 		const handleOpenChange = useCallback(
@@ -104,6 +111,14 @@ const DocumentSelector = React.memo(
 									onSelectionChange={handleSelectionChange}
 									onDone={handleDone}
 									initialSelectedDocuments={selectedDocuments}
+									pageIndex={pageIndex}
+									pageSize={pageSize}
+									onPageIndexChange={setPageIndex}
+									onPageSizeChange={(newSize) => {
+										setPageSize(newSize);
+										setPageIndex(0);
+									}}
+									canNext={documents.length === pageSize}
 								/>
 							) : null}
 						</div>

surfsense_web/components/chat/DocumentsDataTable.tsx

@@ -41,6 +41,11 @@ interface DocumentsDataTableProps {
 	onSelectionChange: (documents: Document[]) => void;
 	onDone: () => void;
 	initialSelectedDocuments?: Document[];
+	pageIndex?: number;
+	pageSize?: number;
+	onPageIndexChange?: (pageIndex: number) => void;
+	onPageSizeChange?: (pageSize: number) => void;
+	canNext?: boolean;
 }
 
 // Combine EnumConnectorName with additional document types
@@ -181,6 +186,11 @@ export function DocumentsDataTable({
 	onSelectionChange,
 	onDone,
 	initialSelectedDocuments = [],
+	pageIndex = 0,
+	pageSize = 100,
+	onPageIndexChange,
+	onPageSizeChange,
+	canNext = false,
 }: DocumentsDataTableProps) {
 	const [sorting, setSorting] = useState<SortingState>([]);
 	const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
@@ -400,35 +410,33 @@ export function DocumentsDataTable({
 			{/* Footer Pagination */}
 			<div className="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-3 text-xs sm:text-sm text-muted-foreground border-t pt-3 md:pt-4 flex-shrink-0">
 				<div className="text-center sm:text-left">
-					Showing {table.getState().pagination.pageIndex * table.getState().pagination.pageSize + 1}{" "}
+					Showing {pageIndex * pageSize + 1} to{" "}
-					to{" "}
+					{Math.min((pageIndex + 1) * pageSize, pageIndex * pageSize + documents.length)} of{" "}
-					{Math.min(
+					{canNext
-						(table.getState().pagination.pageIndex + 1) * table.getState().pagination.pageSize,
+						? `${(pageIndex + 1) * pageSize}+`
-						table.getFilteredRowModel().rows.length
+						: `${pageIndex * pageSize + documents.length}`}{" "}
-					)}{" "}
+					documents
-					of {table.getFilteredRowModel().rows.length} documents
 				</div>
 				<div className="flex items-center justify-center sm:justify-end space-x-2">
 					<Button
 						variant="outline"
 						size="sm"
-						onClick={() => table.previousPage()}
+						onClick={() => onPageIndexChange?.(pageIndex - 1)}
-						disabled={!table.getCanPreviousPage()}
+						disabled={pageIndex === 0}
 						className="text-xs sm:text-sm"
 					>
 						Previous
 					</Button>
 					<div className="flex items-center space-x-1 text-xs sm:text-sm">
 						<span>Page</span>
-						<strong>{table.getState().pagination.pageIndex + 1}</strong>
+						<strong>{pageIndex + 1}</strong>
-						<span>of</span>
+						{canNext && <span>+</span>}
-						<strong>{table.getPageCount()}</strong>
 					</div>
 					<Button
 						variant="outline"
 						size="sm"
-						onClick={() => table.nextPage()}
+						onClick={() => onPageIndexChange?.(pageIndex + 1)}
-						disabled={!table.getCanNextPage()}
+						disabled={!canNext}
 						className="text-xs sm:text-sm"
 					>
 						Next

surfsense_web/hooks/use-documents.ts

@@ -29,7 +29,12 @@ export type DocumentType =
 	| "GOOGLE_GMAIL_CONNECTOR"
 	| "AIRTABLE_CONNECTOR";
 
-export function useDocuments(searchSpaceId: number, lazy: boolean = false) {
+export function useDocuments(
+	searchSpaceId: number,
+	lazy: boolean = false,
+	skip: number = 0,
+	limit: number = 300
+) {
 	const [documents, setDocuments] = useState<Document[]>([]);
 	const [loading, setLoading] = useState(!lazy); // Don't show loading initially for lazy mode
 	const [error, setError] = useState<string | null>(null);
@@ -41,7 +46,7 @@ export function useDocuments(searchSpaceId: number, lazy: boolean = false) {
 		try {
 			setLoading(true);
 			const response = await fetch(
-				`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents?search_space_id=${searchSpaceId}`,
+				`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents?search_space_id=${searchSpaceId}&skip=${skip}&limit=${limit}`,
 				{
 					headers: {
 						Authorization: `Bearer ${localStorage.getItem("surfsense_bearer_token")}`,
@@ -65,13 +70,13 @@ export function useDocuments(searchSpaceId: number, lazy: boolean = false) {
 		} finally {
 			setLoading(false);
 		}
-	}, [searchSpaceId, isLoaded, lazy]);
+	}, [searchSpaceId, skip, limit, isLoaded, lazy]);
 
 	useEffect(() => {
 		if (!lazy && searchSpaceId) {
 			fetchDocuments();
 		}
-	}, [searchSpaceId, lazy, fetchDocuments]);
+	}, [searchSpaceId, skip, limit, lazy, fetchDocuments]);
 
 	// Function to refresh the documents list
 	const refreshDocuments = useCallback(async () => {