feat: implement upload-based folder indexing and synchronization features

surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx

@@ -23,7 +23,9 @@ import { FolderPickerDialog } from "@/components/documents/FolderPickerDialog";
 import { FolderTreeView } from "@/components/documents/FolderTreeView";
 import { VersionHistoryDialog } from "@/components/documents/version-history";
 import { EXPORT_FILE_EXTENSIONS } from "@/components/shared/ExportMenuItems";
-import { FolderWatchDialog, type SelectedFolder } from "@/components/sources/FolderWatchDialog";
+import { DEFAULT_EXCLUDE_PATTERNS, FolderWatchDialog, type SelectedFolder } from "@/components/sources/FolderWatchDialog";
+import { uploadFolderScan } from "@/lib/folder-sync-upload";
+import { getSupportedExtensionsSet } from "@/lib/supported-extensions";
 import {
 	AlertDialog,
 	AlertDialogAction,
@@ -304,14 +306,17 @@ export function DocumentsSidebar({
 			}
 
 			try {
-				await documentsApiService.folderIndex(searchSpaceId, {
-					folder_path: matched.path,
-					folder_name: matched.name,
-					search_space_id: searchSpaceId,
-					root_folder_id: folder.id,
-					file_extensions: matched.fileExtensions ?? undefined,
+				toast.info(`Re-scanning folder: ${matched.name}`);
+				await uploadFolderScan({
+					folderPath: matched.path,
+					folderName: matched.name,
+					searchSpaceId,
+					excludePatterns: matched.excludePatterns ?? DEFAULT_EXCLUDE_PATTERNS,
+					fileExtensions: matched.fileExtensions ?? Array.from(getSupportedExtensionsSet()),
+					enableSummary: false,
+					rootFolderId: folder.id,
 				});
-				toast.success(`Re-scanning folder: ${matched.name}`);
+				toast.success(`Re-scan complete: ${matched.name}`);
 			} catch (err) {
 				toast.error((err as Error)?.message || "Failed to re-scan folder");
 			}

surfsense_web/components/sources/FolderWatchDialog.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import { X } from "lucide-react";
-import { useCallback, useEffect, useMemo, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { toast } from "sonner";
 import { Button } from "@/components/ui/button";
 import {
@@ -13,8 +13,8 @@ import {
 } from "@/components/ui/dialog";
 import { Spinner } from "@/components/ui/spinner";
 import { Switch } from "@/components/ui/switch";
-import { documentsApiService } from "@/lib/apis/documents-api.service";
 import { getSupportedExtensionsSet } from "@/lib/supported-extensions";
+import { type FolderSyncProgress, uploadFolderScan } from "@/lib/folder-sync-upload";
 
 export interface SelectedFolder {
 	path: string;
@@ -29,7 +29,7 @@ interface FolderWatchDialogProps {
 	initialFolder?: SelectedFolder | null;
 }
 
-const DEFAULT_EXCLUDE_PATTERNS = [
+export const DEFAULT_EXCLUDE_PATTERNS = [
 	".git",
 	"node_modules",
 	"__pycache__",
@@ -48,6 +48,8 @@ export function FolderWatchDialog({
 	const [selectedFolder, setSelectedFolder] = useState<SelectedFolder | null>(null);
 	const [shouldSummarize, setShouldSummarize] = useState(false);
 	const [submitting, setSubmitting] = useState(false);
+	const [progress, setProgress] = useState<FolderSyncProgress | null>(null);
+	const abortRef = useRef<AbortController | null>(null);
 
 	useEffect(() => {
 		if (open && initialFolder) {
@@ -68,29 +70,38 @@ export function FolderWatchDialog({
 		setSelectedFolder({ path: folderPath, name: folderName });
 	}, []);
 
+	const handleCancel = useCallback(() => {
+		abortRef.current?.abort();
+	}, []);
+
 	const handleSubmit = useCallback(async () => {
 		if (!selectedFolder) return;
 		const api = window.electronAPI;
 		if (!api) return;
 
+		const controller = new AbortController();
+		abortRef.current = controller;
 		setSubmitting(true);
-		try {
-			const result = await documentsApiService.folderIndex(searchSpaceId, {
-				folder_path: selectedFolder.path,
-				folder_name: selectedFolder.name,
-				search_space_id: searchSpaceId,
-				enable_summary: shouldSummarize,
-				file_extensions: supportedExtensions,
-			});
+		setProgress(null);
 
-			const rootFolderId = (result as { root_folder_id?: number })?.root_folder_id ?? null;
+		try {
+			const rootFolderId = await uploadFolderScan({
+				folderPath: selectedFolder.path,
+				folderName: selectedFolder.name,
+				searchSpaceId,
+				excludePatterns: DEFAULT_EXCLUDE_PATTERNS,
+				fileExtensions: supportedExtensions,
+				enableSummary: shouldSummarize,
+				onProgress: setProgress,
+				signal: controller.signal,
+			});
 
 			await api.addWatchedFolder({
 				path: selectedFolder.path,
 				name: selectedFolder.name,
 				excludePatterns: DEFAULT_EXCLUDE_PATTERNS,
 				fileExtensions: supportedExtensions,
-				rootFolderId,
+				rootFolderId: rootFolderId ?? null,
 				searchSpaceId,
 				active: true,
 			});
@@ -98,12 +109,19 @@ export function FolderWatchDialog({
 			toast.success(`Watching folder: ${selectedFolder.name}`);
 			setSelectedFolder(null);
 			setShouldSummarize(false);
+			setProgress(null);
 			onOpenChange(false);
 			onSuccess?.();
 		} catch (err) {
-			toast.error((err as Error)?.message || "Failed to watch folder");
+			if ((err as Error)?.name === "AbortError") {
+				toast.info("Folder sync cancelled. Partial progress was saved.");
+			} else {
+				toast.error((err as Error)?.message || "Failed to watch folder");
+			}
 		} finally {
+			abortRef.current = null;
 			setSubmitting(false);
+			setProgress(null);
 		}
 	}, [
 		selectedFolder,
@@ -119,12 +137,31 @@ export function FolderWatchDialog({
 			if (!nextOpen && !submitting) {
 				setSelectedFolder(null);
 				setShouldSummarize(false);
+				setProgress(null);
 			}
 			onOpenChange(nextOpen);
 		},
 		[onOpenChange, submitting]
 	);
 
+	const progressLabel = useMemo(() => {
+		if (!progress) return null;
+		switch (progress.phase) {
+			case "listing":
+				return "Scanning folder...";
+			case "checking":
+				return `Checking ${progress.total} file(s)...`;
+			case "uploading":
+				return `Uploading ${progress.uploaded}/${progress.total} file(s)...`;
+			case "finalizing":
+				return "Finalizing...";
+			case "done":
+				return "Done!";
+			default:
+				return null;
+		}
+	}, [progress]);
+
 	return (
 		<Dialog open={open} onOpenChange={handleOpenChange}>
 			<DialogContent className="sm:max-w-md select-none">
@@ -174,14 +211,39 @@ export function FolderWatchDialog({
 								<Switch checked={shouldSummarize} onCheckedChange={setShouldSummarize} />
 							</div>
 
-							<Button className="w-full relative" onClick={handleSubmit} disabled={submitting}>
-								<span className={submitting ? "invisible" : ""}>Start Folder Sync</span>
-								{submitting && (
-									<span className="absolute inset-0 flex items-center justify-center">
-										<Spinner size="sm" />
-									</span>
+							{progressLabel && (
+								<div className="rounded-lg bg-slate-400/5 dark:bg-white/5 px-3 py-2">
+									<p className="text-xs text-muted-foreground">{progressLabel}</p>
+									{progress && progress.phase === "uploading" && progress.total > 0 && (
+										<div className="mt-1.5 h-1.5 w-full rounded-full bg-muted overflow-hidden">
+											<div
+												className="h-full bg-primary rounded-full transition-[width] duration-300"
+												style={{ width: `${Math.round((progress.uploaded / progress.total) * 100)}%` }}
+											/>
+										</div>
+									)}
+								</div>
+							)}
+
+							<div className="flex gap-2">
+								{submitting ? (
+									<>
+										<Button variant="outline" className="flex-1" onClick={handleCancel}>
+											Cancel
+										</Button>
+										<Button className="flex-1 relative" disabled>
+											<span className="invisible">Syncing...</span>
+											<span className="absolute inset-0 flex items-center justify-center">
+												<Spinner size="sm" />
+											</span>
+										</Button>
+									</>
+								) : (
+									<Button className="w-full" onClick={handleSubmit}>
+										Start Folder Sync
+									</Button>
 								)}
-							</Button>
+							</div>
 						</>
 					)}
 				</div>

surfsense_web/hooks/use-folder-sync.ts

@@ -20,12 +20,18 @@ const DEBOUNCE_MS = 2000;
 const MAX_WAIT_MS = 10_000;
 const MAX_BATCH_SIZE = 50;
 
+interface FileEntry {
+	fullPath: string;
+	relativePath: string;
+	action: string;
+}
+
 interface BatchItem {
 	folderPath: string;
 	folderName: string;
 	searchSpaceId: number;
 	rootFolderId: number | null;
-	filePaths: string[];
+	files: FileEntry[];
 	ackIds: string[];
 }
 
@@ -44,18 +50,40 @@ export function useFolderSync() {
 		while (queueRef.current.length > 0) {
 			const batch = queueRef.current.shift()!;
 			try {
-				await documentsApiService.folderIndexFiles(batch.searchSpaceId, {
-					folder_path: batch.folderPath,
-					folder_name: batch.folderName,
-					search_space_id: batch.searchSpaceId,
-					target_file_paths: batch.filePaths,
-					root_folder_id: batch.rootFolderId,
-				});
+				const addChangeFiles = batch.files.filter((f) => f.action === "add" || f.action === "change");
+				const unlinkFiles = batch.files.filter((f) => f.action === "unlink");
+
+				if (addChangeFiles.length > 0 && electronAPI?.readLocalFiles) {
+					const fullPaths = addChangeFiles.map((f) => f.fullPath);
+					const fileDataArr = await electronAPI.readLocalFiles(fullPaths);
+
+					const files: File[] = fileDataArr.map((fd) => {
+						const blob = new Blob([fd.data], { type: fd.mimeType || "application/octet-stream" });
+						return new File([blob], fd.name, { type: blob.type });
+					});
+
+					await documentsApiService.folderUploadFiles(files, {
+						folder_name: batch.folderName,
+						search_space_id: batch.searchSpaceId,
+						relative_paths: addChangeFiles.map((f) => f.relativePath),
+						root_folder_id: batch.rootFolderId,
+					});
+				}
+
+				if (unlinkFiles.length > 0) {
+					await documentsApiService.folderNotifyUnlinked({
+						folder_name: batch.folderName,
+						search_space_id: batch.searchSpaceId,
+						root_folder_id: batch.rootFolderId,
+						relative_paths: unlinkFiles.map((f) => f.relativePath),
+					});
+				}
+
 				if (electronAPI?.acknowledgeFileEvents && batch.ackIds.length > 0) {
 					await electronAPI.acknowledgeFileEvents(batch.ackIds);
 				}
 			} catch (err) {
-				console.error("[FolderSync] Failed to trigger batch re-index:", err);
+				console.error("[FolderSync] Failed to process batch:", err);
 			}
 		}
 		processingRef.current = false;
@@ -68,10 +96,10 @@ export function useFolderSync() {
 		if (!pending) return;
 		pendingByFolder.current.delete(folderKey);
 
-		for (let i = 0; i < pending.filePaths.length; i += MAX_BATCH_SIZE) {
+		for (let i = 0; i < pending.files.length; i += MAX_BATCH_SIZE) {
 			queueRef.current.push({
 				...pending,
-				filePaths: pending.filePaths.slice(i, i + MAX_BATCH_SIZE),
+				files: pending.files.slice(i, i + MAX_BATCH_SIZE),
 				ackIds: i === 0 ? pending.ackIds : [],
 			});
 		}
@@ -83,9 +111,14 @@ export function useFolderSync() {
 		const existing = pendingByFolder.current.get(folderKey);
 
 		if (existing) {
-			const pathSet = new Set(existing.filePaths);
-			pathSet.add(event.fullPath);
-			existing.filePaths = Array.from(pathSet);
+			const pathSet = new Set(existing.files.map((f) => f.fullPath));
+			if (!pathSet.has(event.fullPath)) {
+				existing.files.push({
+					fullPath: event.fullPath,
+					relativePath: event.relativePath,
+					action: event.action,
+				});
+			}
 			if (!existing.ackIds.includes(event.id)) {
 				existing.ackIds.push(event.id);
 			}
@@ -95,7 +128,11 @@ export function useFolderSync() {
 				folderName: event.folderName,
 				searchSpaceId: event.searchSpaceId,
 				rootFolderId: event.rootFolderId,
-				filePaths: [event.fullPath],
+				files: [{
+					fullPath: event.fullPath,
+					relativePath: event.relativePath,
+					action: event.action,
+				}],
 				ackIds: [event.id],
 			});
 			firstEventTime.current.set(folderKey, Date.now());

surfsense_web/lib/apis/documents-api.service.ts

@@ -453,6 +453,76 @@ class DocumentsApiService {
 		return baseApiService.post(`/api/v1/documents/folder-index-files`, undefined, { body });
 	};
 
+	folderMtimeCheck = async (body: {
+		folder_name: string;
+		search_space_id: number;
+		files: { relative_path: string; mtime: number }[];
+	}): Promise<{ files_to_upload: string[] }> => {
+		return baseApiService.post(`/api/v1/documents/folder-mtime-check`, undefined, { body }) as unknown as { files_to_upload: string[] };
+	};
+
+	folderUploadFiles = async (
+		files: File[],
+		metadata: {
+			folder_name: string;
+			search_space_id: number;
+			relative_paths: string[];
+			root_folder_id?: number | null;
+			enable_summary?: boolean;
+		},
+		signal?: AbortSignal,
+	): Promise<{ message: string; status: string; root_folder_id: number; file_count: number }> => {
+		const formData = new FormData();
+		for (const file of files) {
+			formData.append("files", file);
+		}
+		formData.append("folder_name", metadata.folder_name);
+		formData.append("search_space_id", String(metadata.search_space_id));
+		formData.append("relative_paths", JSON.stringify(metadata.relative_paths));
+		if (metadata.root_folder_id != null) {
+			formData.append("root_folder_id", String(metadata.root_folder_id));
+		}
+		formData.append("enable_summary", String(metadata.enable_summary ?? false));
+
+		const totalSize = files.reduce((acc, f) => acc + f.size, 0);
+		const timeoutMs = Math.min(Math.max((totalSize / (1024 * 1024)) * 5000, 30_000), 600_000);
+
+		const controller = new AbortController();
+		const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+
+		if (signal) {
+			signal.addEventListener("abort", () => controller.abort(), { once: true });
+		}
+
+		try {
+			return await baseApiService.postFormData(
+				`/api/v1/documents/folder-upload`,
+				undefined,
+				{ body: formData, signal: controller.signal },
+			) as { message: string; status: string; root_folder_id: number; file_count: number };
+		} finally {
+			clearTimeout(timeoutId);
+		}
+	};
+
+	folderNotifyUnlinked = async (body: {
+		folder_name: string;
+		search_space_id: number;
+		root_folder_id: number | null;
+		relative_paths: string[];
+	}): Promise<{ deleted_count: number }> => {
+		return baseApiService.post(`/api/v1/documents/folder-unlink`, undefined, { body }) as unknown as { deleted_count: number };
+	};
+
+	folderSyncFinalize = async (body: {
+		folder_name: string;
+		search_space_id: number;
+		root_folder_id: number | null;
+		all_relative_paths: string[];
+	}): Promise<{ deleted_count: number }> => {
+		return baseApiService.post(`/api/v1/documents/folder-sync-finalize`, undefined, { body }) as unknown as { deleted_count: number };
+	};
+
 	getWatchedFolders = async (searchSpaceId: number) => {
 		return baseApiService.get(
 			`/api/v1/documents/watched-folders?search_space_id=${searchSpaceId}`,

surfsense_web/lib/folder-sync-upload.ts

@@ -0,0 +1,214 @@
+import { documentsApiService } from "@/lib/apis/documents-api.service";
+
+const MAX_BATCH_SIZE_BYTES = 20 * 1024 * 1024; // 20 MB
+const MAX_BATCH_FILES = 10;
+const UPLOAD_CONCURRENCY = 3;
+
+export interface FolderSyncProgress {
+	phase: "listing" | "checking" | "uploading" | "finalizing" | "done";
+	uploaded: number;
+	total: number;
+}
+
+export interface FolderSyncParams {
+	folderPath: string;
+	folderName: string;
+	searchSpaceId: number;
+	excludePatterns: string[];
+	fileExtensions: string[];
+	enableSummary: boolean;
+	rootFolderId?: number | null;
+	onProgress?: (progress: FolderSyncProgress) => void;
+	signal?: AbortSignal;
+}
+
+function buildBatches(
+	entries: FolderFileEntry[],
+): FolderFileEntry[][] {
+	const batches: FolderFileEntry[][] = [];
+	let currentBatch: FolderFileEntry[] = [];
+	let currentSize = 0;
+
+	for (const entry of entries) {
+		if (entry.size >= MAX_BATCH_SIZE_BYTES) {
+			if (currentBatch.length > 0) {
+				batches.push(currentBatch);
+				currentBatch = [];
+				currentSize = 0;
+			}
+			batches.push([entry]);
+			continue;
+		}
+
+		if (
+			currentBatch.length >= MAX_BATCH_FILES ||
+			currentSize + entry.size > MAX_BATCH_SIZE_BYTES
+		) {
+			batches.push(currentBatch);
+			currentBatch = [];
+			currentSize = 0;
+		}
+
+		currentBatch.push(entry);
+		currentSize += entry.size;
+	}
+
+	if (currentBatch.length > 0) {
+		batches.push(currentBatch);
+	}
+
+	return batches;
+}
+
+async function uploadBatchesWithConcurrency(
+	batches: FolderFileEntry[][],
+	params: {
+		folderName: string;
+		searchSpaceId: number;
+		rootFolderId: number | null;
+		enableSummary: boolean;
+		signal?: AbortSignal;
+		onBatchComplete?: (filesInBatch: number) => void;
+	},
+) {
+	const api = window.electronAPI;
+	if (!api) throw new Error("Electron API not available");
+
+	let batchIdx = 0;
+	const errors: string[] = [];
+
+	async function processNext(): Promise<void> {
+		while (true) {
+			if (params.signal?.aborted) return;
+
+			const idx = batchIdx++;
+			if (idx >= batches.length) return;
+
+			const batch = batches[idx];
+			const fullPaths = batch.map((e) => e.fullPath);
+
+			try {
+				const fileDataArr = await api.readLocalFiles(fullPaths);
+
+				const files: File[] = fileDataArr.map((fd) => {
+					const blob = new Blob([fd.data], { type: fd.mimeType || "application/octet-stream" });
+					return new File([blob], fd.name, { type: blob.type });
+				});
+
+				await documentsApiService.folderUploadFiles(
+					files,
+					{
+						folder_name: params.folderName,
+						search_space_id: params.searchSpaceId,
+						relative_paths: batch.map((e) => e.relativePath),
+						root_folder_id: params.rootFolderId,
+						enable_summary: params.enableSummary,
+					},
+					params.signal,
+				);
+
+				params.onBatchComplete?.(batch.length);
+			} catch (err) {
+				if (params.signal?.aborted) return;
+				const msg = (err as Error)?.message || "Upload failed";
+				errors.push(`Batch ${idx}: ${msg}`);
+			}
+		}
+	}
+
+	const workers = Array.from({ length: Math.min(UPLOAD_CONCURRENCY, batches.length) }, () => processNext());
+	await Promise.all(workers);
+
+	if (errors.length > 0 && !params.signal?.aborted) {
+		console.error("Some batches failed:", errors);
+	}
+}
+
+/**
+ * Run a full upload-based folder scan: list files, mtime-check, upload
+ * changed files in parallel batches, and finalize (delete orphans).
+ *
+ * Returns the root_folder_id to pass to addWatchedFolder.
+ */
+export async function uploadFolderScan(params: FolderSyncParams): Promise<number | null> {
+	const api = window.electronAPI;
+	if (!api) throw new Error("Electron API not available");
+
+	const { folderPath, folderName, searchSpaceId, excludePatterns, fileExtensions, enableSummary, signal } = params;
+	let rootFolderId = params.rootFolderId ?? null;
+
+	params.onProgress?.({ phase: "listing", uploaded: 0, total: 0 });
+
+	if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+
+	const allFiles = await api.listFolderFiles({
+		path: folderPath,
+		name: folderName,
+		excludePatterns,
+		fileExtensions,
+		rootFolderId: rootFolderId ?? null,
+		searchSpaceId,
+		active: true,
+	});
+
+	if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+
+	params.onProgress?.({ phase: "checking", uploaded: 0, total: allFiles.length });
+
+	const mtimeCheckResult = await documentsApiService.folderMtimeCheck({
+		folder_name: folderName,
+		search_space_id: searchSpaceId,
+		files: allFiles.map((f) => ({ relative_path: f.relativePath, mtime: f.mtimeMs / 1000 })),
+	});
+
+	const filesToUpload = mtimeCheckResult.files_to_upload;
+	const uploadSet = new Set(filesToUpload);
+	const entriesToUpload = allFiles.filter((f) => uploadSet.has(f.relativePath));
+
+	if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+
+	if (entriesToUpload.length > 0) {
+		const batches = buildBatches(entriesToUpload);
+
+		let uploaded = 0;
+		params.onProgress?.({ phase: "uploading", uploaded: 0, total: entriesToUpload.length });
+
+		await uploadBatchesWithConcurrency(batches, {
+			folderName,
+			searchSpaceId,
+			rootFolderId: rootFolderId ?? null,
+			enableSummary,
+			signal,
+			onBatchComplete: (count) => {
+				uploaded += count;
+				params.onProgress?.({ phase: "uploading", uploaded, total: entriesToUpload.length });
+			},
+		});
+
+		if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+
+		if (!rootFolderId) {
+			const watchedFolders = await documentsApiService.getWatchedFolders(searchSpaceId);
+			const folderList = watchedFolders as Array<{ id: number; name: string }> | undefined;
+			const matched = folderList?.find((f) => f.name === folderName);
+			if (matched?.id) {
+				rootFolderId = matched.id;
+			}
+		}
+	}
+
+	if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+
+	params.onProgress?.({ phase: "finalizing", uploaded: entriesToUpload.length, total: entriesToUpload.length });
+
+	await documentsApiService.folderSyncFinalize({
+		folder_name: folderName,
+		search_space_id: searchSpaceId,
+		root_folder_id: rootFolderId ?? null,
+		all_relative_paths: allFiles.map((f) => f.relativePath),
+	});
+
+	params.onProgress?.({ phase: "done", uploaded: entriesToUpload.length, total: entriesToUpload.length });
+
+	return rootFolderId;
+}

surfsense_web/types/window.d.ts

@@ -34,6 +34,13 @@ interface LocalFileData {
 	size: number;
 }
 
+interface FolderFileEntry {
+	relativePath: string;
+	fullPath: string;
+	size: number;
+	mtimeMs: number;
+}
+
 interface ElectronAPI {
 	versions: {
 		electron: string;
@@ -82,6 +89,7 @@ interface ElectronAPI {
 	signalRendererReady: () => Promise<void>;
 	getPendingFileEvents: () => Promise<FolderSyncFileChangedEvent[]>;
 	acknowledgeFileEvents: (eventIds: string[]) => Promise<{ acknowledged: number }>;
+	listFolderFiles: (config: WatchedFolderConfig) => Promise<FolderFileEntry[]>;
 	// Browse files/folders via native dialogs
 	browseFiles: () => Promise<string[] | null>;
 	readLocalFiles: (paths: string[]) => Promise<LocalFileData[]>;