Merge commit 'fe6f830eab' into dev_mod

surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx

@@ -78,6 +78,10 @@ import {
 import Loading from "../loading";
 
 const MobileEditorPanel = dynamic(
+	() =>
+		import("@/components/editor-panel/editor-panel").then((m) => ({
+			default: m.MobileEditorPanel,
+		})),
 	() =>
 		import("@/components/editor-panel/editor-panel").then((m) => ({
 			default: m.MobileEditorPanel,
@@ -85,6 +89,10 @@ const MobileEditorPanel = dynamic(
 	{ ssr: false }
 );
 const MobileHitlEditPanel = dynamic(
+	() =>
+		import("@/components/hitl-edit-panel/hitl-edit-panel").then((m) => ({
+			default: m.MobileHitlEditPanel,
+		})),
 	() =>
 		import("@/components/hitl-edit-panel/hitl-edit-panel").then((m) => ({
 			default: m.MobileHitlEditPanel,
@@ -92,6 +100,10 @@ const MobileHitlEditPanel = dynamic(
 	{ ssr: false }
 );
 const MobileReportPanel = dynamic(
+	() =>
+		import("@/components/report-panel/report-panel").then((m) => ({
+			default: m.MobileReportPanel,
+		})),
 	() =>
 		import("@/components/report-panel/report-panel").then((m) => ({
 			default: m.MobileReportPanel,

surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx

@@ -302,12 +302,12 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
 
 	return (
 		<div className="space-y-8">
-			{/* Document/Files Connectors */}
+			{/* File Storage Integrations */}
 			{hasDocumentFileConnectors && (
 				<section>
 					<div className="flex items-center gap-2 mb-4">
 						<h3 className="text-sm font-semibold text-muted-foreground">
-							Document/Files Connectors
+							File Storage Integrations
 						</h3>
 					</div>
 					<div className="grid grid-cols-1 sm:grid-cols-2 gap-3">

surfsense_web/components/assistant-ui/document-upload-popup.tsx

@@ -20,7 +20,13 @@ import { searchSpaceSettingsDialogAtom } from "@/atoms/settings/settings-dialog.
 import { DocumentUploadTab } from "@/components/sources/DocumentUploadTab";
 import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
 import { Button } from "@/components/ui/button";
-import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog";
+import {
+	Dialog,
+	DialogContent,
+	DialogDescription,
+	DialogHeader,
+	DialogTitle,
+} from "@/components/ui/dialog";
 
 // Context for opening the dialog from anywhere
 interface DocumentUploadDialogContextType {
@@ -127,17 +133,15 @@ const DocumentUploadPopupContent: FC<{
 				onEscapeKeyDown={(e) => e.preventDefault()}
 				className="select-none max-w-2xl w-[95vw] sm:w-[640px] h-[min(440px,75dvh)] sm:h-[min(520px,80vh)] flex flex-col p-0 gap-0 overflow-hidden border border-border ring-0 bg-muted dark:bg-muted text-foreground [&>button]:right-3 sm:[&>button]:right-6 [&>button]:top-5 sm:[&>button]:top-8 [&>button]:opacity-80 [&>button]:hover:opacity-100 [&>button]:hover:bg-foreground/10 [&>button]:z-[100] [&>button>svg]:size-4 sm:[&>button>svg]:size-5"
 			>
-				<DialogTitle className="sr-only">Upload Document</DialogTitle>
-
 				<div className="flex-1 min-h-0 overflow-y-auto overscroll-contain">
-					<div className="sticky top-0 z-20 bg-muted px-4 sm:px-6 pt-6 sm:pt-8 pb-10">
-						<div className="flex items-center gap-2 mb-1 pr-8 sm:pr-0">
-							<h2 className="text-xl sm:text-3xl font-semibold tracking-tight">Upload Documents</h2>
-						</div>
-						<p className="text-xs sm:text-base text-muted-foreground/80 line-clamp-1">
+					<DialogHeader className="sticky top-0 z-20 bg-muted px-4 sm:px-6 pt-6 sm:pt-8 pb-10">
+						<DialogTitle className="text-xl sm:text-3xl font-semibold tracking-tight pr-8 sm:pr-0">
+							Upload Documents
+						</DialogTitle>
+						<DialogDescription className="text-xs sm:text-base text-muted-foreground/80 line-clamp-1">
 							Upload and sync your documents to your search space
-						</p>
-					</div>
+						</DialogDescription>
+					</DialogHeader>
 
 					<div className="px-4 sm:px-6 pb-4 sm:pb-6">
 						{!isLoading && !hasDocumentSummaryLLM ? (

surfsense_web/components/assistant-ui/tooltip-icon-button.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import { Slottable } from "@radix-ui/react-slot";
-import { type ComponentPropsWithRef, forwardRef, type ReactNode } from "react";
+import { type ComponentPropsWithRef, forwardRef, type ReactNode, useState } from "react";
 import { Button } from "@/components/ui/button";
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
 import { useMediaQuery } from "@/hooks/use-media-query";
@@ -17,9 +17,13 @@ export const TooltipIconButton = forwardRef<HTMLButtonElement, TooltipIconButton
 	({ children, tooltip, side = "bottom", className, disableTooltip, ...rest }, ref) => {
 		const isTouchDevice = useMediaQuery("(pointer: coarse)");
 		const suppressTooltip = disableTooltip || isTouchDevice;
+		const [tooltipOpen, setTooltipOpen] = useState(false);
 
 		return (
-			<Tooltip open={suppressTooltip ? false : undefined}>
+			<Tooltip
+				open={suppressTooltip ? false : tooltipOpen}
+				onOpenChange={suppressTooltip ? undefined : setTooltipOpen}
+			>
 				<TooltipTrigger asChild>
 					<Button
 						variant="ghost"

surfsense_web/components/documents/FolderNode.tsx

@@ -49,6 +49,7 @@ export interface FolderDisplay {
 	position: string;
 	parentId: number | null;
 	searchSpaceId: number;
+	metadata?: Record<string, unknown> | null;
 }
 
 interface FolderNodeProps {
@@ -354,7 +355,7 @@ export const FolderNode = React.memo(function FolderNode({
 									className="hidden sm:inline-flex h-6 w-6 shrink-0 opacity-0 group-hover:opacity-100 transition-opacity"
 									onClick={(e) => e.stopPropagation()}
 								>
-									<MoreHorizontal className="h-3.5 w-3.5" />
+									<MoreHorizontal className="h-3.5 w-3.5 text-muted-foreground" />
 								</Button>
 							</DropdownMenuTrigger>
 							<DropdownMenuContent align="end" className="w-40">

surfsense_web/components/documents/FolderTreeView.tsx

@@ -168,6 +168,12 @@ export function FolderTreeView({
 		return states;
 	}, [folders, docsByFolder, foldersByParent, mentionedDocIds]);
 
+	const folderMap = useMemo(() => {
+		const map: Record<number, FolderDisplay> = {};
+		for (const f of folders) map[f.id] = f;
+		return map;
+	}, [folders]);
+
 	const folderProcessingStates = useMemo(() => {
 		const states: Record<number, "idle" | "processing" | "failed"> = {};
 
@@ -178,6 +184,11 @@ export function FolderTreeView({
 			);
 			let hasFailed = directDocs.some((d) => d.status?.state === "failed");
 
+			const folder = folderMap[folderId];
+			if (folder?.metadata?.indexing_in_progress) {
+				hasProcessing = true;
+			}
+
 			for (const child of foldersByParent[folderId] ?? []) {
 				const sub = compute(child.id);
 				hasProcessing = hasProcessing || sub.hasProcessing;
@@ -195,7 +206,7 @@ export function FolderTreeView({
 			if (states[f.id] === undefined) compute(f.id);
 		}
 		return states;
-	}, [folders, docsByFolder, foldersByParent]);
+	}, [folders, docsByFolder, foldersByParent, folderMap]);
 
 	function renderLevel(parentId: number | null, depth: number): React.ReactNode[] {
 		const key = parentId ?? "root";
@@ -283,7 +294,7 @@ export function FolderTreeView({
 
 	if (treeNodes.length === 0 && folders.length === 0 && documents.length === 0) {
 		return (
-			<div className="flex flex-1 flex-col items-center justify-center gap-1 px-4 py-12 text-muted-foreground">
+			<div className="flex flex-1 flex-col items-center justify-center gap-1 px-4 py-12 text-muted-foreground select-none">
 				<p className="text-sm font-medium">No documents found</p>
 				<p className="text-xs text-muted-foreground/70">
 					Use the upload button or connect a source above

surfsense_web/components/layout/ui/dialogs/CreateSearchSpaceDialog.tsx

@@ -152,16 +152,10 @@ export function CreateSearchSpaceDialog({ open, onOpenChange }: CreateSearchSpac
 							<Button
 								type="submit"
 								disabled={isSubmitting}
-								className="h-8 sm:h-9 text-xs sm:text-sm"
+								className="h-8 sm:h-9 text-xs sm:text-sm relative"
 							>
-								{isSubmitting ? (
-									<>
-										<Spinner size="sm" className="mr-1.5" />
-										{t("creating")}
-									</>
-								) : (
-									<>{t("create_button")}</>
-								)}
+								<span className={isSubmitting ? "opacity-0" : ""}>{t("create_button")}</span>
+								{isSubmitting && <Spinner size="sm" className="absolute" />}
 							</Button>
 						</DialogFooter>
 					</form>

surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx

@@ -23,7 +23,11 @@ import { FolderPickerDialog } from "@/components/documents/FolderPickerDialog";
 import { FolderTreeView } from "@/components/documents/FolderTreeView";
 import { VersionHistoryDialog } from "@/components/documents/version-history";
 import { EXPORT_FILE_EXTENSIONS } from "@/components/shared/ExportMenuItems";
-import { FolderWatchDialog, type SelectedFolder } from "@/components/sources/FolderWatchDialog";
+import {
+	DEFAULT_EXCLUDE_PATTERNS,
+	FolderWatchDialog,
+	type SelectedFolder,
+} from "@/components/sources/FolderWatchDialog";
 import {
 	AlertDialog,
 	AlertDialogAction,
@@ -46,6 +50,8 @@ import { useElectronAPI } from "@/hooks/use-platform";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
 import { foldersApiService } from "@/lib/apis/folders-api.service";
 import { authenticatedFetch } from "@/lib/auth-utils";
+import { uploadFolderScan } from "@/lib/folder-sync-upload";
+import { getSupportedExtensionsSet } from "@/lib/supported-extensions";
 import { queries } from "@/zero/queries/index";
 import { SidebarSlideOutPanel } from "./SidebarSlideOutPanel";
 
@@ -114,48 +120,48 @@ export function DocumentsSidebar({
 		setFolderWatchOpen(true);
 	}, []);
 
-	useEffect(() => {
+	const refreshWatchedIds = useCallback(async () => {
 		if (!electronAPI?.getWatchedFolders) return;
 		const api = electronAPI;
 
-		async function loadWatchedIds() {
-			const folders = await api.getWatchedFolders();
+		const folders = await api.getWatchedFolders();
 
-			if (folders.length === 0) {
-				try {
-					const backendFolders = await documentsApiService.getWatchedFolders(searchSpaceId);
-					for (const bf of backendFolders) {
-						const meta = bf.metadata as Record<string, unknown> | null;
-						if (!meta?.watched || !meta.folder_path) continue;
-						await api.addWatchedFolder({
-							path: meta.folder_path as string,
-							name: bf.name,
-							rootFolderId: bf.id,
-							searchSpaceId: bf.search_space_id,
-							excludePatterns: (meta.exclude_patterns as string[]) ?? [],
-							fileExtensions: (meta.file_extensions as string[] | null) ?? null,
-							active: true,
-						});
-					}
-					const recovered = await api.getWatchedFolders();
-					const ids = new Set(
-						recovered.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number)
-					);
-					setWatchedFolderIds(ids);
-					return;
-				} catch (err) {
-					console.error("[DocumentsSidebar] Recovery from backend failed:", err);
+		if (folders.length === 0) {
+			try {
+				const backendFolders = await documentsApiService.getWatchedFolders(searchSpaceId);
+				for (const bf of backendFolders) {
+					const meta = bf.metadata as Record<string, unknown> | null;
+					if (!meta?.watched || !meta.folder_path) continue;
+					await api.addWatchedFolder({
+						path: meta.folder_path as string,
+						name: bf.name,
+						rootFolderId: bf.id,
+						searchSpaceId: bf.search_space_id,
+						excludePatterns: (meta.exclude_patterns as string[]) ?? [],
+						fileExtensions: (meta.file_extensions as string[] | null) ?? null,
+						active: true,
+					});
 				}
+				const recovered = await api.getWatchedFolders();
+				const ids = new Set(
+					recovered.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number)
+				);
+				setWatchedFolderIds(ids);
+				return;
+			} catch (err) {
+				console.error("[DocumentsSidebar] Recovery from backend failed:", err);
 			}
-
-			const ids = new Set(
-				folders.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number)
-			);
-			setWatchedFolderIds(ids);
 		}
 
-		loadWatchedIds();
+		const ids = new Set(
+			folders.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number)
+		);
+		setWatchedFolderIds(ids);
 	}, [searchSpaceId, electronAPI]);
+
+	useEffect(() => {
+		refreshWatchedIds();
+	}, [refreshWatchedIds]);
 	const { mutateAsync: deleteDocumentMutation } = useAtomValue(deleteDocumentMutationAtom);
 
 	const [sidebarDocs, setSidebarDocs] = useAtom(sidebarSelectedDocumentsAtom);
@@ -192,6 +198,7 @@ export function DocumentsSidebar({
 				position: f.position,
 				parentId: f.parentId ?? null,
 				searchSpaceId: f.searchSpaceId,
+				metadata: f.metadata as Record<string, unknown> | null | undefined,
 			})),
 		[zeroFolders]
 	);
@@ -304,14 +311,17 @@ export function DocumentsSidebar({
 			}
 
 			try {
-				await documentsApiService.folderIndex(searchSpaceId, {
-					folder_path: matched.path,
-					folder_name: matched.name,
-					search_space_id: searchSpaceId,
-					root_folder_id: folder.id,
-					file_extensions: matched.fileExtensions ?? undefined,
+				toast.info(`Re-scanning folder: ${matched.name}`);
+				await uploadFolderScan({
+					folderPath: matched.path,
+					folderName: matched.name,
+					searchSpaceId,
+					excludePatterns: matched.excludePatterns ?? DEFAULT_EXCLUDE_PATTERNS,
+					fileExtensions: matched.fileExtensions ?? Array.from(getSupportedExtensionsSet()),
+					enableSummary: false,
+					rootFolderId: folder.id,
 				});
-				toast.success(`Re-scanning folder: ${matched.name}`);
+				toast.success(`Re-scan complete: ${matched.name}`);
 			} catch (err) {
 				toast.error((err as Error)?.message || "Failed to re-scan folder");
 			}
@@ -337,8 +347,9 @@ export function DocumentsSidebar({
 				console.error("[DocumentsSidebar] Failed to clear watched metadata:", err);
 			}
 			toast.success(`Stopped watching: ${matched.name}`);
+			refreshWatchedIds();
 		},
-		[electronAPI]
+		[electronAPI, refreshWatchedIds]
 	);
 
 	const handleRenameFolder = useCallback(async (folder: FolderDisplay, newName: string) => {
@@ -867,6 +878,7 @@ export function DocumentsSidebar({
 					}}
 					searchSpaceId={searchSpaceId}
 					initialFolder={watchInitialFolder}
+					onSuccess={refreshWatchedIds}
 				/>
 			)}
 

surfsense_web/components/sources/FolderWatchDialog.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import { X } from "lucide-react";
-import { useCallback, useEffect, useMemo, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { toast } from "sonner";
 import { Button } from "@/components/ui/button";
 import {
@@ -13,7 +13,7 @@ import {
 } from "@/components/ui/dialog";
 import { Spinner } from "@/components/ui/spinner";
 import { Switch } from "@/components/ui/switch";
-import { documentsApiService } from "@/lib/apis/documents-api.service";
+import { type FolderSyncProgress, uploadFolderScan } from "@/lib/folder-sync-upload";
 import { getSupportedExtensionsSet } from "@/lib/supported-extensions";
 
 export interface SelectedFolder {
@@ -29,7 +29,7 @@ interface FolderWatchDialogProps {
 	initialFolder?: SelectedFolder | null;
 }
 
-const DEFAULT_EXCLUDE_PATTERNS = [
+export const DEFAULT_EXCLUDE_PATTERNS = [
 	".git",
 	"node_modules",
 	"__pycache__",
@@ -48,6 +48,8 @@ export function FolderWatchDialog({
 	const [selectedFolder, setSelectedFolder] = useState<SelectedFolder | null>(null);
 	const [shouldSummarize, setShouldSummarize] = useState(false);
 	const [submitting, setSubmitting] = useState(false);
+	const [progress, setProgress] = useState<FolderSyncProgress | null>(null);
+	const abortRef = useRef<AbortController | null>(null);
 
 	useEffect(() => {
 		if (open && initialFolder) {
@@ -64,33 +66,42 @@ export function FolderWatchDialog({
 		const folderPath = await api.selectFolder();
 		if (!folderPath) return;
 
-		const folderName = folderPath.split("/").pop() || folderPath.split("\\").pop() || folderPath;
+		const folderName = folderPath.split(/[/\\]/).pop() || folderPath;
 		setSelectedFolder({ path: folderPath, name: folderName });
 	}, []);
 
+	const handleCancel = useCallback(() => {
+		abortRef.current?.abort();
+	}, []);
+
 	const handleSubmit = useCallback(async () => {
 		if (!selectedFolder) return;
 		const api = window.electronAPI;
 		if (!api) return;
 
+		const controller = new AbortController();
+		abortRef.current = controller;
 		setSubmitting(true);
-		try {
-			const result = await documentsApiService.folderIndex(searchSpaceId, {
-				folder_path: selectedFolder.path,
-				folder_name: selectedFolder.name,
-				search_space_id: searchSpaceId,
-				enable_summary: shouldSummarize,
-				file_extensions: supportedExtensions,
-			});
+		setProgress(null);
 
-			const rootFolderId = (result as { root_folder_id?: number })?.root_folder_id ?? null;
+		try {
+			const rootFolderId = await uploadFolderScan({
+				folderPath: selectedFolder.path,
+				folderName: selectedFolder.name,
+				searchSpaceId,
+				excludePatterns: DEFAULT_EXCLUDE_PATTERNS,
+				fileExtensions: supportedExtensions,
+				enableSummary: shouldSummarize,
+				onProgress: setProgress,
+				signal: controller.signal,
+			});
 
 			await api.addWatchedFolder({
 				path: selectedFolder.path,
 				name: selectedFolder.name,
 				excludePatterns: DEFAULT_EXCLUDE_PATTERNS,
 				fileExtensions: supportedExtensions,
-				rootFolderId,
+				rootFolderId: rootFolderId ?? null,
 				searchSpaceId,
 				active: true,
 			});
@@ -98,12 +109,19 @@ export function FolderWatchDialog({
 			toast.success(`Watching folder: ${selectedFolder.name}`);
 			setSelectedFolder(null);
 			setShouldSummarize(false);
+			setProgress(null);
 			onOpenChange(false);
 			onSuccess?.();
 		} catch (err) {
-			toast.error((err as Error)?.message || "Failed to watch folder");
+			if ((err as Error)?.name === "AbortError") {
+				toast.info("Folder sync cancelled. Partial progress was saved.");
+			} else {
+				toast.error((err as Error)?.message || "Failed to watch folder");
+			}
 		} finally {
+			abortRef.current = null;
 			setSubmitting(false);
+			setProgress(null);
 		}
 	}, [
 		selectedFolder,
@@ -119,21 +137,44 @@ export function FolderWatchDialog({
 			if (!nextOpen && !submitting) {
 				setSelectedFolder(null);
 				setShouldSummarize(false);
+				setProgress(null);
 			}
 			onOpenChange(nextOpen);
 		},
 		[onOpenChange, submitting]
 	);
 
+	const progressLabel = useMemo(() => {
+		if (!progress) return null;
+		switch (progress.phase) {
+			case "listing":
+				return "Scanning folder...";
+			case "checking":
+				return `Checking ${progress.total} file(s)...`;
+			case "uploading":
+				return `Uploading ${progress.uploaded}/${progress.total} file(s)...`;
+			case "finalizing":
+				return "Finalizing...";
+			case "done":
+				return "Done!";
+			default:
+				return null;
+		}
+	}, [progress]);
+
 	return (
 		<Dialog open={open} onOpenChange={handleOpenChange}>
-			<DialogContent className="sm:max-w-md select-none">
-				<DialogHeader>
-					<DialogTitle>Watch Local Folder</DialogTitle>
-					<DialogDescription>Select a folder to sync and watch for changes.</DialogDescription>
+			<DialogContent className="sm:max-w-md select-none p-0 gap-0 overflow-hidden bg-muted dark:bg-muted border border-border [&>button]:opacity-80 [&>button]:hover:opacity-100 [&>button]:hover:bg-foreground/10">
+				<DialogHeader className="px-4 sm:px-6 pt-5 sm:pt-6 pb-3">
+					<DialogTitle className="text-lg sm:text-xl font-semibold tracking-tight">
+						Watch Local Folder
+					</DialogTitle>
+					<DialogDescription className="text-xs sm:text-sm text-muted-foreground/80">
+						Select a folder to sync and watch for changes
+					</DialogDescription>
 				</DialogHeader>
 
-				<div className="space-y-3 pt-2">
+				<div className="flex flex-col gap-3 px-4 sm:px-6 pb-4 sm:pb-6 min-h-[17rem]">
 					{selectedFolder ? (
 						<div className="flex items-center gap-2 py-1.5 pl-4 pr-2 rounded-md bg-slate-400/5 dark:bg-white/5 overflow-hidden">
 							<div className="min-w-0 flex-1 select-text">
@@ -156,7 +197,7 @@ export function FolderWatchDialog({
 						<button
 							type="button"
 							onClick={handleSelectFolder}
-							className="flex w-full items-center justify-center gap-2 rounded-lg border-2 border-dashed border-muted-foreground/30 py-8 text-sm text-muted-foreground transition-colors hover:border-foreground/50 hover:text-foreground"
+							className="flex flex-1 w-full items-center justify-center gap-2 rounded-lg border-2 border-dashed border-muted-foreground/30 text-sm text-muted-foreground transition-colors hover:border-foreground/50 hover:text-foreground"
 						>
 							Browse for a folder
 						</button>
@@ -174,14 +215,41 @@ export function FolderWatchDialog({
 								<Switch checked={shouldSummarize} onCheckedChange={setShouldSummarize} />
 							</div>
 
-							<Button className="w-full relative" onClick={handleSubmit} disabled={submitting}>
-								<span className={submitting ? "invisible" : ""}>Start Folder Sync</span>
-								{submitting && (
-									<span className="absolute inset-0 flex items-center justify-center">
-										<Spinner size="sm" />
-									</span>
+							{progressLabel && (
+								<div className="rounded-lg bg-slate-400/5 dark:bg-white/5 px-3 py-2">
+									<p className="text-xs text-muted-foreground">{progressLabel}</p>
+									{progress && progress.phase === "uploading" && progress.total > 0 && (
+										<div className="mt-1.5 h-1.5 w-full rounded-full bg-muted overflow-hidden">
+											<div
+												className="h-full bg-primary rounded-full transition-[width] duration-300"
+												style={{
+													width: `${Math.round((progress.uploaded / progress.total) * 100)}%`,
+												}}
+											/>
+										</div>
+									)}
+								</div>
+							)}
+
+							<div className="flex gap-2 mt-auto">
+								{submitting ? (
+									<>
+										<Button variant="secondary" className="flex-1" onClick={handleCancel}>
+											Cancel
+										</Button>
+										<Button className="flex-1 relative" disabled>
+											<span className="invisible">Syncing...</span>
+											<span className="absolute inset-0 flex items-center justify-center">
+												<Spinner size="sm" />
+											</span>
+										</Button>
+									</>
+								) : (
+									<Button className="w-full" onClick={handleSubmit}>
+										Start Folder Sync
+									</Button>
 								)}
-							</Button>
+							</div>
 						</>
 					)}
 				</div>

surfsense_web/contracts/types/folder.types.ts

@@ -9,7 +9,7 @@ export const folder = z.object({
 	created_by_id: z.string().nullable().optional(),
 	created_at: z.string(),
 	updated_at: z.string(),
-	metadata: z.record(z.unknown()).nullable().optional(),
+	metadata: z.record(z.string(), z.any()).nullable().optional(),
 });
 
 export const folderCreateRequest = z.object({

surfsense_web/hooks/use-folder-sync.ts

@@ -20,12 +20,18 @@ const DEBOUNCE_MS = 2000;
 const MAX_WAIT_MS = 10_000;
 const MAX_BATCH_SIZE = 50;
 
+interface FileEntry {
+	fullPath: string;
+	relativePath: string;
+	action: string;
+}
+
 interface BatchItem {
 	folderPath: string;
 	folderName: string;
 	searchSpaceId: number;
 	rootFolderId: number | null;
-	filePaths: string[];
+	files: FileEntry[];
 	ackIds: string[];
 }
 
@@ -44,18 +50,42 @@ export function useFolderSync() {
 		while (queueRef.current.length > 0) {
 			const batch = queueRef.current.shift()!;
 			try {
-				await documentsApiService.folderIndexFiles(batch.searchSpaceId, {
-					folder_path: batch.folderPath,
-					folder_name: batch.folderName,
-					search_space_id: batch.searchSpaceId,
-					target_file_paths: batch.filePaths,
-					root_folder_id: batch.rootFolderId,
-				});
+				const addChangeFiles = batch.files.filter(
+					(f) => f.action === "add" || f.action === "change"
+				);
+				const unlinkFiles = batch.files.filter((f) => f.action === "unlink");
+
+				if (addChangeFiles.length > 0 && electronAPI?.readLocalFiles) {
+					const fullPaths = addChangeFiles.map((f) => f.fullPath);
+					const fileDataArr = await electronAPI.readLocalFiles(fullPaths);
+
+					const files: File[] = fileDataArr.map((fd) => {
+						const blob = new Blob([fd.data], { type: fd.mimeType || "application/octet-stream" });
+						return new File([blob], fd.name, { type: blob.type });
+					});
+
+					await documentsApiService.folderUploadFiles(files, {
+						folder_name: batch.folderName,
+						search_space_id: batch.searchSpaceId,
+						relative_paths: addChangeFiles.map((f) => f.relativePath),
+						root_folder_id: batch.rootFolderId,
+					});
+				}
+
+				if (unlinkFiles.length > 0) {
+					await documentsApiService.folderNotifyUnlinked({
+						folder_name: batch.folderName,
+						search_space_id: batch.searchSpaceId,
+						root_folder_id: batch.rootFolderId,
+						relative_paths: unlinkFiles.map((f) => f.relativePath),
+					});
+				}
+
 				if (electronAPI?.acknowledgeFileEvents && batch.ackIds.length > 0) {
 					await electronAPI.acknowledgeFileEvents(batch.ackIds);
 				}
 			} catch (err) {
-				console.error("[FolderSync] Failed to trigger batch re-index:", err);
+				console.error("[FolderSync] Failed to process batch:", err);
 			}
 		}
 		processingRef.current = false;
@@ -68,10 +98,10 @@ export function useFolderSync() {
 		if (!pending) return;
 		pendingByFolder.current.delete(folderKey);
 
-		for (let i = 0; i < pending.filePaths.length; i += MAX_BATCH_SIZE) {
+		for (let i = 0; i < pending.files.length; i += MAX_BATCH_SIZE) {
 			queueRef.current.push({
 				...pending,
-				filePaths: pending.filePaths.slice(i, i + MAX_BATCH_SIZE),
+				files: pending.files.slice(i, i + MAX_BATCH_SIZE),
 				ackIds: i === 0 ? pending.ackIds : [],
 			});
 		}
@@ -83,9 +113,14 @@ export function useFolderSync() {
 		const existing = pendingByFolder.current.get(folderKey);
 
 		if (existing) {
-			const pathSet = new Set(existing.filePaths);
-			pathSet.add(event.fullPath);
-			existing.filePaths = Array.from(pathSet);
+			const pathSet = new Set(existing.files.map((f) => f.fullPath));
+			if (!pathSet.has(event.fullPath)) {
+				existing.files.push({
+					fullPath: event.fullPath,
+					relativePath: event.relativePath,
+					action: event.action,
+				});
+			}
 			if (!existing.ackIds.includes(event.id)) {
 				existing.ackIds.push(event.id);
 			}
@@ -95,7 +130,13 @@ export function useFolderSync() {
 				folderName: event.folderName,
 				searchSpaceId: event.searchSpaceId,
 				rootFolderId: event.rootFolderId,
-				filePaths: [event.fullPath],
+				files: [
+					{
+						fullPath: event.fullPath,
+						relativePath: event.relativePath,
+						action: event.action,
+					},
+				],
 				ackIds: [event.id],
 			});
 			firstEventTime.current.set(folderKey, Date.now());

surfsense_web/lib/apis/documents-api.service.ts

@@ -424,33 +424,79 @@ class DocumentsApiService {
 		return baseApiService.post(`/api/v1/documents/${documentId}/versions/${versionNumber}/restore`);
 	};
 
-	folderIndex = async (
-		searchSpaceId: number,
-		body: {
-			folder_path: string;
-			folder_name: string;
-			search_space_id: number;
-			exclude_patterns?: string[];
-			file_extensions?: string[];
-			root_folder_id?: number;
-			enable_summary?: boolean;
-		}
-	) => {
-		return baseApiService.post(`/api/v1/documents/folder-index`, undefined, { body });
+	folderMtimeCheck = async (body: {
+		folder_name: string;
+		search_space_id: number;
+		files: { relative_path: string; mtime: number }[];
+	}): Promise<{ files_to_upload: string[] }> => {
+		return baseApiService.post(`/api/v1/documents/folder-mtime-check`, undefined, {
+			body,
+		}) as unknown as { files_to_upload: string[] };
 	};
 
-	folderIndexFiles = async (
-		searchSpaceId: number,
-		body: {
-			folder_path: string;
+	folderUploadFiles = async (
+		files: File[],
+		metadata: {
 			folder_name: string;
 			search_space_id: number;
-			target_file_paths: string[];
+			relative_paths: string[];
 			root_folder_id?: number | null;
 			enable_summary?: boolean;
+		},
+		signal?: AbortSignal
+	): Promise<{ message: string; status: string; root_folder_id: number; file_count: number }> => {
+		const formData = new FormData();
+		for (const file of files) {
+			formData.append("files", file);
 		}
-	) => {
-		return baseApiService.post(`/api/v1/documents/folder-index-files`, undefined, { body });
+		formData.append("folder_name", metadata.folder_name);
+		formData.append("search_space_id", String(metadata.search_space_id));
+		formData.append("relative_paths", JSON.stringify(metadata.relative_paths));
+		if (metadata.root_folder_id != null) {
+			formData.append("root_folder_id", String(metadata.root_folder_id));
+		}
+		formData.append("enable_summary", String(metadata.enable_summary ?? false));
+
+		const totalSize = files.reduce((acc, f) => acc + f.size, 0);
+		const timeoutMs = Math.min(Math.max((totalSize / (1024 * 1024)) * 5000, 30_000), 600_000);
+
+		const controller = new AbortController();
+		const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+
+		if (signal) {
+			signal.addEventListener("abort", () => controller.abort(), { once: true });
+		}
+
+		try {
+			return (await baseApiService.postFormData(`/api/v1/documents/folder-upload`, undefined, {
+				body: formData,
+				signal: controller.signal,
+			})) as { message: string; status: string; root_folder_id: number; file_count: number };
+		} finally {
+			clearTimeout(timeoutId);
+		}
+	};
+
+	folderNotifyUnlinked = async (body: {
+		folder_name: string;
+		search_space_id: number;
+		root_folder_id: number | null;
+		relative_paths: string[];
+	}): Promise<{ deleted_count: number }> => {
+		return baseApiService.post(`/api/v1/documents/folder-unlink`, undefined, {
+			body,
+		}) as unknown as { deleted_count: number };
+	};
+
+	folderSyncFinalize = async (body: {
+		folder_name: string;
+		search_space_id: number;
+		root_folder_id: number | null;
+		all_relative_paths: string[];
+	}): Promise<{ deleted_count: number }> => {
+		return baseApiService.post(`/api/v1/documents/folder-sync-finalize`, undefined, {
+			body,
+		}) as unknown as { deleted_count: number };
 	};
 
 	getWatchedFolders = async (searchSpaceId: number) => {

surfsense_web/lib/folder-sync-upload.ts

@@ -0,0 +1,239 @@
+import { documentsApiService } from "@/lib/apis/documents-api.service";
+
+const MAX_BATCH_SIZE_BYTES = 20 * 1024 * 1024; // 20 MB
+const MAX_BATCH_FILES = 10;
+const UPLOAD_CONCURRENCY = 3;
+
+export interface FolderSyncProgress {
+	phase: "listing" | "checking" | "uploading" | "finalizing" | "done";
+	uploaded: number;
+	total: number;
+}
+
+export interface FolderSyncParams {
+	folderPath: string;
+	folderName: string;
+	searchSpaceId: number;
+	excludePatterns: string[];
+	fileExtensions: string[];
+	enableSummary: boolean;
+	rootFolderId?: number | null;
+	onProgress?: (progress: FolderSyncProgress) => void;
+	signal?: AbortSignal;
+}
+
+function buildBatches(entries: FolderFileEntry[]): FolderFileEntry[][] {
+	const batches: FolderFileEntry[][] = [];
+	let currentBatch: FolderFileEntry[] = [];
+	let currentSize = 0;
+
+	for (const entry of entries) {
+		if (entry.size >= MAX_BATCH_SIZE_BYTES) {
+			if (currentBatch.length > 0) {
+				batches.push(currentBatch);
+				currentBatch = [];
+				currentSize = 0;
+			}
+			batches.push([entry]);
+			continue;
+		}
+
+		if (currentBatch.length >= MAX_BATCH_FILES || currentSize + entry.size > MAX_BATCH_SIZE_BYTES) {
+			batches.push(currentBatch);
+			currentBatch = [];
+			currentSize = 0;
+		}
+
+		currentBatch.push(entry);
+		currentSize += entry.size;
+	}
+
+	if (currentBatch.length > 0) {
+		batches.push(currentBatch);
+	}
+
+	return batches;
+}
+
+async function uploadBatchesWithConcurrency(
+	batches: FolderFileEntry[][],
+	params: {
+		folderName: string;
+		searchSpaceId: number;
+		rootFolderId: number | null;
+		enableSummary: boolean;
+		signal?: AbortSignal;
+		onBatchComplete?: (filesInBatch: number) => void;
+	}
+): Promise<number | null> {
+	const api = window.electronAPI;
+	if (!api) throw new Error("Electron API not available");
+
+	let batchIdx = 0;
+	let resolvedRootFolderId = params.rootFolderId;
+	const errors: string[] = [];
+
+	async function processNext(): Promise<void> {
+		while (true) {
+			if (params.signal?.aborted) return;
+
+			const idx = batchIdx++;
+			if (idx >= batches.length) return;
+
+			const batch = batches[idx];
+			const fullPaths = batch.map((e) => e.fullPath);
+
+			try {
+				const fileDataArr = await api.readLocalFiles(fullPaths);
+
+				const files: File[] = fileDataArr.map((fd) => {
+					const blob = new Blob([fd.data], { type: fd.mimeType || "application/octet-stream" });
+					return new File([blob], fd.name, { type: blob.type });
+				});
+
+				const result = await documentsApiService.folderUploadFiles(
+					files,
+					{
+						folder_name: params.folderName,
+						search_space_id: params.searchSpaceId,
+						relative_paths: batch.map((e) => e.relativePath),
+						root_folder_id: resolvedRootFolderId,
+						enable_summary: params.enableSummary,
+					},
+					params.signal
+				);
+
+				if (result.root_folder_id && !resolvedRootFolderId) {
+					resolvedRootFolderId = result.root_folder_id;
+				}
+
+				params.onBatchComplete?.(batch.length);
+			} catch (err) {
+				if (params.signal?.aborted) return;
+				const msg = (err as Error)?.message || "Upload failed";
+				errors.push(`Batch ${idx}: ${msg}`);
+			}
+		}
+	}
+
+	const workers = Array.from({ length: Math.min(UPLOAD_CONCURRENCY, batches.length) }, () =>
+		processNext()
+	);
+	await Promise.all(workers);
+
+	if (errors.length > 0 && !params.signal?.aborted) {
+		console.error("Some batches failed:", errors);
+	}
+
+	return resolvedRootFolderId;
+}
+
+/**
+ * Run a full upload-based folder scan: list files, mtime-check, upload
+ * changed files in parallel batches, and finalize (delete orphans).
+ *
+ * Returns the root_folder_id to pass to addWatchedFolder.
+ */
+export async function uploadFolderScan(params: FolderSyncParams): Promise<number | null> {
+	const api = window.electronAPI;
+	if (!api) throw new Error("Electron API not available");
+
+	const {
+		folderPath,
+		folderName,
+		searchSpaceId,
+		excludePatterns,
+		fileExtensions,
+		enableSummary,
+		signal,
+	} = params;
+	let rootFolderId = params.rootFolderId ?? null;
+
+	params.onProgress?.({ phase: "listing", uploaded: 0, total: 0 });
+
+	if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+
+	const allFiles = await api.listFolderFiles({
+		path: folderPath,
+		name: folderName,
+		excludePatterns,
+		fileExtensions,
+		rootFolderId: rootFolderId ?? null,
+		searchSpaceId,
+		active: true,
+	});
+
+	if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+
+	params.onProgress?.({ phase: "checking", uploaded: 0, total: allFiles.length });
+
+	const mtimeCheckResult = await documentsApiService.folderMtimeCheck({
+		folder_name: folderName,
+		search_space_id: searchSpaceId,
+		files: allFiles.map((f) => ({ relative_path: f.relativePath, mtime: f.mtimeMs / 1000 })),
+	});
+
+	const filesToUpload = mtimeCheckResult.files_to_upload;
+	const uploadSet = new Set(filesToUpload);
+	const entriesToUpload = allFiles.filter((f) => uploadSet.has(f.relativePath));
+
+	if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+
+	if (entriesToUpload.length > 0) {
+		const batches = buildBatches(entriesToUpload);
+
+		let uploaded = 0;
+		params.onProgress?.({ phase: "uploading", uploaded: 0, total: entriesToUpload.length });
+
+		const uploadedRootId = await uploadBatchesWithConcurrency(batches, {
+			folderName,
+			searchSpaceId,
+			rootFolderId: rootFolderId ?? null,
+			enableSummary,
+			signal,
+			onBatchComplete: (count) => {
+				uploaded += count;
+				params.onProgress?.({ phase: "uploading", uploaded, total: entriesToUpload.length });
+			},
+		});
+
+		if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+
+		if (uploadedRootId) {
+			rootFolderId = uploadedRootId;
+		}
+	}
+
+	if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+
+	params.onProgress?.({
+		phase: "finalizing",
+		uploaded: entriesToUpload.length,
+		total: entriesToUpload.length,
+	});
+
+	await documentsApiService.folderSyncFinalize({
+		folder_name: folderName,
+		search_space_id: searchSpaceId,
+		root_folder_id: rootFolderId ?? null,
+		all_relative_paths: allFiles.map((f) => f.relativePath),
+	});
+
+	params.onProgress?.({
+		phase: "done",
+		uploaded: entriesToUpload.length,
+		total: entriesToUpload.length,
+	});
+
+	// Seed the Electron mtime store so the reconciliation scan in
+	// startWatcher won't re-emit events for files we just indexed.
+	if (api.seedFolderMtimes) {
+		const mtimes: Record<string, number> = {};
+		for (const f of allFiles) {
+			mtimes[f.relativePath] = f.mtimeMs;
+		}
+		await api.seedFolderMtimes(folderPath, mtimes);
+	}
+
+	return rootFolderId;
+}

surfsense_web/types/window.d.ts

@@ -34,6 +34,13 @@ interface LocalFileData {
 	size: number;
 }
 
+interface FolderFileEntry {
+	relativePath: string;
+	fullPath: string;
+	size: number;
+	mtimeMs: number;
+}
+
 interface ElectronAPI {
 	versions: {
 		electron: string;
@@ -82,6 +89,8 @@ interface ElectronAPI {
 	signalRendererReady: () => Promise<void>;
 	getPendingFileEvents: () => Promise<FolderSyncFileChangedEvent[]>;
 	acknowledgeFileEvents: (eventIds: string[]) => Promise<{ acknowledged: number }>;
+	listFolderFiles: (config: WatchedFolderConfig) => Promise<FolderFileEntry[]>;
+	seedFolderMtimes: (folderPath: string, mtimes: Record<string, number>) => Promise<void>;
 	// Browse files/folders via native dialogs
 	browseFiles: () => Promise<string[] | null>;
 	readLocalFiles: (paths: string[]) => Promise<LocalFileData[]>;

surfsense_web/zero/schema/folders.ts

@@ -1,4 +1,4 @@
-import { number, string, table } from "@rocicorp/zero";
+import { json, number, string, table } from "@rocicorp/zero";
 
 export const folderTable = table("folders")
 	.columns({
@@ -10,5 +10,6 @@ export const folderTable = table("folders")
 		createdById: string().optional().from("created_by_id"),
 		createdAt: number().from("created_at"),
 		updatedAt: number().from("updated_at"),
+		metadata: json<Record<string, unknown>>().optional().from("metadata"),
 	})
 	.primaryKey("id");