feat: preserve folder structure on web folder upload

surfsense_web/components/sources/DocumentUploadTab.tsx

@@ -26,6 +26,7 @@ import { Progress } from "@/components/ui/progress";
 import { Spinner } from "@/components/ui/spinner";
 import { Switch } from "@/components/ui/switch";
 import { useElectronAPI } from "@/hooks/use-platform";
+import { documentsApiService } from "@/lib/apis/documents-api.service";
 import {
 	trackDocumentUploadFailure,
 	trackDocumentUploadStarted,
@@ -48,6 +49,77 @@ interface FileWithId {
 	file: File;
 }
 
+interface FolderEntry {
+	id: string;
+	file: File;
+	relativePath: string;
+}
+
+interface FolderUploadData {
+	folderName: string;
+	entries: FolderEntry[];
+}
+
+interface FolderTreeNode {
+	name: string;
+	isFolder: boolean;
+	size?: number;
+	children: FolderTreeNode[];
+}
+
+function buildFolderTree(entries: FolderEntry[]): FolderTreeNode[] {
+	const root: FolderTreeNode = { name: "", isFolder: true, children: [] };
+
+	for (const entry of entries) {
+		const parts = entry.relativePath.split("/");
+		let current = root;
+
+		for (let i = 0; i < parts.length - 1; i++) {
+			let child = current.children.find((c) => c.name === parts[i] && c.isFolder);
+			if (!child) {
+				child = { name: parts[i], isFolder: true, children: [] };
+				current.children.push(child);
+			}
+			current = child;
+		}
+
+		current.children.push({
+			name: parts[parts.length - 1],
+			isFolder: false,
+			size: entry.file.size,
+			children: [],
+		});
+	}
+
+	function sortNodes(node: FolderTreeNode) {
+		node.children.sort((a, b) => {
+			if (a.isFolder !== b.isFolder) return a.isFolder ? -1 : 1;
+			return a.name.localeCompare(b.name);
+		});
+		for (const child of node.children) sortNodes(child);
+	}
+	sortNodes(root);
+
+	return root.children;
+}
+
+function flattenTree(
+	nodes: FolderTreeNode[],
+	depth = 0
+): { name: string; isFolder: boolean; depth: number; size?: number }[] {
+	const items: { name: string; isFolder: boolean; depth: number; size?: number }[] = [];
+	for (const node of nodes) {
+		items.push({ name: node.name, isFolder: node.isFolder, depth, size: node.size });
+		if (node.isFolder && node.children.length > 0) {
+			items.push(...flattenTree(node.children, depth + 1));
+		}
+	}
+	return items;
+}
+
+const FOLDER_BATCH_SIZE_BYTES = 20 * 1024 * 1024;
+const FOLDER_BATCH_MAX_FILES = 10;
+
 const MAX_FILE_SIZE_MB = 500;
 const MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024;
 
@@ -69,6 +141,8 @@ export function DocumentUploadTab({
 	const fileInputRef = useRef<HTMLInputElement>(null);
 	const folderInputRef = useRef<HTMLInputElement>(null);
 	const progressIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
+	const [folderUpload, setFolderUpload] = useState<FolderUploadData | null>(null);
+	const [isFolderUploading, setIsFolderUploading] = useState(false);
 
 	useEffect(() => {
 		return () => {
@@ -105,6 +179,7 @@ export function DocumentUploadTab({
 			const valid = incoming.filter((f) => f.size <= MAX_FILE_SIZE_BYTES);
 			if (valid.length === 0) return;
 
+			setFolderUpload(null);
 			setFiles((prev) => {
 				const newEntries = valid.map((f) => ({
 					id: crypto.randomUUID?.() ?? `file-${Date.now()}-${Math.random().toString(36)}`,
@@ -159,6 +234,7 @@ export function DocumentUploadTab({
 				file: new File([fd.data], fd.name, { type: fd.mimeType }),
 			})
 		);
+		setFolderUpload(null);
 		setFiles((prev) => [...prev, ...newFiles]);
 	}, [electronAPI, supportedExtensionsSet, t]);
 
@@ -167,18 +243,35 @@ export function DocumentUploadTab({
 			const fileList = e.target.files;
 			if (!fileList || fileList.length === 0) return;
 
-			const folderFiles = Array.from(fileList).filter((f) => {
+			const allFiles = Array.from(fileList);
-				const ext = f.name.includes(".") ? `.${f.name.split(".").pop()?.toLowerCase()}` : "";
+			const firstPath = allFiles[0]?.webkitRelativePath || "";
-				return ext !== "" && supportedExtensionsSet.has(ext);
+			const folderName = firstPath.split("/")[0];
-			});
 
-			if (folderFiles.length === 0) {
+			if (!folderName) {
+				addFiles(allFiles);
+				e.target.value = "";
+				return;
+			}
+
+			const entries: FolderEntry[] = allFiles
+				.filter((f) => {
+					const ext = f.name.includes(".") ? `.${f.name.split(".").pop()?.toLowerCase()}` : "";
+					return ext !== "" && supportedExtensionsSet.has(ext);
+				})
+				.map((f) => ({
+					id: crypto.randomUUID?.() ?? `file-${Date.now()}-${Math.random().toString(36)}`,
+					file: f,
+					relativePath: f.webkitRelativePath.substring(folderName.length + 1),
+				}));
+
+			if (entries.length === 0) {
 				toast.error(t("no_supported_files_in_folder"));
 				e.target.value = "";
 				return;
 			}
 
-			addFiles(folderFiles);
+			setFiles([]);
+			setFolderUpload({ folderName, entries });
 			e.target.value = "";
 		},
 		[addFiles, supportedExtensionsSet, t]
@@ -192,9 +285,18 @@ export function DocumentUploadTab({
 		return `${parseFloat((bytes / k ** i).toFixed(2))} ${sizes[i]}`;
 	};
 
-	const totalFileSize = files.reduce((total, entry) => total + entry.file.size, 0);
+	const totalFileSize = folderUpload
+		? folderUpload.entries.reduce((total, entry) => total + entry.file.size, 0)
+		: files.reduce((total, entry) => total + entry.file.size, 0);
 
-	const hasContent = files.length > 0;
+	const fileCount = folderUpload ? folderUpload.entries.length : files.length;
+	const hasContent = files.length > 0 || folderUpload !== null;
+	const isAnyUploading = isUploading || isFolderUploading;
+
+	const folderTreeItems = useMemo(() => {
+		if (!folderUpload) return [];
+		return flattenTree(buildFolderTree(folderUpload.entries));
+	}, [folderUpload]);
 
 	const handleAccordionChange = useCallback(
 		(value: string) => {
@@ -204,7 +306,94 @@ export function DocumentUploadTab({
 		[onAccordionStateChange]
 	);
 
+	const handleFolderUpload = async () => {
+		if (!folderUpload) return;
+
+		setUploadProgress(0);
+		setIsFolderUploading(true);
+		const total = folderUpload.entries.length;
+		trackDocumentUploadStarted(Number(searchSpaceId), total, totalFileSize);
+
+		try {
+			const batches: FolderEntry[][] = [];
+			let currentBatch: FolderEntry[] = [];
+			let currentSize = 0;
+
+			for (const entry of folderUpload.entries) {
+				const size = entry.file.size;
+
+				if (size >= FOLDER_BATCH_SIZE_BYTES) {
+					if (currentBatch.length > 0) {
+						batches.push(currentBatch);
+						currentBatch = [];
+						currentSize = 0;
+					}
+					batches.push([entry]);
+					continue;
+				}
+
+				if (
+					currentBatch.length >= FOLDER_BATCH_MAX_FILES ||
+					currentSize + size > FOLDER_BATCH_SIZE_BYTES
+				) {
+					batches.push(currentBatch);
+					currentBatch = [];
+					currentSize = 0;
+				}
+
+				currentBatch.push(entry);
+				currentSize += size;
+			}
+
+			if (currentBatch.length > 0) {
+				batches.push(currentBatch);
+			}
+
+			let rootFolderId: number | null = null;
+			let uploaded = 0;
+
+			for (const batch of batches) {
+				const result = await documentsApiService.folderUploadFiles(
+					batch.map((e) => e.file),
+					{
+						folder_name: folderUpload.folderName,
+						search_space_id: Number(searchSpaceId),
+						relative_paths: batch.map((e) => e.relativePath),
+						root_folder_id: rootFolderId,
+						enable_summary: shouldSummarize,
+					}
+				);
+
+				if (result.root_folder_id && !rootFolderId) {
+					rootFolderId = result.root_folder_id;
+				}
+
+				uploaded += batch.length;
+				setUploadProgress(Math.round((uploaded / total) * 100));
+			}
+
+			trackDocumentUploadSuccess(Number(searchSpaceId), total);
+			toast(t("upload_initiated"), { description: t("upload_initiated_desc") });
+			setFolderUpload(null);
+			onSuccess?.();
+		} catch (error) {
+			const message = error instanceof Error ? error.message : "Upload failed";
+			trackDocumentUploadFailure(Number(searchSpaceId), message);
+			toast(t("upload_error"), {
+				description: `${t("upload_error_desc")}: ${message}`,
+			});
+		} finally {
+			setIsFolderUploading(false);
+			setUploadProgress(0);
+		}
+	};
+
 	const handleUpload = async () => {
+		if (folderUpload) {
+			await handleFolderUpload();
+			return;
+		}
+
 		setUploadProgress(0);
 		trackDocumentUploadStarted(Number(searchSpaceId), files.length, totalFileSize);
 
@@ -398,55 +587,92 @@ export function DocumentUploadTab({
 			</div>
 
 			{/* FILES SELECTED */}
-			{files.length > 0 && (
+			{hasContent && (
 				<div className="rounded-lg border border-border p-3 space-y-2">
 					<div className="flex items-center justify-between">
 						<p className="text-sm font-medium">
-							{t("selected_files", { count: files.length })}
+							{folderUpload ? (
-							<Dot className="inline h-4 w-4" />
+								<>
-							{formatFileSize(totalFileSize)}
+									<FolderOpen className="inline h-4 w-4 mr-1 -mt-0.5" />
+									{folderUpload.folderName}
+									<Dot className="inline h-4 w-4" />
+									{folderUpload.entries.length}{" "}
+									{folderUpload.entries.length === 1 ? "file" : "files"}
+									<Dot className="inline h-4 w-4" />
+									{formatFileSize(totalFileSize)}
+								</>
+							) : (
+								<>
+									{t("selected_files", { count: files.length })}
+									<Dot className="inline h-4 w-4" />
+									{formatFileSize(totalFileSize)}
+								</>
+							)}
 						</p>
 						<Button
 							variant="ghost"
 							size="sm"
 							className="h-7 text-xs text-muted-foreground hover:text-foreground"
-							onClick={() => setFiles([])}
+							onClick={() => {
-							disabled={isUploading}
+								setFiles([]);
+								setFolderUpload(null);
+							}}
+							disabled={isAnyUploading}
 						>
 							{t("clear_all")}
 						</Button>
 					</div>
 
 					<div className="max-h-[160px] sm:max-h-[200px] overflow-y-auto -mx-1">
-						{files.map((entry) => (
+						{folderUpload
-							<div
+							? folderTreeItems.map((item, i) => (
-								key={entry.id}
+									<div
-								className="flex items-center gap-2 py-1.5 px-2 rounded-md hover:bg-slate-400/5 dark:hover:bg-white/5 group"
+										key={`${item.depth}-${i}-${item.name}`}
-							>
+										className="flex items-center gap-1.5 py-0.5 px-2"
-								<span className="text-[10px] font-medium uppercase leading-none bg-muted px-1.5 py-0.5 rounded text-muted-foreground shrink-0">
+										style={{ paddingLeft: `${item.depth * 16 + 8}px` }}
-									{entry.file.name.split(".").pop() || "?"}
+									>
-								</span>
+										{item.isFolder ? (
-								<span className="text-sm truncate flex-1 min-w-0">{entry.file.name}</span>
+											<FolderOpen className="h-3.5 w-3.5 text-blue-400 shrink-0" />
-								<span className="text-xs text-muted-foreground shrink-0">
+										) : (
-									{formatFileSize(entry.file.size)}
+											<FileIcon className="h-3.5 w-3.5 text-muted-foreground shrink-0" />
-								</span>
+										)}
-								<Button
+										<span className="text-sm truncate flex-1 min-w-0">{item.name}</span>
-									variant="ghost"
+										{!item.isFolder && item.size != null && (
-									size="icon"
+											<span className="text-xs text-muted-foreground shrink-0">
-									className="h-6 w-6 shrink-0"
+												{formatFileSize(item.size)}
-									onClick={() => setFiles((prev) => prev.filter((e) => e.id !== entry.id))}
+											</span>
-									disabled={isUploading}
+										)}
-								>
+									</div>
-									<X className="h-3 w-3" />
+								))
-								</Button>
+							: files.map((entry) => (
-							</div>
+									<div
-						))}
+										key={entry.id}
+										className="flex items-center gap-2 py-1.5 px-2 rounded-md hover:bg-slate-400/5 dark:hover:bg-white/5 group"
+									>
+										<span className="text-[10px] font-medium uppercase leading-none bg-muted px-1.5 py-0.5 rounded text-muted-foreground shrink-0">
+											{entry.file.name.split(".").pop() || "?"}
+										</span>
+										<span className="text-sm truncate flex-1 min-w-0">{entry.file.name}</span>
+										<span className="text-xs text-muted-foreground shrink-0">
+											{formatFileSize(entry.file.size)}
+										</span>
+										<Button
+											variant="ghost"
+											size="icon"
+											className="h-6 w-6 shrink-0"
+											onClick={() => setFiles((prev) => prev.filter((e) => e.id !== entry.id))}
+											disabled={isAnyUploading}
+										>
+											<X className="h-3 w-3" />
+										</Button>
+									</div>
+								))}
 					</div>
 
-					{isUploading && (
+					{isAnyUploading && (
 						<div className="space-y-1">
 							<div className="flex items-center justify-between text-xs">
-								<span>{t("uploading_files")}</span>
+								<span>{folderUpload ? "Uploading folder…" : t("uploading_files")}</span>
 								<span>{Math.round(uploadProgress)}%</span>
 							</div>
 							<Progress value={uploadProgress} className="h-1.5" />
@@ -466,16 +692,18 @@ export function DocumentUploadTab({
 					<Button
 						className="w-full"
 						onClick={handleUpload}
-						disabled={isUploading || files.length === 0}
+						disabled={isAnyUploading || fileCount === 0}
 					>
-						{isUploading ? (
+						{isAnyUploading ? (
 							<span className="flex items-center gap-2">
 								<Spinner size="sm" />
 								{t("uploading")}
 							</span>
 						) : (
 							<span className="flex items-center gap-2">
-								{t("upload_button", { count: files.length })}
+								{folderUpload
+									? `Upload Folder (${fileCount} files)`
+									: t("upload_button", { count: fileCount })}
 							</span>
 						)}
 					</Button>