diff --git a/surfsense_web/components/sources/DocumentUploadTab.tsx b/surfsense_web/components/sources/DocumentUploadTab.tsx index 124354a49..5fc8e3fd3 100644 --- a/surfsense_web/components/sources/DocumentUploadTab.tsx +++ b/surfsense_web/components/sources/DocumentUploadTab.tsx @@ -26,6 +26,7 @@ import { Progress } from "@/components/ui/progress"; import { Spinner } from "@/components/ui/spinner"; import { Switch } from "@/components/ui/switch"; import { useElectronAPI } from "@/hooks/use-platform"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; import { trackDocumentUploadFailure, trackDocumentUploadStarted, @@ -48,6 +49,77 @@ interface FileWithId { file: File; } +interface FolderEntry { + id: string; + file: File; + relativePath: string; +} + +interface FolderUploadData { + folderName: string; + entries: FolderEntry[]; +} + +interface FolderTreeNode { + name: string; + isFolder: boolean; + size?: number; + children: FolderTreeNode[]; +} + +function buildFolderTree(entries: FolderEntry[]): FolderTreeNode[] { + const root: FolderTreeNode = { name: "", isFolder: true, children: [] }; + + for (const entry of entries) { + const parts = entry.relativePath.split("/"); + let current = root; + + for (let i = 0; i < parts.length - 1; i++) { + let child = current.children.find((c) => c.name === parts[i] && c.isFolder); + if (!child) { + child = { name: parts[i], isFolder: true, children: [] }; + current.children.push(child); + } + current = child; + } + + current.children.push({ + name: parts[parts.length - 1], + isFolder: false, + size: entry.file.size, + children: [], + }); + } + + function sortNodes(node: FolderTreeNode) { + node.children.sort((a, b) => { + if (a.isFolder !== b.isFolder) return a.isFolder ? -1 : 1; + return a.name.localeCompare(b.name); + }); + for (const child of node.children) sortNodes(child); + } + sortNodes(root); + + return root.children; +} + +function flattenTree( + nodes: FolderTreeNode[], + depth = 0 +): { name: string; isFolder: boolean; depth: number; size?: number }[] { + const items: { name: string; isFolder: boolean; depth: number; size?: number }[] = []; + for (const node of nodes) { + items.push({ name: node.name, isFolder: node.isFolder, depth, size: node.size }); + if (node.isFolder && node.children.length > 0) { + items.push(...flattenTree(node.children, depth + 1)); + } + } + return items; +} + +const FOLDER_BATCH_SIZE_BYTES = 20 * 1024 * 1024; +const FOLDER_BATCH_MAX_FILES = 10; + const MAX_FILE_SIZE_MB = 500; const MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024; @@ -69,6 +141,8 @@ export function DocumentUploadTab({ const fileInputRef = useRef(null); const folderInputRef = useRef(null); const progressIntervalRef = useRef | null>(null); + const [folderUpload, setFolderUpload] = useState(null); + const [isFolderUploading, setIsFolderUploading] = useState(false); useEffect(() => { return () => { @@ -105,6 +179,7 @@ export function DocumentUploadTab({ const valid = incoming.filter((f) => f.size <= MAX_FILE_SIZE_BYTES); if (valid.length === 0) return; + setFolderUpload(null); setFiles((prev) => { const newEntries = valid.map((f) => ({ id: crypto.randomUUID?.() ?? `file-${Date.now()}-${Math.random().toString(36)}`, @@ -159,6 +234,7 @@ export function DocumentUploadTab({ file: new File([fd.data], fd.name, { type: fd.mimeType }), }) ); + setFolderUpload(null); setFiles((prev) => [...prev, ...newFiles]); }, [electronAPI, supportedExtensionsSet, t]); @@ -167,18 +243,35 @@ export function DocumentUploadTab({ const fileList = e.target.files; if (!fileList || fileList.length === 0) return; - const folderFiles = Array.from(fileList).filter((f) => { - const ext = f.name.includes(".") ? `.${f.name.split(".").pop()?.toLowerCase()}` : ""; - return ext !== "" && supportedExtensionsSet.has(ext); - }); + const allFiles = Array.from(fileList); + const firstPath = allFiles[0]?.webkitRelativePath || ""; + const folderName = firstPath.split("/")[0]; - if (folderFiles.length === 0) { + if (!folderName) { + addFiles(allFiles); + e.target.value = ""; + return; + } + + const entries: FolderEntry[] = allFiles + .filter((f) => { + const ext = f.name.includes(".") ? `.${f.name.split(".").pop()?.toLowerCase()}` : ""; + return ext !== "" && supportedExtensionsSet.has(ext); + }) + .map((f) => ({ + id: crypto.randomUUID?.() ?? `file-${Date.now()}-${Math.random().toString(36)}`, + file: f, + relativePath: f.webkitRelativePath.substring(folderName.length + 1), + })); + + if (entries.length === 0) { toast.error(t("no_supported_files_in_folder")); e.target.value = ""; return; } - addFiles(folderFiles); + setFiles([]); + setFolderUpload({ folderName, entries }); e.target.value = ""; }, [addFiles, supportedExtensionsSet, t] @@ -192,9 +285,18 @@ export function DocumentUploadTab({ return `${parseFloat((bytes / k ** i).toFixed(2))} ${sizes[i]}`; }; - const totalFileSize = files.reduce((total, entry) => total + entry.file.size, 0); + const totalFileSize = folderUpload + ? folderUpload.entries.reduce((total, entry) => total + entry.file.size, 0) + : files.reduce((total, entry) => total + entry.file.size, 0); - const hasContent = files.length > 0; + const fileCount = folderUpload ? folderUpload.entries.length : files.length; + const hasContent = files.length > 0 || folderUpload !== null; + const isAnyUploading = isUploading || isFolderUploading; + + const folderTreeItems = useMemo(() => { + if (!folderUpload) return []; + return flattenTree(buildFolderTree(folderUpload.entries)); + }, [folderUpload]); const handleAccordionChange = useCallback( (value: string) => { @@ -204,7 +306,94 @@ export function DocumentUploadTab({ [onAccordionStateChange] ); + const handleFolderUpload = async () => { + if (!folderUpload) return; + + setUploadProgress(0); + setIsFolderUploading(true); + const total = folderUpload.entries.length; + trackDocumentUploadStarted(Number(searchSpaceId), total, totalFileSize); + + try { + const batches: FolderEntry[][] = []; + let currentBatch: FolderEntry[] = []; + let currentSize = 0; + + for (const entry of folderUpload.entries) { + const size = entry.file.size; + + if (size >= FOLDER_BATCH_SIZE_BYTES) { + if (currentBatch.length > 0) { + batches.push(currentBatch); + currentBatch = []; + currentSize = 0; + } + batches.push([entry]); + continue; + } + + if ( + currentBatch.length >= FOLDER_BATCH_MAX_FILES || + currentSize + size > FOLDER_BATCH_SIZE_BYTES + ) { + batches.push(currentBatch); + currentBatch = []; + currentSize = 0; + } + + currentBatch.push(entry); + currentSize += size; + } + + if (currentBatch.length > 0) { + batches.push(currentBatch); + } + + let rootFolderId: number | null = null; + let uploaded = 0; + + for (const batch of batches) { + const result = await documentsApiService.folderUploadFiles( + batch.map((e) => e.file), + { + folder_name: folderUpload.folderName, + search_space_id: Number(searchSpaceId), + relative_paths: batch.map((e) => e.relativePath), + root_folder_id: rootFolderId, + enable_summary: shouldSummarize, + } + ); + + if (result.root_folder_id && !rootFolderId) { + rootFolderId = result.root_folder_id; + } + + uploaded += batch.length; + setUploadProgress(Math.round((uploaded / total) * 100)); + } + + trackDocumentUploadSuccess(Number(searchSpaceId), total); + toast(t("upload_initiated"), { description: t("upload_initiated_desc") }); + setFolderUpload(null); + onSuccess?.(); + } catch (error) { + const message = error instanceof Error ? error.message : "Upload failed"; + trackDocumentUploadFailure(Number(searchSpaceId), message); + toast(t("upload_error"), { + description: `${t("upload_error_desc")}: ${message}`, + }); + } finally { + setIsFolderUploading(false); + setUploadProgress(0); + } + }; + const handleUpload = async () => { + if (folderUpload) { + await handleFolderUpload(); + return; + } + setUploadProgress(0); trackDocumentUploadStarted(Number(searchSpaceId), files.length, totalFileSize); @@ -398,55 +587,92 @@ export function DocumentUploadTab({ {/* FILES SELECTED */} - {files.length > 0 && ( + {hasContent && (

- {t("selected_files", { count: files.length })} - - {formatFileSize(totalFileSize)} + {folderUpload ? ( + <> + + {folderUpload.folderName} + + {folderUpload.entries.length}{" "} + {folderUpload.entries.length === 1 ? "file" : "files"} + + {formatFileSize(totalFileSize)} + + ) : ( + <> + {t("selected_files", { count: files.length })} + + {formatFileSize(totalFileSize)} + + )}

- {files.map((entry) => ( -
- - {entry.file.name.split(".").pop() || "?"} - - {entry.file.name} - - {formatFileSize(entry.file.size)} - - -
- ))} + {folderUpload + ? folderTreeItems.map((item, i) => ( +
+ {item.isFolder ? ( + + ) : ( + + )} + {item.name} + {!item.isFolder && item.size != null && ( + + {formatFileSize(item.size)} + + )} +
+ )) + : files.map((entry) => ( +
+ + {entry.file.name.split(".").pop() || "?"} + + {entry.file.name} + + {formatFileSize(entry.file.size)} + + +
+ ))}
- {isUploading && ( + {isAnyUploading && (
- {t("uploading_files")} + {folderUpload ? "Uploading folder…" : t("uploading_files")} {Math.round(uploadProgress)}%
@@ -466,16 +692,18 @@ export function DocumentUploadTab({