Merge commit 'fe6f830eab' into dev_mod

surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx

@@ -302,12 +302,12 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
 
 	return (
 		<div className="space-y-8">
-			{/* Document/Files Connectors */}
+			{/* File Storage Integrations */}
 			{hasDocumentFileConnectors && (
 				<section>
 					<div className="flex items-center gap-2 mb-4">
 						<h3 className="text-sm font-semibold text-muted-foreground">
-							Document/Files Connectors
+							File Storage Integrations
 						</h3>
 					</div>
 					<div className="grid grid-cols-1 sm:grid-cols-2 gap-3">

surfsense_web/components/assistant-ui/document-upload-popup.tsx

@@ -20,7 +20,13 @@ import { searchSpaceSettingsDialogAtom } from "@/atoms/settings/settings-dialog.
 import { DocumentUploadTab } from "@/components/sources/DocumentUploadTab";
 import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
 import { Button } from "@/components/ui/button";
-import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog";
+import {
+	Dialog,
+	DialogContent,
+	DialogDescription,
+	DialogHeader,
+	DialogTitle,
+} from "@/components/ui/dialog";
 
 // Context for opening the dialog from anywhere
 interface DocumentUploadDialogContextType {
@@ -127,17 +133,15 @@ const DocumentUploadPopupContent: FC<{
 				onEscapeKeyDown={(e) => e.preventDefault()}
 				className="select-none max-w-2xl w-[95vw] sm:w-[640px] h-[min(440px,75dvh)] sm:h-[min(520px,80vh)] flex flex-col p-0 gap-0 overflow-hidden border border-border ring-0 bg-muted dark:bg-muted text-foreground [&>button]:right-3 sm:[&>button]:right-6 [&>button]:top-5 sm:[&>button]:top-8 [&>button]:opacity-80 [&>button]:hover:opacity-100 [&>button]:hover:bg-foreground/10 [&>button]:z-[100] [&>button>svg]:size-4 sm:[&>button>svg]:size-5"
 			>
-				<DialogTitle className="sr-only">Upload Document</DialogTitle>
-
 				<div className="flex-1 min-h-0 overflow-y-auto overscroll-contain">
-					<div className="sticky top-0 z-20 bg-muted px-4 sm:px-6 pt-6 sm:pt-8 pb-10">
-						<div className="flex items-center gap-2 mb-1 pr-8 sm:pr-0">
-							<h2 className="text-xl sm:text-3xl font-semibold tracking-tight">Upload Documents</h2>
-						</div>
-						<p className="text-xs sm:text-base text-muted-foreground/80 line-clamp-1">
+					<DialogHeader className="sticky top-0 z-20 bg-muted px-4 sm:px-6 pt-6 sm:pt-8 pb-10">
+						<DialogTitle className="text-xl sm:text-3xl font-semibold tracking-tight pr-8 sm:pr-0">
+							Upload Documents
+						</DialogTitle>
+						<DialogDescription className="text-xs sm:text-base text-muted-foreground/80 line-clamp-1">
 							Upload and sync your documents to your search space
-						</p>
-					</div>
+						</DialogDescription>
+					</DialogHeader>
 
 					<div className="px-4 sm:px-6 pb-4 sm:pb-6">
 						{!isLoading && !hasDocumentSummaryLLM ? (

surfsense_web/components/assistant-ui/tooltip-icon-button.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import { Slottable } from "@radix-ui/react-slot";
-import { type ComponentPropsWithRef, forwardRef, type ReactNode } from "react";
+import { type ComponentPropsWithRef, forwardRef, type ReactNode, useState } from "react";
 import { Button } from "@/components/ui/button";
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
 import { useMediaQuery } from "@/hooks/use-media-query";
@@ -17,9 +17,13 @@ export const TooltipIconButton = forwardRef<HTMLButtonElement, TooltipIconButton
 	({ children, tooltip, side = "bottom", className, disableTooltip, ...rest }, ref) => {
 		const isTouchDevice = useMediaQuery("(pointer: coarse)");
 		const suppressTooltip = disableTooltip || isTouchDevice;
+		const [tooltipOpen, setTooltipOpen] = useState(false);
 
 		return (
-			<Tooltip open={suppressTooltip ? false : undefined}>
+			<Tooltip
+				open={suppressTooltip ? false : tooltipOpen}
+				onOpenChange={suppressTooltip ? undefined : setTooltipOpen}
+			>
 				<TooltipTrigger asChild>
 					<Button
 						variant="ghost"

surfsense_web/components/documents/FolderNode.tsx

@@ -49,6 +49,7 @@ export interface FolderDisplay {
 	position: string;
 	parentId: number | null;
 	searchSpaceId: number;
+	metadata?: Record<string, unknown> | null;
 }
 
 interface FolderNodeProps {
@@ -354,7 +355,7 @@ export const FolderNode = React.memo(function FolderNode({
 									className="hidden sm:inline-flex h-6 w-6 shrink-0 opacity-0 group-hover:opacity-100 transition-opacity"
 									onClick={(e) => e.stopPropagation()}
 								>
-									<MoreHorizontal className="h-3.5 w-3.5" />
+									<MoreHorizontal className="h-3.5 w-3.5 text-muted-foreground" />
 								</Button>
 							</DropdownMenuTrigger>
 							<DropdownMenuContent align="end" className="w-40">

surfsense_web/components/documents/FolderTreeView.tsx

@@ -168,6 +168,12 @@ export function FolderTreeView({
 		return states;
 	}, [folders, docsByFolder, foldersByParent, mentionedDocIds]);
 
+	const folderMap = useMemo(() => {
+		const map: Record<number, FolderDisplay> = {};
+		for (const f of folders) map[f.id] = f;
+		return map;
+	}, [folders]);
+
 	const folderProcessingStates = useMemo(() => {
 		const states: Record<number, "idle" | "processing" | "failed"> = {};
 
@@ -178,6 +184,11 @@ export function FolderTreeView({
 			);
 			let hasFailed = directDocs.some((d) => d.status?.state === "failed");
 
+			const folder = folderMap[folderId];
+			if (folder?.metadata?.indexing_in_progress) {
+				hasProcessing = true;
+			}
+
 			for (const child of foldersByParent[folderId] ?? []) {
 				const sub = compute(child.id);
 				hasProcessing = hasProcessing || sub.hasProcessing;
@@ -195,7 +206,7 @@ export function FolderTreeView({
 			if (states[f.id] === undefined) compute(f.id);
 		}
 		return states;
-	}, [folders, docsByFolder, foldersByParent]);
+	}, [folders, docsByFolder, foldersByParent, folderMap]);
 
 	function renderLevel(parentId: number | null, depth: number): React.ReactNode[] {
 		const key = parentId ?? "root";
@@ -283,7 +294,7 @@ export function FolderTreeView({
 
 	if (treeNodes.length === 0 && folders.length === 0 && documents.length === 0) {
 		return (
-			<div className="flex flex-1 flex-col items-center justify-center gap-1 px-4 py-12 text-muted-foreground">
+			<div className="flex flex-1 flex-col items-center justify-center gap-1 px-4 py-12 text-muted-foreground select-none">
 				<p className="text-sm font-medium">No documents found</p>
 				<p className="text-xs text-muted-foreground/70">
 					Use the upload button or connect a source above

surfsense_web/components/layout/ui/dialogs/CreateSearchSpaceDialog.tsx

@@ -152,16 +152,10 @@ export function CreateSearchSpaceDialog({ open, onOpenChange }: CreateSearchSpac
 							<Button
 								type="submit"
 								disabled={isSubmitting}
-								className="h-8 sm:h-9 text-xs sm:text-sm"
+								className="h-8 sm:h-9 text-xs sm:text-sm relative"
 							>
-								{isSubmitting ? (
-									<>
-										<Spinner size="sm" className="mr-1.5" />
-										{t("creating")}
-									</>
-								) : (
-									<>{t("create_button")}</>
-								)}
+								<span className={isSubmitting ? "opacity-0" : ""}>{t("create_button")}</span>
+								{isSubmitting && <Spinner size="sm" className="absolute" />}
 							</Button>
 						</DialogFooter>
 					</form>

surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx

@@ -23,7 +23,11 @@ import { FolderPickerDialog } from "@/components/documents/FolderPickerDialog";
 import { FolderTreeView } from "@/components/documents/FolderTreeView";
 import { VersionHistoryDialog } from "@/components/documents/version-history";
 import { EXPORT_FILE_EXTENSIONS } from "@/components/shared/ExportMenuItems";
-import { FolderWatchDialog, type SelectedFolder } from "@/components/sources/FolderWatchDialog";
+import {
+	DEFAULT_EXCLUDE_PATTERNS,
+	FolderWatchDialog,
+	type SelectedFolder,
+} from "@/components/sources/FolderWatchDialog";
 import {
 	AlertDialog,
 	AlertDialogAction,
@@ -46,6 +50,8 @@ import { useElectronAPI } from "@/hooks/use-platform";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
 import { foldersApiService } from "@/lib/apis/folders-api.service";
 import { authenticatedFetch } from "@/lib/auth-utils";
+import { uploadFolderScan } from "@/lib/folder-sync-upload";
+import { getSupportedExtensionsSet } from "@/lib/supported-extensions";
 import { queries } from "@/zero/queries/index";
 import { SidebarSlideOutPanel } from "./SidebarSlideOutPanel";
 
@@ -114,48 +120,48 @@ export function DocumentsSidebar({
 		setFolderWatchOpen(true);
 	}, []);
 
-	useEffect(() => {
+	const refreshWatchedIds = useCallback(async () => {
 		if (!electronAPI?.getWatchedFolders) return;
 		const api = electronAPI;
 
-		async function loadWatchedIds() {
-			const folders = await api.getWatchedFolders();
+		const folders = await api.getWatchedFolders();
 
-			if (folders.length === 0) {
-				try {
-					const backendFolders = await documentsApiService.getWatchedFolders(searchSpaceId);
-					for (const bf of backendFolders) {
-						const meta = bf.metadata as Record<string, unknown> | null;
-						if (!meta?.watched || !meta.folder_path) continue;
-						await api.addWatchedFolder({
-							path: meta.folder_path as string,
-							name: bf.name,
-							rootFolderId: bf.id,
-							searchSpaceId: bf.search_space_id,
-							excludePatterns: (meta.exclude_patterns as string[]) ?? [],
-							fileExtensions: (meta.file_extensions as string[] | null) ?? null,
-							active: true,
-						});
-					}
-					const recovered = await api.getWatchedFolders();
-					const ids = new Set(
-						recovered.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number)
-					);
-					setWatchedFolderIds(ids);
-					return;
-				} catch (err) {
-					console.error("[DocumentsSidebar] Recovery from backend failed:", err);
+		if (folders.length === 0) {
+			try {
+				const backendFolders = await documentsApiService.getWatchedFolders(searchSpaceId);
+				for (const bf of backendFolders) {
+					const meta = bf.metadata as Record<string, unknown> | null;
+					if (!meta?.watched || !meta.folder_path) continue;
+					await api.addWatchedFolder({
+						path: meta.folder_path as string,
+						name: bf.name,
+						rootFolderId: bf.id,
+						searchSpaceId: bf.search_space_id,
+						excludePatterns: (meta.exclude_patterns as string[]) ?? [],
+						fileExtensions: (meta.file_extensions as string[] | null) ?? null,
+						active: true,
+					});
 				}
+				const recovered = await api.getWatchedFolders();
+				const ids = new Set(
+					recovered.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number)
+				);
+				setWatchedFolderIds(ids);
+				return;
+			} catch (err) {
+				console.error("[DocumentsSidebar] Recovery from backend failed:", err);
 			}
-
-			const ids = new Set(
-				folders.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number)
-			);
-			setWatchedFolderIds(ids);
 		}
 
-		loadWatchedIds();
+		const ids = new Set(
+			folders.filter((f) => f.rootFolderId != null).map((f) => f.rootFolderId as number)
+		);
+		setWatchedFolderIds(ids);
 	}, [searchSpaceId, electronAPI]);
+
+	useEffect(() => {
+		refreshWatchedIds();
+	}, [refreshWatchedIds]);
 	const { mutateAsync: deleteDocumentMutation } = useAtomValue(deleteDocumentMutationAtom);
 
 	const [sidebarDocs, setSidebarDocs] = useAtom(sidebarSelectedDocumentsAtom);
@@ -192,6 +198,7 @@ export function DocumentsSidebar({
 				position: f.position,
 				parentId: f.parentId ?? null,
 				searchSpaceId: f.searchSpaceId,
+				metadata: f.metadata as Record<string, unknown> | null | undefined,
 			})),
 		[zeroFolders]
 	);
@@ -304,14 +311,17 @@ export function DocumentsSidebar({
 			}
 
 			try {
-				await documentsApiService.folderIndex(searchSpaceId, {
-					folder_path: matched.path,
-					folder_name: matched.name,
-					search_space_id: searchSpaceId,
-					root_folder_id: folder.id,
-					file_extensions: matched.fileExtensions ?? undefined,
+				toast.info(`Re-scanning folder: ${matched.name}`);
+				await uploadFolderScan({
+					folderPath: matched.path,
+					folderName: matched.name,
+					searchSpaceId,
+					excludePatterns: matched.excludePatterns ?? DEFAULT_EXCLUDE_PATTERNS,
+					fileExtensions: matched.fileExtensions ?? Array.from(getSupportedExtensionsSet()),
+					enableSummary: false,
+					rootFolderId: folder.id,
 				});
-				toast.success(`Re-scanning folder: ${matched.name}`);
+				toast.success(`Re-scan complete: ${matched.name}`);
 			} catch (err) {
 				toast.error((err as Error)?.message || "Failed to re-scan folder");
 			}
@@ -337,8 +347,9 @@ export function DocumentsSidebar({
 				console.error("[DocumentsSidebar] Failed to clear watched metadata:", err);
 			}
 			toast.success(`Stopped watching: ${matched.name}`);
+			refreshWatchedIds();
 		},
-		[electronAPI]
+		[electronAPI, refreshWatchedIds]
 	);
 
 	const handleRenameFolder = useCallback(async (folder: FolderDisplay, newName: string) => {
@@ -867,6 +878,7 @@ export function DocumentsSidebar({
 					}}
 					searchSpaceId={searchSpaceId}
 					initialFolder={watchInitialFolder}
+					onSuccess={refreshWatchedIds}
 				/>
 			)}
 

surfsense_web/components/sources/FolderWatchDialog.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import { X } from "lucide-react";
-import { useCallback, useEffect, useMemo, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { toast } from "sonner";
 import { Button } from "@/components/ui/button";
 import {
@@ -13,7 +13,7 @@ import {
 } from "@/components/ui/dialog";
 import { Spinner } from "@/components/ui/spinner";
 import { Switch } from "@/components/ui/switch";
-import { documentsApiService } from "@/lib/apis/documents-api.service";
+import { type FolderSyncProgress, uploadFolderScan } from "@/lib/folder-sync-upload";
 import { getSupportedExtensionsSet } from "@/lib/supported-extensions";
 
 export interface SelectedFolder {
@@ -29,7 +29,7 @@ interface FolderWatchDialogProps {
 	initialFolder?: SelectedFolder | null;
 }
 
-const DEFAULT_EXCLUDE_PATTERNS = [
+export const DEFAULT_EXCLUDE_PATTERNS = [
 	".git",
 	"node_modules",
 	"__pycache__",
@@ -48,6 +48,8 @@ export function FolderWatchDialog({
 	const [selectedFolder, setSelectedFolder] = useState<SelectedFolder | null>(null);
 	const [shouldSummarize, setShouldSummarize] = useState(false);
 	const [submitting, setSubmitting] = useState(false);
+	const [progress, setProgress] = useState<FolderSyncProgress | null>(null);
+	const abortRef = useRef<AbortController | null>(null);
 
 	useEffect(() => {
 		if (open && initialFolder) {
@@ -64,33 +66,42 @@ export function FolderWatchDialog({
 		const folderPath = await api.selectFolder();
 		if (!folderPath) return;
 
-		const folderName = folderPath.split("/").pop() || folderPath.split("\\").pop() || folderPath;
+		const folderName = folderPath.split(/[/\\]/).pop() || folderPath;
 		setSelectedFolder({ path: folderPath, name: folderName });
 	}, []);
 
+	const handleCancel = useCallback(() => {
+		abortRef.current?.abort();
+	}, []);
+
 	const handleSubmit = useCallback(async () => {
 		if (!selectedFolder) return;
 		const api = window.electronAPI;
 		if (!api) return;
 
+		const controller = new AbortController();
+		abortRef.current = controller;
 		setSubmitting(true);
-		try {
-			const result = await documentsApiService.folderIndex(searchSpaceId, {
-				folder_path: selectedFolder.path,
-				folder_name: selectedFolder.name,
-				search_space_id: searchSpaceId,
-				enable_summary: shouldSummarize,
-				file_extensions: supportedExtensions,
-			});
+		setProgress(null);
 
-			const rootFolderId = (result as { root_folder_id?: number })?.root_folder_id ?? null;
+		try {
+			const rootFolderId = await uploadFolderScan({
+				folderPath: selectedFolder.path,
+				folderName: selectedFolder.name,
+				searchSpaceId,
+				excludePatterns: DEFAULT_EXCLUDE_PATTERNS,
+				fileExtensions: supportedExtensions,
+				enableSummary: shouldSummarize,
+				onProgress: setProgress,
+				signal: controller.signal,
+			});
 
 			await api.addWatchedFolder({
 				path: selectedFolder.path,
 				name: selectedFolder.name,
 				excludePatterns: DEFAULT_EXCLUDE_PATTERNS,
 				fileExtensions: supportedExtensions,
-				rootFolderId,
+				rootFolderId: rootFolderId ?? null,
 				searchSpaceId,
 				active: true,
 			});
@@ -98,12 +109,19 @@ export function FolderWatchDialog({
 			toast.success(`Watching folder: ${selectedFolder.name}`);
 			setSelectedFolder(null);
 			setShouldSummarize(false);
+			setProgress(null);
 			onOpenChange(false);
 			onSuccess?.();
 		} catch (err) {
-			toast.error((err as Error)?.message || "Failed to watch folder");
+			if ((err as Error)?.name === "AbortError") {
+				toast.info("Folder sync cancelled. Partial progress was saved.");
+			} else {
+				toast.error((err as Error)?.message || "Failed to watch folder");
+			}
 		} finally {
+			abortRef.current = null;
 			setSubmitting(false);
+			setProgress(null);
 		}
 	}, [
 		selectedFolder,
@@ -119,21 +137,44 @@ export function FolderWatchDialog({
 			if (!nextOpen && !submitting) {
 				setSelectedFolder(null);
 				setShouldSummarize(false);
+				setProgress(null);
 			}
 			onOpenChange(nextOpen);
 		},
 		[onOpenChange, submitting]
 	);
 
+	const progressLabel = useMemo(() => {
+		if (!progress) return null;
+		switch (progress.phase) {
+			case "listing":
+				return "Scanning folder...";
+			case "checking":
+				return `Checking ${progress.total} file(s)...`;
+			case "uploading":
+				return `Uploading ${progress.uploaded}/${progress.total} file(s)...`;
+			case "finalizing":
+				return "Finalizing...";
+			case "done":
+				return "Done!";
+			default:
+				return null;
+		}
+	}, [progress]);
+
 	return (
 		<Dialog open={open} onOpenChange={handleOpenChange}>
-			<DialogContent className="sm:max-w-md select-none">
-				<DialogHeader>
-					<DialogTitle>Watch Local Folder</DialogTitle>
-					<DialogDescription>Select a folder to sync and watch for changes.</DialogDescription>
+			<DialogContent className="sm:max-w-md select-none p-0 gap-0 overflow-hidden bg-muted dark:bg-muted border border-border [&>button]:opacity-80 [&>button]:hover:opacity-100 [&>button]:hover:bg-foreground/10">
+				<DialogHeader className="px-4 sm:px-6 pt-5 sm:pt-6 pb-3">
+					<DialogTitle className="text-lg sm:text-xl font-semibold tracking-tight">
+						Watch Local Folder
+					</DialogTitle>
+					<DialogDescription className="text-xs sm:text-sm text-muted-foreground/80">
+						Select a folder to sync and watch for changes
+					</DialogDescription>
 				</DialogHeader>
 
-				<div className="space-y-3 pt-2">
+				<div className="flex flex-col gap-3 px-4 sm:px-6 pb-4 sm:pb-6 min-h-[17rem]">
 					{selectedFolder ? (
 						<div className="flex items-center gap-2 py-1.5 pl-4 pr-2 rounded-md bg-slate-400/5 dark:bg-white/5 overflow-hidden">
 							<div className="min-w-0 flex-1 select-text">
@@ -156,7 +197,7 @@ export function FolderWatchDialog({
 						<button
 							type="button"
 							onClick={handleSelectFolder}
-							className="flex w-full items-center justify-center gap-2 rounded-lg border-2 border-dashed border-muted-foreground/30 py-8 text-sm text-muted-foreground transition-colors hover:border-foreground/50 hover:text-foreground"
+							className="flex flex-1 w-full items-center justify-center gap-2 rounded-lg border-2 border-dashed border-muted-foreground/30 text-sm text-muted-foreground transition-colors hover:border-foreground/50 hover:text-foreground"
 						>
 							Browse for a folder
 						</button>
@@ -174,14 +215,41 @@ export function FolderWatchDialog({
 								<Switch checked={shouldSummarize} onCheckedChange={setShouldSummarize} />
 							</div>
 
-							<Button className="w-full relative" onClick={handleSubmit} disabled={submitting}>
-								<span className={submitting ? "invisible" : ""}>Start Folder Sync</span>
-								{submitting && (
-									<span className="absolute inset-0 flex items-center justify-center">
-										<Spinner size="sm" />
-									</span>
+							{progressLabel && (
+								<div className="rounded-lg bg-slate-400/5 dark:bg-white/5 px-3 py-2">
+									<p className="text-xs text-muted-foreground">{progressLabel}</p>
+									{progress && progress.phase === "uploading" && progress.total > 0 && (
+										<div className="mt-1.5 h-1.5 w-full rounded-full bg-muted overflow-hidden">
+											<div
+												className="h-full bg-primary rounded-full transition-[width] duration-300"
+												style={{
+													width: `${Math.round((progress.uploaded / progress.total) * 100)}%`,
+												}}
+											/>
+										</div>
+									)}
+								</div>
+							)}
+
+							<div className="flex gap-2 mt-auto">
+								{submitting ? (
+									<>
+										<Button variant="secondary" className="flex-1" onClick={handleCancel}>
+											Cancel
+										</Button>
+										<Button className="flex-1 relative" disabled>
+											<span className="invisible">Syncing...</span>
+											<span className="absolute inset-0 flex items-center justify-center">
+												<Spinner size="sm" />
+											</span>
+										</Button>
+									</>
+								) : (
+									<Button className="w-full" onClick={handleSubmit}>
+										Start Folder Sync
+									</Button>
 								)}
-							</Button>
+							</div>
 						</>
 					)}
 				</div>