diff --git a/surfsense_web/components/documents/FolderNode.tsx b/surfsense_web/components/documents/FolderNode.tsx index 2ec430871..01b01b8db 100644 --- a/surfsense_web/components/documents/FolderNode.tsx +++ b/surfsense_web/components/documents/FolderNode.tsx @@ -77,7 +77,7 @@ interface FolderNodeProps { contextMenuOpen?: boolean; onContextMenuOpenChange?: (open: boolean) => void; isWatched?: boolean; - onRescan?: (folder: FolderDisplay) => void; + onRescan?: (folder: FolderDisplay) => void | Promise; onStopWatching?: (folder: FolderDisplay) => void; } @@ -124,6 +124,17 @@ export const FolderNode = React.memo(function FolderNode({ const inputRef = useRef(null); const rowRef = useRef(null); const [dropZone, setDropZone] = useState(null); + const [isRescanning, setIsRescanning] = useState(false); + + const handleRescan = useCallback(async () => { + if (isRescanning) return; + setIsRescanning(true); + try { + await onRescan?.(folder); + } finally { + setIsRescanning(false); + } + }, [folder, onRescan, isRescanning]); const [{ isDragging }, drag] = useDrag( () => ({ @@ -347,17 +358,17 @@ export const FolderNode = React.memo(function FolderNode({ - {isWatched && onRescan && ( - { - e.stopPropagation(); - onRescan(folder); - }} - > - - Re-scan - - )} + {isWatched && onRescan && ( + { + e.stopPropagation(); + handleRescan(); + }} + > + + Re-scan + + )} {isWatched && onStopWatching && ( { @@ -396,16 +407,15 @@ export const FolderNode = React.memo(function FolderNode({ Move to... - { - e.stopPropagation(); - onDelete(folder); - }} - > - - Delete - + { + e.stopPropagation(); + onDelete(folder); + }} + > + + Delete + )} @@ -414,12 +424,12 @@ export const FolderNode = React.memo(function FolderNode({ {!isRenaming && contextMenuOpen && ( - {isWatched && onRescan && ( - onRescan(folder)}> - - Re-scan - - )} + {isWatched && onRescan && ( + handleRescan()}> + + Re-scan + + )} {isWatched && onStopWatching && ( onStopWatching(folder)}> @@ -438,13 +448,10 @@ export const FolderNode = React.memo(function FolderNode({ Move to... - onDelete(folder)} - > - - Delete - + onDelete(folder)}> + + Delete + )} diff --git a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx index 64da1e832..3d69e40c6 100644 --- a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx @@ -2,7 +2,7 @@ import { useQuery } from "@rocicorp/zero/react"; import { useAtom, useAtomValue, useSetAtom } from "jotai"; -import { ChevronLeft, ChevronRight, Trash2, Unplug } from "lucide-react"; +import { ChevronLeft, ChevronRight, FolderOpen, Trash2, Unplug } from "lucide-react"; import { useParams } from "next/navigation"; import { useTranslations } from "next-intl"; import { useCallback, useEffect, useMemo, useState } from "react"; @@ -20,6 +20,7 @@ import { CreateFolderDialog } from "@/components/documents/CreateFolderDialog"; import type { DocumentNodeDoc } from "@/components/documents/DocumentNode"; import type { FolderDisplay } from "@/components/documents/FolderNode"; import { FolderPickerDialog } from "@/components/documents/FolderPickerDialog"; +import { FolderWatchDialog } from "@/components/sources/FolderWatchDialog"; import { FolderTreeView } from "@/components/documents/FolderTreeView"; import { VersionHistoryDialog } from "@/components/documents/version-history"; import { EXPORT_FILE_EXTENSIONS } from "@/components/shared/ExportMenuItems"; @@ -95,6 +96,8 @@ export function DocumentsSidebar({ const debouncedSearch = useDebouncedValue(search, 250); const [activeTypes, setActiveTypes] = useState([]); const [watchedFolderIds, setWatchedFolderIds] = useState>(new Set()); + const [folderWatchOpen, setFolderWatchOpen] = useState(false); + const isElectron = typeof window !== "undefined" && !!window.electronAPI; useEffect(() => { const api = typeof window !== "undefined" ? window.electronAPI : null; @@ -292,6 +295,7 @@ export function DocumentsSidebar({ folder_name: matched.name, search_space_id: searchSpaceId, root_folder_id: folder.id, + file_extensions: matched.fileExtensions ?? undefined, }); toast.success(`Re-scanning folder: ${matched.name}`); } catch (err) { @@ -747,6 +751,17 @@ export function DocumentsSidebar({ + {isElectron && ( + + )} +
)} + {isElectron && ( + + )} + void; onAccordionStateChange?: (isExpanded: boolean) => void; } -const audioFileTypes = { - "audio/mpeg": [".mp3", ".mpeg", ".mpga"], - "audio/mp4": [".mp4", ".m4a"], - "audio/wav": [".wav"], - "audio/webm": [".webm"], - "text/markdown": [".md", ".markdown"], - "text/plain": [".txt"], -}; - -const commonTypes = { - "application/pdf": [".pdf"], - "application/vnd.openxmlformats-officedocument.wordprocessingml.document": [".docx"], - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [".xlsx"], - "application/vnd.openxmlformats-officedocument.presentationml.presentation": [".pptx"], - "text/html": [".html", ".htm"], - "text/csv": [".csv"], - "text/tab-separated-values": [".tsv"], - "image/jpeg": [".jpg", ".jpeg"], - "image/png": [".png"], - "image/bmp": [".bmp"], - "image/webp": [".webp"], - "image/tiff": [".tiff"], -}; - -const FILE_TYPE_CONFIG: Record> = { - LLAMACLOUD: { - ...commonTypes, - "application/msword": [".doc"], - "application/vnd.ms-word.document.macroEnabled.12": [".docm"], - "application/msword-template": [".dot"], - "application/vnd.ms-word.template.macroEnabled.12": [".dotm"], - "application/vnd.ms-powerpoint": [".ppt"], - "application/vnd.ms-powerpoint.template.macroEnabled.12": [".pptm"], - "application/vnd.ms-powerpoint.template": [".pot"], - "application/vnd.openxmlformats-officedocument.presentationml.template": [".potx"], - "application/vnd.ms-excel": [".xls"], - "application/vnd.ms-excel.sheet.macroEnabled.12": [".xlsm"], - "application/vnd.ms-excel.sheet.binary.macroEnabled.12": [".xlsb"], - "application/vnd.ms-excel.workspace": [".xlw"], - "application/rtf": [".rtf"], - "application/xml": [".xml"], - "application/epub+zip": [".epub"], - "image/gif": [".gif"], - "image/svg+xml": [".svg"], - ...audioFileTypes, - }, - DOCLING: { - ...commonTypes, - "text/asciidoc": [".adoc", ".asciidoc"], - "text/html": [".html", ".htm", ".xhtml"], - "image/tiff": [".tiff", ".tif"], - ...audioFileTypes, - }, - default: { - ...commonTypes, - "application/msword": [".doc"], - "message/rfc822": [".eml"], - "application/epub+zip": [".epub"], - "image/heic": [".heic"], - "application/vnd.ms-outlook": [".msg"], - "application/vnd.oasis.opendocument.text": [".odt"], - "text/x-org": [".org"], - "application/pkcs7-signature": [".p7s"], - "application/vnd.ms-powerpoint": [".ppt"], - "text/x-rst": [".rst"], - "application/rtf": [".rtf"], - "application/vnd.ms-excel": [".xls"], - "application/xml": [".xml"], - ...audioFileTypes, - }, -}; - interface FileWithId { id: string; file: File; @@ -150,24 +73,16 @@ export function DocumentUploadTab({ }; }, []); - const [selectedFolder, setSelectedFolder] = useState(null); - const [watchFolder, setWatchFolder] = useState(true); - const [folderSubmitting, setFolderSubmitting] = useState(false); const isElectron = typeof window !== "undefined" && !!window.electronAPI?.browseFiles; - const acceptedFileTypes = useMemo(() => { - const etlService = process.env.NEXT_PUBLIC_ETL_SERVICE; - return FILE_TYPE_CONFIG[etlService || "default"] || FILE_TYPE_CONFIG.default; - }, []); - + const acceptedFileTypes = useMemo(() => getAcceptedFileTypes(), []); const supportedExtensions = useMemo( - () => Array.from(new Set(Object.values(acceptedFileTypes).flat())).sort(), + () => getSupportedExtensions(acceptedFileTypes), [acceptedFileTypes] ); - const supportedExtensionsSet = useMemo( - () => new Set(supportedExtensions.map((ext) => ext.toLowerCase())), - [supportedExtensions] + () => getSupportedExtensionsSet(acceptedFileTypes), + [acceptedFileTypes] ); const addFiles = useCallback( @@ -197,7 +112,6 @@ export function DocumentUploadTab({ const onDrop = useCallback( (acceptedFiles: File[]) => { - setSelectedFolder(null); addFiles(acceptedFiles); }, [addFiles] @@ -221,27 +135,23 @@ export function DocumentUploadTab({ const paths = await api.browseFiles(); if (!paths || paths.length === 0) return; - setSelectedFolder(null); const fileDataList = await api.readLocalFiles(paths); - const newFiles: FileWithId[] = fileDataList.map((fd) => ({ + const filtered = fileDataList.filter((fd) => { + const ext = fd.name.includes(".") ? `.${fd.name.split(".").pop()?.toLowerCase()}` : ""; + return ext !== "" && supportedExtensionsSet.has(ext); + }); + + if (filtered.length === 0) { + toast.error(t("no_supported_files_in_folder")); + return; + } + + const newFiles: FileWithId[] = filtered.map((fd) => ({ id: crypto.randomUUID?.() ?? `file-${Date.now()}-${Math.random().toString(36)}`, file: new File([fd.data], fd.name, { type: fd.mimeType }), })); setFiles((prev) => [...prev, ...newFiles]); - }, []); - - const handleBrowseFolder = useCallback(async () => { - const api = window.electronAPI; - if (!api?.selectFolder) return; - - const folderPath = await api.selectFolder(); - if (!folderPath) return; - - const folderName = folderPath.split("/").pop() || folderPath.split("\\").pop() || folderPath; - setFiles([]); - setSelectedFolder({ path: folderPath, name: folderName }); - setWatchFolder(true); - }, []); + }, [supportedExtensionsSet, t]); const handleFolderChange = useCallback( (e: ChangeEvent) => { @@ -275,7 +185,7 @@ export function DocumentUploadTab({ const totalFileSize = files.reduce((total, entry) => total + entry.file.size, 0); - const hasContent = files.length > 0 || selectedFolder !== null; + const hasContent = files.length > 0; const handleAccordionChange = useCallback( (value: string) => { @@ -285,54 +195,6 @@ export function DocumentUploadTab({ [onAccordionStateChange] ); - const handleFolderSubmit = useCallback(async () => { - if (!selectedFolder) return; - const api = window.electronAPI; - if (!api) return; - - setFolderSubmitting(true); - try { - const numericSpaceId = Number(searchSpaceId); - const result = await documentsApiService.folderIndex(numericSpaceId, { - folder_path: selectedFolder.path, - folder_name: selectedFolder.name, - search_space_id: numericSpaceId, - enable_summary: shouldSummarize, - }); - - const rootFolderId = (result as { root_folder_id?: number })?.root_folder_id ?? null; - - if (watchFolder) { - await api.addWatchedFolder({ - path: selectedFolder.path, - name: selectedFolder.name, - excludePatterns: [ - ".git", - "node_modules", - "__pycache__", - ".DS_Store", - ".obsidian", - ".trash", - ], - fileExtensions: null, - rootFolderId, - searchSpaceId: Number(searchSpaceId), - active: true, - }); - toast.success(`Watching folder: ${selectedFolder.name}`); - } else { - toast.success(`Syncing folder: ${selectedFolder.name}`); - } - - setSelectedFolder(null); - onSuccess?.(); - } catch (err) { - toast.error((err as Error)?.message || "Failed to process folder"); - } finally { - setFolderSubmitting(false); - } - }, [selectedFolder, watchFolder, searchSpaceId, shouldSummarize, onSuccess]); - const handleUpload = async () => { setUploadProgress(0); trackDocumentUploadStarted(Number(searchSpaceId), files.length, totalFileSize); @@ -392,14 +254,14 @@ export function DocumentUploadTab({ className="dark:bg-neutral-800" onClick={(e) => e.stopPropagation()} > - - - Files - - - - Folder - + + + Files + + folderInputRef.current?.click()}> + + Folder + ); @@ -457,9 +319,8 @@ export function DocumentUploadTab({ {/* MOBILE DROP ZONE */}
- {hasContent ? ( - !selectedFolder && - (isElectron ? ( + {hasContent ? ( + isElectron ? (
{renderBrowseButton({ compact: true, fullWidth: true })}
) : ( - )) + ) ) : ( -
- -
-
-
-

Watch folder

-

Auto-sync when files change

-
- -
-
-
-

Enable AI Summary

-

- Improves search quality but adds latency -

-
- -
-
- - -
- )} - {/* FILES SELECTED */} {files.length > 0 && (
diff --git a/surfsense_web/components/sources/FolderWatchDialog.tsx b/surfsense_web/components/sources/FolderWatchDialog.tsx new file mode 100644 index 000000000..85e1fd1c0 --- /dev/null +++ b/surfsense_web/components/sources/FolderWatchDialog.tsx @@ -0,0 +1,196 @@ +"use client"; + +import { FolderOpen, X } from "lucide-react"; +import { useCallback, useMemo, useState } from "react"; +import { toast } from "sonner"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Spinner } from "@/components/ui/spinner"; +import { Switch } from "@/components/ui/switch"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; +import { getSupportedExtensionsSet } from "@/lib/supported-extensions"; + +interface SelectedFolder { + path: string; + name: string; +} + +interface FolderWatchDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + searchSpaceId: number; + onSuccess?: () => void; +} + +const DEFAULT_EXCLUDE_PATTERNS = [ + ".git", + "node_modules", + "__pycache__", + ".DS_Store", + ".obsidian", + ".trash", +]; + +export function FolderWatchDialog({ + open, + onOpenChange, + searchSpaceId, + onSuccess, +}: FolderWatchDialogProps) { + const [selectedFolder, setSelectedFolder] = useState(null); + const [shouldSummarize, setShouldSummarize] = useState(false); + const [submitting, setSubmitting] = useState(false); + + const supportedExtensions = useMemo( + () => Array.from(getSupportedExtensionsSet()), + [] + ); + + const handleSelectFolder = useCallback(async () => { + const api = window.electronAPI; + if (!api?.selectFolder) return; + + const folderPath = await api.selectFolder(); + if (!folderPath) return; + + const folderName = + folderPath.split("/").pop() || folderPath.split("\\").pop() || folderPath; + setSelectedFolder({ path: folderPath, name: folderName }); + }, []); + + const handleSubmit = useCallback(async () => { + if (!selectedFolder) return; + const api = window.electronAPI; + if (!api) return; + + setSubmitting(true); + try { + const result = await documentsApiService.folderIndex(searchSpaceId, { + folder_path: selectedFolder.path, + folder_name: selectedFolder.name, + search_space_id: searchSpaceId, + enable_summary: shouldSummarize, + file_extensions: supportedExtensions, + }); + + const rootFolderId = + (result as { root_folder_id?: number })?.root_folder_id ?? null; + + await api.addWatchedFolder({ + path: selectedFolder.path, + name: selectedFolder.name, + excludePatterns: DEFAULT_EXCLUDE_PATTERNS, + fileExtensions: supportedExtensions, + rootFolderId, + searchSpaceId, + active: true, + }); + + toast.success(`Watching folder: ${selectedFolder.name}`); + setSelectedFolder(null); + setShouldSummarize(false); + onOpenChange(false); + onSuccess?.(); + } catch (err) { + toast.error((err as Error)?.message || "Failed to watch folder"); + } finally { + setSubmitting(false); + } + }, [selectedFolder, searchSpaceId, shouldSummarize, supportedExtensions, onOpenChange, onSuccess]); + + const handleOpenChange = useCallback( + (nextOpen: boolean) => { + if (!nextOpen && !submitting) { + setSelectedFolder(null); + setShouldSummarize(false); + } + onOpenChange(nextOpen); + }, + [onOpenChange, submitting] + ); + + return ( + + + + Watch Local Folder + + Select a folder to sync and watch for changes. + + + +
+ {selectedFolder ? ( +
+ +
+

+ {selectedFolder.name} +

+

+ {selectedFolder.path} +

+
+ +
+ ) : ( + + )} + + {selectedFolder && ( + <> +
+
+

Enable AI Summary

+

+ Improves search quality but adds latency +

+
+ +
+ + + + )} +
+ + + ); +} diff --git a/surfsense_web/lib/supported-extensions.ts b/surfsense_web/lib/supported-extensions.ts new file mode 100644 index 000000000..a50a28cc6 --- /dev/null +++ b/surfsense_web/lib/supported-extensions.ts @@ -0,0 +1,94 @@ +const audioFileTypes: Record = { + "audio/mpeg": [".mp3", ".mpeg", ".mpga"], + "audio/mp4": [".mp4", ".m4a"], + "audio/wav": [".wav"], + "audio/webm": [".webm"], + "text/markdown": [".md", ".markdown"], + "text/plain": [".txt"], +}; + +const commonTypes: Record = { + "application/pdf": [".pdf"], + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": [".docx"], + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [".xlsx"], + "application/vnd.openxmlformats-officedocument.presentationml.presentation": [".pptx"], + "text/html": [".html", ".htm"], + "text/csv": [".csv"], + "text/tab-separated-values": [".tsv"], + "image/jpeg": [".jpg", ".jpeg"], + "image/png": [".png"], + "image/bmp": [".bmp"], + "image/webp": [".webp"], + "image/tiff": [".tiff"], +}; + +export const FILE_TYPE_CONFIG: Record> = { + LLAMACLOUD: { + ...commonTypes, + "application/msword": [".doc"], + "application/vnd.ms-word.document.macroEnabled.12": [".docm"], + "application/msword-template": [".dot"], + "application/vnd.ms-word.template.macroEnabled.12": [".dotm"], + "application/vnd.ms-powerpoint": [".ppt"], + "application/vnd.ms-powerpoint.template.macroEnabled.12": [".pptm"], + "application/vnd.ms-powerpoint.template": [".pot"], + "application/vnd.openxmlformats-officedocument.presentationml.template": [".potx"], + "application/vnd.ms-excel": [".xls"], + "application/vnd.ms-excel.sheet.macroEnabled.12": [".xlsm"], + "application/vnd.ms-excel.sheet.binary.macroEnabled.12": [".xlsb"], + "application/vnd.ms-excel.workspace": [".xlw"], + "application/rtf": [".rtf"], + "application/xml": [".xml"], + "application/epub+zip": [".epub"], + "image/gif": [".gif"], + "image/svg+xml": [".svg"], + ...audioFileTypes, + }, + DOCLING: { + ...commonTypes, + "text/asciidoc": [".adoc", ".asciidoc"], + "text/html": [".html", ".htm", ".xhtml"], + "image/tiff": [".tiff", ".tif"], + ...audioFileTypes, + }, + AZURE_DI: { + ...commonTypes, + "image/heic": [".heic"], + ...audioFileTypes, + }, + default: { + ...commonTypes, + "application/msword": [".doc"], + "message/rfc822": [".eml"], + "application/epub+zip": [".epub"], + "image/heic": [".heic"], + "application/vnd.ms-outlook": [".msg"], + "application/vnd.oasis.opendocument.text": [".odt"], + "text/x-org": [".org"], + "application/pkcs7-signature": [".p7s"], + "application/vnd.ms-powerpoint": [".ppt"], + "text/x-rst": [".rst"], + "application/rtf": [".rtf"], + "application/vnd.ms-excel": [".xls"], + "application/xml": [".xml"], + ...audioFileTypes, + }, +}; + +export function getAcceptedFileTypes(): Record { + const etlService = process.env.NEXT_PUBLIC_ETL_SERVICE; + return FILE_TYPE_CONFIG[etlService || "default"] || FILE_TYPE_CONFIG.default; +} + +export function getSupportedExtensions( + acceptedFileTypes?: Record +): string[] { + const types = acceptedFileTypes ?? getAcceptedFileTypes(); + return Array.from(new Set(Object.values(types).flat())).sort(); +} + +export function getSupportedExtensionsSet( + acceptedFileTypes?: Record +): Set { + return new Set(getSupportedExtensions(acceptedFileTypes).map((ext) => ext.toLowerCase())); +}