diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon.tsx index 43108c745..5c03d96fa 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon.tsx @@ -35,6 +35,7 @@ export function getDocumentTypeLabel(type: string): string { BOOKSTACK_CONNECTOR: "BookStack", CIRCLEBACK: "Circleback", OBSIDIAN_CONNECTOR: "Obsidian", + LOCAL_FOLDER_FILE: "Local Folder", SURFSENSE_DOCS: "SurfSense Docs", NOTE: "Note", COMPOSIO_GOOGLE_DRIVE_CONNECTOR: "Composio Google Drive", diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/local-folder-connect-form.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/local-folder-connect-form.tsx new file mode 100644 index 000000000..2e893c1c0 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/local-folder-connect-form.tsx @@ -0,0 +1,272 @@ +"use client"; + +import { zodResolver } from "@hookform/resolvers/zod"; +import { FolderSync, Info } from "lucide-react"; +import type { FC } from "react"; +import { useRef } from "react"; +import { useForm } from "react-hook-form"; +import * as z from "zod"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Button } from "@/components/ui/button"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { EnumConnectorName } from "@/contracts/enums/connector"; +import { getConnectorBenefits } from "../connector-benefits"; +import type { ConnectFormProps } from "../index"; + +const localFolderFormSchema = z.object({ + name: z.string().min(3, { + message: "Connector name must be at least 3 characters.", + }), + folder_path: z.string().min(1, { + message: "Folder path is required.", + }), + folder_name: z.string().min(1, { + message: "Folder name is required.", + }), + exclude_patterns: z.string().optional(), + file_extensions: z.string().optional(), +}); + +type LocalFolderFormValues = z.infer; + +export const LocalFolderConnectForm: FC = ({ onSubmit, isSubmitting }) => { + const isSubmittingRef = useRef(false); + const isElectron = typeof window !== "undefined" && !!window.electronAPI; + + const form = useForm({ + resolver: zodResolver(localFolderFormSchema), + defaultValues: { + name: "Local Folder", + folder_path: "", + folder_name: "", + exclude_patterns: "node_modules,.git,.DS_Store", + file_extensions: "", + }, + }); + + const handleBrowse = async () => { + if (!isElectron) return; + const selected = await window.electronAPI!.selectFolder(); + if (selected) { + form.setValue("folder_path", selected); + const folderName = selected.split(/[\\/]/).pop() || "folder"; + if (!form.getValues("folder_name")) { + form.setValue("folder_name", folderName); + } + if (form.getValues("name") === "Local Folder") { + form.setValue("name", folderName); + } + } + }; + + const handleSubmit = async (values: LocalFolderFormValues) => { + if (isSubmittingRef.current || isSubmitting) return; + isSubmittingRef.current = true; + + try { + const excludePatterns = values.exclude_patterns + ? values.exclude_patterns + .split(",") + .map((p) => p.trim()) + .filter(Boolean) + : []; + + const fileExtensions = values.file_extensions + ? values.file_extensions + .split(",") + .map((e) => { + const ext = e.trim(); + return ext.startsWith(".") ? ext : `.${ext}`; + }) + .filter(Boolean) + : null; + + await onSubmit({ + name: values.name, + connector_type: EnumConnectorName.LOCAL_FOLDER_CONNECTOR, + config: { + folder_path: values.folder_path, + folder_name: values.folder_name, + exclude_patterns: excludePatterns, + file_extensions: fileExtensions, + }, + is_indexable: true, + is_active: true, + last_indexed_at: null, + periodic_indexing_enabled: false, + indexing_frequency_minutes: null, + next_scheduled_at: null, + }); + } finally { + isSubmittingRef.current = false; + } + }; + + return ( +
+ + + Desktop App Required + + Real-time file watching is powered by the SurfSense desktop app. Files are + automatically synced whenever changes are detected. + + + +
+
+ + ( + + Connector Name + + + + + + )} + /> + + ( + + Folder Path +
+ + + + {isElectron && ( + + )} +
+ + The absolute path to the folder to watch and sync. + + + + )} + /> + + ( + + Display Name + + + + + A friendly name shown in the documents sidebar. + + + + )} + /> + + ( + + Exclude Patterns + + + + + Comma-separated patterns of directories/files to exclude. + + + + )} + /> + + ( + + File Extensions (optional) + + + + + Leave empty to index all supported files, or specify comma-separated extensions. + + + + )} + /> + + + +
+ + {getConnectorBenefits(EnumConnectorName.LOCAL_FOLDER_CONNECTOR) && ( +
+

+ What you get with Local Folder sync: +

+
    + {getConnectorBenefits(EnumConnectorName.LOCAL_FOLDER_CONNECTOR)?.map( + (benefit) =>
  • {benefit}
    • + )} +
+
+ )} +
+ ); +}; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/connector-benefits.ts b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/connector-benefits.ts index 0dc093100..40c6a7fdd 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/connector-benefits.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/connector-benefits.ts @@ -111,6 +111,14 @@ export function getConnectorBenefits(connectorType: string): string[] | null { "Incremental sync - only changed files are re-indexed", "Full support for your vault's folder structure", ], + LOCAL_FOLDER_CONNECTOR: [ + "Watch local folders for real-time changes via the desktop app", + "Automatic change detection — only modified files are re-indexed", + "Version history with up to 20 snapshots per document", + "Mirrors your folder structure in the SurfSense sidebar", + "Supports any text-based file format", + "Works as a periodic sync fallback when the desktop app is not running", + ], }; return benefits[connectorType] || null; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx index 37d4ad5d8..116893399 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx @@ -7,6 +7,7 @@ import { GithubConnectForm } from "./components/github-connect-form"; import { LinkupApiConnectForm } from "./components/linkup-api-connect-form"; import { LumaConnectForm } from "./components/luma-connect-form"; import { MCPConnectForm } from "./components/mcp-connect-form"; +import { LocalFolderConnectForm } from "./components/local-folder-connect-form"; import { ObsidianConnectForm } from "./components/obsidian-connect-form"; import { TavilyApiConnectForm } from "./components/tavily-api-connect-form"; @@ -58,7 +59,8 @@ export function getConnectFormComponent(connectorType: string): ConnectFormCompo return MCPConnectForm; case "OBSIDIAN_CONNECTOR": return ObsidianConnectForm; - // Add other connector types here as needed + case "LOCAL_FOLDER_CONNECTOR": + return LocalFolderConnectForm; default: return null; } diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/local-folder-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/local-folder-config.tsx new file mode 100644 index 000000000..cb4295079 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/local-folder-config.tsx @@ -0,0 +1,163 @@ +"use client"; + +import type { FC } from "react"; +import { useState } from "react"; +import { FolderSync } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import type { ConnectorConfigProps } from "../index"; + +export const LocalFolderConfig: FC = ({ + connector, + onConfigChange, + onNameChange, +}) => { + const isElectron = typeof window !== "undefined" && !!window.electronAPI; + + const [folderPath, setFolderPath] = useState( + (connector.config?.folder_path as string) || "" + ); + const [folderName, setFolderName] = useState( + (connector.config?.folder_name as string) || "" + ); + const [excludePatterns, setExcludePatterns] = useState(() => { + const patterns = connector.config?.exclude_patterns; + if (Array.isArray(patterns)) { + return patterns.join(", "); + } + return (patterns as string) || "node_modules, .git, .DS_Store"; + }); + const [fileExtensions, setFileExtensions] = useState(() => { + const exts = connector.config?.file_extensions; + if (Array.isArray(exts)) { + return exts.join(", "); + } + return (exts as string) || ""; + }); + const [name, setName] = useState(connector.name || ""); + + const handleFolderPathChange = (value: string) => { + setFolderPath(value); + onConfigChange?.({ ...connector.config, folder_path: value }); + }; + + const handleFolderNameChange = (value: string) => { + setFolderName(value); + onConfigChange?.({ ...connector.config, folder_name: value }); + }; + + const handleExcludePatternsChange = (value: string) => { + setExcludePatterns(value); + const arr = value + .split(",") + .map((p) => p.trim()) + .filter(Boolean); + onConfigChange?.({ ...connector.config, exclude_patterns: arr }); + }; + + const handleFileExtensionsChange = (value: string) => { + setFileExtensions(value); + const arr = value + ? value + .split(",") + .map((e) => { + const ext = e.trim(); + return ext.startsWith(".") ? ext : `.${ext}`; + }) + .filter(Boolean) + : null; + onConfigChange?.({ ...connector.config, file_extensions: arr }); + }; + + const handleNameChange = (value: string) => { + setName(value); + onNameChange?.(value); + }; + + const handleBrowse = async () => { + if (!isElectron) return; + const selected = await window.electronAPI!.selectFolder(); + if (selected) { + handleFolderPathChange(selected); + const autoName = selected.split(/[\\/]/).pop() || "folder"; + if (!folderName) handleFolderNameChange(autoName); + } + }; + + return ( +
+
+
+ + handleNameChange(e.target.value)} + placeholder="Local Folder" + className="border-slate-400/20 focus-visible:border-slate-400/40" + /> +
+
+ +
+

Folder Configuration

+ +
+
+ +
+ handleFolderPathChange(e.target.value)} + placeholder="/path/to/your/folder" + className="border-slate-400/20 focus-visible:border-slate-400/40 font-mono flex-1" + /> + {isElectron && ( + + )} +
+
+ +
+ + handleFolderNameChange(e.target.value)} + placeholder="My Notes" + className="border-slate-400/20 focus-visible:border-slate-400/40" + /> +
+ +
+ + handleExcludePatternsChange(e.target.value)} + placeholder="node_modules, .git, .DS_Store" + className="border-slate-400/20 focus-visible:border-slate-400/40 font-mono" + /> +

+ Comma-separated patterns of directories/files to exclude. +

+
+ +
+ + handleFileExtensionsChange(e.target.value)} + placeholder=".md, .txt, .rst" + className="border-slate-400/20 focus-visible:border-slate-400/40 font-mono" + /> +

+ Leave empty to index all supported files. +

+
+
+
+
+ ); +}; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx index a63435260..3dc1891c8 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx @@ -19,6 +19,7 @@ import { JiraConfig } from "./components/jira-config"; import { LinkupApiConfig } from "./components/linkup-api-config"; import { LumaConfig } from "./components/luma-config"; import { MCPConfig } from "./components/mcp-config"; +import { LocalFolderConfig } from "./components/local-folder-config"; import { ObsidianConfig } from "./components/obsidian-config"; import { OneDriveConfig } from "./components/onedrive-config"; import { SlackConfig } from "./components/slack-config"; @@ -82,6 +83,8 @@ export function getConnectorConfigComponent( return MCPConfig; case "OBSIDIAN_CONNECTOR": return ObsidianConfig; + case "LOCAL_FOLDER_CONNECTOR": + return LocalFolderConfig; case "COMPOSIO_GOOGLE_DRIVE_CONNECTOR": return ComposioDriveConfig; case "COMPOSIO_GMAIL_CONNECTOR": diff --git a/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts b/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts index f924bb15f..dd5978002 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts @@ -29,6 +29,7 @@ export const CONNECTOR_TO_DOCUMENT_TYPE: Record = { BOOKSTACK_CONNECTOR: "BOOKSTACK_CONNECTOR", CIRCLEBACK_CONNECTOR: "CIRCLEBACK", OBSIDIAN_CONNECTOR: "OBSIDIAN_CONNECTOR", + LOCAL_FOLDER_CONNECTOR: "LOCAL_FOLDER_FILE", // Special mappings (connector type differs from document type) GOOGLE_DRIVE_CONNECTOR: "GOOGLE_DRIVE_FILE", diff --git a/surfsense_web/components/documents/version-history.tsx b/surfsense_web/components/documents/version-history.tsx new file mode 100644 index 000000000..29740e079 --- /dev/null +++ b/surfsense_web/components/documents/version-history.tsx @@ -0,0 +1,185 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import { Clock, RotateCcw } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { + Sheet, + SheetContent, + SheetHeader, + SheetTitle, + SheetTrigger, +} from "@/components/ui/sheet"; +import { Spinner } from "@/components/ui/spinner"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; +import { toast } from "sonner"; + +interface DocumentVersionSummary { + version_number: number; + title: string; + content_hash: string; + created_at: string | null; +} + +interface VersionHistoryProps { + documentId: number; + documentType: string; +} + +export function VersionHistoryButton({ documentId, documentType }: VersionHistoryProps) { + const showVersionHistory = documentType === "LOCAL_FOLDER_FILE" || documentType === "OBSIDIAN_CONNECTOR"; + if (!showVersionHistory) return null; + + return ( + + + + + + + Version History + + + + + ); +} + +function VersionHistoryPanel({ documentId }: { documentId: number }) { + const [versions, setVersions] = useState([]); + const [loading, setLoading] = useState(true); + const [selectedVersion, setSelectedVersion] = useState(null); + const [versionContent, setVersionContent] = useState(""); + const [contentLoading, setContentLoading] = useState(false); + const [restoring, setRestoring] = useState(false); + + const loadVersions = useCallback(async () => { + setLoading(true); + try { + const data = await documentsApiService.listDocumentVersions(documentId); + setVersions(data as DocumentVersionSummary[]); + } catch { + toast.error("Failed to load version history"); + } finally { + setLoading(false); + } + }, [documentId]); + + useEffect(() => { + loadVersions(); + }, [loadVersions]); + + const handleSelectVersion = async (versionNumber: number) => { + setSelectedVersion(versionNumber); + setContentLoading(true); + try { + const data = (await documentsApiService.getDocumentVersion( + documentId, + versionNumber + )) as { source_markdown: string }; + setVersionContent(data.source_markdown || ""); + } catch { + toast.error("Failed to load version content"); + } finally { + setContentLoading(false); + } + }; + + const handleRestore = async (versionNumber: number) => { + setRestoring(true); + try { + await documentsApiService.restoreDocumentVersion(documentId, versionNumber); + toast.success(`Restored version ${versionNumber}`); + await loadVersions(); + } catch { + toast.error("Failed to restore version"); + } finally { + setRestoring(false); + } + }; + + if (loading) { + return ( +
+ +
+ ); + } + + if (versions.length === 0) { + return ( +
+ +

No version history available yet.

+

Versions are created when file content changes.

+
+ ); + } + + return ( +
+
+ {versions.map((v) => ( +
handleSelectVersion(v.version_number)} + > +
+
+

Version {v.version_number}

+ {v.created_at && ( +

+ {new Date(v.created_at).toLocaleString()} +

+ )} + {v.title && ( +

+ {v.title} +

+ )} +
+ +
+
+ ))} +
+ + {selectedVersion !== null && ( +
+

+ Preview — Version {selectedVersion} +

+ {contentLoading ? ( +
+ +
+ ) : ( + 
+							{versionContent || "(empty)"}
+
+ )} +
+ )} +
+ ); +} diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts index 1a3326bae..c663d6115 100644 --- a/surfsense_web/contracts/types/document.types.ts +++ b/surfsense_web/contracts/types/document.types.ts @@ -26,6 +26,7 @@ export const documentTypeEnum = z.enum([ "BOOKSTACK_CONNECTOR", "CIRCLEBACK", "OBSIDIAN_CONNECTOR", + "LOCAL_FOLDER_FILE", "SURFSENSE_DOCS", "NOTE", "COMPOSIO_GOOGLE_DRIVE_CONNECTOR", diff --git a/surfsense_web/hooks/use-folder-sync.ts b/surfsense_web/hooks/use-folder-sync.ts new file mode 100644 index 000000000..a35faf98f --- /dev/null +++ b/surfsense_web/hooks/use-folder-sync.ts @@ -0,0 +1,41 @@ +"use client"; + +import { useEffect, useRef } from "react"; +import { connectorsApiService } from "@/lib/apis/connectors-api.service"; + +const DEBOUNCE_MS = 2000; + +export function useFolderSync() { + const pendingRef = useRef>>(new Map()); + + useEffect(() => { + const api = typeof window !== "undefined" ? window.electronAPI : null; + if (!api?.onFileChanged) return; + + const cleanup = api.onFileChanged((event) => { + const key = `${event.connectorId}:${event.fullPath}`; + + const existing = pendingRef.current.get(key); + if (existing) clearTimeout(existing); + + const timeout = setTimeout(async () => { + pendingRef.current.delete(key); + try { + await connectorsApiService.indexFile(event.connectorId, event.fullPath); + } catch (err) { + console.error("[FolderSync] Failed to trigger re-index:", err); + } + }, DEBOUNCE_MS); + + pendingRef.current.set(key, timeout); + }); + + return () => { + cleanup(); + for (const timeout of pendingRef.current.values()) { + clearTimeout(timeout); + } + pendingRef.current.clear(); + }; + }, []); +} diff --git a/surfsense_web/lib/apis/connectors-api.service.ts b/surfsense_web/lib/apis/connectors-api.service.ts index abd16c7a7..f2722df70 100644 --- a/surfsense_web/lib/apis/connectors-api.service.ts +++ b/surfsense_web/lib/apis/connectors-api.service.ts @@ -404,6 +404,18 @@ class ConnectorsApiService { listDiscordChannelsResponse ); }; + + // ============================================================================= + // Local Folder Connector Methods + // ============================================================================= + + indexFile = async (connectorId: number, filePath: string) => { + return baseApiService.post( + `/api/v1/search-source-connectors/${connectorId}/index-file`, + undefined, + { body: { file_path: filePath } } + ); + }; } export type { SlackChannel, DiscordChannel }; diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts index 14a247032..d4a80f8a0 100644 --- a/surfsense_web/lib/apis/documents-api.service.ts +++ b/surfsense_web/lib/apis/documents-api.service.ts @@ -379,6 +379,22 @@ class DocumentsApiService { }); }; + listDocumentVersions = async (documentId: number) => { + return baseApiService.get(`/api/v1/documents/${documentId}/versions`); + }; + + getDocumentVersion = async (documentId: number, versionNumber: number) => { + return baseApiService.get( + `/api/v1/documents/${documentId}/versions/${versionNumber}` + ); + }; + + restoreDocumentVersion = async (documentId: number, versionNumber: number) => { + return baseApiService.post( + `/api/v1/documents/${documentId}/versions/${versionNumber}/restore` + ); + }; + /** * Delete a document */ diff --git a/surfsense_web/lib/connectors/utils.ts b/surfsense_web/lib/connectors/utils.ts index 90f7f5d21..6ce78be67 100644 --- a/surfsense_web/lib/connectors/utils.ts +++ b/surfsense_web/lib/connectors/utils.ts @@ -30,6 +30,7 @@ export const getConnectorTypeDisplay = (type: string): string => { YOUTUBE_CONNECTOR: "YouTube", CIRCLEBACK_CONNECTOR: "Circleback", OBSIDIAN_CONNECTOR: "Obsidian", + LOCAL_FOLDER_CONNECTOR: "Local Folder", DROPBOX_CONNECTOR: "Dropbox", MCP_CONNECTOR: "MCP Server", }; diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index 9cf1aa596..921449b41 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -1,5 +1,30 @@ import type { PostHog } from "posthog-js"; +interface WatchedFolderConfig { + path: string; + name: string; + excludePatterns: string[]; + fileExtensions: string[] | null; + connectorId: number; + searchSpaceId: number; + active: boolean; +} + +interface FolderSyncFileChangedEvent { + connectorId: number; + searchSpaceId: number; + folderPath: string; + relativePath: string; + fullPath: string; + action: "add" | "change" | "unlink"; + timestamp: number; +} + +interface FolderSyncWatcherReadyEvent { + connectorId: number; + folderPath: string; +} + interface ElectronAPI { versions: { electron: string; @@ -14,6 +39,16 @@ interface ElectronAPI { setQuickAskMode: (mode: string) => Promise; getQuickAskMode: () => Promise; replaceText: (text: string) => Promise; + // Folder sync + selectFolder: () => Promise; + addWatchedFolder: (config: WatchedFolderConfig) => Promise; + removeWatchedFolder: (folderPath: string) => Promise; + getWatchedFolders: () => Promise; + getWatcherStatus: () => Promise<{ path: string; active: boolean; watching: boolean }[]>; + onFileChanged: (callback: (data: FolderSyncFileChangedEvent) => void) => () => void; + onWatcherReady: (callback: (data: FolderSyncWatcherReadyEvent) => void) => () => void; + pauseWatcher: () => Promise; + resumeWatcher: () => Promise; } declare global {