feat: implement local folder synchronization and versioning with new metadata handling and document_versions table

surfsense_backend/alembic/versions/117_add_local_folder_sync_and_versioning.py

@@ -1,4 +1,4 @@
-"""Add LOCAL_FOLDER_FILE document type and document_versions table
+"""Add LOCAL_FOLDER_FILE document type, folder metadata, and document_versions table
 
 Revision ID: 117
 Revises: 116
@@ -38,6 +38,19 @@ def upgrade() -> None:
     """
     )
 
+    # Add JSONB metadata column to folders table
+    col_exists = conn.execute(
+        sa.text(
+            "SELECT 1 FROM information_schema.columns "
+            "WHERE table_name = 'folders' AND column_name = 'metadata'"
+        )
+    ).fetchone()
+    if not col_exists:
+        op.add_column(
+            "folders",
+            sa.Column("metadata", sa.dialects.postgresql.JSONB, nullable=True),
+        )
+
     # Create document_versions table
     table_exists = conn.execute(
         sa.text(
@@ -124,3 +137,13 @@ def downgrade() -> None:
     op.execute("DROP INDEX IF EXISTS ix_document_versions_created_at")
     op.execute("DROP INDEX IF EXISTS ix_document_versions_document_id")
     op.execute("DROP TABLE IF EXISTS document_versions")
+
+    # Drop metadata column from folders
+    col_exists = conn.execute(
+        sa.text(
+            "SELECT 1 FROM information_schema.columns "
+            "WHERE table_name = 'folders' AND column_name = 'metadata'"
+        )
+    ).fetchone()
+    if col_exists:
+        op.drop_column("folders", "metadata")

surfsense_backend/app/db.py

@@ -956,6 +956,7 @@ class Folder(BaseModel, TimestampMixin):
         onupdate=lambda: datetime.now(UTC),
         index=True,
     )
+    folder_metadata = Column("metadata", JSONB, nullable=True)
 
     parent = relationship("Folder", remote_side="Folder.id", backref="children")
     search_space = relationship("SearchSpace", back_populates="folders")

surfsense_backend/app/routes/documents_routes.py

@@ -1310,6 +1310,13 @@ async def folder_index(
         "You don't have permission to create documents in this search space",
     )
 
+    watched_metadata = {
+        "watched": True,
+        "folder_path": request.folder_path,
+        "exclude_patterns": request.exclude_patterns,
+        "file_extensions": request.file_extensions,
+    }
+
     root_folder_id = request.root_folder_id
     if root_folder_id:
         existing = (
@@ -1319,6 +1326,9 @@ async def folder_index(
         ).scalar_one_or_none()
         if not existing:
             root_folder_id = None
+        else:
+            existing.folder_metadata = watched_metadata
+            await session.commit()
 
     if not root_folder_id:
         root_folder = Folder(
@@ -1326,6 +1336,7 @@ async def folder_index(
             search_space_id=request.search_space_id,
             created_by_id=str(user.id),
             position="a0",
+            folder_metadata=watched_metadata,
         )
         session.add(root_folder)
         await session.flush()
@@ -1403,3 +1414,34 @@ async def folder_index_file(
         "message": "File indexing started",
         "status": "processing",
     }
+
+
+@router.get("/documents/watched-folders", response_model=list["FolderRead"])
+async def get_watched_folders(
+    search_space_id: int,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    """Return root folders that are marked as watched (metadata->>'watched' = 'true')."""
+    from app.schemas import FolderRead  # noqa: F811
+
+    await check_permission(
+        session,
+        user,
+        search_space_id,
+        Permission.DOCUMENTS_READ.value,
+        "You don't have permission to read documents in this search space",
+    )
+
+    folders = (
+        await session.execute(
+            select(Folder).where(
+                Folder.search_space_id == search_space_id,
+                Folder.parent_id.is_(None),
+                Folder.folder_metadata.isnot(None),
+                Folder.folder_metadata["watched"].astext == "true",
+            )
+        )
+    ).scalars().all()
+
+    return folders

surfsense_backend/app/routes/folders_routes.py

@@ -192,6 +192,33 @@ async def get_folder_breadcrumb(
         ) from e
 
 
+@router.patch("/folders/{folder_id}/watched")
+async def stop_watching_folder(
+    folder_id: int,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    """Clear the watched flag from a folder's metadata."""
+    folder = await session.get(Folder, folder_id)
+    if not folder:
+        raise HTTPException(status_code=404, detail="Folder not found")
+
+    await check_permission(
+        session,
+        user,
+        folder.search_space_id,
+        Permission.DOCUMENTS_UPDATE.value,
+        "You don't have permission to update folders in this search space",
+    )
+
+    if folder.folder_metadata and isinstance(folder.folder_metadata, dict):
+        updated = {**folder.folder_metadata, "watched": False}
+        folder.folder_metadata = updated
+    await session.commit()
+
+    return {"message": "Folder watch status updated"}
+
+
 @router.put("/folders/{folder_id}", response_model=FolderRead)
 async def update_folder(
     folder_id: int,

surfsense_backend/app/schemas/folders.py

@@ -3,6 +3,8 @@
 from datetime import datetime
 from uuid import UUID
 
+from typing import Any
+
 from pydantic import BaseModel, ConfigDict, Field
 
 
@@ -34,6 +36,7 @@ class FolderRead(BaseModel):
     created_by_id: UUID | None
     created_at: datetime
     updated_at: datetime
+    metadata: dict[str, Any] | None = Field(default=None, validation_alias="folder_metadata")
 
     model_config = ConfigDict(from_attributes=True)
 

surfsense_web/components/documents/FolderNode.tsx

@@ -76,6 +76,7 @@ interface FolderNodeProps {
 	isWatched?: boolean;
 	onRescan?: (folder: FolderDisplay) => void;
 	onStopWatching?: (folder: FolderDisplay) => void;
+	onViewMetadata?: (folder: FolderDisplay) => void;
 }
 
 function getDropZone(
@@ -116,6 +117,7 @@ export const FolderNode = React.memo(function FolderNode({
 	isWatched,
 	onRescan,
 	onStopWatching,
+	onViewMetadata,
 }: FolderNodeProps) {
 	const [renameValue, setRenameValue] = useState(folder.name);
 	const inputRef = useRef<HTMLInputElement>(null);
@@ -251,13 +253,21 @@ export const FolderNode = React.memo(function FolderNode({
 						isOver && !canDrop && "cursor-not-allowed"
 					)}
 					style={{ paddingLeft: `${depth * 16 + 4}px` }}
-					onClick={() => onToggleExpand(folder.id)}
+				onClick={(e) => {
-					onKeyDown={(e) => {
+					if ((e.ctrlKey || e.metaKey) && onViewMetadata) {
-						if (e.key === "Enter" || e.key === " ") {
+						e.preventDefault();
-							e.preventDefault();
+						e.stopPropagation();
-							onToggleExpand(folder.id);
+						onViewMetadata(folder);
-						}
+						return;
-					}}
+					}
+					onToggleExpand(folder.id);
+				}}
+				onKeyDown={(e) => {
+					if (e.key === "Enter" || e.key === " ") {
+						e.preventDefault();
+						onToggleExpand(folder.id);
+					}
+				}}
 					onDoubleClick={(e) => {
 						e.stopPropagation();
 						startRename();

surfsense_web/components/documents/FolderTreeView.tsx

@@ -43,6 +43,7 @@ interface FolderTreeViewProps {
 	watchedFolderIds?: Set<number>;
 	onRescanFolder?: (folder: FolderDisplay) => void;
 	onStopWatchingFolder?: (folder: FolderDisplay) => void;
+	onViewFolderMetadata?: (folder: FolderDisplay) => void;
 }
 
 function groupBy<T>(items: T[], keyFn: (item: T) => string | number): Record<string | number, T[]> {
@@ -79,6 +80,7 @@ export function FolderTreeView({
 	watchedFolderIds,
 	onRescanFolder,
 	onStopWatchingFolder,
+	onViewFolderMetadata,
 }: FolderTreeViewProps) {
 	const foldersByParent = useMemo(() => groupBy(folders, (f) => f.parentId ?? "root"), [folders]);
 
@@ -210,10 +212,11 @@ export function FolderTreeView({
 					siblingPositions={siblingPositions}
 					contextMenuOpen={openContextMenuId === `folder-${f.id}`}
 					onContextMenuOpenChange={(open) => setOpenContextMenuId(open ? `folder-${f.id}` : null)}
-					isWatched={watchedFolderIds?.has(f.id)}
+				isWatched={watchedFolderIds?.has(f.id)}
-					onRescan={onRescanFolder}
+				onRescan={onRescanFolder}
-					onStopWatching={onStopWatchingFolder}
+				onStopWatching={onStopWatchingFolder}
-				/>
+				onViewMetadata={onViewFolderMetadata}
+			/>
 			);
 
 			if (isExpanded) {

surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx

@@ -21,6 +21,7 @@ import type { DocumentNodeDoc } from "@/components/documents/DocumentNode";
 import type { FolderDisplay } from "@/components/documents/FolderNode";
 import { FolderPickerDialog } from "@/components/documents/FolderPickerDialog";
 import { FolderTreeView } from "@/components/documents/FolderTreeView";
+import { JsonMetadataViewer } from "@/components/json-metadata-viewer";
 import { EXPORT_FILE_EXTENSIONS } from "@/components/shared/ExportMenuItems";
 import {
 	AlertDialog,
@@ -95,12 +96,46 @@ export function DocumentsSidebar({
 	const [activeTypes, setActiveTypes] = useState<DocumentTypeEnum[]>([]);
 	const [watchedFolderIds, setWatchedFolderIds] = useState<Set<number>>(new Set());
 
+	const [metadataFolder, setMetadataFolder] = useState<FolderDisplay | null>(null);
+	const [metadataJson, setMetadataJson] = useState<Record<string, unknown> | null>(null);
+	const [metadataLoading, setMetadataLoading] = useState(false);
+
 	useEffect(() => {
 		const api = typeof window !== "undefined" ? window.electronAPI : null;
 		if (!api?.getWatchedFolders) return;
 
 		async function loadWatchedIds() {
 			const folders = await api!.getWatchedFolders();
+
+			if (folders.length === 0) {
+				try {
+					const backendFolders = await documentsApiService.getWatchedFolders(searchSpaceId);
+					for (const bf of backendFolders) {
+						const meta = bf.metadata as Record<string, unknown> | null;
+						if (!meta?.watched || !meta.folder_path) continue;
+						await api!.addWatchedFolder({
+							path: meta.folder_path as string,
+							name: bf.name,
+							rootFolderId: bf.id,
+							searchSpaceId: bf.search_space_id,
+							excludePatterns: (meta.exclude_patterns as string[]) ?? [],
+							fileExtensions: (meta.file_extensions as string[] | null) ?? null,
+							active: true,
+						});
+					}
+					const recovered = await api!.getWatchedFolders();
+					const ids = new Set(
+						recovered
+							.filter((f) => f.rootFolderId != null)
+							.map((f) => f.rootFolderId as number)
+					);
+					setWatchedFolderIds(ids);
+					return;
+				} catch (err) {
+					console.error("[DocumentsSidebar] Recovery from backend failed:", err);
+				}
+			}
+
 			const ids = new Set(
 				folders
 					.filter((f) => f.rootFolderId != null)
@@ -110,7 +145,7 @@ export function DocumentsSidebar({
 		}
 
 		loadWatchedIds();
-	}, []);
+	}, [searchSpaceId]);
 	const { mutateAsync: deleteDocumentMutation } = useAtomValue(deleteDocumentMutationAtom);
 
 	const [sidebarDocs, setSidebarDocs] = useAtom(sidebarSelectedDocumentsAtom);
@@ -318,11 +353,30 @@ export function DocumentsSidebar({
 			}
 
 			await api.removeWatchedFolder(matched.path);
+			try {
+				await foldersApiService.stopWatching(folder.id);
+			} catch (err) {
+				console.error("[DocumentsSidebar] Failed to clear watched metadata:", err);
+			}
 			toast.success(`Stopped watching: ${matched.name}`);
 		},
 		[]
 	);
 
+	const handleViewFolderMetadata = useCallback(async (folder: FolderDisplay) => {
+		setMetadataFolder(folder);
+		setMetadataLoading(true);
+		try {
+			const fullFolder = await foldersApiService.getFolder(folder.id);
+			setMetadataJson((fullFolder.metadata as Record<string, unknown>) ?? {});
+		} catch (err) {
+			console.error("[DocumentsSidebar] Failed to fetch folder metadata:", err);
+			setMetadataJson({ error: "Failed to load folder metadata" });
+		} finally {
+			setMetadataLoading(false);
+		}
+	}, []);
+
 	const handleRenameFolder = useCallback(async (folder: FolderDisplay, newName: string) => {
 		try {
 			await foldersApiService.updateFolder(folder.id, { name: newName });
@@ -801,11 +855,26 @@ export function DocumentsSidebar({
 				onReorderFolder={handleReorderFolder}
 				watchedFolderIds={watchedFolderIds}
 				onRescanFolder={handleRescanFolder}
-				onStopWatchingFolder={handleStopWatching}
+			onStopWatchingFolder={handleStopWatching}
-			/>
+			onViewFolderMetadata={handleViewFolderMetadata}
-			</div>
+		/>
+		</div>
 
-			<FolderPickerDialog
+		<JsonMetadataViewer
+			title={metadataFolder?.name ?? "Folder"}
+			metadata={metadataJson}
+			loading={metadataLoading}
+			open={!!metadataFolder}
+			onOpenChange={(open) => {
+				if (!open) {
+					setMetadataFolder(null);
+					setMetadataJson(null);
+					setMetadataLoading(false);
+				}
+			}}
+		/>
+
+		<FolderPickerDialog
 				open={folderPickerOpen}
 				onOpenChange={setFolderPickerOpen}
 				folders={treeFolders}

surfsense_web/contracts/types/folder.types.ts

@@ -9,6 +9,7 @@ export const folder = z.object({
 	created_by_id: z.string().nullable().optional(),
 	created_at: z.string(),
 	updated_at: z.string(),
+	metadata: z.record(z.unknown()).nullable().optional(),
 });
 
 export const folderCreateRequest = z.object({

surfsense_web/lib/apis/documents-api.service.ts

@@ -37,6 +37,7 @@ import {
 	uploadDocumentRequest,
 	uploadDocumentResponse,
 } from "@/contracts/types/document.types";
+import { folderListResponse } from "@/contracts/types/folder.types";
 import { ValidationError } from "../error";
 import { baseApiService } from "./base-api.service";
 
@@ -403,6 +404,10 @@ class DocumentsApiService {
 		return baseApiService.post(`/api/v1/documents/folder-index-file`, undefined, { body });
 	};
 
+	getWatchedFolders = async (searchSpaceId: number) => {
+		return baseApiService.get(`/api/v1/documents/watched-folders?search_space_id=${searchSpaceId}`, folderListResponse);
+	};
+
 	/**
 	 * Delete a document
 	 */

surfsense_web/lib/apis/folders-api.service.ts

@@ -85,6 +85,10 @@ class FoldersApiService {
 		return baseApiService.delete(`/api/v1/folders/${folderId}`, folderDeleteResponse);
 	};
 
+	stopWatching = async (folderId: number) => {
+		return baseApiService.patch(`/api/v1/folders/${folderId}/watched`, undefined);
+	};
+
 	moveDocument = async (documentId: number, request: DocumentMoveRequest) => {
 		const parsed = documentMoveRequest.safeParse(request);
 		if (!parsed.success) {