feat: implement local folder synchronization and versioning with new metadata handling and document_versions table

surfsense_backend/alembic/versions/117_add_local_folder_sync_and_versioning.py

@@ -1,4 +1,4 @@
-"""Add LOCAL_FOLDER_FILE document type and document_versions table
+"""Add LOCAL_FOLDER_FILE document type, folder metadata, and document_versions table
 
 Revision ID: 117
 Revises: 116
@@ -38,6 +38,19 @@ def upgrade() -> None:
     """
     )
 
+    # Add JSONB metadata column to folders table
+    col_exists = conn.execute(
+        sa.text(
+            "SELECT 1 FROM information_schema.columns "
+            "WHERE table_name = 'folders' AND column_name = 'metadata'"
+        )
+    ).fetchone()
+    if not col_exists:
+        op.add_column(
+            "folders",
+            sa.Column("metadata", sa.dialects.postgresql.JSONB, nullable=True),
+        )
+
     # Create document_versions table
     table_exists = conn.execute(
         sa.text(
@@ -124,3 +137,13 @@ def downgrade() -> None:
     op.execute("DROP INDEX IF EXISTS ix_document_versions_created_at")
     op.execute("DROP INDEX IF EXISTS ix_document_versions_document_id")
     op.execute("DROP TABLE IF EXISTS document_versions")
+
+    # Drop metadata column from folders
+    col_exists = conn.execute(
+        sa.text(
+            "SELECT 1 FROM information_schema.columns "
+            "WHERE table_name = 'folders' AND column_name = 'metadata'"
+        )
+    ).fetchone()
+    if col_exists:
+        op.drop_column("folders", "metadata")

surfsense_backend/app/db.py

@@ -956,6 +956,7 @@ class Folder(BaseModel, TimestampMixin):
         onupdate=lambda: datetime.now(UTC),
         index=True,
     )
+    folder_metadata = Column("metadata", JSONB, nullable=True)
 
     parent = relationship("Folder", remote_side="Folder.id", backref="children")
     search_space = relationship("SearchSpace", back_populates="folders")

surfsense_backend/app/routes/documents_routes.py

@@ -1310,6 +1310,13 @@ async def folder_index(
         "You don't have permission to create documents in this search space",
     )
 
+    watched_metadata = {
+        "watched": True,
+        "folder_path": request.folder_path,
+        "exclude_patterns": request.exclude_patterns,
+        "file_extensions": request.file_extensions,
+    }
+
     root_folder_id = request.root_folder_id
     if root_folder_id:
         existing = (
@@ -1319,6 +1326,9 @@ async def folder_index(
         ).scalar_one_or_none()
         if not existing:
             root_folder_id = None
+        else:
+            existing.folder_metadata = watched_metadata
+            await session.commit()
 
     if not root_folder_id:
         root_folder = Folder(
@@ -1326,6 +1336,7 @@ async def folder_index(
             search_space_id=request.search_space_id,
             created_by_id=str(user.id),
             position="a0",
+            folder_metadata=watched_metadata,
         )
         session.add(root_folder)
         await session.flush()
@@ -1403,3 +1414,34 @@ async def folder_index_file(
         "message": "File indexing started",
         "status": "processing",
     }
+
+
+@router.get("/documents/watched-folders", response_model=list["FolderRead"])
+async def get_watched_folders(
+    search_space_id: int,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    """Return root folders that are marked as watched (metadata->>'watched' = 'true')."""
+    from app.schemas import FolderRead  # noqa: F811
+
+    await check_permission(
+        session,
+        user,
+        search_space_id,
+        Permission.DOCUMENTS_READ.value,
+        "You don't have permission to read documents in this search space",
+    )
+
+    folders = (
+        await session.execute(
+            select(Folder).where(
+                Folder.search_space_id == search_space_id,
+                Folder.parent_id.is_(None),
+                Folder.folder_metadata.isnot(None),
+                Folder.folder_metadata["watched"].astext == "true",
+            )
+        )
+    ).scalars().all()
+
+    return folders

surfsense_backend/app/routes/folders_routes.py

@@ -192,6 +192,33 @@ async def get_folder_breadcrumb(
         ) from e
 
 
+@router.patch("/folders/{folder_id}/watched")
+async def stop_watching_folder(
+    folder_id: int,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    """Clear the watched flag from a folder's metadata."""
+    folder = await session.get(Folder, folder_id)
+    if not folder:
+        raise HTTPException(status_code=404, detail="Folder not found")
+
+    await check_permission(
+        session,
+        user,
+        folder.search_space_id,
+        Permission.DOCUMENTS_UPDATE.value,
+        "You don't have permission to update folders in this search space",
+    )
+
+    if folder.folder_metadata and isinstance(folder.folder_metadata, dict):
+        updated = {**folder.folder_metadata, "watched": False}
+        folder.folder_metadata = updated
+    await session.commit()
+
+    return {"message": "Folder watch status updated"}
+
+
 @router.put("/folders/{folder_id}", response_model=FolderRead)
 async def update_folder(
     folder_id: int,

surfsense_backend/app/schemas/folders.py

@@ -3,6 +3,8 @@
 from datetime import datetime
 from uuid import UUID
 
+from typing import Any
+
 from pydantic import BaseModel, ConfigDict, Field
 
 
@@ -34,6 +36,7 @@ class FolderRead(BaseModel):
     created_by_id: UUID | None
     created_at: datetime
     updated_at: datetime
+    metadata: dict[str, Any] | None = Field(default=None, validation_alias="folder_metadata")
 
     model_config = ConfigDict(from_attributes=True)