Merge remote-tracking branch 'upstream/dev' into feat/whatsapp-gateway-integration

surfsense_backend/app/routes/__init__.py

@@ -1,5 +1,7 @@
 from fastapi import APIRouter
 
+from app.automations.api import router as automations_router
+
 from .agent_action_log_route import router as agent_action_log_router
 from .agent_flags_route import router as agent_flags_router
 from .agent_permissions_route import router as agent_permissions_router
@@ -56,7 +58,6 @@ from .search_source_connectors_routes import router as search_source_connectors_
 from .search_spaces_routes import router as search_spaces_router
 from .slack_add_connector_route import router as slack_add_connector_router
 from .stripe_routes import router as stripe_router
-from .surfsense_docs_routes import router as surfsense_docs_router
 from .team_memory_routes import router as team_memory_router
 from .teams_add_connector_route import router as teams_add_connector_router
 from .video_presentations_routes import router as video_presentations_router
@@ -112,7 +113,6 @@ router.include_router(new_llm_config_router)  # LLM configs with prompt configur
 router.include_router(model_list_router)  # Dynamic model catalogue from OpenRouter
 router.include_router(logs_router)
 router.include_router(circleback_webhook_router)  # Circleback meeting webhooks
-router.include_router(surfsense_docs_router)  # Surfsense documentation for citations
 router.include_router(notifications_router)  # Notifications with Zero sync
 router.include_router(
     mcp_oauth_router
@@ -125,3 +125,4 @@ router.include_router(youtube_router)  # YouTube playlist resolution
 router.include_router(prompts_router)
 router.include_router(memory_router)  # User personal memory (memory.md style)
 router.include_router(team_memory_router)  # Search-space team memory
+router.include_router(automations_router)  # Automations CRUD + run history

surfsense_backend/app/routes/anonymous_chat_routes.py

@@ -351,10 +351,9 @@ async def stream_anonymous_chat(
     async def _generate():
         from langchain_core.messages import AIMessage, HumanMessage
 
-        from app.agents.new_chat.chat_deepagent import create_surfsense_deep_agent
+        from app.agents.new_chat.anonymous_agent import create_anonymous_chat_agent
         from app.agents.new_chat.checkpointer import get_checkpointer
         from app.db import shielded_async_session
-        from app.services.connector_service import ConnectorService
         from app.services.new_streaming_service import VercelStreamingService
         from app.services.token_tracking_service import start_turn
         from app.tasks.chat.stream_new_chat import StreamResult, _stream_agent_events
@@ -363,24 +362,23 @@ async def stream_anonymous_chat(
         streaming_service = VercelStreamingService()
 
         try:
-            async with shielded_async_session() as session:
-                connector_service = ConnectorService(session, search_space_id=None)
+            async with shielded_async_session():
                 checkpointer = await get_checkpointer()
 
                 anon_thread_id = f"anon-{session_id}-{request_id}"
 
-                agent = await create_surfsense_deep_agent(
+                # Load the optional uploaded document as read-only context.
+                anon_doc = await _load_anon_document(session_id)
+
+                # Minimal Q/A agent: web_search only (when enabled), no
+                # filesystem / persistence / subagents. The uploaded document
+                # is injected into the system prompt as read-only context.
+                agent = await create_anonymous_chat_agent(
                     llm=llm,
-                    search_space_id=0,
-                    db_session=session,
-                    connector_service=connector_service,
                     checkpointer=checkpointer,
-                    user_id=None,
-                    thread_id=None,
-                    agent_config=agent_config,
-                    enabled_tools=list(enabled_for_agent),
-                    disabled_tools=None,
                     anon_session_id=session_id,
+                    anon_doc=anon_doc,
+                    enable_web_search="web_search" in enabled_for_agent,
                 )
 
                 langchain_messages = []
@@ -396,7 +394,6 @@ async def stream_anonymous_chat(
 
                 input_state = {
                     "messages": langchain_messages,
-                    "search_space_id": 0,
                 }
 
                 langgraph_config = {
@@ -500,6 +497,38 @@ ANON_ALLOWED_EXTENSIONS = PLAINTEXT_EXTENSIONS | DIRECT_CONVERT_EXTENSIONS
 ANON_DOC_REDIS_PREFIX = "anon:doc:"
 
 
+async def _load_anon_document(session_id: str) -> dict[str, Any] | None:
+    """Read the anonymous session's uploaded document from Redis.
+
+    Returns ``{"title", "content"}`` for read-only injection into the agent's
+    system prompt, or ``None`` when nothing was uploaded for this session.
+    """
+    import json as _json
+
+    import redis.asyncio as aioredis
+
+    redis_client = aioredis.from_url(config.REDIS_APP_URL, decode_responses=True)
+    redis_key = f"{ANON_DOC_REDIS_PREFIX}{session_id}"
+    try:
+        data = await redis_client.get(redis_key)
+        if not data:
+            return None
+        payload = _json.loads(data)
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.warning("Failed to load anonymous document from Redis: %s", exc)
+        return None
+    finally:
+        await redis_client.aclose()
+
+    content = str(payload.get("content") or "")
+    if not content:
+        return None
+    return {
+        "title": str(payload.get("filename") or "uploaded_document"),
+        "content": content,
+    }
+
+
 class AnonDocResponse(BaseModel):
     filename: str
     size_bytes: int

surfsense_backend/app/routes/folders_routes.py

@@ -525,11 +525,8 @@ async def bulk_move_documents(
                     detail="Cannot move documents to a folder in a different search space",
                 )
 
-        await session.execute(
-            Document.__table__.update()
-            .where(Document.id.in_(request.document_ids))
-            .values(folder_id=request.folder_id)
-        )
+        for doc in documents:
+            doc.folder_id = request.folder_id
         await session.commit()
         return {"message": f"{len(request.document_ids)} documents moved successfully"}
 

surfsense_backend/app/routes/new_chat_routes.py

@@ -1785,7 +1785,6 @@ async def handle_new_chat(
                 user_id=str(user.id),
                 llm_config_id=llm_config_id,
                 mentioned_document_ids=request.mentioned_document_ids,
-                mentioned_surfsense_doc_ids=request.mentioned_surfsense_doc_ids,
                 mentioned_folder_ids=request.mentioned_folder_ids,
                 mentioned_connector_ids=request.mentioned_connector_ids,
                 mentioned_connectors=mentioned_connectors_payload,
@@ -2278,7 +2277,6 @@ async def regenerate_response(
                     user_id=str(user.id),
                     llm_config_id=llm_config_id,
                     mentioned_document_ids=request.mentioned_document_ids,
-                    mentioned_surfsense_doc_ids=request.mentioned_surfsense_doc_ids,
                     mentioned_folder_ids=request.mentioned_folder_ids,
                     mentioned_connector_ids=request.mentioned_connector_ids,
                     mentioned_connectors=mentioned_connectors_payload,

surfsense_backend/app/routes/rbac_routes.py

@@ -107,6 +107,12 @@ PERMISSION_DESCRIPTIONS = {
     "settings:view": "View search space settings",
     "settings:update": "Modify search space settings",
     "settings:delete": "Delete the entire search space",
+    # Automations
+    "automations:create": "Create automations from chat or JSON",
+    "automations:read": "View automations, their triggers, and run history",
+    "automations:update": "Edit automations and manage their triggers",
+    "automations:delete": "Remove automations from the search space",
+    "automations:execute": "Manually fire automations",
     # Full access
     "*": "Full access to all features and settings",
 }

surfsense_backend/app/routes/surfsense_docs_routes.py

@@ -1,172 +0,0 @@
-"""
-Routes for Surfsense documentation.
-
-These endpoints support the citation system for Surfsense docs,
-allowing the frontend to fetch document details when a user clicks
-on a [citation:doc-XXX] link.
-"""
-
-from fastapi import APIRouter, Depends, HTTPException
-from sqlalchemy import func, select
-from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import selectinload
-
-from app.db import (
-    SurfsenseDocsChunk,
-    SurfsenseDocsDocument,
-    User,
-    get_async_session,
-)
-from app.schemas import PaginatedResponse
-from app.schemas.surfsense_docs import (
-    SurfsenseDocsChunkRead,
-    SurfsenseDocsDocumentRead,
-    SurfsenseDocsDocumentWithChunksRead,
-)
-from app.users import current_active_user
-from app.utils.surfsense_docs import surfsense_docs_public_url
-
-router = APIRouter()
-
-
-@router.get(
-    "/surfsense-docs/by-chunk/{chunk_id}",
-    response_model=SurfsenseDocsDocumentWithChunksRead,
-)
-async def get_surfsense_doc_by_chunk_id(
-    chunk_id: int,
-    session: AsyncSession = Depends(get_async_session),
-    user: User = Depends(current_active_user),
-):
-    """
-    Retrieves a Surfsense documentation document based on a chunk ID.
-
-    This endpoint is used by the frontend to resolve [citation:doc-XXX] links.
-    """
-    try:
-        # Get the chunk
-        chunk_result = await session.execute(
-            select(SurfsenseDocsChunk).filter(SurfsenseDocsChunk.id == chunk_id)
-        )
-        chunk = chunk_result.scalars().first()
-
-        if not chunk:
-            raise HTTPException(
-                status_code=404,
-                detail=f"Surfsense docs chunk with id {chunk_id} not found",
-            )
-
-        # Get the associated document with all its chunks
-        document_result = await session.execute(
-            select(SurfsenseDocsDocument)
-            .options(selectinload(SurfsenseDocsDocument.chunks))
-            .filter(SurfsenseDocsDocument.id == chunk.document_id)
-        )
-        document = document_result.scalars().first()
-
-        if not document:
-            raise HTTPException(
-                status_code=404,
-                detail="Surfsense docs document not found",
-            )
-
-        # Sort chunks by ID
-        sorted_chunks = sorted(document.chunks, key=lambda x: x.id)
-
-        return SurfsenseDocsDocumentWithChunksRead(
-            id=document.id,
-            title=document.title,
-            source=document.source,
-            public_url=surfsense_docs_public_url(document.source),
-            content=document.content,
-            chunks=[
-                SurfsenseDocsChunkRead(id=c.id, content=c.content)
-                for c in sorted_chunks
-            ],
-        )
-    except HTTPException:
-        raise
-    except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail=f"Failed to retrieve Surfsense documentation: {e!s}",
-        ) from e
-
-
-@router.get(
-    "/surfsense-docs",
-    response_model=PaginatedResponse[SurfsenseDocsDocumentRead],
-)
-async def list_surfsense_docs(
-    page: int = 0,
-    page_size: int = 50,
-    title: str | None = None,
-    session: AsyncSession = Depends(get_async_session),
-    user: User = Depends(current_active_user),
-):
-    """
-    List all Surfsense documentation documents.
-
-    Args:
-        page: Zero-based page index.
-        page_size: Number of items per page (default: 50).
-        title: Optional title filter (case-insensitive substring match).
-        session: Database session (injected).
-        user: Current authenticated user (injected).
-
-    Returns:
-        PaginatedResponse[SurfsenseDocsDocumentRead]: Paginated list of Surfsense docs.
-    """
-    try:
-        # Base query
-        query = select(SurfsenseDocsDocument)
-        count_query = select(func.count()).select_from(SurfsenseDocsDocument)
-
-        # Filter by title if provided
-        if title and title.strip():
-            query = query.filter(SurfsenseDocsDocument.title.ilike(f"%{title}%"))
-            count_query = count_query.filter(
-                SurfsenseDocsDocument.title.ilike(f"%{title}%")
-            )
-
-        # Get total count
-        total_result = await session.execute(count_query)
-        total = total_result.scalar() or 0
-
-        # Calculate offset
-        offset = page * page_size
-
-        # Get paginated results
-        result = await session.execute(
-            query.order_by(SurfsenseDocsDocument.title).offset(offset).limit(page_size)
-        )
-        docs = result.scalars().all()
-
-        # Convert to response format
-        items = [
-            SurfsenseDocsDocumentRead(
-                id=doc.id,
-                title=doc.title,
-                source=doc.source,
-                public_url=surfsense_docs_public_url(doc.source),
-                content=doc.content,
-                created_at=doc.created_at,
-                updated_at=doc.updated_at,
-            )
-            for doc in docs
-        ]
-
-        has_more = (offset + len(items)) < total
-
-        return PaginatedResponse(
-            items=items,
-            total=total,
-            page=page,
-            page_size=page_size,
-            has_more=has_more,
-        )
-    except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail=f"Failed to list Surfsense documentation: {e!s}",
-        ) from e