Merge branch 'dev' into google-drive-connector

Merge in dev

.dockerignore

@@ -75,7 +75,6 @@ surfsense_backend/lib64/
 
 # Logs
 **/*.log
-**/logs/
 
 # Temporary files
 **/tmp/

surfsense_backend/app/agents/new_chat/chat_deepagent.py

@@ -50,6 +50,9 @@ def create_surfsense_deep_agent(
     - display_image: Display images in chat
     - scrape_webpage: Extract content from webpages
 
+    The agent also includes TodoListMiddleware by default (via create_deep_agent) which provides:
+    - write_todos: Create and update planning/todo lists for complex tasks
+
     The system prompt can be configured via agent_config:
     - Custom system instructions (or use defaults)
     - Citation toggle (enable/disable citation requirements)
@@ -138,6 +141,7 @@ def create_surfsense_deep_agent(
         system_prompt = build_surfsense_system_prompt()
 
     # Create the deep agent with system prompt and checkpointer
+    # Note: TodoListMiddleware (write_todos) is included by default in create_deep_agent
     agent = create_deep_agent(
         model=llm,
         tools=tools,

surfsense_backend/app/agents/new_chat/system_prompt.py

@@ -64,18 +64,23 @@ You have access to the following tools:
   - The preview card will automatically be displayed in the chat.
 
 4. display_image: Display an image in the chat with metadata.
-  - Use this tool when you want to show an image from a URL to the user.
+  - Use this tool ONLY when you have a valid public HTTP/HTTPS image URL to show.
   - This displays the image with an optional title, description, and source attribution.
-  - Common use cases:
-    * Showing an image from a URL mentioned in the conversation
-    * Displaying a diagram, chart, or illustration you're referencing
-    * Showing visual examples when explaining concepts
-  - IMPORTANT: Do NOT use this tool for user-uploaded image attachments!
-    * User attachments are already visible in the chat UI - the user can see them
-    * This tool requires a valid HTTP/HTTPS URL, not a local file path
-    * When a user uploads an image, just analyze it and respond - don't try to display it again
+  - Valid use cases:
+    * Showing an image from a URL the user explicitly mentioned in their message
+    * Displaying images found in scraped webpage content (from scrape_webpage tool)
+    * Showing a publicly accessible diagram or chart from a known URL
+  
+  CRITICAL - NEVER USE THIS TOOL FOR USER-UPLOADED ATTACHMENTS:
+  When a user uploads/attaches an image file to their message:
+    * The image is ALREADY VISIBLE in the chat UI as a thumbnail on their message
+    * You do NOT have a URL for their uploaded image - only extracted text/description
+    * Calling display_image will FAIL and show "Image not available" error
+    * Simply analyze the image content and respond with your analysis - DO NOT try to display it
+    * The user can already see their own uploaded image - they don't need you to show it again
+  
   - Args:
-    - src: The URL of the image to display (must be a valid HTTP/HTTPS image URL, not a local path)
+    - src: The URL of the image (MUST be a valid public HTTP/HTTPS URL that you know exists)
     - alt: Alternative text describing the image (for accessibility)
     - title: Optional title to display below the image
     - description: Optional description providing context about the image
@@ -104,6 +109,20 @@ You have access to the following tools:
     * This makes your response more visual and engaging.
     * Prioritize showing: diagrams, charts, infographics, key illustrations, or images that help explain the content.
     * Don't show every image - just the most relevant 1-3 images that enhance understanding.
+
+6. write_todos: Create and update a planning/todo list to break down complex tasks.
+  - IMPORTANT: Use this tool when the user asks you to create a plan, break down a task, or explain something in structured steps.
+  - This tool creates a visual plan with progress tracking that the user can see in the UI.
+  - When to use:
+    * User asks to "create a plan" or "break down" a task
+    * User asks for "steps" to do something
+    * User asks you to "explain" something in sections
+    * Any multi-step task that would benefit from structured planning
+  - Args:
+    - todos: List of todo items, each with:
+      * content: Description of the task (required)
+      * status: "pending", "in_progress", or "completed" (required)
+  - The tool automatically adds IDs and formats the output for the UI.
 </tools>
 <tool_call_examples>
 - User: "Fetch all my notes and what's in them?"
@@ -134,8 +153,15 @@ You have access to the following tools:
 - User: "Show me this image: https://example.com/image.png"
   - Call: `display_image(src="https://example.com/image.png", alt="User shared image")`
 
-- User: "Can you display a diagram of a neural network?"
-  - Call: `display_image(src="https://example.com/neural-network.png", alt="Neural network diagram", title="Neural Network Architecture", description="A visual representation of a neural network with input, hidden, and output layers")`
+- User uploads an image file and asks: "What is this image about?"
+  - DO NOT call display_image! The user's uploaded image is already visible in the chat.
+  - Simply analyze the image content (which you receive as extracted text/description) and respond.
+  - WRONG: `display_image(src="...", ...)` - This will fail with "Image not available"
+  - CORRECT: Just provide your analysis directly: "Based on the image you shared, this appears to be..."
+
+- User uploads a screenshot and asks: "Can you explain what's in this image?"
+  - DO NOT call display_image! Just analyze and respond directly.
+  - The user can already see their screenshot - they don't need you to display it again.
 
 - User: "Read this article and summarize it for me: https://example.com/blog/ai-trends"
   - Call: `scrape_webpage(url="https://example.com/blog/ai-trends")`
@@ -154,6 +180,34 @@ You have access to the following tools:
   - Then, if the content contains useful diagrams/images like `![Neural Network Diagram](https://example.com/nn-diagram.png)`:
     - Call: `display_image(src="https://example.com/nn-diagram.png", alt="Neural Network Diagram", title="Neural Network Architecture")`
   - Then provide your explanation, referencing the displayed image
+
+- User: "Create a plan for building a user authentication system"
+  - Call: `write_todos(todos=[{"content": "Design database schema for users and sessions", "status": "in_progress"}, {"content": "Implement registration and login endpoints", "status": "pending"}, {"content": "Add password reset functionality", "status": "pending"}])`
+  - Then explain each step in detail as you work through them
+
+- User: "Break down how to build a REST API into steps"
+  - Call: `write_todos(todos=[{"content": "Design API endpoints and data models", "status": "in_progress"}, {"content": "Set up server framework and routing", "status": "pending"}, {"content": "Implement CRUD operations", "status": "pending"}, {"content": "Add authentication and error handling", "status": "pending"}])`
+  - Then provide detailed explanations for each step
+
+- User: "Help me plan my trip to Japan"
+  - Call: `write_todos(todos=[{"content": "Research best time to visit and book flights", "status": "in_progress"}, {"content": "Plan itinerary for cities to visit", "status": "pending"}, {"content": "Book accommodations", "status": "pending"}, {"content": "Prepare travel documents and currency", "status": "pending"}])`
+  - Then provide travel preparation guidance
+
+- User: "Break down how to learn guitar"
+  - Call: `write_todos(todos=[{"content": "Learn basic chords and finger positioning", "status": "in_progress"}, {"content": "Practice strumming patterns", "status": "pending"}, {"content": "Learn to read tabs and sheet music", "status": "pending"}, {"content": "Master simple songs", "status": "pending"}])`
+  - Then provide learning milestones and tips
+
+- User: "Plan my workout routine for the week"
+  - Call: `write_todos(todos=[{"content": "Monday: Upper body strength training", "status": "in_progress"}, {"content": "Tuesday: Cardio and core workout", "status": "pending"}, {"content": "Wednesday: Rest or light stretching", "status": "pending"}, {"content": "Thursday: Lower body strength training", "status": "pending"}, {"content": "Friday: Full body HIIT session", "status": "pending"}])`
+  - Then provide exercise details and tips
+
+- User: "Help me organize my home renovation project"
+  - Call: `write_todos(todos=[{"content": "Define scope and create budget", "status": "in_progress"}, {"content": "Research and hire contractors", "status": "pending"}, {"content": "Obtain necessary permits", "status": "pending"}, {"content": "Order materials and fixtures", "status": "pending"}, {"content": "Execute renovation phases", "status": "pending"}])`
+  - Then provide detailed renovation guidance
+
+- User: "What steps should I take to start a podcast?"
+  - Call: `write_todos(todos=[{"content": "Define podcast concept and target audience", "status": "in_progress"}, {"content": "Set up recording equipment and software", "status": "pending"}, {"content": "Plan episode structure and content", "status": "pending"}, {"content": "Record and edit first episodes", "status": "pending"}, {"content": "Choose hosting platform and publish", "status": "pending"}])`
+  - Then provide podcast launch guidance
 </tool_call_examples>
 """
 

surfsense_backend/app/agents/new_chat/tools/link_preview.py

@@ -14,8 +14,8 @@ from urllib.parse import urlparse
 import httpx
 import trafilatura
 from fake_useragent import UserAgent
-from langchain_community.document_loaders import AsyncChromiumLoader
 from langchain_core.tools import tool
+from playwright.async_api import async_playwright
 
 logger = logging.getLogger(__name__)
 
@@ -170,7 +170,7 @@ def _make_absolute_url(image_url: str, base_url: str) -> str:
 
 async def fetch_with_chromium(url: str) -> dict[str, Any] | None:
     """
-    Fetch page content using headless Chromium browser.
+    Fetch page content using headless Chromium browser via Playwright.
     Used as a fallback when simple HTTP requests are blocked (403, etc.).
 
     Args:
@@ -186,18 +186,17 @@ async def fetch_with_chromium(url: str) -> dict[str, Any] | None:
         ua = UserAgent()
         user_agent = ua.random
 
-        # Use AsyncChromiumLoader to fetch the page
-        crawl_loader = AsyncChromiumLoader(
-            urls=[url], headless=True, user_agent=user_agent
-        )
-        documents = await crawl_loader.aload()
+        # Use Playwright to fetch the page
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            context = await browser.new_context(user_agent=user_agent)
+            page = await context.new_page()
 
-        if not documents:
-            logger.warning(f"[link_preview] Chromium returned no documents for {url}")
-            return None
-
-        doc = documents[0]
-        raw_html = doc.page_content
+            try:
+                await page.goto(url, wait_until="domcontentloaded", timeout=30000)
+                raw_html = await page.content()
+            finally:
+                await browser.close()
 
         if not raw_html or len(raw_html.strip()) == 0:
             logger.warning(f"[link_preview] Chromium returned empty content for {url}")
@@ -280,15 +279,18 @@ def create_link_preview_tool():
             url = f"https://{url}"
 
         try:
+            # Generate a random User-Agent to avoid bot detection
+            ua = UserAgent()
+            user_agent = ua.random
+
             # Use a browser-like User-Agent to fetch Open Graph metadata.
-            # This is the same approach used by Slack, Discord, Twitter, etc. for link previews.
             # We're only fetching publicly available metadata (title, description, thumbnail)
             # that websites intentionally expose via OG tags for link preview purposes.
             async with httpx.AsyncClient(
                 timeout=10.0,
                 follow_redirects=True,
                 headers={
-                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+                    "User-Agent": user_agent,
                     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
                     "Accept-Language": "en-US,en;q=0.9",
                     "Accept-Encoding": "gzip, deflate, br",

surfsense_backend/app/agents/new_chat/tools/registry.py

@@ -125,6 +125,7 @@ BUILTIN_TOOLS: list[ToolDefinition] = [
         ),
         requires=[],  # firecrawl_api_key is optional
     ),
+    # Note: write_todos is now provided by TodoListMiddleware from deepagents
     # =========================================================================
     # ADD YOUR CUSTOM TOOLS BELOW
     # =========================================================================

surfsense_backend/app/connectors/webcrawler_connector.py

@@ -1,7 +1,7 @@
 """
 WebCrawler Connector Module
 
-A module for crawling web pages and extracting content using Firecrawl or AsyncChromiumLoader.
+A module for crawling web pages and extracting content using Firecrawl or Playwright.
 Provides a unified interface for web scraping.
 """
 
@@ -12,7 +12,7 @@ import trafilatura
 import validators
 from fake_useragent import UserAgent
 from firecrawl import AsyncFirecrawlApp
-from langchain_community.document_loaders import AsyncChromiumLoader
+from playwright.async_api import async_playwright
 
 logger = logging.getLogger(__name__)
 
@@ -25,7 +25,9 @@ class WebCrawlerConnector:
         Initialize the WebCrawlerConnector class.
 
         Args:
-            firecrawl_api_key: Firecrawl API key (optional, will use AsyncChromiumLoader if not provided)
+            firecrawl_api_key: Firecrawl API key (optional). If provided, Firecrawl will be tried first
+                             and Chromium will be used as fallback if Firecrawl fails. If not provided,
+                             Chromium will be used directly.
         """
         self.firecrawl_api_key = firecrawl_api_key
         self.use_firecrawl = bool(firecrawl_api_key)
@@ -46,6 +48,9 @@ class WebCrawlerConnector:
         """
         Crawl a single URL and extract its content.
 
+        If Firecrawl API key is provided, tries Firecrawl first and falls back to Chromium
+        if Firecrawl fails. If no Firecrawl API key is provided, uses Chromium directly.
+
         Args:
             url: URL to crawl
             formats: List of formats to extract (e.g., ["markdown", "html"]) - only for Firecrawl
@@ -56,19 +61,37 @@ class WebCrawlerConnector:
                 - content: Extracted content (markdown or HTML)
                 - metadata: Page metadata (title, description, etc.)
                 - source: Original URL
-                - crawler_type: Type of crawler used
+                - crawler_type: Type of crawler used ("firecrawl" or "chromium")
         """
         try:
             # Validate URL
             if not validators.url(url):
                 return None, f"Invalid URL: {url}"
 
+            # Try Firecrawl first if API key is provided
             if self.use_firecrawl:
-                result = await self._crawl_with_firecrawl(url, formats)
+                try:
+                    logger.info(f"[webcrawler] Using Firecrawl for: {url}")
+                    result = await self._crawl_with_firecrawl(url, formats)
+                    return result, None
+                except Exception as firecrawl_error:
+                    # Firecrawl failed, fallback to Chromium
+                    logger.warning(
+                        f"[webcrawler] Firecrawl failed, falling back to Chromium+Trafilatura for: {url}"
+                    )
+                    try:
+                        result = await self._crawl_with_chromium(url)
+                        return result, None
+                    except Exception as chromium_error:
+                        return (
+                            None,
+                            f"Both Firecrawl and Chromium failed. Firecrawl error: {firecrawl_error!s}, Chromium error: {chromium_error!s}",
+                        )
             else:
+                # No Firecrawl API key, use Chromium directly
+                logger.info(f"[webcrawler] Using Chromium+Trafilatura for: {url}")
                 result = await self._crawl_with_chromium(url)
-
-            return result, None
+                return result, None
 
         except Exception as e:
             return None, f"Error crawling URL {url}: {e!s}"
@@ -126,7 +149,7 @@ class WebCrawlerConnector:
 
     async def _crawl_with_chromium(self, url: str) -> dict[str, Any]:
         """
-        Crawl URL using AsyncChromiumLoader with Trafilatura for content extraction.
+        Crawl URL using Playwright with Trafilatura for content extraction.
         Falls back to raw HTML if Trafilatura extraction fails.
 
         Args:
@@ -142,30 +165,30 @@ class WebCrawlerConnector:
         ua = UserAgent()
         user_agent = ua.random
 
-        # Pass User-Agent to AsyncChromiumLoader
-        crawl_loader = AsyncChromiumLoader(
-            urls=[url], headless=True, user_agent=user_agent
-        )
-        documents = await crawl_loader.aload()
+        # Use Playwright to fetch the page
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            context = await browser.new_context(user_agent=user_agent)
+            page = await context.new_page()
 
-        if not documents:
+            try:
+                await page.goto(url, wait_until="domcontentloaded", timeout=30000)
+                raw_html = await page.content()
+                page_title = await page.title()
+            finally:
+                await browser.close()
+
+        if not raw_html:
             raise ValueError(f"Failed to load content from {url}")
 
-        doc = documents[0]
-        raw_html = doc.page_content
-
-        # Extract basic metadata from the document
-        base_metadata = doc.metadata if doc.metadata else {}
+        # Extract basic metadata from the page
+        base_metadata = {"title": page_title} if page_title else {}
 
         # Try to extract main content using Trafilatura
         extracted_content = None
         trafilatura_metadata = None
 
         try:
-            logger.info(
-                f"Attempting to extract main content from {url} using Trafilatura"
-            )
-
             # Extract main content as markdown
             extracted_content = trafilatura.extract(
                 raw_html,
@@ -179,23 +202,10 @@ class WebCrawlerConnector:
             # Extract metadata using Trafilatura
             trafilatura_metadata = trafilatura.extract_metadata(raw_html)
 
-            if extracted_content and len(extracted_content.strip()) > 0:
-                logger.info(
-                    f"Successfully extracted main content from {url} using Trafilatura "
-                    f"({len(extracted_content)} chars vs {len(raw_html)} chars raw HTML)"
-                )
-            else:
-                logger.warning(
-                    f"Trafilatura extraction returned empty content for {url}, "
-                    "falling back to raw HTML"
-                )
+            if not extracted_content or len(extracted_content.strip()) == 0:
                 extracted_content = None
 
-        except Exception as e:
-            logger.warning(
-                f"Trafilatura extraction failed for {url}: {e}. "
-                "Falling back to raw HTML"
-            )
+        except Exception:
             extracted_content = None
 
         # Build metadata, preferring Trafilatura metadata when available

surfsense_backend/app/routes/logs_routes.py

@@ -319,6 +319,9 @@ async def get_logs_summary(
                     if log.log_metadata
                     else "Unknown"
                 )
+                document_id = (
+                    log.log_metadata.get("document_id") if log.log_metadata else None
+                )
                 summary["active_tasks"].append(
                     {
                         "id": log.id,
@@ -326,6 +329,7 @@ async def get_logs_summary(
                         "message": log.message,
                         "started_at": log.created_at,
                         "source": log.source,
+                        "document_id": document_id,
                     }
                 )
 

surfsense_backend/app/tasks/chat/stream_new_chat.py

@@ -69,6 +69,30 @@ def format_mentioned_documents_as_context(documents: list[Document]) -> str:
     return "\n".join(context_parts)
 
 
+def extract_todos_from_deepagents(command_output) -> dict:
+    """
+    Extract todos from deepagents' TodoListMiddleware Command output.
+
+    deepagents returns a Command object with:
+    - Command.update['todos'] = [{'content': '...', 'status': '...'}]
+
+    Returns the todos directly (no transformation needed - UI matches deepagents format).
+    """
+    todos_data = []
+    if hasattr(command_output, "update"):
+        # It's a Command object from deepagents
+        update = command_output.update
+        todos_data = update.get("todos", [])
+    elif isinstance(command_output, dict):
+        # Already a dict - check if it has todos directly or in update
+        if "todos" in command_output:
+            todos_data = command_output.get("todos", [])
+        elif "update" in command_output and isinstance(command_output["update"], dict):
+            todos_data = command_output["update"].get("todos", [])
+
+    return {"todos": todos_data}
+
+
 async def stream_new_chat(
     user_query: str,
     search_space_id: int,
@@ -146,6 +170,16 @@ async def stream_new_chat(
         # Create connector service
         connector_service = ConnectorService(session, search_space_id=search_space_id)
 
+        # Get Firecrawl API key from webcrawler connector if configured
+        from app.db import SearchSourceConnectorType
+
+        firecrawl_api_key = None
+        webcrawler_connector = await connector_service.get_connector_by_type(
+            SearchSourceConnectorType.WEBCRAWLER_CONNECTOR, search_space_id
+        )
+        if webcrawler_connector and webcrawler_connector.config:
+            firecrawl_api_key = webcrawler_connector.config.get("FIRECRAWL_API_KEY")
+
         # Get the PostgreSQL checkpointer for persistent conversation memory
         checkpointer = await get_checkpointer()
 
@@ -157,6 +191,7 @@ async def stream_new_chat(
             connector_service=connector_service,
             checkpointer=checkpointer,
             agent_config=agent_config,  # Pass prompt configuration
+            firecrawl_api_key=firecrawl_api_key,  # Pass Firecrawl API key if configured
         )
 
         # Build input with message history from frontend
@@ -211,7 +246,8 @@ async def stream_new_chat(
         config = {
             "configurable": {
                 "thread_id": str(chat_id),
-            }
+            },
+            "recursion_limit": 80,  # Increase from default 25 to allow more tool iterations
         }
 
         # Start the message stream
@@ -233,6 +269,8 @@ async def stream_new_chat(
         completed_step_ids: set[str] = set()
         # Track if we just finished a tool (text flows silently after tools)
         just_finished_tool: bool = False
+        # Track write_todos calls to show "Creating plan" vs "Updating plan"
+        write_todos_call_count: int = 0
 
         def next_thinking_step_id() -> str:
             nonlocal thinking_step_counter
@@ -441,6 +479,60 @@ async def stream_new_chat(
                         status="in_progress",
                         items=last_active_step_items,
                     )
+                elif tool_name == "write_todos":
+                    # Track write_todos calls for better messaging
+                    write_todos_call_count += 1
+                    todos = (
+                        tool_input.get("todos", [])
+                        if isinstance(tool_input, dict)
+                        else []
+                    )
+                    todo_count = len(todos) if isinstance(todos, list) else 0
+
+                    if write_todos_call_count == 1:
+                        # First call - creating the plan
+                        last_active_step_title = "Creating plan"
+                        last_active_step_items = [f"Defining {todo_count} tasks..."]
+                    else:
+                        # Subsequent calls - updating the plan
+                        # Try to provide context about what's being updated
+                        in_progress_count = (
+                            sum(
+                                1
+                                for t in todos
+                                if isinstance(t, dict)
+                                and t.get("status") == "in_progress"
+                            )
+                            if isinstance(todos, list)
+                            else 0
+                        )
+                        completed_count = (
+                            sum(
+                                1
+                                for t in todos
+                                if isinstance(t, dict)
+                                and t.get("status") == "completed"
+                            )
+                            if isinstance(todos, list)
+                            else 0
+                        )
+
+                        last_active_step_title = "Updating progress"
+                        last_active_step_items = (
+                            [
+                                f"Progress: {completed_count}/{todo_count} completed",
+                                f"In progress: {in_progress_count} tasks",
+                            ]
+                            if completed_count > 0
+                            else [f"Working on {todo_count} tasks"]
+                        )
+
+                    yield streaming_service.format_thinking_step(
+                        step_id=tool_step_id,
+                        title=last_active_step_title,
+                        status="in_progress",
+                        items=last_active_step_items,
+                    )
                 elif tool_name == "generate_podcast":
                     podcast_title = (
                         tool_input.get("podcast_title", "SurfSense Podcast")
@@ -465,6 +557,15 @@ async def stream_new_chat(
                         status="in_progress",
                         items=last_active_step_items,
                     )
+                # elif tool_name == "ls":
+                #     last_active_step_title = "Exploring files"
+                #     last_active_step_items = []
+                #     yield streaming_service.format_thinking_step(
+                #         step_id=tool_step_id,
+                #         title="Exploring files",
+                #         status="in_progress",
+                #         items=None,
+                #     )
                 else:
                     last_active_step_title = f"Using {tool_name.replace('_', ' ')}"
                     last_active_step_items = []
@@ -546,9 +647,11 @@ async def stream_new_chat(
                 tool_name = event.get("name", "unknown_tool")
                 raw_output = event.get("data", {}).get("output", "")
 
-                # Extract content from ToolMessage if needed
-                # LangGraph may return a ToolMessage object instead of raw dict
-                if hasattr(raw_output, "content"):
+                # Handle deepagents' write_todos Command object specially
+                if tool_name == "write_todos" and hasattr(raw_output, "update"):
+                    # deepagents returns a Command object - extract todos directly
+                    tool_output = extract_todos_from_deepagents(raw_output)
+                elif hasattr(raw_output, "content"):
                     # It's a ToolMessage object - extract the content
                     content = raw_output.content
                     # If content is a string that looks like JSON, try to parse it
@@ -707,6 +810,104 @@ async def stream_new_chat(
                         status="completed",
                         items=completed_items,
                     )
+                elif tool_name == "write_todos":
+                    # Build completion items for planning/updating
+                    if isinstance(tool_output, dict):
+                        todos = tool_output.get("todos", [])
+                        todo_count = len(todos) if isinstance(todos, list) else 0
+                        completed_count = (
+                            sum(
+                                1
+                                for t in todos
+                                if isinstance(t, dict)
+                                and t.get("status") == "completed"
+                            )
+                            if isinstance(todos, list)
+                            else 0
+                        )
+                        in_progress_count = (
+                            sum(
+                                1
+                                for t in todos
+                                if isinstance(t, dict)
+                                and t.get("status") == "in_progress"
+                            )
+                            if isinstance(todos, list)
+                            else 0
+                        )
+
+                        # Use context-aware completion message
+                        if last_active_step_title == "Creating plan":
+                            completed_items = [f"Created {todo_count} tasks"]
+                        else:
+                            # Updating progress - show stats
+                            completed_items = [
+                                f"Progress: {completed_count}/{todo_count} completed",
+                            ]
+                            if in_progress_count > 0:
+                                # Find the currently in-progress task name
+                                in_progress_task = next(
+                                    (
+                                        t.get("content", "")[:40]
+                                        for t in todos
+                                        if isinstance(t, dict)
+                                        and t.get("status") == "in_progress"
+                                    ),
+                                    None,
+                                )
+                                if in_progress_task:
+                                    completed_items.append(
+                                        f"Current: {in_progress_task}..."
+                                    )
+                    else:
+                        completed_items = ["Plan updated"]
+                    yield streaming_service.format_thinking_step(
+                        step_id=original_step_id,
+                        title=last_active_step_title,
+                        status="completed",
+                        items=completed_items,
+                    )
+                elif tool_name == "ls":
+                    # Build completion items showing file names found
+                    if isinstance(tool_output, dict):
+                        result = tool_output.get("result", "")
+                    elif isinstance(tool_output, str):
+                        result = tool_output
+                    else:
+                        result = str(tool_output) if tool_output else ""
+
+                    # Parse file paths and extract just the file names
+                    file_names = []
+                    if result:
+                        # The ls tool returns paths, extract just the file/folder names
+                        for line in result.strip().split("\n"):
+                            line = line.strip()
+                            if line:
+                                # Get just the filename from the path
+                                name = line.rstrip("/").split("/")[-1]
+                                if name and len(name) <= 40:
+                                    file_names.append(name)
+                                elif name:
+                                    file_names.append(name[:37] + "...")
+
+                    # Build display items - wrap file names in brackets for icon rendering
+                    if file_names:
+                        if len(file_names) <= 5:
+                            # Wrap each file name in brackets for styled tile rendering
+                            completed_items = [f"[{name}]" for name in file_names]
+                        else:
+                            # Show first few with brackets and count
+                            completed_items = [f"[{name}]" for name in file_names[:4]]
+                            completed_items.append(f"(+{len(file_names) - 4} more)")
+                    else:
+                        completed_items = ["No files found"]
+
+                    yield streaming_service.format_thinking_step(
+                        step_id=original_step_id,
+                        title="Exploring files",
+                        status="completed",
+                        items=completed_items,
+                    )
                 else:
                     yield streaming_service.format_thinking_step(
                         step_id=original_step_id,
@@ -843,6 +1044,27 @@ async def stream_new_chat(
                     yield streaming_service.format_terminal_info(
                         "Knowledge base search completed", "success"
                     )
+                elif tool_name == "write_todos":
+                    # Stream the full write_todos result so frontend can render the Plan component
+                    yield streaming_service.format_tool_output_available(
+                        tool_call_id,
+                        tool_output
+                        if isinstance(tool_output, dict)
+                        else {"result": tool_output},
+                    )
+                    # Send terminal message with plan info
+                    if isinstance(tool_output, dict):
+                        todos = tool_output.get("todos", [])
+                        todo_count = len(todos) if isinstance(todos, list) else 0
+                        yield streaming_service.format_terminal_info(
+                            f"Plan created ({todo_count} tasks)",
+                            "success",
+                        )
+                    else:
+                        yield streaming_service.format_terminal_info(
+                            "Plan created",
+                            "success",
+                        )
                 else:
                     # Default handling for other tools
                     yield streaming_service.format_tool_output_available(

surfsense_backend/pyproject.toml

@@ -7,6 +7,8 @@ requires-python = ">=3.12"
 dependencies = [
     "alembic>=1.13.0",
     "asyncpg>=0.30.0",
+    "datasets>=2.21.0",
+    "pyarrow>=15.0.0,<19.0.0",
     "discord-py>=2.5.2",
     "docling>=2.15.0",
     "fastapi>=0.115.8",

surfsense_backend/uv.lock

@@ -1148,6 +1148,30 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686 },
 ]
 
+[[package]]
+name = "datasets"
+version = "4.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "dill" },
+    { name = "filelock" },
+    { name = "fsspec", extra = ["http"] },
+    { name = "huggingface-hub" },
+    { name = "multiprocess" },
+    { name = "numpy" },
+    { name = "packaging" },
+    { name = "pandas" },
+    { name = "pyarrow" },
+    { name = "pyyaml" },
+    { name = "requests" },
+    { name = "tqdm" },
+    { name = "xxhash" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e3/9d/348ed92110ba5f9b70b51ca1078d4809767a835aa2b7ce7e74ad2b98323d/datasets-4.0.0.tar.gz", hash = "sha256:9657e7140a9050db13443ba21cb5de185af8af944479b00e7ff1e00a61c8dbf1", size = 569566 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/eb/62/eb8157afb21bd229c864521c1ab4fa8e9b4f1b06bafdd8c4668a7a31b5dd/datasets-4.0.0-py3-none-any.whl", hash = "sha256:7ef95e62025fd122882dbce6cb904c8cd3fbc829de6669a5eb939c77d50e203d", size = 494825 },
+]
+
 [[package]]
 name = "dateparser"
 version = "1.2.2"
@@ -1213,11 +1237,11 @@ wheels = [
 
 [[package]]
 name = "dill"
-version = "0.4.0"
+version = "0.3.8"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/12/80/630b4b88364e9a8c8c5797f4602d0f76ef820909ee32f0bacb9f90654042/dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", size = 186976 }
+sdist = { url = "https://files.pythonhosted.org/packages/17/4d/ac7ffa80c69ea1df30a8aa11b3578692a5118e7cd1aa157e3ef73b092d15/dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca", size = 184847 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668 },
+    { url = "https://files.pythonhosted.org/packages/c9/7a/cef76fd8438a42f96db64ddaa85280485a9c395e7df3db8158cfec1eee34/dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7", size = 116252 },
 ]
 
 [[package]]
@@ -1875,11 +1899,16 @@ wheels = [
 
 [[package]]
 name = "fsspec"
-version = "2025.5.1"
+version = "2025.3.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/00/f7/27f15d41f0ed38e8fcc488584b57e902b331da7f7c6dcda53721b15838fc/fsspec-2025.5.1.tar.gz", hash = "sha256:2e55e47a540b91843b755e83ded97c6e897fa0942b11490113f09e9c443c2475", size = 303033 }
+sdist = { url = "https://files.pythonhosted.org/packages/34/f4/5721faf47b8c499e776bc34c6a8fc17efdf7fdef0b00f398128bc5dcb4ac/fsspec-2025.3.0.tar.gz", hash = "sha256:a935fd1ea872591f2b5148907d103488fc523295e6c64b835cfad8c3eca44972", size = 298491 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/bb/61/78c7b3851add1481b048b5fdc29067397a1784e2910592bc81bb3f608635/fsspec-2025.5.1-py3-none-any.whl", hash = "sha256:24d3a2e663d5fc735ab256263c4075f374a174c3410c0b25e5bd1970bceaa462", size = 199052 },
+    { url = "https://files.pythonhosted.org/packages/56/53/eb690efa8513166adef3e0669afd31e95ffde69fb3c52ec2ac7223ed6018/fsspec-2025.3.0-py3-none-any.whl", hash = "sha256:efb87af3efa9103f94ca91a7f8cb7a4df91af9f74fc106c9c7ea0efd7277c1b3", size = 193615 },
+]
+
+[package.optional-dependencies]
+http = [
+    { name = "aiohttp" },
 ]
 
 [[package]]
@@ -3711,19 +3740,18 @@ wheels = [
 
 [[package]]
 name = "multiprocess"
-version = "0.70.18"
+version = "0.70.16"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "dill" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/72/fd/2ae3826f5be24c6ed87266bc4e59c46ea5b059a103f3d7e7eb76a52aeecb/multiprocess-0.70.18.tar.gz", hash = "sha256:f9597128e6b3e67b23956da07cf3d2e5cba79e2f4e0fba8d7903636663ec6d0d", size = 1798503 }
+sdist = { url = "https://files.pythonhosted.org/packages/b5/ae/04f39c5d0d0def03247c2893d6f2b83c136bf3320a2154d7b8858f2ba72d/multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1", size = 1772603 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ba/d8/0cba6cf51a1a31f20471fbc823a716170c73012ddc4fb85d706630ed6e8f/multiprocess-0.70.18-py310-none-any.whl", hash = "sha256:60c194974c31784019c1f459d984e8f33ee48f10fcf42c309ba97b30d9bd53ea", size = 134948 },
-    { url = "https://files.pythonhosted.org/packages/4b/88/9039f2fed1012ef584751d4ceff9ab4a51e5ae264898f0b7cbf44340a859/multiprocess-0.70.18-py311-none-any.whl", hash = "sha256:5aa6eef98e691281b3ad923be2832bf1c55dd2c859acd73e5ec53a66aae06a1d", size = 144462 },
-    { url = "https://files.pythonhosted.org/packages/bf/b6/5f922792be93b82ec6b5f270bbb1ef031fd0622847070bbcf9da816502cc/multiprocess-0.70.18-py312-none-any.whl", hash = "sha256:9b78f8e5024b573730bfb654783a13800c2c0f2dfc0c25e70b40d184d64adaa2", size = 150287 },
-    { url = "https://files.pythonhosted.org/packages/ee/25/7d7e78e750bc1aecfaf0efbf826c69a791d2eeaf29cf20cba93ff4cced78/multiprocess-0.70.18-py313-none-any.whl", hash = "sha256:871743755f43ef57d7910a38433cfe41319e72be1bbd90b79c7a5ac523eb9334", size = 151917 },
-    { url = "https://files.pythonhosted.org/packages/3b/c3/ca84c19bd14cdfc21c388fdcebf08b86a7a470ebc9f5c3c084fc2dbc50f7/multiprocess-0.70.18-py38-none-any.whl", hash = "sha256:dbf705e52a154fe5e90fb17b38f02556169557c2dd8bb084f2e06c2784d8279b", size = 132636 },
-    { url = "https://files.pythonhosted.org/packages/6c/28/dd72947e59a6a8c856448a5e74da6201cb5502ddff644fbc790e4bd40b9a/multiprocess-0.70.18-py39-none-any.whl", hash = "sha256:e78ca805a72b1b810c690b6b4cc32579eba34f403094bbbae962b7b5bf9dfcb8", size = 133478 },
+    { url = "https://files.pythonhosted.org/packages/bc/f7/7ec7fddc92e50714ea3745631f79bd9c96424cb2702632521028e57d3a36/multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02", size = 134824 },
+    { url = "https://files.pythonhosted.org/packages/50/15/b56e50e8debaf439f44befec5b2af11db85f6e0f344c3113ae0be0593a91/multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a", size = 143519 },
+    { url = "https://files.pythonhosted.org/packages/0a/7d/a988f258104dcd2ccf1ed40fdc97e26c4ac351eeaf81d76e266c52d84e2f/multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e", size = 146741 },
+    { url = "https://files.pythonhosted.org/packages/ea/89/38df130f2c799090c978b366cfdf5b96d08de5b29a4a293df7f7429fa50b/multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435", size = 132628 },
+    { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351 },
 ]
 
 [[package]]
@@ -4931,6 +4959,34 @@ bcrypt = [
     { name = "bcrypt" },
 ]
 
+[[package]]
+name = "pyarrow"
+version = "18.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/7f/7b/640785a9062bb00314caa8a387abce547d2a420cf09bd6c715fe659ccffb/pyarrow-18.1.0.tar.gz", hash = "sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73", size = 1118671 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/6a/50/12829e7111b932581e51dda51d5cb39207a056c30fe31ef43f14c63c4d7e/pyarrow-18.1.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:9f3a76670b263dc41d0ae877f09124ab96ce10e4e48f3e3e4257273cee61ad0d", size = 29514620 },
+    { url = "https://files.pythonhosted.org/packages/d1/41/468c944eab157702e96abab3d07b48b8424927d4933541ab43788bb6964d/pyarrow-18.1.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:da31fbca07c435be88a0c321402c4e31a2ba61593ec7473630769de8346b54ee", size = 30856494 },
+    { url = "https://files.pythonhosted.org/packages/68/f9/29fb659b390312a7345aeb858a9d9c157552a8852522f2c8bad437c29c0a/pyarrow-18.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:543ad8459bc438efc46d29a759e1079436290bd583141384c6f7a1068ed6f992", size = 39203624 },
+    { url = "https://files.pythonhosted.org/packages/6e/f6/19360dae44200e35753c5c2889dc478154cd78e61b1f738514c9f131734d/pyarrow-18.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0743e503c55be0fdb5c08e7d44853da27f19dc854531c0570f9f394ec9671d54", size = 40139341 },
+    { url = "https://files.pythonhosted.org/packages/bb/e6/9b3afbbcf10cc724312e824af94a2e993d8ace22994d823f5c35324cebf5/pyarrow-18.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d4b3d2a34780645bed6414e22dda55a92e0fcd1b8a637fba86800ad737057e33", size = 38618629 },
+    { url = "https://files.pythonhosted.org/packages/3a/2e/3b99f8a3d9e0ccae0e961978a0d0089b25fb46ebbcfb5ebae3cca179a5b3/pyarrow-18.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c52f81aa6f6575058d8e2c782bf79d4f9fdc89887f16825ec3a66607a5dd8e30", size = 40078661 },
+    { url = "https://files.pythonhosted.org/packages/76/52/f8da04195000099d394012b8d42c503d7041b79f778d854f410e5f05049a/pyarrow-18.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ad4892617e1a6c7a551cfc827e072a633eaff758fa09f21c4ee548c30bcaf99", size = 25092330 },
+    { url = "https://files.pythonhosted.org/packages/cb/87/aa4d249732edef6ad88899399047d7e49311a55749d3c373007d034ee471/pyarrow-18.1.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:84e314d22231357d473eabec709d0ba285fa706a72377f9cc8e1cb3c8013813b", size = 29497406 },
+    { url = "https://files.pythonhosted.org/packages/3c/c7/ed6adb46d93a3177540e228b5ca30d99fc8ea3b13bdb88b6f8b6467e2cb7/pyarrow-18.1.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:f591704ac05dfd0477bb8f8e0bd4b5dc52c1cadf50503858dce3a15db6e46ff2", size = 30835095 },
+    { url = "https://files.pythonhosted.org/packages/41/d7/ed85001edfb96200ff606943cff71d64f91926ab42828676c0fc0db98963/pyarrow-18.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acb7564204d3c40babf93a05624fc6a8ec1ab1def295c363afc40b0c9e66c191", size = 39194527 },
+    { url = "https://files.pythonhosted.org/packages/59/16/35e28eab126342fa391593415d79477e89582de411bb95232f28b131a769/pyarrow-18.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74de649d1d2ccb778f7c3afff6085bd5092aed4c23df9feeb45dd6b16f3811aa", size = 40131443 },
+    { url = "https://files.pythonhosted.org/packages/0c/95/e855880614c8da20f4cd74fa85d7268c725cf0013dc754048593a38896a0/pyarrow-18.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f96bd502cb11abb08efea6dab09c003305161cb6c9eafd432e35e76e7fa9b90c", size = 38608750 },
+    { url = "https://files.pythonhosted.org/packages/54/9d/f253554b1457d4fdb3831b7bd5f8f00f1795585a606eabf6fec0a58a9c38/pyarrow-18.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:36ac22d7782554754a3b50201b607d553a8d71b78cdf03b33c1125be4b52397c", size = 40066690 },
+    { url = "https://files.pythonhosted.org/packages/2f/58/8912a2563e6b8273e8aa7b605a345bba5a06204549826f6493065575ebc0/pyarrow-18.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:25dbacab8c5952df0ca6ca0af28f50d45bd31c1ff6fcf79e2d120b4a65ee7181", size = 25081054 },
+    { url = "https://files.pythonhosted.org/packages/82/f9/d06ddc06cab1ada0c2f2fd205ac8c25c2701182de1b9c4bf7a0a44844431/pyarrow-18.1.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6a276190309aba7bc9d5bd2933230458b3521a4317acfefe69a354f2fe59f2bc", size = 29525542 },
+    { url = "https://files.pythonhosted.org/packages/ab/94/8917e3b961810587ecbdaa417f8ebac0abb25105ae667b7aa11c05876976/pyarrow-18.1.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:ad514dbfcffe30124ce655d72771ae070f30bf850b48bc4d9d3b25993ee0e386", size = 30829412 },
+    { url = "https://files.pythonhosted.org/packages/5e/e3/3b16c3190f3d71d3b10f6758d2d5f7779ef008c4fd367cedab3ed178a9f7/pyarrow-18.1.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aebc13a11ed3032d8dd6e7171eb6e86d40d67a5639d96c35142bd568b9299324", size = 39119106 },
+    { url = "https://files.pythonhosted.org/packages/1d/d6/5d704b0d25c3c79532f8c0639f253ec2803b897100f64bcb3f53ced236e5/pyarrow-18.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6cf5c05f3cee251d80e98726b5c7cc9f21bab9e9783673bac58e6dfab57ecc8", size = 40090940 },
+    { url = "https://files.pythonhosted.org/packages/37/29/366bc7e588220d74ec00e497ac6710c2833c9176f0372fe0286929b2d64c/pyarrow-18.1.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:11b676cd410cf162d3f6a70b43fb9e1e40affbc542a1e9ed3681895f2962d3d9", size = 38548177 },
+    { url = "https://files.pythonhosted.org/packages/c8/11/fabf6ecabb1fe5b7d96889228ca2a9158c4c3bb732e3b8ee3f7f6d40b703/pyarrow-18.1.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:b76130d835261b38f14fc41fdfb39ad8d672afb84c447126b84d5472244cfaba", size = 40043567 },
+]
+
 [[package]]
 name = "pyasn1"
 version = "0.6.1"
@@ -6353,7 +6409,7 @@ wheels = [
 
 [[package]]
 name = "surf-new-backend"
-version = "0.0.8"
+version = "0.0.9"
 source = { virtual = "." }
 dependencies = [
     { name = "alembic" },
@@ -6361,6 +6417,7 @@ dependencies = [
     { name = "boto3" },
     { name = "celery", extra = ["redis"] },
     { name = "chonkie", extra = ["all"] },
+    { name = "datasets" },
     { name = "deepagents" },
     { name = "discord-py" },
     { name = "docling" },
@@ -6391,6 +6448,7 @@ dependencies = [
     { name = "pgvector" },
     { name = "playwright" },
     { name = "psycopg", extra = ["binary", "pool"] },
+    { name = "pyarrow" },
     { name = "pypdf" },
     { name = "python-ffmpeg" },
     { name = "redis" },
@@ -6421,6 +6479,7 @@ requires-dist = [
     { name = "boto3", specifier = ">=1.35.0" },
     { name = "celery", extras = ["redis"], specifier = ">=5.5.3" },
     { name = "chonkie", extras = ["all"], specifier = ">=1.5.0" },
+    { name = "datasets", specifier = ">=2.21.0" },
     { name = "deepagents", specifier = ">=0.3.0" },
     { name = "discord-py", specifier = ">=2.5.2" },
     { name = "docling", specifier = ">=2.15.0" },
@@ -6451,6 +6510,7 @@ requires-dist = [
     { name = "pgvector", specifier = ">=0.3.6" },
     { name = "playwright", specifier = ">=1.50.0" },
     { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.3.2" },
+    { name = "pyarrow", specifier = ">=15.0.0,<19.0.0" },
     { name = "pypdf", specifier = ">=5.1.0" },
     { name = "python-ffmpeg", specifier = ">=2.0.12" },
     { name = "redis", specifier = ">=5.2.1" },

surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { format } from "date-fns";
+import { useAtomValue } from "jotai";
 import {
 	Calendar as CalendarIcon,
 	Clock,
@@ -18,6 +19,12 @@ import { useParams, useRouter } from "next/navigation";
 import { useTranslations } from "next-intl";
 import { useEffect, useState } from "react";
 import { toast } from "sonner";
+import {
+	deleteConnectorMutationAtom,
+	indexConnectorMutationAtom,
+	updateConnectorMutationAtom,
+} from "@/atoms/connectors/connector-mutation.atoms";
+import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms";
 import {
 	AlertDialog,
 	AlertDialogAction,
@@ -62,7 +69,6 @@ import {
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 import { cn } from "@/lib/utils";
 import { authenticatedFetch } from "@/lib/auth-utils";
 import { GoogleDriveFolderTree } from "@/components/connectors/google-drive-folder-tree";
@@ -94,8 +100,12 @@ export default function ConnectorsPage() {
 	const searchSpaceId = params.search_space_id as string;
 	const today = new Date();
 
-	const { connectors, isLoading, error, deleteConnector, indexConnector, updateConnector } =
-		useSearchSourceConnectors(false, parseInt(searchSpaceId));
+	const { data: connectors = [], isLoading, error } = useAtomValue(connectorsAtom);
+
+	const { mutateAsync: deleteConnector } = useAtomValue(deleteConnectorMutationAtom);
+	const { mutateAsync: indexConnector } = useAtomValue(indexConnectorMutationAtom);
+	const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom);
+
 	const [connectorToDelete, setConnectorToDelete] = useState<number | null>(null);
 	const [indexingConnectorId, setIndexingConnectorId] = useState<number | null>(null);
 	const [datePickerOpen, setDatePickerOpen] = useState(false);
@@ -133,11 +143,9 @@ export default function ConnectorsPage() {
 		if (connectorToDelete === null) return;
 
 		try {
-			await deleteConnector(connectorToDelete);
-			toast.success(t("delete_success"));
+			await deleteConnector({ id: connectorToDelete });
 		} catch (error) {
 			console.error("Error deleting connector:", error);
-			toast.error(t("delete_failed"));
 		} finally {
 			setConnectorToDelete(null);
 		}
@@ -228,7 +236,14 @@ export default function ConnectorsPage() {
 			const startDateStr = startDate ? format(startDate, "yyyy-MM-dd") : undefined;
 			const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined;
 
-			await indexConnector(selectedConnectorForIndexing, searchSpaceId, startDateStr, endDateStr);
+			await indexConnector({
+				connector_id: selectedConnectorForIndexing,
+				queryParams: {
+					search_space_id: searchSpaceId,
+					start_date: startDateStr,
+					end_date: endDateStr,
+				},
+			});
 			toast.success(t("indexing_started"));
 		} catch (error) {
 			console.error("Error indexing connector content:", error);
@@ -245,7 +260,12 @@ export default function ConnectorsPage() {
 	const handleQuickIndexConnector = async (connectorId: number) => {
 		setIndexingConnectorId(connectorId);
 		try {
-			await indexConnector(connectorId, searchSpaceId);
+			await indexConnector({
+				connector_id: connectorId,
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 			toast.success(t("indexing_started"));
 		} catch (error) {
 			console.error("Error indexing connector content:", error);
@@ -305,9 +325,12 @@ export default function ConnectorsPage() {
 				}
 			}
 
-			await updateConnector(selectedConnectorForPeriodic, {
-				periodic_indexing_enabled: periodicEnabled,
-				indexing_frequency_minutes: frequency,
+			await updateConnector({
+				id: selectedConnectorForPeriodic,
+				data: {
+					periodic_indexing_enabled: periodicEnabled,
+					indexing_frequency_minutes: frequency,
+				},
 			});
 
 			toast.success(

surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams, useRouter } from "next/navigation";
@@ -8,6 +9,8 @@ import { useEffect, useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { updateConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
+import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms";
 import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
 import { Button } from "@/components/ui/button";
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
@@ -21,10 +24,8 @@ import {
 	FormMessage,
 } from "@/components/ui/form";
 import { Input } from "@/components/ui/input";
-import {
-	type SearchSourceConnector,
-	useSearchSourceConnectors,
-} from "@/hooks/use-search-source-connectors";
+import type { EnumConnectorName } from "@/contracts/enums/connector";
+import type { SearchSourceConnector } from "@/hooks/use-search-source-connectors";
 
 // Define the form schema with Zod
 const apiConnectorFormSchema = z.object({
@@ -85,7 +86,8 @@ export default function EditConnectorPage() {
 	const searchSpaceId = params.search_space_id as string;
 	const connectorId = parseInt(params.connector_id as string, 10);
 
-	const { connectors, updateConnector } = useSearchSourceConnectors(false, parseInt(searchSpaceId));
+	const { data: connectors = [] } = useAtomValue(connectorsAtom);
+	const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom);
 	const [connector, setConnector] = useState<SearchSourceConnector | null>(null);
 	const [isLoading, setIsLoading] = useState(true);
 	const [isSubmitting, setIsSubmitting] = useState(false);
@@ -99,14 +101,12 @@ export default function EditConnectorPage() {
 		},
 	});
 
-	// Find connector in the list
 	useEffect(() => {
 		const currentConnector = connectors.find((c) => c.id === connectorId);
 
 		if (currentConnector) {
 			setConnector(currentConnector);
 
-			// Check if connector type is supported
 			const apiKeyField = getApiKeyFieldName(currentConnector.connector_type);
 			if (apiKeyField) {
 				form.reset({
@@ -114,14 +114,12 @@ export default function EditConnectorPage() {
 					api_key: currentConnector.config[apiKeyField] || "",
 				});
 			} else {
-				// Redirect if not a supported connector type
 				toast.error("This connector type is not supported for editing");
 				router.push(`/dashboard/${searchSpaceId}/connectors`);
 			}
 
 			setIsLoading(false);
 		} else if (!isLoading && connectors.length > 0) {
-			// If connectors are loaded but this one isn't found
 			toast.error("Connector not found");
 			router.push(`/dashboard/${searchSpaceId}/connectors`);
 		}
@@ -135,18 +133,20 @@ export default function EditConnectorPage() {
 		try {
 			const apiKeyField = getApiKeyFieldName(connector.connector_type);
 
-			// Only update the API key if a new one was provided
 			const updatedConfig = { ...connector.config };
 			if (values.api_key) {
 				updatedConfig[apiKeyField] = values.api_key;
 			}
 
-			await updateConnector(connectorId, {
-				name: values.name,
-				connector_type: connector.connector_type,
-				config: updatedConfig,
-				is_indexable: connector.is_indexable,
-				last_indexed_at: connector.last_indexed_at,
+			await updateConnector({
+				id: connectorId,
+				data: {
+					name: values.name,
+					connector_type: connector.connector_type as EnumConnectorName,
+					config: updatedConfig,
+					is_indexable: connector.is_indexable,
+					last_indexed_at: connector.last_indexed_at,
+				},
 			});
 
 			toast.success("Connector updated successfully!");

surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx

@@ -1,11 +1,13 @@
 "use client";
 
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, ExternalLink, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import Link from "next/link";
 import { useParams, useRouter } from "next/navigation";
 import { useEffect, useState } from "react";
 import { toast } from "sonner";
+import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms";
 import { Button } from "@/components/ui/button";
 import {
 	Card,
@@ -18,10 +20,7 @@ import {
 import { EnumConnectorName } from "@/contracts/enums/connector";
 // import { IconBrandAirtable } from "@tabler/icons-react";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import {
-	type SearchSourceConnector,
-	useSearchSourceConnectors,
-} from "@/hooks/use-search-source-connectors";
+import type { SearchSourceConnector } from "@/contracts/types/connector.types";
 import { authenticatedFetch } from "@/lib/auth-utils";
 
 export default function AirtableConnectorPage() {
@@ -31,11 +30,12 @@ export default function AirtableConnectorPage() {
 	const [isConnecting, setIsConnecting] = useState(false);
 	const [doesConnectorExist, setDoesConnectorExist] = useState(false);
 
-	const { fetchConnectors } = useSearchSourceConnectors(true, parseInt(searchSpaceId));
+	const { refetch: fetchConnectors } = useAtomValue(connectorsAtom);
 
 	useEffect(() => {
-		fetchConnectors(parseInt(searchSpaceId)).then((data) => {
-			const connector = data.find(
+		fetchConnectors().then((data) => {
+			const connectors = data.data || [];
+			const connector = connectors.find(
 				(c: SearchSourceConnector) => c.connector_type === EnumConnectorName.AIRTABLE_CONNECTOR
 			);
 			if (connector) {

surfsense_web/app/dashboard/[search_space_id]/connectors/add/baidu-search-api/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams, useRouter } from "next/navigation";
@@ -8,6 +9,7 @@ import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
 import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
 import { Button } from "@/components/ui/button";
 import {
@@ -38,7 +40,6 @@ import {
 import { Switch } from "@/components/ui/switch";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 
 // Define the form schema with Zod
 const baiduSearchApiFormSchema = z.object({
@@ -61,7 +62,7 @@ export default function BaiduSearchApiPage() {
 	const params = useParams();
 	const searchSpaceId = params.search_space_id as string;
 	const [isSubmitting, setIsSubmitting] = useState(false);
-	const { createConnector } = useSearchSourceConnectors();
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	// Initialize the form
 	const form = useForm<BaiduSearchApiFormValues>({
@@ -95,8 +96,8 @@ export default function BaiduSearchApiPage() {
 				config.BAIDU_ENABLE_DEEP_SEARCH = values.enable_deep_search;
 			}
 
-			await createConnector(
-				{
+			await createConnector({
+				data: {
 					name: values.name,
 					connector_type: EnumConnectorName.BAIDU_SEARCH_API,
 					config,
@@ -106,8 +107,10 @@ export default function BaiduSearchApiPage() {
 					indexing_frequency_minutes: null,
 					next_scheduled_at: null,
 				},
-				parseInt(searchSpaceId)
-			);
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("Baidu Search connector created successfully!");
 

surfsense_web/app/dashboard/[search_space_id]/connectors/add/bookstack-connector/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams, useRouter } from "next/navigation";
@@ -8,6 +9,7 @@ import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
 import { Alert, AlertDescription } from "@/components/ui/alert";
 import { Button } from "@/components/ui/button";
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
@@ -24,7 +26,6 @@ import { Input } from "@/components/ui/input";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 
 // Define the form schema with Zod
 const bookstackConnectorFormSchema = z.object({
@@ -50,7 +51,7 @@ export default function BookStackConnectorPage() {
 	const params = useParams();
 	const searchSpaceId = params.search_space_id as string;
 	const [isSubmitting, setIsSubmitting] = useState(false);
-	const { createConnector } = useSearchSourceConnectors();
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	// Initialize the form
 	const form = useForm<BookStackConnectorFormValues>({
@@ -67,8 +68,8 @@ export default function BookStackConnectorPage() {
 	const onSubmit = async (values: BookStackConnectorFormValues) => {
 		setIsSubmitting(true);
 		try {
-			await createConnector(
-				{
+			await createConnector({
+				data: {
 					name: values.name,
 					connector_type: EnumConnectorName.BOOKSTACK_CONNECTOR,
 					config: {
@@ -82,8 +83,10 @@ export default function BookStackConnectorPage() {
 					indexing_frequency_minutes: null,
 					next_scheduled_at: null,
 				},
-				parseInt(searchSpaceId)
-			);
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("BookStack connector created successfully!");
 

surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, ExternalLink, Eye, EyeOff } from "lucide-react";
 import Link from "next/link";
 import { useParams, useRouter } from "next/navigation";
@@ -8,6 +9,7 @@ import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
 import { Button } from "@/components/ui/button";
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
 import {
@@ -22,7 +24,6 @@ import {
 import { Input } from "@/components/ui/input";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 
 // Define the form schema with Zod
 const clickupConnectorFormSchema = z.object({
@@ -41,7 +42,7 @@ export default function ClickUpConnectorPage() {
 	const router = useRouter();
 	const params = useParams();
 	const searchSpaceId = params.search_space_id as string;
-	const { createConnector } = useSearchSourceConnectors();
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 	const [isLoading, setIsLoading] = useState(false);
 	const [showApiToken, setShowApiToken] = useState(false);
 
@@ -59,20 +60,23 @@ export default function ClickUpConnectorPage() {
 		setIsLoading(true);
 
 		try {
-			const connectorData = {
-				name: values.name,
-				connector_type: EnumConnectorName.CLICKUP_CONNECTOR,
-				is_indexable: true,
-				config: {
-					CLICKUP_API_TOKEN: values.api_token,
+			await createConnector({
+				data: {
+					name: values.name,
+					connector_type: EnumConnectorName.CLICKUP_CONNECTOR,
+					is_indexable: true,
+					config: {
+						CLICKUP_API_TOKEN: values.api_token,
+					},
+					last_indexed_at: null,
+					periodic_indexing_enabled: false,
+					indexing_frequency_minutes: null,
+					next_scheduled_at: null,
 				},
-				last_indexed_at: null,
-				periodic_indexing_enabled: false,
-				indexing_frequency_minutes: null,
-				next_scheduled_at: null,
-			};
-
-			await createConnector(connectorData, parseInt(searchSpaceId));
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("ClickUp connector created successfully!");
 			router.push(`/dashboard/${searchSpaceId}/connectors`);

surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams, useRouter } from "next/navigation";
@@ -8,6 +9,7 @@ import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
 import { Alert, AlertDescription } from "@/components/ui/alert";
 import { Button } from "@/components/ui/button";
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
@@ -24,7 +26,6 @@ import { Input } from "@/components/ui/input";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 
 // Define the form schema with Zod
 const confluenceConnectorFormSchema = z.object({
@@ -60,7 +61,7 @@ export default function ConfluenceConnectorPage() {
 	const params = useParams();
 	const searchSpaceId = params.search_space_id as string;
 	const [isSubmitting, setIsSubmitting] = useState(false);
-	const { createConnector } = useSearchSourceConnectors();
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	// Initialize the form
 	const form = useForm<ConfluenceConnectorFormValues>({
@@ -77,8 +78,8 @@ export default function ConfluenceConnectorPage() {
 	const onSubmit = async (values: ConfluenceConnectorFormValues) => {
 		setIsSubmitting(true);
 		try {
-			await createConnector(
-				{
+			await createConnector({
+				data: {
 					name: values.name,
 					connector_type: EnumConnectorName.CONFLUENCE_CONNECTOR,
 					config: {
@@ -92,8 +93,10 @@ export default function ConfluenceConnectorPage() {
 					indexing_frequency_minutes: null,
 					next_scheduled_at: null,
 				},
-				parseInt(searchSpaceId)
-			);
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("Confluence connector created successfully!");
 

surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams, useRouter } from "next/navigation";
@@ -8,6 +9,7 @@ import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
 import {
 	Accordion,
 	AccordionContent,
@@ -37,7 +39,6 @@ import { Input } from "@/components/ui/input";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 
 // Define the form schema with Zod
 const discordConnectorFormSchema = z.object({
@@ -58,7 +59,7 @@ export default function DiscordConnectorPage() {
 	const params = useParams();
 	const searchSpaceId = params.search_space_id as string;
 	const [isSubmitting, setIsSubmitting] = useState(false);
-	const { createConnector } = useSearchSourceConnectors();
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	// Initialize the form
 	const form = useForm<DiscordConnectorFormValues>({
@@ -73,8 +74,8 @@ export default function DiscordConnectorPage() {
 	const onSubmit = async (values: DiscordConnectorFormValues) => {
 		setIsSubmitting(true);
 		try {
-			await createConnector(
-				{
+			await createConnector({
+				data: {
 					name: values.name,
 					connector_type: EnumConnectorName.DISCORD_CONNECTOR,
 					config: {
@@ -86,8 +87,10 @@ export default function DiscordConnectorPage() {
 					indexing_frequency_minutes: null,
 					next_scheduled_at: null,
 				},
-				parseInt(searchSpaceId)
-			);
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("Discord connector created successfully!");
 			router.push(`/dashboard/${searchSpaceId}/connectors`);

surfsense_web/app/dashboard/[search_space_id]/connectors/add/elasticsearch-connector/page.tsx

@@ -2,6 +2,7 @@
 
 import { zodResolver } from "@hookform/resolvers/zod";
 import * as RadioGroup from "@radix-ui/react-radio-group";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams, useRouter, useSearchParams } from "next/navigation";
@@ -9,7 +10,7 @@ import { useId, useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
-
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
 import {
 	Accordion,
 	AccordionContent,
@@ -40,10 +41,8 @@ import { Input } from "@/components/ui/input";
 import { Label } from "@/components/ui/label";
 import { Separator } from "@/components/ui/separator";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
-
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 
 // Define the form schema with Zod
 const elasticsearchConnectorFormSchema = z
@@ -91,7 +90,7 @@ export default function ElasticsearchConnectorPage() {
 	const authBasicId = useId();
 	const authApiKeyId = useId();
 
-	const { createConnector } = useSearchSourceConnectors();
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	// Initialize the form
 	const form = useForm<ElasticsearchConnectorFormValues>({
@@ -173,19 +172,21 @@ export default function ElasticsearchConnectorPage() {
 				config.ELASTICSEARCH_MAX_DOCUMENTS = values.max_documents;
 			}
 
-			const connectorPayload = {
-				name: values.name,
-				connector_type: EnumConnectorName.ELASTICSEARCH_CONNECTOR,
-				is_indexable: true,
-				last_indexed_at: null,
-				periodic_indexing_enabled: false,
-				indexing_frequency_minutes: null,
-				next_scheduled_at: null,
-				config,
-			};
-
-			// Use existing hook method
-			await createConnector(connectorPayload, searchSpaceIdNum);
+			await createConnector({
+				data: {
+					name: values.name,
+					connector_type: EnumConnectorName.ELASTICSEARCH_CONNECTOR,
+					is_indexable: true,
+					last_indexed_at: null,
+					periodic_indexing_enabled: false,
+					indexing_frequency_minutes: null,
+					next_scheduled_at: null,
+					config,
+				},
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("Elasticsearch connector created successfully!");
 			router.push(`/dashboard/${searchSpaceId}/connectors`);

surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, CircleAlert, Github, Info, ListChecks, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams, useRouter } from "next/navigation";
@@ -8,6 +9,7 @@ import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
 import {
 	Accordion,
 	AccordionContent,
@@ -38,8 +40,6 @@ import { Input } from "@/components/ui/input";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-// Assuming useSearchSourceConnectors hook exists and works similarly
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 import { authenticatedFetch, redirectToLogin } from "@/lib/auth-utils";
 
 // Define the form schema with Zod for GitHub PAT entry step
@@ -85,7 +85,7 @@ export default function GithubConnectorPage() {
 	const [connectorName, setConnectorName] = useState<string>("GitHub Connector");
 	const [validatedPat, setValidatedPat] = useState<string>(""); // Store the validated PAT
 
-	const { createConnector } = useSearchSourceConnectors();
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	// Initialize the form for PAT entry
 	const form = useForm<GithubPatFormValues>({
@@ -141,8 +141,8 @@ export default function GithubConnectorPage() {
 
 		setIsCreatingConnector(true);
 		try {
-			await createConnector(
-				{
+			await createConnector({
+				data: {
 					name: connectorName, // Use the stored name
 					connector_type: EnumConnectorName.GITHUB_CONNECTOR,
 					config: {
@@ -155,8 +155,10 @@ export default function GithubConnectorPage() {
 					indexing_frequency_minutes: null,
 					next_scheduled_at: null,
 				},
-				parseInt(searchSpaceId)
-			);
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("GitHub connector created successfully!");
 			router.push(`/dashboard/${searchSpaceId}/connectors`);

surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, ExternalLink, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import Link from "next/link";
@@ -9,6 +10,7 @@ import { useEffect, useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import { z } from "zod";
+import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms";
 import { Button } from "@/components/ui/button";
 import {
 	Card,
@@ -20,10 +22,7 @@ import {
 } from "@/components/ui/card";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import {
-	type SearchSourceConnector,
-	useSearchSourceConnectors,
-} from "@/hooks/use-search-source-connectors";
+import type { SearchSourceConnector } from "@/contracts/types/connector.types";
 import { authenticatedFetch } from "@/lib/auth-utils";
 
 export default function GoogleCalendarConnectorPage() {
@@ -33,11 +32,12 @@ export default function GoogleCalendarConnectorPage() {
 	const [isConnecting, setIsConnecting] = useState(false);
 	const [doesConnectorExist, setDoesConnectorExist] = useState(false);
 
-	const { fetchConnectors } = useSearchSourceConnectors(true, parseInt(searchSpaceId));
+	const { refetch: fetchConnectors } = useAtomValue(connectorsAtom);
 
 	useEffect(() => {
-		fetchConnectors(parseInt(searchSpaceId)).then((data) => {
-			const connector = data.find(
+		fetchConnectors().then((data) => {
+			const connectors = data.data || [];
+			const connector = connectors.find(
 				(c: SearchSourceConnector) =>
 					c.connector_type === EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR
 			);

surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, ExternalLink, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import Link from "next/link";
@@ -9,6 +10,7 @@ import { useEffect, useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import { z } from "zod";
+import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms";
 import { Button } from "@/components/ui/button";
 import {
 	Card,
@@ -20,10 +22,7 @@ import {
 } from "@/components/ui/card";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import {
-	type SearchSourceConnector,
-	useSearchSourceConnectors,
-} from "@/hooks/use-search-source-connectors";
+import type { SearchSourceConnector } from "@/contracts/types/connector.types";
 import { authenticatedFetch } from "@/lib/auth-utils";
 
 export default function GoogleGmailConnectorPage() {
@@ -33,11 +32,12 @@ export default function GoogleGmailConnectorPage() {
 	const [isConnecting, setIsConnecting] = useState(false);
 	const [doesConnectorExist, setDoesConnectorExist] = useState(false);
 
-	const { fetchConnectors } = useSearchSourceConnectors(true, parseInt(searchSpaceId));
+	const { refetch: fetchConnectors } = useAtomValue(connectorsAtom);
 
 	useEffect(() => {
-		fetchConnectors(parseInt(searchSpaceId)).then((data) => {
-			const connector = data.find(
+		fetchConnectors().then((data) => {
+			const connectors = data.data || [];
+			const connector = connectors.find(
 				(c: SearchSourceConnector) => c.connector_type === EnumConnectorName.GOOGLE_GMAIL_CONNECTOR
 			);
 			if (connector) {

surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams, useRouter } from "next/navigation";
@@ -8,6 +9,7 @@ import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
 import {
 	Accordion,
 	AccordionContent,
@@ -37,7 +39,6 @@ import { Input } from "@/components/ui/input";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 
 // Define the form schema with Zod
 const jiraConnectorFormSchema = z.object({
@@ -73,7 +74,7 @@ export default function JiraConnectorPage() {
 	const params = useParams();
 	const searchSpaceId = params.search_space_id as string;
 	const [isSubmitting, setIsSubmitting] = useState(false);
-	const { createConnector } = useSearchSourceConnectors();
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	// Initialize the form
 	const form = useForm<JiraConnectorFormValues>({
@@ -90,8 +91,8 @@ export default function JiraConnectorPage() {
 	const onSubmit = async (values: JiraConnectorFormValues) => {
 		setIsSubmitting(true);
 		try {
-			await createConnector(
-				{
+			await createConnector({
+				data: {
 					name: values.name,
 					connector_type: EnumConnectorName.JIRA_CONNECTOR,
 					config: {
@@ -105,8 +106,10 @@ export default function JiraConnectorPage() {
 					indexing_frequency_minutes: null,
 					next_scheduled_at: null,
 				},
-				parseInt(searchSpaceId)
-			);
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("Jira connector created successfully!");
 

surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams, useRouter } from "next/navigation";
@@ -8,6 +9,7 @@ import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
 import {
 	Accordion,
 	AccordionContent,
@@ -37,7 +39,6 @@ import { Input } from "@/components/ui/input";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 
 // Define the form schema with Zod
 const linearConnectorFormSchema = z.object({
@@ -62,7 +63,7 @@ export default function LinearConnectorPage() {
 	const params = useParams();
 	const searchSpaceId = params.search_space_id as string;
 	const [isSubmitting, setIsSubmitting] = useState(false);
-	const { createConnector } = useSearchSourceConnectors();
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	// Initialize the form
 	const form = useForm<LinearConnectorFormValues>({
@@ -77,8 +78,8 @@ export default function LinearConnectorPage() {
 	const onSubmit = async (values: LinearConnectorFormValues) => {
 		setIsSubmitting(true);
 		try {
-			await createConnector(
-				{
+			await createConnector({
+				data: {
 					name: values.name,
 					connector_type: EnumConnectorName.LINEAR_CONNECTOR,
 					config: {
@@ -90,8 +91,10 @@ export default function LinearConnectorPage() {
 					indexing_frequency_minutes: null,
 					next_scheduled_at: null,
 				},
-				parseInt(searchSpaceId)
-			);
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("Linear connector created successfully!");
 

surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams, useRouter } from "next/navigation";
@@ -8,6 +9,7 @@ import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
 import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
 import { Button } from "@/components/ui/button";
 import {
@@ -30,7 +32,6 @@ import {
 import { Input } from "@/components/ui/input";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 
 // Define the form schema with Zod
 const linkupApiFormSchema = z.object({
@@ -50,7 +51,7 @@ export default function LinkupApiPage() {
 	const params = useParams();
 	const searchSpaceId = params.search_space_id as string;
 	const [isSubmitting, setIsSubmitting] = useState(false);
-	const { createConnector } = useSearchSourceConnectors();
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	// Initialize the form
 	const form = useForm<LinkupApiFormValues>({
@@ -65,8 +66,8 @@ export default function LinkupApiPage() {
 	const onSubmit = async (values: LinkupApiFormValues) => {
 		setIsSubmitting(true);
 		try {
-			await createConnector(
-				{
+			await createConnector({
+				data: {
 					name: values.name,
 					connector_type: EnumConnectorName.LINKUP_API,
 					config: {
@@ -78,8 +79,10 @@ export default function LinkupApiPage() {
 					indexing_frequency_minutes: null,
 					next_scheduled_at: null,
 				},
-				parseInt(searchSpaceId)
-			);
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("Linkup API connector created successfully!");
 

surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Key, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import Link from "next/link";
@@ -9,6 +10,8 @@ import { useEffect, useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
+import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms";
 import { Button } from "@/components/ui/button";
 import {
 	Card,
@@ -30,10 +33,7 @@ import {
 import { Input } from "@/components/ui/input";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import {
-	type SearchSourceConnector,
-	useSearchSourceConnectors,
-} from "@/hooks/use-search-source-connectors";
+import type { SearchSourceConnector } from "@/contracts/types/connector.types";
 
 // Define the form schema with Zod
 const lumaConnectorFormSchema = z.object({
@@ -55,10 +55,8 @@ export default function LumaConnectorPage() {
 	const [isSubmitting, setIsSubmitting] = useState(false);
 	const [doesConnectorExist, setDoesConnectorExist] = useState(false);
 
-	const { fetchConnectors, createConnector } = useSearchSourceConnectors(
-		true,
-		parseInt(searchSpaceId)
-	);
+	const { data: connectors } = useAtomValue(connectorsAtom);
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	// Initialize the form
 	const form = useForm<LumaConnectorFormValues>({
@@ -69,29 +67,26 @@ export default function LumaConnectorPage() {
 		},
 	});
 
+	const { refetch: fetchConnectors } = useAtomValue(connectorsAtom);
+
 	useEffect(() => {
-		fetchConnectors(parseInt(searchSpaceId))
-			.then((data) => {
-				if (data && Array.isArray(data)) {
-					const connector = data.find(
-						(c: SearchSourceConnector) => c.connector_type === EnumConnectorName.LUMA_CONNECTOR
-					);
-					if (connector) {
-						setDoesConnectorExist(true);
-					}
-				}
-			})
-			.catch((error) => {
-				console.error("Error fetching connectors:", error);
-			});
-	}, [fetchConnectors, searchSpaceId]);
+		fetchConnectors().then((data) => {
+			const connectors = data.data || [];
+			const connector = connectors.find(
+				(c: SearchSourceConnector) => c.connector_type === EnumConnectorName.LUMA_CONNECTOR
+			);
+			if (connector) {
+				setDoesConnectorExist(true);
+			}
+		});
+	}, []);
 
 	// Handle form submission
 	const onSubmit = async (values: LumaConnectorFormValues) => {
 		setIsSubmitting(true);
 		try {
-			await createConnector(
-				{
+			await createConnector({
+				data: {
 					name: values.name,
 					connector_type: EnumConnectorName.LUMA_CONNECTOR,
 					config: {
@@ -103,8 +98,10 @@ export default function LumaConnectorPage() {
 					indexing_frequency_minutes: null,
 					next_scheduled_at: null,
 				},
-				parseInt(searchSpaceId)
-			);
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("Luma connector created successfully!");
 

surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams, useRouter } from "next/navigation";
@@ -8,6 +9,7 @@ import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
 import {
 	Accordion,
 	AccordionContent,
@@ -37,7 +39,6 @@ import { Input } from "@/components/ui/input";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 
 // Define the form schema with Zod
 const notionConnectorFormSchema = z.object({
@@ -57,7 +58,7 @@ export default function NotionConnectorPage() {
 	const params = useParams();
 	const searchSpaceId = params.search_space_id as string;
 	const [isSubmitting, setIsSubmitting] = useState(false);
-	const { createConnector } = useSearchSourceConnectors();
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	// Initialize the form
 	const form = useForm<NotionConnectorFormValues>({
@@ -72,8 +73,8 @@ export default function NotionConnectorPage() {
 	const onSubmit = async (values: NotionConnectorFormValues) => {
 		setIsSubmitting(true);
 		try {
-			await createConnector(
-				{
+			await createConnector({
+				data: {
 					name: values.name,
 					connector_type: EnumConnectorName.NOTION_CONNECTOR,
 					config: {
@@ -85,8 +86,10 @@ export default function NotionConnectorPage() {
 					indexing_frequency_minutes: null,
 					next_scheduled_at: null,
 				},
-				parseInt(searchSpaceId)
-			);
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("Notion connector created successfully!");
 

surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams, useRouter } from "next/navigation";
@@ -8,6 +9,7 @@ import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
 import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
 import { Button } from "@/components/ui/button";
 import {
@@ -31,7 +33,6 @@ import { Input } from "@/components/ui/input";
 import { Switch } from "@/components/ui/switch";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 
 const searxngFormSchema = z.object({
 	name: z.string().min(3, {
@@ -67,7 +68,7 @@ export default function SearxngConnectorPage() {
 	const params = useParams();
 	const searchSpaceId = params.search_space_id as string;
 	const [isSubmitting, setIsSubmitting] = useState(false);
-	const { createConnector } = useSearchSourceConnectors();
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	const form = useForm<SearxngFormValues>({
 		resolver: zodResolver(searxngFormSchema),
@@ -115,8 +116,8 @@ export default function SearxngConnectorPage() {
 				config.SEARXNG_VERIFY_SSL = false;
 			}
 
-			await createConnector(
-				{
+			await createConnector({
+				data: {
 					name: values.name,
 					connector_type: EnumConnectorName.SEARXNG_API,
 					config,
@@ -126,8 +127,10 @@ export default function SearxngConnectorPage() {
 					indexing_frequency_minutes: null,
 					next_scheduled_at: null,
 				},
-				parseInt(searchSpaceId)
-			);
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("SearxNG connector created successfully!");
 			router.push(`/dashboard/${searchSpaceId}/connectors`);

surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams, useRouter } from "next/navigation";
@@ -8,6 +9,7 @@ import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
 import {
 	Accordion,
 	AccordionContent,
@@ -37,7 +39,6 @@ import { Input } from "@/components/ui/input";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 
 // Define the form schema with Zod
 const slackConnectorFormSchema = z.object({
@@ -57,7 +58,7 @@ export default function SlackConnectorPage() {
 	const params = useParams();
 	const searchSpaceId = params.search_space_id as string;
 	const [isSubmitting, setIsSubmitting] = useState(false);
-	const { createConnector } = useSearchSourceConnectors();
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	// Initialize the form
 	const form = useForm<SlackConnectorFormValues>({
@@ -72,8 +73,8 @@ export default function SlackConnectorPage() {
 	const onSubmit = async (values: SlackConnectorFormValues) => {
 		setIsSubmitting(true);
 		try {
-			await createConnector(
-				{
+			await createConnector({
+				data: {
 					name: values.name,
 					connector_type: EnumConnectorName.SLACK_CONNECTOR,
 					config: {
@@ -85,8 +86,10 @@ export default function SlackConnectorPage() {
 					indexing_frequency_minutes: null,
 					next_scheduled_at: null,
 				},
-				parseInt(searchSpaceId)
-			);
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("Slack connector created successfully!");
 

surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams, useRouter } from "next/navigation";
@@ -8,6 +9,7 @@ import { useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
 import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
 import { Button } from "@/components/ui/button";
 import {
@@ -30,7 +32,6 @@ import {
 import { Input } from "@/components/ui/input";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 
 // Define the form schema with Zod
 const tavilyApiFormSchema = z.object({
@@ -50,7 +51,7 @@ export default function TavilyApiPage() {
 	const params = useParams();
 	const searchSpaceId = params.search_space_id as string;
 	const [isSubmitting, setIsSubmitting] = useState(false);
-	const { createConnector } = useSearchSourceConnectors();
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	// Initialize the form
 	const form = useForm<TavilyApiFormValues>({
@@ -65,8 +66,8 @@ export default function TavilyApiPage() {
 	const onSubmit = async (values: TavilyApiFormValues) => {
 		setIsSubmitting(true);
 		try {
-			await createConnector(
-				{
+			await createConnector({
+				data: {
 					name: values.name,
 					connector_type: EnumConnectorName.TAVILY_API,
 					config: {
@@ -78,8 +79,10 @@ export default function TavilyApiPage() {
 					indexing_frequency_minutes: null,
 					next_scheduled_at: null,
 				},
-				parseInt(searchSpaceId)
-			);
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("Tavily API connector created successfully!");
 

surfsense_web/app/dashboard/[search_space_id]/connectors/add/webcrawler-connector/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { ArrowLeft, Check, Globe, Loader2 } from "lucide-react";
 import { motion } from "motion/react";
 import Link from "next/link";
@@ -9,6 +10,8 @@ import { useEffect, useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import * as z from "zod";
+import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
+import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms";
 import { Button } from "@/components/ui/button";
 import {
 	Card,
@@ -31,10 +34,7 @@ import { Input } from "@/components/ui/input";
 import { Textarea } from "@/components/ui/textarea";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import {
-	type SearchSourceConnector,
-	useSearchSourceConnectors,
-} from "@/hooks/use-search-source-connectors";
+import type { SearchSourceConnector } from "@/contracts/types/connector.types";
 
 // Define the form schema with Zod
 const webcrawlerConnectorFormSchema = z.object({
@@ -55,10 +55,8 @@ export default function WebcrawlerConnectorPage() {
 	const [isSubmitting, setIsSubmitting] = useState(false);
 	const [doesConnectorExist, setDoesConnectorExist] = useState(false);
 
-	const { fetchConnectors, createConnector } = useSearchSourceConnectors(
-		true,
-		parseInt(searchSpaceId)
-	);
+	const { refetch: fetchConnectors } = useAtomValue(connectorsAtom);
+	const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
 
 	// Initialize the form
 	const form = useForm<WebcrawlerConnectorFormValues>({
@@ -71,22 +69,16 @@ export default function WebcrawlerConnectorPage() {
 	});
 
 	useEffect(() => {
-		fetchConnectors(parseInt(searchSpaceId))
-			.then((data) => {
-				if (data && Array.isArray(data)) {
-					const connector = data.find(
-						(c: SearchSourceConnector) =>
-							c.connector_type === EnumConnectorName.WEBCRAWLER_CONNECTOR
-					);
-					if (connector) {
-						setDoesConnectorExist(true);
-					}
-				}
-			})
-			.catch((error) => {
-				console.error("Error fetching connectors:", error);
-			});
-	}, [fetchConnectors, searchSpaceId]);
+		fetchConnectors().then((data) => {
+			const connectors = data.data || [];
+			const connector = connectors.find(
+				(c: SearchSourceConnector) => c.connector_type === EnumConnectorName.WEBCRAWLER_CONNECTOR
+			);
+			if (connector) {
+				setDoesConnectorExist(true);
+			}
+		});
+	}, []);
 
 	// Handle form submission
 	const onSubmit = async (values: WebcrawlerConnectorFormValues) => {
@@ -104,8 +96,8 @@ export default function WebcrawlerConnectorPage() {
 				config.INITIAL_URLS = values.initial_urls;
 			}
 
-			await createConnector(
-				{
+			await createConnector({
+				data: {
 					name: values.name,
 					connector_type: EnumConnectorName.WEBCRAWLER_CONNECTOR,
 					config: config,
@@ -115,8 +107,10 @@ export default function WebcrawlerConnectorPage() {
 					indexing_frequency_minutes: null,
 					next_scheduled_at: null,
 				},
-				parseInt(searchSpaceId)
-			);
+				queryParams: {
+					search_space_id: searchSpaceId,
+				},
+			});
 
 			toast.success("Webcrawler connector created successfully!");
 

surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/ProcessingIndicator.tsx

@@ -0,0 +1,43 @@
+"use client";
+
+import { Loader2 } from "lucide-react";
+import { AnimatePresence, motion } from "motion/react";
+import { useTranslations } from "next-intl";
+import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
+
+interface ProcessingIndicatorProps {
+	activeTasksCount: number;
+}
+
+export function ProcessingIndicator({ activeTasksCount }: ProcessingIndicatorProps) {
+	const t = useTranslations("documents");
+
+	if (activeTasksCount === 0) return null;
+
+	return (
+		<AnimatePresence>
+			<motion.div
+				initial={{ opacity: 0, height: 0, marginBottom: 0 }}
+				animate={{ opacity: 1, height: "auto", marginBottom: 24 }}
+				exit={{ opacity: 0, height: 0, marginBottom: 0 }}
+				transition={{ duration: 0.3 }}
+			>
+				<Alert className="border-border bg-primary/5">
+					<div className="flex items-center gap-4">
+						<div className="flex h-10 w-10 items-center justify-center rounded-full bg-primary/10">
+							<Loader2 className="h-5 w-5 animate-spin text-primary" />
+						</div>
+						<div className="flex-1">
+							<AlertTitle className="text-primary font-semibold">
+								{t("processing_documents")}
+							</AlertTitle>
+							<AlertDescription className="text-muted-foreground">
+								{t("active_tasks_count", { count: activeTasksCount })}
+							</AlertDescription>
+						</div>
+					</div>
+				</Alert>
+			</motion.div>
+		</AnimatePresence>
+	);
+}

surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx

@@ -2,19 +2,23 @@
 
 import { useQuery } from "@tanstack/react-query";
 import { useAtomValue } from "jotai";
+import { RefreshCw } from "lucide-react";
 import { motion } from "motion/react";
 import { useParams } from "next/navigation";
 import { useTranslations } from "next-intl";
-import { useCallback, useEffect, useId, useMemo, useState } from "react";
+import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react";
 import { toast } from "sonner";
 import { deleteDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms";
 import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
+import { Button } from "@/components/ui/button";
 import type { DocumentTypeEnum } from "@/contracts/types/document.types";
+import { useLogsSummary } from "@/hooks/use-logs";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
 import { cacheKeys } from "@/lib/query-client/cache-keys";
 import { DocumentsFilters } from "./components/DocumentsFilters";
 import { DocumentsTableShell, type SortKey } from "./components/DocumentsTableShell";
 import { PaginationControls } from "./components/PaginationControls";
+import { ProcessingIndicator } from "./components/ProcessingIndicator";
 import type { ColumnVisibility } from "./components/types";
 
 function useDebounced<T>(value: T, delay = 250) {
@@ -127,7 +131,22 @@ export default function DocumentsTable() {
 		} else {
 			await refetchDocuments();
 		}
-	}, [debouncedSearch, refetchSearch, refetchDocuments]);
+		toast.success(t("refresh_success") || "Documents refreshed");
+	}, [debouncedSearch, refetchSearch, refetchDocuments, t]);
+
+	// Set up polling for active tasks
+	const { summary } = useLogsSummary(searchSpaceId, 24, { refetchInterval: 5000 });
+	const activeTasksCount = summary?.active_tasks.length || 0;
+	const prevActiveTasksCount = useRef(activeTasksCount);
+
+	// Auto-refresh when a task finishes
+	useEffect(() => {
+		if (prevActiveTasksCount.current > activeTasksCount) {
+			// A task has finished!
+			refreshCurrentView();
+		}
+		prevActiveTasksCount.current = activeTasksCount;
+	}, [activeTasksCount, refreshCurrentView]);
 
 	// Create a delete function for single document deletion
 	const deleteDocument = useCallback(
@@ -189,8 +208,26 @@ export default function DocumentsTable() {
 			initial={{ opacity: 0, y: 20 }}
 			animate={{ opacity: 1, y: 0 }}
 			transition={{ duration: 0.3 }}
-			className="w-full px-6 py-4 min-h-[calc(100vh-64px)]"
+			className="w-full px-6 py-4 space-y-6 min-h-[calc(100vh-64px)]"
 		>
+			<motion.div
+				className="flex items-center justify-between"
+				initial={{ opacity: 0, y: 10 }}
+				animate={{ opacity: 1, y: 0 }}
+				transition={{ delay: 0.1 }}
+			>
+				<div>
+					<h2 className="text-2xl font-bold tracking-tight">{t("title")}</h2>
+					<p className="text-muted-foreground">{t("subtitle")}</p>
+				</div>
+				<Button onClick={refreshCurrentView} variant="outline" size="sm">
+					<RefreshCw className="w-4 h-4 mr-2" />
+					{t("refresh")}
+				</Button>
+			</motion.div>
+
+			<ProcessingIndicator activeTasksCount={activeTasksCount} />
+
 			<DocumentsFilters
 				typeCounts={typeCounts ?? {}}
 				selectedIds={selectedIds}

surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx

@@ -15,6 +15,7 @@ import {
 	useReactTable,
 	type VisibilityState,
 } from "@tanstack/react-table";
+import { useAtomValue } from "jotai";
 import {
 	Activity,
 	AlertCircle,
@@ -44,8 +45,13 @@ import {
 import { AnimatePresence, motion, type Variants } from "motion/react";
 import { useParams } from "next/navigation";
 import { useTranslations } from "next-intl";
-import React, { useContext, useId, useMemo, useRef, useState } from "react";
+import React, { useCallback, useContext, useId, useMemo, useRef, useState } from "react";
 import { toast } from "sonner";
+import {
+	createLogMutationAtom,
+	deleteLogMutationAtom,
+	updateLogMutationAtom,
+} from "@/atoms/logs/log-mutation.atoms";
 import { JsonMetadataViewer } from "@/components/json-metadata-viewer";
 import {
 	AlertDialog,
@@ -89,7 +95,8 @@ import {
 	TableHeader,
 	TableRow,
 } from "@/components/ui/table";
-import { type Log, type LogLevel, type LogStatus, useLogs, useLogsSummary } from "@/hooks/use-logs";
+import type { CreateLogRequest, Log, UpdateLogRequest } from "@/contracts/types/log.types";
+import { type LogLevel, type LogStatus, useLogs, useLogsSummary } from "@/hooks/use-logs";
 import { cn } from "@/lib/utils";
 
 // Define animation variants for reuse
@@ -334,13 +341,50 @@ export default function LogsManagePage() {
 	const params = useParams();
 	const searchSpaceId = Number(params.search_space_id);
 
-	const {
-		logs,
-		loading: logsLoading,
-		error: logsError,
-		refreshLogs,
-		deleteLog,
-	} = useLogs(searchSpaceId);
+	const { mutateAsync: deleteLogMutation } = useAtomValue(deleteLogMutationAtom);
+	const { mutateAsync: updateLogMutation } = useAtomValue(updateLogMutationAtom);
+	const { mutateAsync: createLogMutation } = useAtomValue(createLogMutationAtom);
+
+	const createLog = useCallback(
+		async (data: CreateLogRequest) => {
+			try {
+				await createLogMutation(data);
+				return true;
+			} catch (error) {
+				console.error("Failed to create log:", error);
+				return false;
+			}
+		},
+		[createLogMutation]
+	);
+
+	const updateLog = useCallback(
+		async (logId: number, data: UpdateLogRequest) => {
+			try {
+				await updateLogMutation({ logId, data });
+				return true;
+			} catch (error) {
+				console.error("Failed to update log:", error);
+				return false;
+			}
+		},
+		[updateLogMutation]
+	);
+
+	const deleteLog = useCallback(
+		async (id: number) => {
+			try {
+				await deleteLogMutation({ id });
+				return true;
+			} catch (error) {
+				console.error("Failed to delete log:", error);
+				return false;
+			}
+		},
+		[deleteLogMutation]
+	);
+
+	const { logs, loading: logsLoading, error: logsError, refreshLogs } = useLogs(searchSpaceId);
 	const {
 		summary,
 		loading: summaryLoading,
@@ -408,7 +452,7 @@ export default function LogsManagePage() {
 			return;
 		}
 
-		const deletePromises = selectedRows.map((row) => deleteLog(row.original.id));
+		const deletePromises = selectedRows.map((row) => deleteLog(row.original.id)); // Already passes { id } via wrapper
 
 		try {
 			const results = await Promise.all(deletePromises);
@@ -437,7 +481,7 @@ export default function LogsManagePage() {
 		<LogsContext.Provider
 			value={{
 				deleteLog: deleteLog || (() => Promise.resolve(false)),
-				refreshLogs: refreshLogs || (() => Promise.resolve()),
+				refreshLogs: () => refreshLogs().then(() => void 0),
 			}}
 		>
 			<motion.div
@@ -524,7 +568,7 @@ export default function LogsManagePage() {
 					table={table}
 					logs={logs}
 					loading={logsLoading}
-					error={logsError}
+					error={logsError?.message ?? null}
 					onRefresh={refreshLogs}
 					id={id}
 					t={t}

surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx

@@ -6,6 +6,7 @@ import {
 	type ThreadMessageLike,
 	useExternalStoreRuntime,
 } from "@assistant-ui/react";
+import { useQueryClient } from "@tanstack/react-query";
 import { useAtomValue, useSetAtom } from "jotai";
 import { useParams } from "next/navigation";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
@@ -17,6 +18,11 @@ import {
 	mentionedDocumentsAtom,
 	messageDocumentsMapAtom,
 } from "@/atoms/chat/mentioned-documents.atom";
+import {
+	clearPlanOwnerRegistry,
+	extractWriteTodosFromContent,
+	hydratePlanStateAtom,
+} from "@/atoms/chat/plan-state.atom";
 import { Thread } from "@/components/assistant-ui/thread";
 import { ChatHeader } from "@/components/new-chat/chat-header";
 import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking";
@@ -24,6 +30,7 @@ import { DisplayImageToolUI } from "@/components/tool-ui/display-image";
 import { GeneratePodcastToolUI } from "@/components/tool-ui/generate-podcast";
 import { LinkPreviewToolUI } from "@/components/tool-ui/link-preview";
 import { ScrapeWebpageToolUI } from "@/components/tool-ui/scrape-webpage";
+import { WriteTodosToolUI } from "@/components/tool-ui/write-todos";
 import { getBearerToken } from "@/lib/auth-utils";
 import { createAttachmentAdapter, extractAttachmentContent } from "@/lib/chat/attachment-adapter";
 import {
@@ -91,9 +98,45 @@ function extractMentionedDocuments(content: unknown): MentionedDocumentInfo[] {
 	return [];
 }
 
+/**
+ * Zod schema for persisted attachment info
+ */
+const PersistedAttachmentSchema = z.object({
+	id: z.string(),
+	name: z.string(),
+	type: z.string(),
+	contentType: z.string().optional(),
+	imageDataUrl: z.string().optional(),
+	extractedContent: z.string().optional(),
+});
+
+const AttachmentsPartSchema = z.object({
+	type: z.literal("attachments"),
+	items: z.array(PersistedAttachmentSchema),
+});
+
+type PersistedAttachment = z.infer<typeof PersistedAttachmentSchema>;
+
+/**
+ * Extract persisted attachments from message content (type-safe with Zod)
+ */
+function extractPersistedAttachments(content: unknown): PersistedAttachment[] {
+	if (!Array.isArray(content)) return [];
+
+	for (const part of content) {
+		const result = AttachmentsPartSchema.safeParse(part);
+		if (result.success) {
+			return result.data.items;
+		}
+	}
+
+	return [];
+}
+
 /**
  * Convert backend message to assistant-ui ThreadMessageLike format
  * Filters out 'thinking-steps' part as it's handled separately via messageThinkingSteps
+ * Restores attachments for user messages from persisted data
  */
 function convertToThreadMessage(msg: MessageRecord): ThreadMessageLike {
 	let content: ThreadMessageLike["content"];
@@ -105,8 +148,12 @@ function convertToThreadMessage(msg: MessageRecord): ThreadMessageLike {
 		const filteredContent = msg.content.filter((part: unknown) => {
 			if (typeof part !== "object" || part === null || !("type" in part)) return true;
 			const partType = (part as { type: string }).type;
-			// Filter out thinking-steps and mentioned-documents
-			return partType !== "thinking-steps" && partType !== "mentioned-documents";
+			// Filter out thinking-steps, mentioned-documents, and attachments
+			return (
+				partType !== "thinking-steps" &&
+				partType !== "mentioned-documents" &&
+				partType !== "attachments"
+			);
 		});
 		content =
 			filteredContent.length > 0
@@ -116,11 +163,31 @@ function convertToThreadMessage(msg: MessageRecord): ThreadMessageLike {
 		content = [{ type: "text", text: String(msg.content) }];
 	}
 
+	// Restore attachments for user messages
+	let attachments: ThreadMessageLike["attachments"];
+	if (msg.role === "user") {
+		const persistedAttachments = extractPersistedAttachments(msg.content);
+		if (persistedAttachments.length > 0) {
+			attachments = persistedAttachments.map((att) => ({
+				id: att.id,
+				name: att.name,
+				type: att.type as "document" | "image" | "file",
+				contentType: att.contentType || "application/octet-stream",
+				status: { type: "complete" as const },
+				content: [],
+				// Custom fields for our ChatAttachment interface
+				imageDataUrl: att.imageDataUrl,
+				extractedContent: att.extractedContent,
+			}));
+		}
+	}
+
 	return {
 		id: `msg-${msg.id}`,
 		role: msg.role,
 		content,
 		createdAt: new Date(msg.created_at),
+		attachments,
 	};
 }
 
@@ -132,6 +199,7 @@ const TOOLS_WITH_UI = new Set([
 	"link_preview",
 	"display_image",
 	"scrape_webpage",
+	"write_todos",
 ]);
 
 /**
@@ -146,6 +214,7 @@ interface ThinkingStepData {
 
 export default function NewChatPage() {
 	const params = useParams();
+	const queryClient = useQueryClient();
 	const [isInitializing, setIsInitializing] = useState(true);
 	const [threadId, setThreadId] = useState<number | null>(null);
 	const [messages, setMessages] = useState<ThreadMessageLike[]>([]);
@@ -163,6 +232,7 @@ export default function NewChatPage() {
 	const setMentionedDocumentIds = useSetAtom(mentionedDocumentIdsAtom);
 	const setMentionedDocuments = useSetAtom(mentionedDocumentsAtom);
 	const setMessageDocumentsMap = useSetAtom(messageDocumentsMapAtom);
+	const hydratePlanState = useSetAtom(hydratePlanStateAtom);
 
 	// Create the attachment adapter for file processing
 	const attachmentAdapter = useMemo(() => createAttachmentAdapter(), []);
@@ -198,6 +268,7 @@ export default function NewChatPage() {
 		setMentionedDocumentIds([]);
 		setMentionedDocuments([]);
 		setMessageDocumentsMap({});
+		clearPlanOwnerRegistry(); // Reset plan ownership for new chat
 
 		try {
 			if (urlChatId > 0) {
@@ -219,6 +290,11 @@ export default function NewChatPage() {
 							if (steps.length > 0) {
 								restoredThinkingSteps.set(`msg-${msg.id}`, steps);
 							}
+							// Hydrate write_todos plan state from persisted tool calls
+							const writeTodosCalls = extractWriteTodosFromContent(msg.content);
+							for (const todoData of writeTodosCalls) {
+								hydratePlanState(todoData);
+							}
 						}
 						if (msg.role === "user") {
 							const docs = extractMentionedDocuments(msg.content);
@@ -247,7 +323,13 @@ export default function NewChatPage() {
 		} finally {
 			setIsInitializing(false);
 		}
-	}, [urlChatId, setMessageDocumentsMap, setMentionedDocumentIds, setMentionedDocuments]);
+	}, [
+		urlChatId,
+		setMessageDocumentsMap,
+		setMentionedDocumentIds,
+		setMentionedDocuments,
+		hydratePlanState,
+	]);
 
 	// Initialize on mount
 	useEffect(() => {
@@ -306,6 +388,7 @@ export default function NewChatPage() {
 
 			// Lazy thread creation: create thread on first message if it doesn't exist
 			let currentThreadId = threadId;
+			let isNewThread = false;
 			if (!currentThreadId) {
 				try {
 					const newThread = await createThread(searchSpaceId, "New Chat");
@@ -315,6 +398,7 @@ export default function NewChatPage() {
 					// Track chat creation
 					trackChatCreated(searchSpaceId, currentThreadId);
 
+					isNewThread = true;
 					// Update URL silently using browser API (not router.replace) to avoid
 					// interrupting the ongoing fetch/streaming with React navigation
 					window.history.replaceState(
@@ -361,25 +445,50 @@ export default function NewChatPage() {
 				}));
 			}
 
-			// Persist user message with mentioned documents (don't await, fire and forget)
-			const persistContent =
-				mentionedDocuments.length > 0
-					? [
-							...message.content,
-							{
-								type: "mentioned-documents",
-								documents: mentionedDocuments.map((doc) => ({
-									id: doc.id,
-									title: doc.title,
-									document_type: doc.document_type,
-								})),
-							},
-						]
-					: message.content;
+			// Persist user message with mentioned documents and attachments (don't await, fire and forget)
+			const persistContent: unknown[] = [...message.content];
+
+			// Add mentioned documents for persistence
+			if (mentionedDocuments.length > 0) {
+				persistContent.push({
+					type: "mentioned-documents",
+					documents: mentionedDocuments.map((doc) => ({
+						id: doc.id,
+						title: doc.title,
+						document_type: doc.document_type,
+					})),
+				});
+			}
+
+			// Add attachments for persistence (so they survive page reload)
+			if (message.attachments && message.attachments.length > 0) {
+				persistContent.push({
+					type: "attachments",
+					items: message.attachments.map((att) => ({
+						id: att.id,
+						name: att.name,
+						type: att.type,
+						contentType: (att as { contentType?: string }).contentType,
+						// Include imageDataUrl for images so they can be displayed after reload
+						imageDataUrl: (att as { imageDataUrl?: string }).imageDataUrl,
+						// Include extractedContent for context (already extracted, no re-processing needed)
+						extractedContent: (att as { extractedContent?: string }).extractedContent,
+					})),
+				});
+			}
+
 			appendMessage(currentThreadId, {
 				role: "user",
 				content: persistContent,
-			}).catch((err) => console.error("Failed to persist user message:", err));
+			})
+				.then(() => {
+					// For new threads, the backend updates the title from the first user message
+					// Invalidate threads query so sidebar shows the updated title in real-time
+					if (isNewThread) {
+						queryClient.invalidateQueries({ queryKey: ["threads", String(searchSpaceId)] });
+					}
+				})
+				.catch((err) => console.error("Failed to persist user message:", err));
 
 			// Start streaming response
 			setIsRunning(true);
@@ -676,7 +785,19 @@ export default function NewChatPage() {
 				}
 			} catch (error) {
 				if (error instanceof Error && error.name === "AbortError") {
-					// Request was cancelled
+					// Request was cancelled by user - persist partial response if any content was received
+					const hasContent = contentParts.some(
+						(part) =>
+							(part.type === "text" && part.text.length > 0) ||
+							(part.type === "tool-call" && TOOLS_WITH_UI.has(part.toolName))
+					);
+					if (hasContent && currentThreadId) {
+						const partialContent = buildContentForPersistence();
+						appendMessage(currentThreadId, {
+							role: "assistant",
+							content: partialContent,
+						}).catch((err) => console.error("Failed to persist partial assistant message:", err));
+					}
 					return;
 				}
 				console.error("[NewChatPage] Chat error:", error);
@@ -720,6 +841,7 @@ export default function NewChatPage() {
 			setMentionedDocumentIds,
 			setMentionedDocuments,
 			setMessageDocumentsMap,
+			queryClient,
 		]
 	);
 
@@ -789,6 +911,7 @@ export default function NewChatPage() {
 			<LinkPreviewToolUI />
 			<DisplayImageToolUI />
 			<ScrapeWebpageToolUI />
+			<WriteTodosToolUI />
 			<div className="flex flex-col h-[calc(100vh-64px)] overflow-hidden">
 				<Thread
 					messageThinkingSteps={messageThinkingSteps}

surfsense_web/app/dashboard/[search_space_id]/page.tsx

@@ -8,8 +8,8 @@ export default function SearchSpaceDashboardPage() {
 	const { search_space_id } = useParams();
 
 	useEffect(() => {
-		router.push(`/dashboard/${search_space_id}/chats`);
-	}, []);
+		router.push(`/dashboard/${search_space_id}/new-chat`);
+	}, [router, search_space_id]);
 
 	return <></>;
 }

surfsense_web/app/layout.tsx

@@ -1,6 +1,5 @@
 import type { Metadata } from "next";
 import "./globals.css";
-import { GoogleAnalytics } from "@next/third-parties/google";
 import { RootProvider } from "fumadocs-ui/provider/next";
 import { Roboto } from "next/font/google";
 import { I18nProvider } from "@/components/providers/I18nProvider";
@@ -92,7 +91,6 @@ export default function RootLayout({
 	// Locale state is managed by LocaleContext and persisted in localStorage
 	return (
 		<html lang="en" suppressHydrationWarning>
-			<GoogleAnalytics gaId="G-T4CHE7W3TE" />
 			<body className={cn(roboto.className, "bg-white dark:bg-black antialiased h-full w-full ")}>
 				<PostHogProvider>
 					<LocaleProvider>

surfsense_web/atoms/chat/plan-state.atom.ts

@@ -0,0 +1,224 @@
+/**
+ * Plan State Atom
+ *
+ * Tracks the latest state of each plan by title.
+ * When write_todos is called multiple times with the same title,
+ * only the FIRST component renders (stays fixed in position),
+ * subsequent calls just update the shared state.
+ */
+
+import { atom } from "jotai";
+
+export interface PlanTodo {
+	id: string;
+	content: string;
+	status: "pending" | "in_progress" | "completed" | "cancelled";
+}
+
+export interface PlanState {
+	id: string;
+	title: string;
+	todos: PlanTodo[];
+	lastUpdated: number;
+	/** The toolCallId of the first component that rendered this plan */
+	ownerToolCallId: string;
+}
+
+/**
+ * SYNCHRONOUS ownership registry - prevents race conditions
+ * Only ONE plan allowed per conversation - first plan wins
+ */
+let firstPlanOwner: { toolCallId: string; title: string } | null = null;
+
+/**
+ * Register as owner of a plan SYNCHRONOUSLY
+ * ONE PLAN PER CONVERSATION: Only the first write_todos call renders.
+ * All subsequent calls update the state but don't render their own card.
+ */
+export function registerPlanOwner(title: string, toolCallId: string): boolean {
+	if (!firstPlanOwner) {
+		// First plan in this conversation - claim ownership
+		firstPlanOwner = { toolCallId, title };
+		return true;
+	}
+
+	// Check if we're the owner
+	return firstPlanOwner.toolCallId === toolCallId;
+}
+
+/**
+ * Get the canonical title for a plan
+ * Returns the first plan's title if one exists, otherwise the provided title
+ */
+export function getCanonicalPlanTitle(title: string): string {
+	return firstPlanOwner?.title || title;
+}
+
+/**
+ * Check if a plan already exists in this conversation
+ */
+export function hasPlan(): boolean {
+	return firstPlanOwner !== null;
+}
+
+/**
+ * Get the first plan's info
+ */
+export function getFirstPlanInfo(): { toolCallId: string; title: string } | null {
+	return firstPlanOwner;
+}
+
+/**
+ * Check if a toolCallId is the owner of the plan SYNCHRONOUSLY
+ */
+export function isPlanOwner(toolCallId: string): boolean {
+	return !firstPlanOwner || firstPlanOwner.toolCallId === toolCallId;
+}
+
+/**
+ * Clear ownership registry (call when starting a new chat)
+ */
+export function clearPlanOwnerRegistry(): void {
+	firstPlanOwner = null;
+}
+
+/**
+ * Map of plan title -> latest plan state
+ * Using title as key since it stays constant across updates
+ */
+export const planStatesAtom = atom<Map<string, PlanState>>(new Map());
+
+/**
+ * Input type for updating plan state
+ */
+export interface UpdatePlanInput {
+	id: string;
+	title: string;
+	todos: PlanTodo[];
+	toolCallId: string;
+}
+
+/**
+ * Helper atom to update a plan state
+ */
+export const updatePlanStateAtom = atom(null, (get, set, plan: UpdatePlanInput) => {
+	const states = new Map(get(planStatesAtom));
+
+	// Register ownership synchronously if not already done
+	registerPlanOwner(plan.title, plan.toolCallId);
+
+	// Get the actual owner from the first plan
+	const ownerToolCallId = firstPlanOwner?.toolCallId || plan.toolCallId;
+
+	// Always use the canonical (first) title for the plan key
+	const canonicalTitle = getCanonicalPlanTitle(plan.title);
+
+	states.set(canonicalTitle, {
+		id: plan.id,
+		title: canonicalTitle,
+		todos: plan.todos,
+		lastUpdated: Date.now(),
+		ownerToolCallId,
+	});
+	set(planStatesAtom, states);
+});
+
+/**
+ * Helper atom to get the latest plan state by title
+ */
+export const getPlanStateAtom = atom((get) => {
+	const states = get(planStatesAtom);
+	return (title: string) => states.get(title);
+});
+
+/**
+ * Helper atom to clear all plan states (useful when starting a new chat)
+ */
+export const clearPlanStatesAtom = atom(null, (get, set) => {
+	clearPlanOwnerRegistry();
+	set(planStatesAtom, new Map());
+});
+
+/**
+ * Hydrate plan state from persisted message content
+ * Call this when loading messages from the database to restore plan state
+ */
+export interface HydratePlanInput {
+	toolCallId: string;
+	result: {
+		id?: string;
+		title?: string;
+		todos?: Array<{
+			id?: string;
+			content: string;
+			status: "pending" | "in_progress" | "completed" | "cancelled";
+		}>;
+	};
+}
+
+export const hydratePlanStateAtom = atom(null, (get, set, plan: HydratePlanInput) => {
+	if (!plan.result?.todos || plan.result.todos.length === 0) return;
+
+	const states = new Map(get(planStatesAtom));
+	const title = plan.result.title || "Plan";
+
+	// Register this as the owner if no plan exists yet
+	registerPlanOwner(title, plan.toolCallId);
+
+	// Get the canonical title
+	const canonicalTitle = getCanonicalPlanTitle(title);
+	const ownerToolCallId = firstPlanOwner?.toolCallId || plan.toolCallId;
+
+	// Only set if this is newer or doesn't exist
+	const existing = states.get(canonicalTitle);
+	if (!existing) {
+		states.set(canonicalTitle, {
+			id: plan.result.id || `plan-${Date.now()}`,
+			title: canonicalTitle,
+			todos: plan.result.todos.map((t, i) => ({
+				id: t.id || `todo-${i}`,
+				content: t.content,
+				status: t.status,
+			})),
+			lastUpdated: Date.now(),
+			ownerToolCallId,
+		});
+		set(planStatesAtom, states);
+	}
+});
+
+/**
+ * Extract write_todos tool call data from message content
+ * Returns an array of { toolCallId, result } for each write_todos call found
+ */
+export function extractWriteTodosFromContent(content: unknown): HydratePlanInput[] {
+	if (!Array.isArray(content)) return [];
+
+	const results: HydratePlanInput[] = [];
+
+	for (const part of content) {
+		if (
+			typeof part === "object" &&
+			part !== null &&
+			"type" in part &&
+			(part as { type: string }).type === "tool-call" &&
+			"toolName" in part &&
+			(part as { toolName: string }).toolName === "write_todos" &&
+			"toolCallId" in part &&
+			"result" in part
+		) {
+			const toolCall = part as {
+				toolCallId: string;
+				result: HydratePlanInput["result"];
+			};
+			if (toolCall.result) {
+				results.push({
+					toolCallId: toolCall.toolCallId,
+					result: toolCall.result,
+				});
+			}
+		}
+	}
+
+	return results;
+}

surfsense_web/atoms/connectors/connector-mutation.atoms.ts

@@ -0,0 +1,100 @@
+import { atomWithMutation } from "jotai-tanstack-query";
+import { toast } from "sonner";
+import type {
+	CreateConnectorRequest,
+	DeleteConnectorRequest,
+	GetConnectorsResponse,
+	IndexConnectorRequest,
+	IndexConnectorResponse,
+	UpdateConnectorRequest,
+} from "@/contracts/types/connector.types";
+import { connectorsApiService } from "@/lib/apis/connectors-api.service";
+import { cacheKeys } from "@/lib/query-client/cache-keys";
+import { queryClient } from "@/lib/query-client/client";
+import { activeSearchSpaceIdAtom } from "../search-spaces/search-space-query.atoms";
+
+export const createConnectorMutationAtom = atomWithMutation((get) => {
+	const searchSpaceId = get(activeSearchSpaceIdAtom);
+
+	return {
+		mutationKey: cacheKeys.connectors.all(searchSpaceId!),
+		enabled: !!searchSpaceId,
+		mutationFn: async (request: CreateConnectorRequest) => {
+			return connectorsApiService.createConnector(request);
+		},
+
+		onSuccess: () => {
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.connectors.all(searchSpaceId!),
+			});
+		},
+	};
+});
+
+export const updateConnectorMutationAtom = atomWithMutation((get) => {
+	const searchSpaceId = get(activeSearchSpaceIdAtom);
+
+	return {
+		mutationKey: cacheKeys.connectors.all(searchSpaceId!),
+		enabled: !!searchSpaceId,
+		mutationFn: async (request: UpdateConnectorRequest) => {
+			return connectorsApiService.updateConnector(request);
+		},
+
+		onSuccess: (_, request: UpdateConnectorRequest) => {
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.connectors.all(searchSpaceId!),
+			});
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.connectors.byId(String(request.id)),
+			});
+		},
+	};
+});
+
+export const deleteConnectorMutationAtom = atomWithMutation((get) => {
+	const searchSpaceId = get(activeSearchSpaceIdAtom);
+
+	return {
+		mutationKey: cacheKeys.connectors.all(searchSpaceId!),
+		enabled: !!searchSpaceId,
+		mutationFn: async (request: DeleteConnectorRequest) => {
+			return connectorsApiService.deleteConnector(request);
+		},
+
+		onSuccess: (_, request: DeleteConnectorRequest) => {
+			queryClient.setQueryData(
+				cacheKeys.connectors.all(searchSpaceId!),
+				(oldData: GetConnectorsResponse | undefined) => {
+					if (!oldData) return oldData;
+					return oldData.filter((connector) => connector.id !== request.id);
+				}
+			);
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.connectors.byId(String(request.id)),
+			});
+		},
+	};
+});
+
+export const indexConnectorMutationAtom = atomWithMutation((get) => {
+	const searchSpaceId = get(activeSearchSpaceIdAtom);
+
+	return {
+		mutationKey: cacheKeys.connectors.index(),
+		enabled: !!searchSpaceId,
+		mutationFn: async (request: IndexConnectorRequest) => {
+			return connectorsApiService.indexConnector(request);
+		},
+
+		onSuccess: (response: IndexConnectorResponse) => {
+			toast.success(response.message);
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.connectors.all(searchSpaceId!),
+			});
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.connectors.byId(String(response.connector_id)),
+			});
+		},
+	};
+});

surfsense_web/atoms/connectors/connector-query.atoms.ts

@@ -0,0 +1,21 @@
+import { atomWithQuery } from "jotai-tanstack-query";
+import { connectorsApiService } from "@/lib/apis/connectors-api.service";
+import { cacheKeys } from "@/lib/query-client/cache-keys";
+import { activeSearchSpaceIdAtom } from "../search-spaces/search-space-query.atoms";
+
+export const connectorsAtom = atomWithQuery((get) => {
+	const searchSpaceId = get(activeSearchSpaceIdAtom);
+
+	return {
+		queryKey: cacheKeys.connectors.all(searchSpaceId!),
+		enabled: !!searchSpaceId,
+		staleTime: 5 * 60 * 1000, // 5 minutes
+		queryFn: async () => {
+			return connectorsApiService.getConnectors({
+				queryParams: {
+					search_space_id: searchSpaceId!,
+				},
+			});
+		},
+	};
+});

surfsense_web/atoms/connectors/ui.atoms.ts

@@ -0,0 +1,7 @@
+import { atom } from "jotai";
+import type { GetConnectorsRequest } from "@/contracts/types/connector.types";
+
+export const globalConnectorsQueryParamsAtom = atom<GetConnectorsRequest["queryParams"]>({
+	skip: 0,
+	limit: 10,
+});

surfsense_web/atoms/logs/log-mutation.atoms.ts

@@ -0,0 +1,68 @@
+import { atomWithMutation } from "jotai-tanstack-query";
+import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
+import type {
+	CreateLogRequest,
+	DeleteLogRequest,
+	UpdateLogRequest,
+} from "@/contracts/types/log.types";
+import { logsApiService } from "@/lib/apis/logs-api.service";
+import { cacheKeys } from "@/lib/query-client/cache-keys";
+import { queryClient } from "@/lib/query-client/client";
+
+/**
+ * Create Log Mutation
+ */
+export const createLogMutationAtom = atomWithMutation((get) => {
+	const searchSpaceId = get(activeSearchSpaceIdAtom);
+	return {
+		mutationKey: cacheKeys.logs.list(searchSpaceId ?? undefined),
+		enabled: !!searchSpaceId,
+		mutationFn: async (request: CreateLogRequest) => logsApiService.createLog(request),
+		onSuccess: () => {
+			queryClient.invalidateQueries({ queryKey: cacheKeys.logs.list(searchSpaceId ?? undefined) });
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.logs.summary(searchSpaceId ?? undefined),
+			});
+		},
+	};
+});
+
+/**
+ * Update Log Mutation
+ */
+export const updateLogMutationAtom = atomWithMutation((get) => {
+	const searchSpaceId = get(activeSearchSpaceIdAtom);
+	return {
+		mutationKey: cacheKeys.logs.list(searchSpaceId ?? undefined),
+		enabled: !!searchSpaceId,
+		mutationFn: async ({ logId, data }: { logId: number; data: UpdateLogRequest }) =>
+			logsApiService.updateLog(logId, data),
+		onSuccess: (_data, variables) => {
+			queryClient.invalidateQueries({ queryKey: cacheKeys.logs.detail(variables.logId) });
+			queryClient.invalidateQueries({ queryKey: cacheKeys.logs.list(searchSpaceId ?? undefined) });
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.logs.summary(searchSpaceId ?? undefined),
+			});
+		},
+	};
+});
+
+/**
+ * Delete Log Mutation
+ */
+export const deleteLogMutationAtom = atomWithMutation((get) => {
+	const searchSpaceId = get(activeSearchSpaceIdAtom);
+	return {
+		mutationKey: cacheKeys.logs.list(searchSpaceId ?? undefined),
+		enabled: !!searchSpaceId,
+		mutationFn: async (request: DeleteLogRequest) => logsApiService.deleteLog(request),
+		onSuccess: (_data, request) => {
+			queryClient.invalidateQueries({ queryKey: cacheKeys.logs.list(searchSpaceId ?? undefined) });
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.logs.summary(searchSpaceId ?? undefined),
+			});
+			if (request?.id)
+				queryClient.invalidateQueries({ queryKey: cacheKeys.logs.detail(request.id) });
+		},
+	};
+});

surfsense_web/components/assistant-ui/thread.tsx

@@ -15,8 +15,6 @@ import {
 	AlertCircle,
 	ArrowDownIcon,
 	ArrowUpIcon,
-	Brain,
-	CheckCircle2,
 	CheckIcon,
 	ChevronLeftIcon,
 	ChevronRightIcon,
@@ -28,8 +26,6 @@ import {
 	Plug2,
 	Plus,
 	RefreshCwIcon,
-	Search,
-	Sparkles,
 	SquareIcon,
 } from "lucide-react";
 import Link from "next/link";
@@ -75,13 +71,8 @@ import {
 	DocumentMentionPicker,
 	type DocumentMentionPickerRef,
 } from "@/components/new-chat/document-mention-picker";
-import {
-	ChainOfThought,
-	ChainOfThoughtContent,
-	ChainOfThoughtItem,
-	ChainOfThoughtStep,
-	ChainOfThoughtTrigger,
-} from "@/components/prompt-kit/chain-of-thought";
+import { ChainOfThoughtItem } from "@/components/prompt-kit/chain-of-thought";
+import { TextShimmerLoader } from "@/components/prompt-kit/loader";
 import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking";
 import { Button } from "@/components/ui/button";
 import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
@@ -103,124 +94,146 @@ interface ThreadProps {
 const ThinkingStepsContext = createContext<Map<string, ThinkingStep[]>>(new Map());
 
 /**
- * Get icon based on step status and title
- */
-function getStepIcon(status: "pending" | "in_progress" | "completed", title: string) {
-	const titleLower = title.toLowerCase();
-
-	if (status === "in_progress") {
-		return <Loader2 className="size-4 animate-spin text-primary" />;
-	}
-
-	if (status === "completed") {
-		return <CheckCircle2 className="size-4 text-emerald-500" />;
-	}
-
-	if (titleLower.includes("search") || titleLower.includes("knowledge")) {
-		return <Search className="size-4 text-muted-foreground" />;
-	}
-
-	if (titleLower.includes("analy") || titleLower.includes("understand")) {
-		return <Brain className="size-4 text-muted-foreground" />;
-	}
-
-	return <Sparkles className="size-4 text-muted-foreground" />;
-}
-
-/**
- * Chain of thought display component with smart expand/collapse behavior
+ * Chain of thought display component - single collapsible dropdown design
  */
 const ThinkingStepsDisplay: FC<{ steps: ThinkingStep[]; isThreadRunning?: boolean }> = ({
 	steps,
 	isThreadRunning = true,
 }) => {
-	// Track which steps the user has manually toggled (overrides auto behavior)
-	const [manualOverrides, setManualOverrides] = useState<Record<string, boolean>>({});
-	// Track previous step statuses to detect changes
-	const prevStatusesRef = useRef<Record<string, string>>({});
+	const [isOpen, setIsOpen] = useState(true);
 
-	// Derive effective status: if thread stopped and step is in_progress, treat as completed
-	const getEffectiveStatus = (step: ThinkingStep): "pending" | "in_progress" | "completed" => {
-		if (step.status === "in_progress" && !isThreadRunning) {
-			return "completed"; // Thread was stopped, so mark as completed
-		}
-		return step.status;
-	};
-
-	// Clear manual overrides when a step's status changes
-	useEffect(() => {
-		const currentStatuses: Record<string, string> = {};
-		steps.forEach((step) => {
-			currentStatuses[step.id] = step.status;
-			// If status changed, clear any manual override for this step
-			if (prevStatusesRef.current[step.id] && prevStatusesRef.current[step.id] !== step.status) {
-				setManualOverrides((prev) => {
-					const next = { ...prev };
-					delete next[step.id];
-					return next;
-				});
+	// Derive effective status for each step
+	const getEffectiveStatus = useCallback(
+		(step: ThinkingStep): "pending" | "in_progress" | "completed" => {
+			if (step.status === "in_progress" && !isThreadRunning) {
+				return "completed";
 			}
-		});
-		prevStatusesRef.current = currentStatuses;
-	}, [steps]);
+			return step.status;
+		},
+		[isThreadRunning]
+	);
+
+	// Calculate summary info
+	const completedSteps = steps.filter((s) => getEffectiveStatus(s) === "completed").length;
+	const inProgressStep = steps.find((s) => getEffectiveStatus(s) === "in_progress");
+	const allCompleted = completedSteps === steps.length && steps.length > 0 && !isThreadRunning;
+	const isProcessing = isThreadRunning && !allCompleted;
+
+	// Auto-collapse when all tasks are completed
+	useEffect(() => {
+		if (allCompleted) {
+			setIsOpen(false);
+		}
+	}, [allCompleted]);
 
 	if (steps.length === 0) return null;
 
-	const getStepOpenState = (step: ThinkingStep): boolean => {
-		const effectiveStatus = getEffectiveStatus(step);
-		// If user has manually toggled, respect that
-		if (manualOverrides[step.id] !== undefined) {
-			return manualOverrides[step.id];
+	// Generate header text
+	const getHeaderText = () => {
+		if (allCompleted) {
+			return `Reviewed ${completedSteps} ${completedSteps === 1 ? "step" : "steps"}`;
 		}
-		// Auto behavior: open if in progress
-		if (effectiveStatus === "in_progress") {
-			return true;
+		if (inProgressStep) {
+			return inProgressStep.title;
 		}
-		// Default: collapsed (all steps collapse when processing is done)
-		return false;
-	};
-
-	const handleToggle = (stepId: string, currentOpen: boolean) => {
-		setManualOverrides((prev) => ({
-			...prev,
-			[stepId]: !currentOpen,
-		}));
+		if (isProcessing) {
+			return `Processing ${completedSteps}/${steps.length} steps`;
+		}
+		return `Reviewed ${completedSteps} ${completedSteps === 1 ? "step" : "steps"}`;
 	};
 
 	return (
 		<div className="mx-auto w-full max-w-(--thread-max-width) px-2 py-2">
-			<ChainOfThought>
-				{steps.map((step) => {
-					const effectiveStatus = getEffectiveStatus(step);
-					const icon = getStepIcon(effectiveStatus, step.title);
-					const isOpen = getStepOpenState(step);
-					return (
-						<ChainOfThoughtStep
-							key={step.id}
-							open={isOpen}
-							onOpenChange={() => handleToggle(step.id, isOpen)}
-						>
-							<ChainOfThoughtTrigger
-								leftIcon={icon}
-								swapIconOnHover={effectiveStatus !== "in_progress"}
-								className={cn(
-									effectiveStatus === "in_progress" && "text-foreground font-medium",
-									effectiveStatus === "completed" && "text-muted-foreground"
-								)}
-							>
-								{step.title}
-							</ChainOfThoughtTrigger>
-							{step.items && step.items.length > 0 && (
-								<ChainOfThoughtContent>
-									{step.items.map((item, idx) => (
-										<ChainOfThoughtItem key={`${step.id}-item-${idx}`}>{item}</ChainOfThoughtItem>
-									))}
-								</ChainOfThoughtContent>
-							)}
-						</ChainOfThoughtStep>
-					);
-				})}
-			</ChainOfThought>
+			<div className="rounded-lg">
+				{/* Main collapsible header */}
+				<button
+					type="button"
+					onClick={() => setIsOpen(!isOpen)}
+					className={cn(
+						"flex w-full items-center gap-1.5 text-left text-sm transition-colors",
+						"text-muted-foreground hover:text-foreground"
+					)}
+				>
+					{/* Header text with shimmer if processing or has in-progress step */}
+					{isProcessing || inProgressStep ? (
+						<TextShimmerLoader text={getHeaderText()} size="sm" />
+					) : (
+						<span>{getHeaderText()}</span>
+					)}
+
+					{/* Chevron */}
+					<ChevronRightIcon
+						className={cn("size-4 transition-transform duration-200", isOpen && "rotate-90")}
+					/>
+				</button>
+
+				{/* Collapsible content with CSS grid animation */}
+				<div
+					className={cn(
+						"grid transition-[grid-template-rows] duration-300 ease-out",
+						isOpen ? "grid-rows-[1fr]" : "grid-rows-[0fr]"
+					)}
+				>
+					<div className="overflow-hidden">
+						<div className="mt-3 pl-1">
+							{steps.map((step, index) => {
+								const effectiveStatus = getEffectiveStatus(step);
+								const isLast = index === steps.length - 1;
+
+								return (
+									<div key={step.id} className="relative flex gap-3">
+										{/* Dot and line column */}
+										<div className="relative flex flex-col items-center w-2">
+											{/* Vertical connection line - extends to next dot */}
+											{!isLast && (
+												<div className="absolute left-1/2 top-[15px] -bottom-[7px] w-px -translate-x-1/2 bg-muted-foreground/30" />
+											)}
+											{/* Step dot - on top of line */}
+											<div className="relative z-10 mt-[7px] flex shrink-0 items-center justify-center">
+												{effectiveStatus === "in_progress" ? (
+													<span className="size-2 rounded-full bg-muted-foreground/30" />
+												) : (
+													<span className="size-2 rounded-full bg-muted-foreground/30" />
+												)}
+											</div>
+										</div>
+
+										{/* Step content */}
+										<div className="flex-1 min-w-0 pb-4">
+											{/* Step title */}
+											<div
+												className={cn(
+													"text-sm leading-5",
+													effectiveStatus === "in_progress" && "text-foreground font-medium",
+													effectiveStatus === "completed" && "text-muted-foreground",
+													effectiveStatus === "pending" && "text-muted-foreground/60"
+												)}
+											>
+												{effectiveStatus === "in_progress" ? (
+													<TextShimmerLoader text={step.title} size="sm" />
+												) : (
+													step.title
+												)}
+											</div>
+
+											{/* Step items (sub-content) */}
+											{step.items && step.items.length > 0 && (
+												<div className="mt-1 space-y-0.5">
+													{step.items.map((item, idx) => (
+														<ChainOfThoughtItem key={`${step.id}-item-${idx}`} className="text-xs">
+															{item}
+														</ChainOfThoughtItem>
+													))}
+												</div>
+											)}
+										</div>
+									</div>
+								);
+							})}
+						</div>
+					</div>
+				</div>
+			</div>
 		</div>
 	);
 };
@@ -286,7 +299,7 @@ export const Thread: FC<ThreadProps> = ({ messageThinkingSteps = new Map(), head
 			>
 				<ThreadPrimitive.Viewport
 					turnAnchor="top"
-					className="aui-thread-viewport relative flex flex-1 min-h-0 flex-col overflow-x-auto overflow-y-scroll scroll-smooth px-4 pt-4"
+					className="aui-thread-viewport relative flex flex-1 min-h-0 flex-col overflow-y-auto px-4 pt-4"
 				>
 					{/* Optional sticky header for model selector etc. */}
 					{header && <div className="sticky top-0 z-10 mb-4">{header}</div>}
@@ -428,13 +441,6 @@ const Composer: FC = () => {
 		}
 	}, [isThreadEmpty]);
 
-	// Reset auto-focus flag when thread becomes non-empty (user sent a message)
-	useEffect(() => {
-		if (!isThreadEmpty) {
-			hasAutoFocusedRef.current = false;
-		}
-	}, [isThreadEmpty]);
-
 	// Sync mentioned document IDs to atom for use in chat request
 	useEffect(() => {
 		setMentionedDocumentIds(mentionedDocuments.map((doc) => doc.id));
@@ -561,7 +567,7 @@ const Composer: FC = () => {
 				<div ref={editorContainerRef} className="aui-composer-input-wrapper px-3 pt-3 pb-6">
 					<InlineMentionEditor
 						ref={editorRef}
-						placeholder="Ask SurfSense (type @ to mention docs)"
+						placeholder="Ask SurfSense or @mention docs"
 						onMentionTrigger={handleMentionTrigger}
 						onMentionClose={handleMentionClose}
 						onChange={handleEditorChange}
@@ -683,14 +689,10 @@ const ConnectorIndicator: FC = () => {
 					) : (
 						<>
 							<Plug2 className="size-4" />
-							{totalSourceCount > 0 ? (
+							{totalSourceCount > 0 && (
 								<span className="absolute -top-0.5 -right-0.5 flex items-center justify-center min-w-[16px] h-4 px-1 text-[10px] font-medium rounded-full bg-primary text-primary-foreground shadow-sm">
 									{totalSourceCount > 99 ? "99+" : totalSourceCount}
 								</span>
-							) : (
-								<span className="absolute -top-0.5 -right-0.5 flex items-center justify-center size-3 rounded-full bg-muted-foreground/30 border border-background">
-									<span className="size-1.5 rounded-full bg-muted-foreground/60" />
-								</span>
 							)}
 						</>
 					)}
@@ -917,7 +919,7 @@ const AssistantMessageInner: FC = () => {
 				<MessageError />
 			</div>
 
-			<div className="aui-assistant-message-footer mt-1 ml-2 flex">
+			<div className="aui-assistant-message-footer mt-1 mb-5 ml-2 flex">
 				<BranchPicker />
 				<AssistantActionBar />
 			</div>

surfsense_web/components/prompt-kit/loader.tsx

@@ -0,0 +1,66 @@
+"use client";
+
+import { cn } from "@/lib/utils";
+
+export interface LoaderProps {
+	variant?: "text-shimmer";
+	size?: "sm" | "md" | "lg";
+	text?: string;
+	className?: string;
+}
+
+const textSizes = {
+	sm: "text-xs",
+	md: "text-sm",
+	lg: "text-base",
+} as const;
+
+/**
+ * TextShimmerLoader - A text loader with a shimmer gradient animation
+ * Used for in-progress states in write_todos and chain-of-thought
+ */
+export function TextShimmerLoader({
+	text = "Thinking",
+	className,
+	size = "md",
+}: {
+	text?: string;
+	className?: string;
+	size?: "sm" | "md" | "lg";
+}) {
+	return (
+		<>
+			<style>
+				{`
+          @keyframes shimmer {
+            0% { background-position: 200% 50%; }
+            100% { background-position: -200% 50%; }
+          }
+        `}
+			</style>
+			<span
+				className={cn(
+					"bg-[linear-gradient(to_right,var(--muted-foreground)_40%,var(--foreground)_60%,var(--muted-foreground)_80%)]",
+					"bg-[length:200%_auto] bg-clip-text font-medium text-transparent",
+					"animate-[shimmer_4s_infinite_linear]",
+					textSizes[size],
+					className
+				)}
+			>
+				{text}
+			</span>
+		</>
+	);
+}
+
+/**
+ * Loader component - currently only supports text-shimmer variant
+ * Can be extended with more variants if needed in the future
+ */
+export function Loader({ variant = "text-shimmer", size = "md", text, className }: LoaderProps) {
+	switch (variant) {
+		case "text-shimmer":
+		default:
+			return <TextShimmerLoader text={text} size={size} className={className} />;
+	}
+}

surfsense_web/components/sidebar/AppSidebarProvider.tsx

@@ -3,7 +3,7 @@
 import { useQuery, useQueryClient } from "@tanstack/react-query";
 import { useAtomValue, useSetAtom } from "jotai";
 import { Trash2 } from "lucide-react";
-import { useRouter } from "next/navigation";
+import { useParams, useRouter } from "next/navigation";
 import { useTranslations } from "next-intl";
 import { useCallback, useMemo, useState } from "react";
 import { hasUnsavedEditorChangesAtom, pendingEditorNavigationAtom } from "@/atoms/editor/ui.atoms";
@@ -50,7 +50,13 @@ export function AppSidebarProvider({
 	const t = useTranslations("dashboard");
 	const tCommon = useTranslations("common");
 	const router = useRouter();
+	const params = useParams();
 	const queryClient = useQueryClient();
+
+	// Get current chat ID from URL params
+	const currentChatId = params?.chat_id
+		? Number(Array.isArray(params.chat_id) ? params.chat_id[0] : params.chat_id)
+		: null;
 	const [isDeletingThread, setIsDeletingThread] = useState(false);
 
 	// Editor state for handling unsaved changes
@@ -61,7 +67,6 @@ export function AppSidebarProvider({
 	const {
 		data: threadsData,
 		error: threadError,
-		isLoading: isLoadingThreads,
 		refetch: refetchThreads,
 	} = useQuery({
 		queryKey: ["threads", searchSpaceId],
@@ -73,7 +78,6 @@ export function AppSidebarProvider({
 		data: searchSpace,
 		isLoading: isLoadingSearchSpace,
 		error: searchSpaceError,
-		refetch: fetchSearchSpace,
 	} = useQuery({
 		queryKey: cacheKeys.searchSpaces.detail(searchSpaceId),
 		queryFn: () => searchSpacesApiService.getSearchSpace({ id: Number(searchSpaceId) }),
@@ -83,12 +87,7 @@ export function AppSidebarProvider({
 	const { data: user } = useAtomValue(currentUserAtom);
 
 	// Fetch notes
-	const {
-		data: notesData,
-		error: notesError,
-		isLoading: isLoadingNotes,
-		refetch: refetchNotes,
-	} = useQuery({
+	const { data: notesData, refetch: refetchNotes } = useQuery({
 		queryKey: ["notes", searchSpaceId],
 		queryFn: () =>
 			notesApiService.getNotes({
@@ -108,11 +107,6 @@ export function AppSidebarProvider({
 	} | null>(null);
 	const [isDeletingNote, setIsDeletingNote] = useState(false);
 
-	// Retry function
-	const retryFetch = useCallback(() => {
-		fetchSearchSpace();
-	}, [fetchSearchSpace]);
-
 	// Transform threads to the format expected by AppSidebar
 	const recentChats = useMemo(() => {
 		if (!threadsData?.threads) return [];
@@ -149,6 +143,10 @@ export function AppSidebarProvider({
 			await deleteThread(threadToDelete.id);
 			// Invalidate threads query to refresh the list
 			queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] });
+			// Only navigate to new-chat if the deleted chat is currently open
+			if (currentChatId === threadToDelete.id) {
+				router.push(`/dashboard/${searchSpaceId}/new-chat`);
+			}
 		} catch (error) {
 			console.error("Error deleting thread:", error);
 		} finally {
@@ -156,7 +154,7 @@ export function AppSidebarProvider({
 			setShowDeleteDialog(false);
 			setThreadToDelete(null);
 		}
-	}, [threadToDelete, queryClient, searchSpaceId]);
+	}, [threadToDelete, queryClient, searchSpaceId, router, currentChatId]);
 
 	// Handle delete note with confirmation
 	const handleDeleteNote = useCallback(async () => {

surfsense_web/components/sidebar/all-chats-sidebar.tsx

@@ -13,7 +13,7 @@ import {
 	X,
 } from "lucide-react";
 import { AnimatePresence, motion } from "motion/react";
-import { useRouter } from "next/navigation";
+import { useParams, useRouter } from "next/navigation";
 import { useTranslations } from "next-intl";
 import { useCallback, useEffect, useState } from "react";
 import { createPortal } from "react-dom";
@@ -47,7 +47,15 @@ interface AllChatsSidebarProps {
 export function AllChatsSidebar({ open, onOpenChange, searchSpaceId }: AllChatsSidebarProps) {
 	const t = useTranslations("sidebar");
 	const router = useRouter();
+	const params = useParams();
 	const queryClient = useQueryClient();
+
+	// Get the current chat ID from URL to check if user is deleting the currently open chat
+	const currentChatId = Array.isArray(params.chat_id)
+		? Number(params.chat_id[0])
+		: params.chat_id
+			? Number(params.chat_id)
+			: null;
 	const [deletingThreadId, setDeletingThreadId] = useState<number | null>(null);
 	const [archivingThreadId, setArchivingThreadId] = useState<number | null>(null);
 	const [searchQuery, setSearchQuery] = useState("");
@@ -126,6 +134,15 @@ export function AllChatsSidebar({ open, onOpenChange, searchSpaceId }: AllChatsS
 				queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] });
 				queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] });
 				queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] });
+
+				// If the deleted chat is currently open, close sidebar first then redirect
+				if (currentChatId === threadId) {
+					onOpenChange(false);
+					// Wait for sidebar close animation to complete before navigating
+					setTimeout(() => {
+						router.push(`/dashboard/${searchSpaceId}/new-chat`);
+					}, 250);
+				}
 			} catch (error) {
 				console.error("Error deleting thread:", error);
 				toast.error(t("error_deleting_chat") || "Failed to delete chat");
@@ -133,7 +150,7 @@ export function AllChatsSidebar({ open, onOpenChange, searchSpaceId }: AllChatsS
 				setDeletingThreadId(null);
 			}
 		},
-		[queryClient, searchSpaceId, t]
+		[queryClient, searchSpaceId, t, currentChatId, router, onOpenChange]
 	);
 
 	// Handle thread archive/unarchive
@@ -293,6 +310,7 @@ export function AllChatsSidebar({ open, onOpenChange, searchSpaceId }: AllChatsS
 										const isDeleting = deletingThreadId === thread.id;
 										const isArchiving = archivingThreadId === thread.id;
 										const isBusy = isDeleting || isArchiving;
+										const isActive = currentChatId === thread.id;
 
 										return (
 											<div
@@ -301,6 +319,7 @@ export function AllChatsSidebar({ open, onOpenChange, searchSpaceId }: AllChatsS
 													"group flex items-center gap-2 rounded-md px-2 py-1.5 text-sm",
 													"hover:bg-accent hover:text-accent-foreground",
 													"transition-colors cursor-pointer",
+													isActive && "bg-accent text-accent-foreground",
 													isBusy && "opacity-50 pointer-events-none"
 												)}
 											>

surfsense_web/components/sidebar/all-notes-sidebar.tsx

@@ -4,7 +4,7 @@ import { useQuery, useQueryClient } from "@tanstack/react-query";
 import { format } from "date-fns";
 import { FileText, Loader2, MoreHorizontal, Plus, Search, Trash2, X } from "lucide-react";
 import { AnimatePresence, motion } from "motion/react";
-import { useRouter } from "next/navigation";
+import { useParams, useRouter } from "next/navigation";
 import { useTranslations } from "next-intl";
 import { useCallback, useEffect, useMemo, useState } from "react";
 import { createPortal } from "react-dom";
@@ -37,7 +37,11 @@ export function AllNotesSidebar({
 }: AllNotesSidebarProps) {
 	const t = useTranslations("sidebar");
 	const router = useRouter();
+	const params = useParams();
 	const queryClient = useQueryClient();
+
+	// Get the current note ID from URL to highlight the open note
+	const currentNoteId = params.note_id ? Number(params.note_id) : null;
 	const [deletingNoteId, setDeletingNoteId] = useState<number | null>(null);
 	const [searchQuery, setSearchQuery] = useState("");
 	const [mounted, setMounted] = useState(false);
@@ -208,7 +212,7 @@ export function AllNotesSidebar({
 						aria-label={t("all_notes") || "All Notes"}
 					>
 						{/* Header */}
-						<div className="flex-shrink-0 p-4 pb-3 space-y-3 border-b">
+						<div className="flex-shrink-0 p-4 pb-3 space-y-3">
 							<div className="flex items-center justify-between">
 								<h2 className="text-lg font-semibold">{t("all_notes") || "All Notes"}</h2>
 								<Button
@@ -260,6 +264,7 @@ export function AllNotesSidebar({
 								<div className="space-y-1">
 									{notes.map((note) => {
 										const isDeleting = deletingNoteId === note.id;
+										const isActive = currentNoteId === note.id;
 
 										return (
 											<div
@@ -268,6 +273,7 @@ export function AllNotesSidebar({
 													"group flex items-center gap-2 rounded-md px-2 py-1.5 text-sm",
 													"hover:bg-accent hover:text-accent-foreground",
 													"transition-colors cursor-pointer",
+													isActive && "bg-accent text-accent-foreground",
 													isDeleting && "opacity-50 pointer-events-none"
 												)}
 											>
@@ -370,7 +376,7 @@ export function AllNotesSidebar({
 
 						{/* Footer with Add Note button */}
 						{onAddNote && notes.length > 0 && (
-							<div className="flex-shrink-0 p-3 border-t">
+							<div className="flex-shrink-0 p-3">
 								<Button
 									onClick={() => {
 										onAddNote();

surfsense_web/components/sidebar/nav-chats.tsx

@@ -10,7 +10,7 @@ import {
 	RefreshCw,
 	Trash2,
 } from "lucide-react";
-import { useRouter } from "next/navigation";
+import { usePathname, useRouter } from "next/navigation";
 import { useTranslations } from "next-intl";
 import { useCallback, useEffect, useState } from "react";
 import { Button } from "@/components/ui/button";
@@ -71,6 +71,7 @@ export function NavChats({
 }: NavChatsProps) {
 	const t = useTranslations("sidebar");
 	const router = useRouter();
+	const pathname = usePathname();
 	const isMobile = useIsMobile();
 	const [isDeleting, setIsDeleting] = useState<number | null>(null);
 	const [isOpen, setIsOpen] = useState(defaultOpen);
@@ -142,6 +143,7 @@ export function NavChats({
 							<SidebarMenu>
 								{chats.map((chat) => {
 									const isDeletingChat = isDeleting === chat.id;
+									const isActive = pathname === chat.url;
 
 									return (
 										<SidebarMenuItem key={chat.id || chat.name} className="group/chat">
@@ -151,6 +153,7 @@ export function NavChats({
 												disabled={isDeletingChat}
 												className={cn(
 													"pr-8", // Make room for the action button
+													isActive && "bg-sidebar-accent text-sidebar-accent-foreground",
 													isDeletingChat && "opacity-50"
 												)}
 											>

surfsense_web/components/sidebar/nav-main.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { ChevronRight, type LucideIcon } from "lucide-react";
+import { usePathname } from "next/navigation";
 import { useTranslations } from "next-intl";
 import { useCallback, useMemo, useState } from "react";
 
@@ -35,6 +36,7 @@ interface NavMainProps {
 
 export function NavMain({ items, onSourcesExpandedChange }: NavMainProps) {
 	const t = useTranslations("nav_menu");
+	const pathname = usePathname();
 
 	// Translation function that handles both exact matches and fallback to original
 	const translateTitle = (title: string): string => {
@@ -55,6 +57,35 @@ export function NavMain({ items, onSourcesExpandedChange }: NavMainProps) {
 		return key ? t(key) : title;
 	};
 
+	// Check if an item is active based on pathname
+	const isItemActive = useCallback(
+		(item: NavItem): boolean => {
+			if (!pathname) return false;
+
+			// For items without sub-items, check if pathname matches or starts with the URL
+			if (!item.items?.length) {
+				// Chat item: active ONLY when on new-chat page without a specific chat ID
+				// (i.e., exactly /dashboard/{id}/new-chat, not /dashboard/{id}/new-chat/123)
+				if (item.url.includes("/new-chat")) {
+					// Match exactly the new-chat base URL (ends with /new-chat)
+					return pathname.endsWith("/new-chat");
+				}
+				// Logs item: active when on logs page
+				if (item.url.includes("/logs")) {
+					return pathname.includes("/logs");
+				}
+				// Check exact match or prefix match
+				return pathname === item.url || pathname.startsWith(`${item.url}/`);
+			}
+
+			// For items with sub-items (like Sources), check if any sub-item URL matches
+			return item.items.some(
+				(subItem) => pathname === subItem.url || pathname.startsWith(subItem.url)
+			);
+		},
+		[pathname]
+	);
+
 	// Memoize items to prevent unnecessary re-renders
 	const memoizedItems = useMemo(() => items, [items]);
 
@@ -88,14 +119,15 @@ export function NavMain({ items, onSourcesExpandedChange }: NavMainProps) {
 				{memoizedItems.map((item, index) => {
 					const translatedTitle = translateTitle(item.title);
 					const hasSub = !!item.items?.length;
-					const isItemOpen = expandedItems[item.title] ?? item.isActive ?? false;
+					const isActive = isItemActive(item);
+					const isItemOpen = expandedItems[item.title] ?? isActive ?? false;
 					return (
 						<Collapsible
 							key={`${item.title}-${index}`}
 							asChild
 							open={hasSub ? isItemOpen : undefined}
 							onOpenChange={hasSub ? (open) => handleOpenChange(item.title, open) : undefined}
-							defaultOpen={!hasSub ? item.isActive : undefined}
+							defaultOpen={!hasSub ? isActive : undefined}
 						>
 							<SidebarMenuItem>
 								{hasSub ? (
@@ -105,7 +137,7 @@ export function NavMain({ items, onSourcesExpandedChange }: NavMainProps) {
 											<SidebarMenuButton
 												asChild
 												tooltip={translatedTitle}
-												isActive={item.isActive}
+												isActive={isActive}
 												aria-label={`${translatedTitle} with submenu`}
 											>
 												<button type="button" className="flex items-center gap-2 w-full text-left">
@@ -147,7 +179,7 @@ export function NavMain({ items, onSourcesExpandedChange }: NavMainProps) {
 									<SidebarMenuButton
 										asChild
 										tooltip={translatedTitle}
-										isActive={item.isActive}
+										isActive={isActive}
 										aria-label={translatedTitle}
 									>
 										<a href={item.url}>

surfsense_web/components/sidebar/nav-notes.tsx

@@ -10,9 +10,9 @@ import {
 	Plus,
 	Trash2,
 } from "lucide-react";
-import { useRouter } from "next/navigation";
+import { usePathname, useRouter } from "next/navigation";
 import { useTranslations } from "next-intl";
-import { useCallback, useEffect, useState } from "react";
+import { useCallback, useEffect, useMemo, useState } from "react";
 import { Button } from "@/components/ui/button";
 import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
 import {
@@ -29,6 +29,7 @@ import {
 	SidebarMenuButton,
 	SidebarMenuItem,
 } from "@/components/ui/sidebar";
+import { useLogsSummary } from "@/hooks/use-logs";
 import { useIsMobile } from "@/hooks/use-mobile";
 import { cn } from "@/lib/utils";
 import { AllNotesSidebar } from "./all-notes-sidebar";
@@ -72,11 +73,27 @@ export function NavNotes({
 }: NavNotesProps) {
 	const t = useTranslations("sidebar");
 	const router = useRouter();
+	const pathname = usePathname();
 	const isMobile = useIsMobile();
 	const [isDeleting, setIsDeleting] = useState<number | null>(null);
 	const [isOpen, setIsOpen] = useState(defaultOpen);
 	const [isAllNotesSidebarOpen, setIsAllNotesSidebarOpen] = useState(false);
 
+	// Poll for active reindexing tasks to show inline loading indicators
+	const { summary } = useLogsSummary(searchSpaceId ? Number(searchSpaceId) : 0, 24, {
+		refetchInterval: 2000,
+	});
+
+	// Create a Set of document IDs that are currently being reindexed
+	const reindexingDocumentIds = useMemo(() => {
+		if (!summary?.active_tasks) return new Set<number>();
+		return new Set(
+			summary.active_tasks
+				.filter((task) => task.document_id != null)
+				.map((task) => task.document_id as number)
+		);
+	}, [summary?.active_tasks]);
+
 	// Auto-collapse on smaller screens when Sources is expanded
 	useEffect(() => {
 		if (isSourcesExpanded && isMobile) {
@@ -157,6 +174,8 @@ export function NavNotes({
 							{notes.length > 0 ? (
 								notes.map((note) => {
 									const isDeletingNote = isDeleting === note.id;
+									const isActive = pathname === note.url;
+									const isReindexing = note.id ? reindexingDocumentIds.has(note.id) : false;
 
 									return (
 										<SidebarMenuItem key={note.id || note.name} className="group/note">
@@ -166,10 +185,15 @@ export function NavNotes({
 												disabled={isDeletingNote}
 												className={cn(
 													"pr-8", // Make room for the action button
+													isActive && "bg-sidebar-accent text-sidebar-accent-foreground",
 													isDeletingNote && "opacity-50"
 												)}
 											>
-												<note.icon className="h-4 w-4 shrink-0" />
+												{isReindexing ? (
+													<Loader2 className="h-4 w-4 shrink-0 animate-spin text-primary" />
+												) : (
+													<note.icon className="h-4 w-4 shrink-0" />
+												)}
 												<span className="truncate">{note.name}</span>
 											</SidebarMenuButton>
 

surfsense_web/components/sidebar/nav-secondary.tsx

@@ -36,12 +36,21 @@ export function NavSecondary({
 			<SidebarMenu>
 				{memoizedItems.map((item, index) => (
 					<SidebarMenuItem key={`${item.title}-${index}`}>
-						<SidebarMenuButton asChild size="sm" aria-label={item.title}>
-							<a href={item.url}>
-								<item.icon />
-								<span>{item.title}</span>
-							</a>
-						</SidebarMenuButton>
+						{item.url === "#" ? (
+							// Non-interactive display item (e.g., search space name)
+							<div className="flex h-7 w-full items-center gap-2 rounded-md px-2 text-xs text-sidebar-foreground">
+								<item.icon className="h-4 w-4 shrink-0" />
+								<span className="truncate">{item.title}</span>
+							</div>
+						) : (
+							// Interactive link item
+							<SidebarMenuButton asChild size="sm" aria-label={item.title}>
+								<a href={item.url}>
+									<item.icon />
+									<span>{item.title}</span>
+								</a>
+							</SidebarMenuButton>
+						)}
 					</SidebarMenuItem>
 				))}
 			</SidebarMenu>

surfsense_web/components/tool-ui/display-image.tsx

@@ -2,6 +2,7 @@
 
 import { makeAssistantToolUI } from "@assistant-ui/react";
 import { AlertCircleIcon, ImageIcon } from "lucide-react";
+import { z } from "zod";
 import {
 	Image,
 	ImageErrorBoundary,
@@ -9,27 +10,41 @@ import {
 	parseSerializableImage,
 } from "@/components/tool-ui/image";
 
-/**
- * Type definitions for the display_image tool
- */
-interface DisplayImageArgs {
-	src: string;
-	alt?: string;
-	title?: string;
-	description?: string;
-}
+// ============================================================================
+// Zod Schemas
+// ============================================================================
 
-interface DisplayImageResult {
-	id: string;
-	assetId: string;
-	src: string;
-	alt?: string; // Made optional - parseSerializableImage provides fallback
-	title?: string;
-	description?: string;
-	domain?: string;
-	ratio?: string;
-	error?: string;
-}
+/**
+ * Schema for display_image tool arguments
+ */
+const DisplayImageArgsSchema = z.object({
+	src: z.string(),
+	alt: z.string().nullish(),
+	title: z.string().nullish(),
+	description: z.string().nullish(),
+});
+
+/**
+ * Schema for display_image tool result
+ */
+const DisplayImageResultSchema = z.object({
+	id: z.string(),
+	assetId: z.string(),
+	src: z.string(),
+	alt: z.string().nullish(),
+	title: z.string().nullish(),
+	description: z.string().nullish(),
+	domain: z.string().nullish(),
+	ratio: z.string().nullish(),
+	error: z.string().nullish(),
+});
+
+// ============================================================================
+// Types
+// ============================================================================
+
+type DisplayImageArgs = z.infer<typeof DisplayImageArgsSchema>;
+type DisplayImageResult = z.infer<typeof DisplayImageResultSchema>;
 
 /**
  * Error state component shown when image display fails
@@ -142,4 +157,9 @@ export const DisplayImageToolUI = makeAssistantToolUI<DisplayImageArgs, DisplayI
 	},
 });
 
-export type { DisplayImageArgs, DisplayImageResult };
+export {
+	DisplayImageArgsSchema,
+	DisplayImageResultSchema,
+	type DisplayImageArgs,
+	type DisplayImageResult,
+};

surfsense_web/components/tool-ui/index.ts

@@ -25,7 +25,9 @@ export {
 } from "./deepagent-thinking";
 export {
 	type DisplayImageArgs,
+	DisplayImageArgsSchema,
 	type DisplayImageResult,
+	DisplayImageResultSchema,
 	DisplayImageToolUI,
 } from "./display-image";
 export { GeneratePodcastToolUI } from "./generate-podcast";
@@ -40,10 +42,14 @@ export {
 } from "./image";
 export {
 	type LinkPreviewArgs,
+	LinkPreviewArgsSchema,
 	type LinkPreviewResult,
+	LinkPreviewResultSchema,
 	LinkPreviewToolUI,
 	type MultiLinkPreviewArgs,
+	MultiLinkPreviewArgsSchema,
 	type MultiLinkPreviewResult,
+	MultiLinkPreviewResultSchema,
 	MultiLinkPreviewToolUI,
 } from "./link-preview";
 export {
@@ -55,8 +61,20 @@ export {
 	parseSerializableMediaCard,
 	type SerializableMediaCard,
 } from "./media-card";
+export {
+	Plan,
+	PlanErrorBoundary,
+	type PlanProps,
+	type PlanTodo,
+	parseSerializablePlan,
+	type SerializablePlan,
+	type TodoStatus,
+} from "./plan";
 export {
 	type ScrapeWebpageArgs,
+	ScrapeWebpageArgsSchema,
 	type ScrapeWebpageResult,
+	ScrapeWebpageResultSchema,
 	ScrapeWebpageToolUI,
 } from "./scrape-webpage";
+export { type WriteTodosData, WriteTodosSchema, WriteTodosToolUI } from "./write-todos";

surfsense_web/components/tool-ui/link-preview.tsx

@@ -2,6 +2,7 @@
 
 import { makeAssistantToolUI } from "@assistant-ui/react";
 import { AlertCircleIcon, ExternalLinkIcon, LinkIcon } from "lucide-react";
+import { z } from "zod";
 import {
 	MediaCard,
 	MediaCardErrorBoundary,
@@ -10,25 +11,39 @@ import {
 	type SerializableMediaCard,
 } from "@/components/tool-ui/media-card";
 
-/**
- * Type definitions for the link_preview tool
- */
-interface LinkPreviewArgs {
-	url: string;
-	title?: string;
-}
+// ============================================================================
+// Zod Schemas
+// ============================================================================
 
-interface LinkPreviewResult {
-	id: string;
-	assetId: string;
-	kind: "link";
-	href: string;
-	title: string;
-	description?: string;
-	thumb?: string;
-	domain?: string;
-	error?: string;
-}
+/**
+ * Schema for link_preview tool arguments
+ */
+const LinkPreviewArgsSchema = z.object({
+	url: z.string(),
+	title: z.string().nullish(),
+});
+
+/**
+ * Schema for link_preview tool result
+ */
+const LinkPreviewResultSchema = z.object({
+	id: z.string(),
+	assetId: z.string(),
+	kind: z.literal("link"),
+	href: z.string(),
+	title: z.string(),
+	description: z.string().nullish(),
+	thumb: z.string().nullish(),
+	domain: z.string().nullish(),
+	error: z.string().nullish(),
+});
+
+// ============================================================================
+// Types
+// ============================================================================
+
+type LinkPreviewArgs = z.infer<typeof LinkPreviewArgsSchema>;
+type LinkPreviewResult = z.infer<typeof LinkPreviewResultSchema>;
 
 /**
  * Error state component shown when link preview fails
@@ -150,20 +165,35 @@ export const LinkPreviewToolUI = makeAssistantToolUI<LinkPreviewArgs, LinkPrevie
 	},
 });
 
-/**
- * Multiple Link Previews Tool UI Component
- *
- * This component handles cases where multiple links need to be previewed.
- * It renders a grid of link preview cards.
- */
-interface MultiLinkPreviewArgs {
-	urls: string[];
-}
+// ============================================================================
+// Multi Link Preview Schemas
+// ============================================================================
 
-interface MultiLinkPreviewResult {
-	previews: LinkPreviewResult[];
-	errors?: { url: string; error: string }[];
-}
+/**
+ * Schema for multi_link_preview tool arguments
+ */
+const MultiLinkPreviewArgsSchema = z.object({
+	urls: z.array(z.string()),
+});
+
+/**
+ * Schema for error items in multi_link_preview result
+ */
+const MultiLinkPreviewErrorSchema = z.object({
+	url: z.string(),
+	error: z.string(),
+});
+
+/**
+ * Schema for multi_link_preview tool result
+ */
+const MultiLinkPreviewResultSchema = z.object({
+	previews: z.array(LinkPreviewResultSchema),
+	errors: z.array(MultiLinkPreviewErrorSchema).nullish(),
+});
+
+type MultiLinkPreviewArgs = z.infer<typeof MultiLinkPreviewArgsSchema>;
+type MultiLinkPreviewResult = z.infer<typeof MultiLinkPreviewResultSchema>;
 
 export const MultiLinkPreviewToolUI = makeAssistantToolUI<
 	MultiLinkPreviewArgs,
@@ -217,4 +247,13 @@ export const MultiLinkPreviewToolUI = makeAssistantToolUI<
 	},
 });
 
-export type { LinkPreviewArgs, LinkPreviewResult, MultiLinkPreviewArgs, MultiLinkPreviewResult };
+export {
+	LinkPreviewArgsSchema,
+	LinkPreviewResultSchema,
+	MultiLinkPreviewArgsSchema,
+	MultiLinkPreviewResultSchema,
+	type LinkPreviewArgs,
+	type LinkPreviewResult,
+	type MultiLinkPreviewArgs,
+	type MultiLinkPreviewResult,
+};

surfsense_web/components/tool-ui/plan/index.tsx

@@ -0,0 +1,52 @@
+"use client";
+
+import { Component, type ReactNode } from "react";
+import { Card, CardContent } from "@/components/ui/card";
+
+export * from "./plan";
+export * from "./schema";
+
+// ============================================================================
+// Error Boundary
+// ============================================================================
+
+interface PlanErrorBoundaryProps {
+	children: ReactNode;
+	fallback?: ReactNode;
+}
+
+interface PlanErrorBoundaryState {
+	hasError: boolean;
+	error?: Error;
+}
+
+export class PlanErrorBoundary extends Component<PlanErrorBoundaryProps, PlanErrorBoundaryState> {
+	constructor(props: PlanErrorBoundaryProps) {
+		super(props);
+		this.state = { hasError: false };
+	}
+
+	static getDerivedStateFromError(error: Error): PlanErrorBoundaryState {
+		return { hasError: true, error };
+	}
+
+	render() {
+		if (this.state.hasError) {
+			if (this.props.fallback) {
+				return this.props.fallback;
+			}
+
+			return (
+				<Card className="w-full max-w-xl border-destructive/50">
+					<CardContent className="pt-6">
+						<div className="flex items-center gap-2 text-destructive">
+							<span className="text-sm">Failed to render plan</span>
+						</div>
+					</CardContent>
+				</Card>
+			);
+		}
+
+		return this.props.children;
+	}
+}

surfsense_web/components/tool-ui/plan/plan.tsx

@@ -0,0 +1,229 @@
+"use client";
+
+import { CheckCircle2, Circle, CircleDashed, ListTodo, PartyPopper, XCircle } from "lucide-react";
+import type { FC } from "react";
+import { useMemo, useState } from "react";
+import { TextShimmerLoader } from "@/components/prompt-kit/loader";
+import { Button } from "@/components/ui/button";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
+import { Progress } from "@/components/ui/progress";
+import { cn } from "@/lib/utils";
+import type { Action, ActionsConfig } from "../shared/schema";
+import type { TodoStatus } from "./schema";
+
+// ============================================================================
+// Status Icon Component
+// ============================================================================
+
+interface StatusIconProps {
+	status: TodoStatus;
+	className?: string;
+	/** When false, in_progress items show as static (no spinner) */
+	isStreaming?: boolean;
+}
+
+const StatusIcon: FC<StatusIconProps> = ({ status, className, isStreaming = true }) => {
+	const baseClass = cn("size-4 shrink-0", className);
+
+	switch (status) {
+		case "completed":
+			return <CheckCircle2 className={cn(baseClass, "text-emerald-500")} />;
+		case "in_progress":
+			// Only animate the spinner if we're actively streaming
+			// When streaming is stopped, show as a static dashed circle
+			return (
+				<CircleDashed
+					className={cn(baseClass, "text-primary", isStreaming && "animate-spin")}
+					style={isStreaming ? { animationDuration: "3s" } : undefined}
+				/>
+			);
+		case "cancelled":
+			return <XCircle className={cn(baseClass, "text-destructive")} />;
+		case "pending":
+		default:
+			return <Circle className={cn(baseClass, "text-muted-foreground")} />;
+	}
+};
+
+// ============================================================================
+// Todo Item Component
+// ============================================================================
+
+interface TodoItemProps {
+	todo: { id: string; content: string; status: TodoStatus };
+	/** When false, in_progress items show as static (no spinner/pulse) */
+	isStreaming?: boolean;
+}
+
+const TodoItem: FC<TodoItemProps> = ({ todo, isStreaming = true }) => {
+	const isStrikethrough = todo.status === "completed" || todo.status === "cancelled";
+	// Only show shimmer animation if streaming and in progress
+	const isShimmer = todo.status === "in_progress" && isStreaming;
+
+	// Render the content with optional shimmer effect
+	const renderContent = () => {
+		if (isShimmer) {
+			return <TextShimmerLoader text={todo.content} size="md" />;
+		}
+		return (
+			<span className={cn("text-sm text-muted-foreground", isStrikethrough && "line-through")}>
+				{todo.content}
+			</span>
+		);
+	};
+
+	return (
+		<div className="flex items-center gap-2 py-2">
+			<StatusIcon status={todo.status} isStreaming={isStreaming} />
+			{renderContent()}
+		</div>
+	);
+};
+
+// ============================================================================
+// Plan Component
+// ============================================================================
+
+export interface PlanProps {
+	id: string;
+	title: string;
+	todos: Array<{ id: string; content: string; status: TodoStatus }>;
+	maxVisibleTodos?: number;
+	showProgress?: boolean;
+	/** When false, in_progress items show as static (no spinner/pulse animations) */
+	isStreaming?: boolean;
+	responseActions?: Action[] | ActionsConfig;
+	className?: string;
+	onResponseAction?: (actionId: string) => void;
+	onBeforeResponseAction?: (actionId: string) => boolean;
+}
+
+export const Plan: FC<PlanProps> = ({
+	id,
+	title,
+	todos,
+	maxVisibleTodos = 4,
+	showProgress = true,
+	isStreaming = true,
+	responseActions,
+	className,
+	onResponseAction,
+	onBeforeResponseAction,
+}) => {
+	const [isExpanded, setIsExpanded] = useState(false);
+
+	// Calculate progress
+	const progress = useMemo(() => {
+		const completed = todos.filter((t) => t.status === "completed").length;
+		const total = todos.filter((t) => t.status !== "cancelled").length;
+		return { completed, total, percentage: total > 0 ? (completed / total) * 100 : 0 };
+	}, [todos]);
+
+	const isAllComplete = progress.completed === progress.total && progress.total > 0;
+
+	// Split todos for collapsible display
+	const visibleTodos = todos.slice(0, maxVisibleTodos);
+	const hiddenTodos = todos.slice(maxVisibleTodos);
+	const hasHiddenTodos = hiddenTodos.length > 0;
+
+	// Handle action click
+	const handleAction = (actionId: string) => {
+		if (onBeforeResponseAction && !onBeforeResponseAction(actionId)) {
+			return;
+		}
+		onResponseAction?.(actionId);
+	};
+
+	// Normalize actions to array
+	const actionArray: Action[] = useMemo(() => {
+		if (!responseActions) return [];
+		if (Array.isArray(responseActions)) return responseActions;
+		return [
+			responseActions.confirm && { ...responseActions.confirm, id: "confirm" },
+			responseActions.cancel && { ...responseActions.cancel, id: "cancel" },
+		].filter(Boolean) as Action[];
+	}, [responseActions]);
+
+	const TodoList: FC<{ items: typeof todos }> = ({ items }) => {
+		return (
+			<div className="space-y-0">
+				{items.map((todo) => (
+					<TodoItem key={todo.id} todo={todo} isStreaming={isStreaming} />
+				))}
+			</div>
+		);
+	};
+
+	return (
+		<Card id={id} className={cn("w-full max-w-xl", className)}>
+			<CardHeader className="pb-3">
+				<div className="flex items-start justify-between gap-2">
+					<div className="flex-1 min-w-0 flex items-center gap-2">
+						<ListTodo className="size-5 text-muted-foreground shrink-0" />
+						<CardTitle className="text-base font-semibold text-muted-foreground">{title}</CardTitle>
+					</div>
+					{isAllComplete && (
+						<div className="flex items-center gap-1 text-emerald-500">
+							<PartyPopper className="size-5" />
+						</div>
+					)}
+				</div>
+
+				{showProgress && (
+					<div className="mt-3 space-y-1.5">
+						<div className="flex items-center justify-between text-xs text-muted-foreground">
+							<span>
+								{progress.completed} of {progress.total} complete
+							</span>
+							<span>{Math.round(progress.percentage)}%</span>
+						</div>
+						<Progress
+							value={progress.percentage}
+							className="h-1.5 bg-muted [&>div]:bg-muted-foreground"
+						/>
+					</div>
+				)}
+			</CardHeader>
+
+			<CardContent className="pt-0">
+				<TodoList items={visibleTodos} />
+
+				{hasHiddenTodos && (
+					<Collapsible open={isExpanded} onOpenChange={setIsExpanded}>
+						<CollapsibleTrigger asChild>
+							<Button
+								variant="ghost"
+								size="sm"
+								className="w-full mt-2 text-xs text-muted-foreground hover:text-foreground"
+							>
+								{isExpanded
+									? "Show less"
+									: `Show ${hiddenTodos.length} more ${hiddenTodos.length === 1 ? "task" : "tasks"}`}
+							</Button>
+						</CollapsibleTrigger>
+						<CollapsibleContent>
+							<TodoList items={hiddenTodos} />
+						</CollapsibleContent>
+					</Collapsible>
+				)}
+
+				{actionArray.length > 0 && (
+					<div className="flex flex-wrap gap-2 pt-4 mt-2 border-t">
+						{actionArray.map((action) => (
+							<Button
+								key={action.id}
+								variant={action.variant || "default"}
+								size="sm"
+								disabled={action.disabled}
+								onClick={() => handleAction(action.id)}
+							>
+								{action.label}
+							</Button>
+						))}
+					</div>
+				)}
+			</CardContent>
+		</Card>
+	);
+};

surfsense_web/components/tool-ui/plan/schema.ts

@@ -0,0 +1,91 @@
+import { z } from "zod";
+
+/**
+ * Todo item status
+ */
+export const TodoStatusSchema = z.enum(["pending", "in_progress", "completed", "cancelled"]);
+export type TodoStatus = z.infer<typeof TodoStatusSchema>;
+
+/**
+ * Single todo item in a plan
+ * Matches deepagents TodoListMiddleware output: { content, status }
+ * id is auto-generated if not provided
+ */
+export const PlanTodoSchema = z.object({
+	id: z.string().optional(),
+	content: z.string(),
+	status: TodoStatusSchema,
+});
+
+export type PlanTodo = z.infer<typeof PlanTodoSchema>;
+
+/**
+ * Serializable plan schema for tool results
+ * Matches deepagents TodoListMiddleware output format
+ * id/title are auto-generated if not provided
+ */
+export const SerializablePlanSchema = z.object({
+	id: z.string().optional(),
+	title: z.string().optional(),
+	todos: z.array(PlanTodoSchema).min(1),
+	maxVisibleTodos: z.number().optional(),
+	showProgress: z.boolean().optional(),
+});
+
+export type SerializablePlan = z.infer<typeof SerializablePlanSchema>;
+
+/**
+ * Normalized plan with required fields (after auto-generation)
+ */
+export interface NormalizedPlan {
+	id: string;
+	title: string;
+	todos: Array<{ id: string; content: string; status: TodoStatus }>;
+	maxVisibleTodos?: number;
+	showProgress?: boolean;
+}
+
+/**
+ * Parse and normalize a plan from tool result
+ * Auto-generates id/title if not provided (for deepagents compatibility)
+ */
+export function parseSerializablePlan(data: unknown): NormalizedPlan {
+	const result = SerializablePlanSchema.safeParse(data);
+
+	if (!result.success) {
+		console.warn("Invalid plan data:", result.error.issues);
+
+		// Try to extract basic info for fallback
+		const obj = (data && typeof data === "object" ? data : {}) as Record<string, unknown>;
+
+		return {
+			id: typeof obj.id === "string" ? obj.id : `plan-${Date.now()}`,
+			title: typeof obj.title === "string" ? obj.title : "Plan",
+			todos: Array.isArray(obj.todos)
+				? obj.todos.map((t: unknown, i: number) => {
+						const todo = t as Record<string, unknown>;
+						return {
+							id: typeof todo?.id === "string" ? todo.id : `todo-${i}`,
+							content: typeof todo?.content === "string" ? todo.content : "Task",
+							status: TodoStatusSchema.safeParse(todo?.status).success
+								? (todo.status as TodoStatus)
+								: ("pending" as const),
+						};
+					})
+				: [{ id: "1", content: "No tasks", status: "pending" as const }],
+		};
+	}
+
+	// Normalize: add id/title if missing
+	return {
+		id: result.data.id || `plan-${Date.now()}`,
+		title: result.data.title || "Plan",
+		todos: result.data.todos.map((t, i) => ({
+			id: t.id || `todo-${i}`,
+			content: t.content,
+			status: t.status,
+		})),
+		maxVisibleTodos: result.data.maxVisibleTodos,
+		showProgress: result.data.showProgress,
+	};
+}

surfsense_web/components/tool-ui/scrape-webpage.tsx

@@ -2,6 +2,7 @@
 
 import { makeAssistantToolUI } from "@assistant-ui/react";
 import { AlertCircleIcon, FileTextIcon } from "lucide-react";
+import { z } from "zod";
 import {
 	Article,
 	ArticleErrorBoundary,
@@ -9,30 +10,44 @@ import {
 	parseSerializableArticle,
 } from "@/components/tool-ui/article";
 
-/**
- * Type definitions for the scrape_webpage tool
- */
-interface ScrapeWebpageArgs {
-	url: string;
-	max_length?: number;
-}
+// ============================================================================
+// Zod Schemas
+// ============================================================================
 
-interface ScrapeWebpageResult {
-	id: string;
-	assetId: string;
-	kind: "article";
-	href: string;
-	title: string;
-	description?: string;
-	content?: string;
-	domain?: string;
-	author?: string;
-	date?: string;
-	word_count?: number;
-	was_truncated?: boolean;
-	crawler_type?: string;
-	error?: string;
-}
+/**
+ * Schema for scrape_webpage tool arguments
+ */
+const ScrapeWebpageArgsSchema = z.object({
+	url: z.string(),
+	max_length: z.number().nullish(),
+});
+
+/**
+ * Schema for scrape_webpage tool result
+ */
+const ScrapeWebpageResultSchema = z.object({
+	id: z.string(),
+	assetId: z.string(),
+	kind: z.literal("article"),
+	href: z.string(),
+	title: z.string(),
+	description: z.string().nullish(),
+	content: z.string().nullish(),
+	domain: z.string().nullish(),
+	author: z.string().nullish(),
+	date: z.string().nullish(),
+	word_count: z.number().nullish(),
+	was_truncated: z.boolean().nullish(),
+	crawler_type: z.string().nullish(),
+	error: z.string().nullish(),
+});
+
+// ============================================================================
+// Types
+// ============================================================================
+
+type ScrapeWebpageArgs = z.infer<typeof ScrapeWebpageArgsSchema>;
+type ScrapeWebpageResult = z.infer<typeof ScrapeWebpageResultSchema>;
 
 /**
  * Error state component shown when webpage scraping fails
@@ -154,4 +169,9 @@ export const ScrapeWebpageToolUI = makeAssistantToolUI<ScrapeWebpageArgs, Scrape
 	},
 });
 
-export type { ScrapeWebpageArgs, ScrapeWebpageResult };
+export {
+	ScrapeWebpageArgsSchema,
+	ScrapeWebpageResultSchema,
+	type ScrapeWebpageArgs,
+	type ScrapeWebpageResult,
+};

surfsense_web/components/tool-ui/shared/action-buttons.tsx

@@ -0,0 +1,41 @@
+"use client";
+
+import type { FC } from "react";
+import { Button } from "@/components/ui/button";
+import type { Action, ActionsConfig } from "./schema";
+
+interface ActionButtonsProps {
+	actions?: Action[] | ActionsConfig;
+	onAction?: (actionId: string) => void;
+	disabled?: boolean;
+}
+
+export const ActionButtons: FC<ActionButtonsProps> = ({ actions, onAction, disabled }) => {
+	if (!actions) return null;
+
+	// Normalize actions to array format
+	const actionArray: Action[] = Array.isArray(actions)
+		? actions
+		: ([
+				actions.confirm && { ...actions.confirm, id: "confirm" },
+				actions.cancel && { ...actions.cancel, id: "cancel" },
+			].filter(Boolean) as Action[]);
+
+	if (actionArray.length === 0) return null;
+
+	return (
+		<div className="flex flex-wrap gap-2 pt-3">
+			{actionArray.map((action) => (
+				<Button
+					key={action.id}
+					variant={action.variant || "default"}
+					size="sm"
+					disabled={disabled || action.disabled}
+					onClick={() => onAction?.(action.id)}
+				>
+					{action.label}
+				</Button>
+			))}
+		</div>
+	);
+};

surfsense_web/components/tool-ui/shared/index.ts

@@ -0,0 +1,2 @@
+export * from "./action-buttons";
+export * from "./schema";

surfsense_web/components/tool-ui/shared/schema.ts

@@ -0,0 +1,23 @@
+import { z } from "zod";
+
+/**
+ * Shared action schema for tool UI components
+ */
+export const ActionSchema = z.object({
+	id: z.string(),
+	label: z.string(),
+	variant: z.enum(["default", "secondary", "destructive", "outline", "ghost", "link"]).optional(),
+	disabled: z.boolean().optional(),
+});
+
+export type Action = z.infer<typeof ActionSchema>;
+
+/**
+ * Actions configuration schema
+ */
+export const ActionsConfigSchema = z.object({
+	confirm: ActionSchema.optional(),
+	cancel: ActionSchema.optional(),
+});
+
+export type ActionsConfig = z.infer<typeof ActionsConfigSchema>;

surfsense_web/components/tool-ui/write-todos.tsx

@@ -0,0 +1,158 @@
+"use client";
+
+import { makeAssistantToolUI, useAssistantState } from "@assistant-ui/react";
+import { useAtomValue, useSetAtom } from "jotai";
+import { Loader2 } from "lucide-react";
+import { useEffect, useMemo } from "react";
+import { z } from "zod";
+import {
+	getCanonicalPlanTitle,
+	planStatesAtom,
+	registerPlanOwner,
+	updatePlanStateAtom,
+} from "@/atoms/chat/plan-state.atom";
+import { Plan, PlanErrorBoundary, parseSerializablePlan, TodoStatusSchema } from "./plan";
+
+// ============================================================================
+// Zod Schemas - Matching deepagents TodoListMiddleware output
+// ============================================================================
+
+/**
+ * Schema for a single todo item (matches deepagents output)
+ */
+const TodoItemSchema = z.object({
+	content: z.string(),
+	status: TodoStatusSchema,
+});
+
+/**
+ * Schema for write_todos tool args/result (matches deepagents output)
+ * deepagents provides: { todos: [{ content, status }] }
+ */
+const WriteTodosSchema = z.object({
+	todos: z.array(TodoItemSchema).nullish(),
+});
+
+// ============================================================================
+// Types
+// ============================================================================
+
+type WriteTodosData = z.infer<typeof WriteTodosSchema>;
+
+/**
+ * Loading state component
+ */
+function WriteTodosLoading() {
+	return (
+		<div className="my-4 w-full max-w-xl rounded-2xl border bg-card/60 px-5 py-4 shadow-sm">
+			<div className="flex items-center gap-3">
+				<Loader2 className="size-5 animate-spin text-primary" />
+				<span className="text-sm text-muted-foreground">Creating plan...</span>
+			</div>
+		</div>
+	);
+}
+
+/**
+ * WriteTodos Tool UI Component
+ *
+ * Displays the agent's planning/todo list with a beautiful UI.
+ * Uses deepagents TodoListMiddleware output directly: { todos: [{ content, status }] }
+ *
+ * FIXED POSITION: When multiple write_todos calls happen in a conversation,
+ * only the FIRST component renders. Subsequent updates just update the
+ * shared state, and the first component reads from it.
+ */
+export const WriteTodosToolUI = makeAssistantToolUI<WriteTodosData, WriteTodosData>({
+	toolName: "write_todos",
+	render: function WriteTodosUI({ args, result, status, toolCallId }) {
+		const updatePlanState = useSetAtom(updatePlanStateAtom);
+		const planStates = useAtomValue(planStatesAtom);
+
+		// Check if the THREAD is running
+		const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning);
+
+		// Use result if available, otherwise args (for streaming)
+		const data = result || args;
+		const hasTodos = data?.todos && data.todos.length > 0;
+
+		// Fixed title for all plans in conversation
+		const planTitle = "Plan";
+
+		// SYNCHRONOUS ownership check
+		const isOwner = useMemo(() => {
+			return registerPlanOwner(planTitle, toolCallId);
+		}, [planTitle, toolCallId]);
+
+		// Get canonical title
+		const canonicalTitle = useMemo(() => getCanonicalPlanTitle(planTitle), [planTitle]);
+
+		// Register/update the plan state
+		useEffect(() => {
+			if (hasTodos) {
+				const normalizedPlan = parseSerializablePlan({ todos: data.todos });
+				updatePlanState({
+					id: normalizedPlan.id,
+					title: canonicalTitle,
+					todos: normalizedPlan.todos,
+					toolCallId,
+				});
+			}
+		}, [data, hasTodos, canonicalTitle, updatePlanState, toolCallId]);
+
+		// Get the current plan state
+		const currentPlanState = planStates.get(canonicalTitle);
+
+		// If we're NOT the owner, render nothing
+		if (!isOwner) {
+			return null;
+		}
+
+		// Loading state
+		if (status.type === "running" || status.type === "requires-action") {
+			if (hasTodos) {
+				const plan = parseSerializablePlan({ todos: data.todos });
+				return (
+					<div className="my-4">
+						<PlanErrorBoundary>
+							<Plan {...plan} showProgress={true} isStreaming={isThreadRunning} />
+						</PlanErrorBoundary>
+					</div>
+				);
+			}
+			return <WriteTodosLoading />;
+		}
+
+		// Incomplete/cancelled state
+		if (status.type === "incomplete") {
+			if (currentPlanState || hasTodos) {
+				const plan = currentPlanState || parseSerializablePlan({ todos: data?.todos || [] });
+				return (
+					<div className="my-4">
+						<PlanErrorBoundary>
+							<Plan {...plan} showProgress={true} isStreaming={isThreadRunning} />
+						</PlanErrorBoundary>
+					</div>
+				);
+			}
+			return null;
+		}
+
+		// Success - render the plan
+		const planToRender =
+			currentPlanState || (hasTodos ? parseSerializablePlan({ todos: data.todos }) : null);
+		if (!planToRender) {
+			return <WriteTodosLoading />;
+		}
+
+		return (
+			<div className="my-4">
+				<PlanErrorBoundary>
+					<Plan {...planToRender} showProgress={true} isStreaming={isThreadRunning} />
+				</PlanErrorBoundary>
+			</div>
+		);
+	},
+});
+
+export { WriteTodosSchema, type WriteTodosData };

surfsense_web/components/ui/sidebar.tsx

@@ -449,7 +449,7 @@ function SidebarMenuItem({ className, ...props }: React.ComponentProps<"li">) {
 }
 
 const sidebarMenuButtonVariants = cva(
-	"peer/menu-button flex w-full items-center gap-2 overflow-hidden rounded-md p-2 text-left text-sm outline-hidden ring-sidebar-ring transition-[width,height,padding] hover:bg-sidebar-accent hover:text-sidebar-accent-foreground focus-visible:ring-2 active:bg-sidebar-accent active:text-sidebar-accent-foreground disabled:pointer-events-none disabled:opacity-50 group-has-data-[sidebar=menu-action]/menu-item:pr-8 aria-disabled:pointer-events-none aria-disabled:opacity-50 data-[active=true]:bg-sidebar-accent data-[active=true]:font-medium data-[active=true]:text-sidebar-accent-foreground data-[state=open]:hover:bg-sidebar-accent data-[state=open]:hover:text-sidebar-accent-foreground group-data-[collapsible=icon]:size-8! group-data-[collapsible=icon]:p-2! [&>span:last-child]:truncate [&>svg]:size-4 [&>svg]:shrink-0",
+	"peer/menu-button flex w-full items-center gap-2 overflow-hidden rounded-md p-2 text-left text-sm outline-hidden ring-sidebar-ring transition-[width,height,padding] hover:bg-sidebar-accent hover:text-sidebar-accent-foreground focus-visible:ring-2 active:bg-sidebar-accent active:text-sidebar-accent-foreground disabled:pointer-events-none disabled:opacity-50 group-has-data-[sidebar=menu-action]/menu-item:pr-8 aria-disabled:pointer-events-none aria-disabled:opacity-50 data-[active=true]:bg-sidebar-accent data-[active=true]:text-sidebar-accent-foreground data-[state=open]:hover:bg-sidebar-accent data-[state=open]:hover:text-sidebar-accent-foreground group-data-[collapsible=icon]:size-8! group-data-[collapsible=icon]:p-2! [&>span:last-child]:truncate [&>svg]:size-4 [&>svg]:shrink-0",
 	{
 		variants: {
 			variant: {

surfsense_web/contracts/types/connector.types.ts

@@ -0,0 +1,159 @@
+import { z } from "zod";
+import { paginationQueryParams } from ".";
+
+export const searchSourceConnectorTypeEnum = z.enum([
+	"SERPER_API",
+	"TAVILY_API",
+	"SEARXNG_API",
+	"LINKUP_API",
+	"BAIDU_SEARCH_API",
+	"SLACK_CONNECTOR",
+	"NOTION_CONNECTOR",
+	"GITHUB_CONNECTOR",
+	"LINEAR_CONNECTOR",
+	"DISCORD_CONNECTOR",
+	"JIRA_CONNECTOR",
+	"CONFLUENCE_CONNECTOR",
+	"CLICKUP_CONNECTOR",
+	"GOOGLE_CALENDAR_CONNECTOR",
+	"GOOGLE_GMAIL_CONNECTOR",
+	"AIRTABLE_CONNECTOR",
+	"LUMA_CONNECTOR",
+	"ELASTICSEARCH_CONNECTOR",
+	"WEBCRAWLER_CONNECTOR",
+	"BOOKSTACK_CONNECTOR",
+]);
+
+export const searchSourceConnector = z.object({
+	id: z.number(),
+	name: z.string(),
+	connector_type: searchSourceConnectorTypeEnum,
+	is_indexable: z.boolean(),
+	last_indexed_at: z.string().nullable(),
+	config: z.record(z.string(), z.any()),
+	periodic_indexing_enabled: z.boolean(),
+	indexing_frequency_minutes: z.number().nullable(),
+	next_scheduled_at: z.string().nullable(),
+	search_space_id: z.number(),
+	user_id: z.string(),
+	created_at: z.string(),
+});
+
+/**
+ * Get connectors
+ */
+export const getConnectorsRequest = z.object({
+	queryParams: paginationQueryParams
+		.pick({ skip: true, limit: true })
+		.extend({
+			search_space_id: z.number().or(z.string()).nullish(),
+		})
+		.nullish(),
+});
+
+export const getConnectorsResponse = z.array(searchSourceConnector);
+
+/**
+ * Get connector
+ */
+export const getConnectorRequest = searchSourceConnector.pick({ id: true });
+
+export const getConnectorResponse = searchSourceConnector;
+
+/**
+ * Create connector
+ */
+export const createConnectorRequest = z.object({
+	data: searchSourceConnector.pick({
+		name: true,
+		connector_type: true,
+		is_indexable: true,
+		last_indexed_at: true,
+		config: true,
+		periodic_indexing_enabled: true,
+		indexing_frequency_minutes: true,
+		next_scheduled_at: true,
+	}),
+	queryParams: z.object({
+		search_space_id: z.number().or(z.string()),
+	}),
+});
+
+export const createConnectorResponse = searchSourceConnector;
+
+/**
+ * Update connector
+ */
+export const updateConnectorRequest = z.object({
+	id: z.number(),
+	data: searchSourceConnector
+		.pick({
+			name: true,
+			connector_type: true,
+			is_indexable: true,
+			last_indexed_at: true,
+			config: true,
+			periodic_indexing_enabled: true,
+			indexing_frequency_minutes: true,
+			next_scheduled_at: true,
+		})
+		.partial(),
+});
+
+export const updateConnectorResponse = searchSourceConnector;
+
+/**
+ * Delete connector
+ */
+export const deleteConnectorRequest = searchSourceConnector.pick({ id: true });
+
+export const deleteConnectorResponse = z.object({
+	message: z.literal("Search source connector deleted successfully"),
+});
+
+/**
+ * Index connector
+ */
+export const indexConnectorRequest = z.object({
+	connector_id: z.number(),
+	queryParams: z.object({
+		search_space_id: z.number().or(z.string()),
+		start_date: z.string().optional(),
+		end_date: z.string().optional(),
+	}),
+});
+
+export const indexConnectorResponse = z.object({
+	message: z.string(),
+	connector_id: z.number(),
+	search_space_id: z.number(),
+	indexing_from: z.string(),
+	indexing_to: z.string(),
+});
+
+/**
+ * List GitHub repositories
+ */
+export const listGitHubRepositoriesRequest = z.object({
+	github_pat: z.string(),
+});
+
+export const listGitHubRepositoriesResponse = z.array(z.record(z.string(), z.any()));
+
+// Inferred types
+export type SearchSourceConnectorType = z.infer<typeof searchSourceConnectorTypeEnum>;
+export type SearchSourceConnector = z.infer<typeof searchSourceConnector>;
+export type GetConnectorsRequest = z.infer<typeof getConnectorsRequest>;
+export type GetConnectorsResponse = z.infer<typeof getConnectorsResponse>;
+export type GetConnectorRequest = z.infer<typeof getConnectorRequest>;
+export type GetConnectorResponse = z.infer<typeof getConnectorResponse>;
+export type CreateConnectorRequest = z.infer<typeof createConnectorRequest>;
+export type CreateConnectorResponse = z.infer<typeof createConnectorResponse>;
+export type UpdateConnectorRequest = z.infer<typeof updateConnectorRequest>;
+export type UpdateConnectorResponse = z.infer<typeof updateConnectorResponse>;
+export type DeleteConnectorRequest = z.infer<typeof deleteConnectorRequest>;
+export type DeleteConnectorResponse = z.infer<typeof deleteConnectorResponse>;
+export type IndexConnectorRequest = z.infer<typeof indexConnectorRequest>;
+export type IndexConnectorResponse = z.infer<typeof indexConnectorResponse>;
+export type ListGitHubRepositoriesRequest = z.infer<typeof listGitHubRepositoriesRequest>;
+export type ListGitHubRepositoriesResponse = z.infer<typeof listGitHubRepositoriesResponse>;

surfsense_web/contracts/types/log.types.ts

@@ -0,0 +1,134 @@
+import { z } from "zod";
+import { paginationQueryParams } from ".";
+
+/**
+ * ENUMS
+ */
+export const logLevelEnum = z.enum(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]);
+
+export const logStatusEnum = z.enum(["IN_PROGRESS", "SUCCESS", "FAILED"]);
+
+/**
+ * Base log schema
+ */
+export const log = z.object({
+	id: z.number(),
+	level: logLevelEnum,
+	status: logStatusEnum,
+	message: z.string(),
+	source: z.string().nullable().optional(),
+	log_metadata: z.record(z.string(), z.any()).nullable().optional(),
+	created_at: z.string(),
+	search_space_id: z.number(),
+});
+
+export const logBase = log.omit({ id: true, created_at: true });
+
+/**
+ * Create log
+ */
+export const createLogRequest = logBase.extend({ search_space_id: z.number() });
+export const createLogResponse = log;
+
+/**
+ * Update log
+ */
+export const updateLogRequest = logBase.partial();
+export const updateLogResponse = log;
+
+/**
+ * Delete log
+ */
+export const deleteLogRequest = z.object({ id: z.number() });
+export const deleteLogResponse = z.object({
+	message: z.string().default("Log deleted successfully"),
+});
+
+/**
+ * Get logs (list)
+ */
+export const logFilters = z.object({
+	search_space_id: z.number().optional(),
+	level: logLevelEnum.optional(),
+	status: logStatusEnum.optional(),
+	source: z.string().optional(),
+	start_date: z.string().optional(),
+	end_date: z.string().optional(),
+});
+
+export const getLogsRequest = z.object({
+	queryParams: paginationQueryParams
+		.extend({
+			search_space_id: z.number().optional(),
+			level: logLevelEnum.optional(),
+			status: logStatusEnum.optional(),
+			source: z.string().optional(),
+			start_date: z.string().optional(),
+			end_date: z.string().optional(),
+		})
+		.nullish(),
+});
+export const getLogsResponse = z.array(log);
+
+/**
+ * Get single log
+ */
+export const getLogRequest = z.object({ id: z.number() });
+export const getLogResponse = log;
+
+/**
+ * Log summary (used for summary dashboard)
+ */
+export const logActiveTask = z.object({
+	id: z.number(),
+	task_name: z.string(),
+	message: z.string(),
+	started_at: z.string(),
+	source: z.string().nullable().optional(),
+	document_id: z.number().nullable().optional(),
+});
+export const logFailure = z.object({
+	id: z.number(),
+	task_name: z.string(),
+	message: z.string(),
+	failed_at: z.string(),
+	source: z.string().nullable().optional(),
+	error_details: z.string().nullable().optional(),
+});
+export const logSummary = z.object({
+	total_logs: z.number(),
+	time_window_hours: z.number(),
+	by_status: z.record(z.string(), z.number()),
+	by_level: z.record(z.string(), z.number()),
+	by_source: z.record(z.string(), z.number()),
+	active_tasks: z.array(logActiveTask),
+	recent_failures: z.array(logFailure),
+});
+export const getLogSummaryRequest = z.object({
+	search_space_id: z.number(),
+	hours: z.number().optional(),
+});
+export const getLogSummaryResponse = logSummary;
+
+/**
+ * Typescript types
+ */
+export type Log = z.infer<typeof log>;
+export type LogLevelEnum = z.infer<typeof logLevelEnum>;
+export type LogStatusEnum = z.infer<typeof logStatusEnum>;
+export type LogFilters = z.infer<typeof logFilters>;
+export type CreateLogRequest = z.infer<typeof createLogRequest>;
+export type CreateLogResponse = z.infer<typeof createLogResponse>;
+export type UpdateLogRequest = z.infer<typeof updateLogRequest>;
+export type UpdateLogResponse = z.infer<typeof updateLogResponse>;
+export type DeleteLogRequest = z.infer<typeof deleteLogRequest>;
+export type DeleteLogResponse = z.infer<typeof deleteLogResponse>;
+export type GetLogsRequest = z.infer<typeof getLogsRequest>;
+export type GetLogsResponse = z.infer<typeof getLogsResponse>;
+export type GetLogRequest = z.infer<typeof getLogRequest>;
+export type GetLogResponse = z.infer<typeof getLogResponse>;
+export type LogSummary = z.infer<typeof logSummary>;
+export type LogFailure = z.infer<typeof logFailure>;
+export type LogActiveTask = z.infer<typeof logActiveTask>;
+export type GetLogSummaryRequest = z.infer<typeof getLogSummaryRequest>;
+export type GetLogSummaryResponse = z.infer<typeof getLogSummaryResponse>;

surfsense_web/hooks/use-connector-edit-page.ts

@@ -1,8 +1,11 @@
 import { zodResolver } from "@hookform/resolvers/zod";
+import { useAtomValue } from "jotai";
 import { useRouter } from "next/navigation";
 import { useCallback, useEffect, useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
+import { updateConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
+import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms";
 import {
 	type EditConnectorFormValues,
 	type EditMode,
@@ -11,10 +14,8 @@ import {
 	type GithubRepo,
 	githubPatSchema,
 } from "@/components/editConnector/types";
-import {
-	type SearchSourceConnector,
-	useSearchSourceConnectors,
-} from "@/hooks/use-search-source-connectors";
+import type { EnumConnectorName } from "@/contracts/enums/connector";
+import type { SearchSourceConnector } from "@/hooks/use-search-source-connectors";
 import { authenticatedFetch } from "@/lib/auth-utils";
 
 const normalizeListInput = (value: unknown): string[] => {
@@ -51,11 +52,8 @@ const normalizeBoolean = (value: unknown): boolean | null => {
 
 export function useConnectorEditPage(connectorId: number, searchSpaceId: string) {
 	const router = useRouter();
-	const {
-		connectors,
-		updateConnector,
-		isLoading: connectorsLoading,
-	} = useSearchSourceConnectors(false, parseInt(searchSpaceId));
+	const { data: connectors = [], isLoading: connectorsLoading } = useAtomValue(connectorsAtom);
+	const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom);
 
 	// State managed by the hook
 	const [connector, setConnector] = useState<SearchSourceConnector | null>(null);
@@ -532,7 +530,13 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
 			}
 
 			try {
-				await updateConnector(connectorId, updatePayload);
+				await updateConnector({
+					id: connectorId,
+					data: {
+						...updatePayload,
+						connector_type: connector.connector_type as EnumConnectorName,
+					},
+				});
 				toast.success("Connector updated!");
 				const newlySavedConfig = updatePayload.config || originalConfig;
 				setOriginalConfig(newlySavedConfig);

surfsense_web/hooks/use-connectors.ts

@@ -1,114 +0,0 @@
-import { authenticatedFetch } from "@/lib/auth-utils";
-
-// Types for connector API
-export interface ConnectorConfig {
-	[key: string]: string;
-}
-
-export interface Connector {
-	id: number;
-	name: string;
-	connector_type: string;
-	config: ConnectorConfig;
-	created_at: string;
-	user_id: string;
-}
-
-export interface CreateConnectorRequest {
-	name: string;
-	connector_type: string;
-	config: ConnectorConfig;
-}
-
-// Get connector type display name
-export const getConnectorTypeDisplay = (type: string): string => {
-	const typeMap: Record<string, string> = {
-		TAVILY_API: "Tavily API",
-		SEARXNG_API: "SearxNG",
-	};
-	return typeMap[type] || type;
-};
-
-// API service for connectors
-export const ConnectorService = {
-	// Create a new connector
-	async createConnector(data: CreateConnectorRequest): Promise<Connector> {
-		const response = await authenticatedFetch(
-			`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors`,
-			{
-				method: "POST",
-				headers: { "Content-Type": "application/json" },
-				body: JSON.stringify(data),
-			}
-		);
-
-		if (!response.ok) {
-			const errorData = await response.json();
-			throw new Error(errorData.detail || "Failed to create connector");
-		}
-
-		return response.json();
-	},
-
-	// Get all connectors
-	async getConnectors(skip = 0, limit = 100): Promise<Connector[]> {
-		const response = await authenticatedFetch(
-			`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors?skip=${skip}&limit=${limit}`,
-			{ method: "GET" }
-		);
-
-		if (!response.ok) {
-			const errorData = await response.json();
-			throw new Error(errorData.detail || "Failed to fetch connectors");
-		}
-
-		return response.json();
-	},
-
-	// Get a specific connector
-	async getConnector(connectorId: number): Promise<Connector> {
-		const response = await authenticatedFetch(
-			`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/${connectorId}`,
-			{ method: "GET" }
-		);
-
-		if (!response.ok) {
-			const errorData = await response.json();
-			throw new Error(errorData.detail || "Failed to fetch connector");
-		}
-
-		return response.json();
-	},
-
-	// Update a connector
-	async updateConnector(connectorId: number, data: CreateConnectorRequest): Promise<Connector> {
-		const response = await authenticatedFetch(
-			`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/${connectorId}`,
-			{
-				method: "PUT",
-				headers: { "Content-Type": "application/json" },
-				body: JSON.stringify(data),
-			}
-		);
-
-		if (!response.ok) {
-			const errorData = await response.json();
-			throw new Error(errorData.detail || "Failed to update connector");
-		}
-
-		return response.json();
-	},
-
-	// Delete a connector
-	async deleteConnector(connectorId: number): Promise<void> {
-		const response = await authenticatedFetch(
-			`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/${connectorId}`,
-			{ method: "DELETE" }
-		);
-
-		if (!response.ok) {
-			const errorData = await response.json();
-			throw new Error(errorData.detail || "Failed to delete connector");
-		}
-	},
-};

surfsense_web/hooks/use-logs.ts

@@ -1,7 +1,8 @@
 "use client";
-import { useCallback, useEffect, useMemo, useState } from "react";
-import { toast } from "sonner";
-import { authenticatedFetch } from "@/lib/auth-utils";
+import { useQuery } from "@tanstack/react-query";
+import { useCallback, useMemo } from "react";
+import { logsApiService } from "@/lib/apis/logs-api.service";
+import { cacheKeys } from "@/lib/query-client/cache-keys";
 
 export type LogLevel = "DEBUG" | "INFO" | "WARNING" | "ERROR" | "CRITICAL";
 export type LogStatus = "IN_PROGRESS" | "SUCCESS" | "FAILED";
@@ -38,6 +39,7 @@ export interface LogSummary {
 		message: string;
 		started_at: string;
 		source?: string;
+		document_id?: number;
 	}>;
 	recent_failures: Array<{
 		id: number;
@@ -50,267 +52,96 @@ export interface LogSummary {
 }
 
 export function useLogs(searchSpaceId?: number, filters: LogFilters = {}) {
-	const [logs, setLogs] = useState<Log[]>([]);
-	const [loading, setLoading] = useState(true);
-	const [error, setError] = useState<string | null>(null);
-
 	// Memoize filters to prevent infinite re-renders
 	const memoizedFilters = useMemo(() => filters, [JSON.stringify(filters)]);
 
 	const buildQueryParams = useCallback(
 		(customFilters: LogFilters = {}) => {
-			const params = new URLSearchParams();
+			const params: Record<string, string> = {};
 
 			const allFilters = { ...memoizedFilters, ...customFilters };
 
 			if (allFilters.search_space_id) {
-				params.append("search_space_id", allFilters.search_space_id.toString());
+				params["search_space_id"] = allFilters.search_space_id.toString();
 			}
 			if (allFilters.level) {
-				params.append("level", allFilters.level);
+				params["level"] = allFilters.level;
 			}
 			if (allFilters.status) {
-				params.append("status", allFilters.status);
+				params["status"] = allFilters.status;
 			}
 			if (allFilters.source) {
-				params.append("source", allFilters.source);
+				params["source"] = allFilters.source;
 			}
 			if (allFilters.start_date) {
-				params.append("start_date", allFilters.start_date);
+				params["start_date"] = allFilters.start_date;
 			}
 			if (allFilters.end_date) {
-				params.append("end_date", allFilters.end_date);
+				params["end_date"] = allFilters.end_date;
 			}
 
-			return params.toString();
+			return params;
 		},
 		[memoizedFilters]
 	);
 
-	const fetchLogs = useCallback(
-		async (customFilters: LogFilters = {}, options: { skip?: number; limit?: number } = {}) => {
-			try {
-				setLoading(true);
-
-				const params = new URLSearchParams(buildQueryParams(customFilters));
-				if (options.skip !== undefined) params.append("skip", options.skip.toString());
-				if (options.limit !== undefined) params.append("limit", options.limit.toString());
-
-				const response = await authenticatedFetch(
-					`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/logs?${params}`,
-					{ method: "GET" }
-				);
-
-				if (!response.ok) {
-					const errorData = await response.json().catch(() => ({}));
-					throw new Error(errorData.detail || "Failed to fetch logs");
-				}
-
-				const data = await response.json();
-				setLogs(data);
-				setError(null);
-				return data;
-			} catch (err: any) {
-				setError(err.message || "Failed to fetch logs");
-				console.error("Error fetching logs:", err);
-				throw err;
-			} finally {
-				setLoading(false);
-			}
-		},
-		[buildQueryParams]
-	);
-
-	// Initial fetch
-	useEffect(() => {
-		const initialFilters = searchSpaceId
-			? { ...memoizedFilters, search_space_id: searchSpaceId }
-			: memoizedFilters;
-		fetchLogs(initialFilters);
-	}, [searchSpaceId, fetchLogs, memoizedFilters]);
-
-	// Function to refresh the logs list
-	const refreshLogs = useCallback(
-		async (customFilters: LogFilters = {}) => {
-			const finalFilters = searchSpaceId
-				? { ...customFilters, search_space_id: searchSpaceId }
-				: customFilters;
-			return await fetchLogs(finalFilters);
-		},
-		[searchSpaceId, fetchLogs]
-	);
-
-	// Function to create a new log
-	// Use silent: true to suppress toast notifications (for internal/background operations)
-	const createLog = useCallback(
-		async (logData: Omit<Log, "id" | "created_at">, options?: { silent?: boolean }) => {
-			const { silent = false } = options || {};
-			try {
-				const response = await authenticatedFetch(
-					`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/logs`,
-					{
-						headers: { "Content-Type": "application/json" },
-						method: "POST",
-						body: JSON.stringify(logData),
-					}
-				);
-
-				if (!response.ok) {
-					const errorData = await response.json().catch(() => ({}));
-					throw new Error(errorData.detail || "Failed to create log");
-				}
-
-				const newLog = await response.json();
-				setLogs((prevLogs) => [newLog, ...prevLogs]);
-				// Only show toast if not silent
-				if (!silent) {
-					toast.success("Log created successfully");
-				}
-				return newLog;
-			} catch (err: any) {
-				// Only show error toast if not silent
-				if (!silent) {
-					toast.error(err.message || "Failed to create log");
-				}
-				console.error("Error creating log:", err);
-				throw err;
-			}
-		},
-		[]
-	);
-
-	// Function to update a log
-	const updateLog = useCallback(
-		async (
-			logId: number,
-			updateData: Partial<Omit<Log, "id" | "created_at" | "search_space_id">>
-		) => {
-			try {
-				const response = await authenticatedFetch(
-					`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/logs/${logId}`,
-					{
-						headers: { "Content-Type": "application/json" },
-						method: "PUT",
-						body: JSON.stringify(updateData),
-					}
-				);
-
-				if (!response.ok) {
-					const errorData = await response.json().catch(() => ({}));
-					throw new Error(errorData.detail || "Failed to update log");
-				}
-
-				const updatedLog = await response.json();
-				setLogs((prevLogs) => prevLogs.map((log) => (log.id === logId ? updatedLog : log)));
-				toast.success("Log updated successfully");
-				return updatedLog;
-			} catch (err: any) {
-				toast.error(err.message || "Failed to update log");
-				console.error("Error updating log:", err);
-				throw err;
-			}
-		},
-		[]
-	);
-
-	// Function to delete a log
-	const deleteLog = useCallback(async (logId: number) => {
-		try {
-			const response = await authenticatedFetch(
-				`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/logs/${logId}`,
-				{ method: "DELETE" }
-			);
-
-			if (!response.ok) {
-				const errorData = await response.json().catch(() => ({}));
-				throw new Error(errorData.detail || "Failed to delete log");
-			}
-
-			setLogs((prevLogs) => prevLogs.filter((log) => log.id !== logId));
-			toast.success("Log deleted successfully");
-			return true;
-		} catch (err: any) {
-			toast.error(err.message || "Failed to delete log");
-			console.error("Error deleting log:", err);
-			return false;
-		}
-	}, []);
-
-	// Function to get a single log
-	const getLog = useCallback(async (logId: number) => {
-		try {
-			const response = await authenticatedFetch(
-				`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/logs/${logId}`,
-				{ method: "GET" }
-			);
-
-			if (!response.ok) {
-				const errorData = await response.json().catch(() => ({}));
-				throw new Error(errorData.detail || "Failed to fetch log");
-			}
-
-			return await response.json();
-		} catch (err: any) {
-			toast.error(err.message || "Failed to fetch log");
-			console.error("Error fetching log:", err);
-			throw err;
-		}
-	}, []);
+	const {
+		data: logs,
+		isLoading: loading,
+		error,
+		refetch,
+	} = useQuery({
+		queryKey: cacheKeys.logs.withQueryParams({
+			search_space_id: searchSpaceId,
+			skip: 0,
+			limit: 5,
+			...buildQueryParams(filters ?? {}),
+		}),
+		queryFn: () =>
+			logsApiService.getLogs({
+				queryParams: {
+					search_space_id: searchSpaceId,
+					skip: 0,
+					limit: 5,
+					...buildQueryParams(filters ?? {}),
+				},
+			}),
+		enabled: !!searchSpaceId,
+		staleTime: 3 * 60 * 1000,
+	});
 
 	return {
-		logs,
+		logs: logs ?? [],
 		loading,
 		error,
-		refreshLogs,
-		createLog,
-		updateLog,
-		deleteLog,
-		getLog,
-		fetchLogs,
+		refreshLogs: refetch,
 	};
 }
 
-// Separate hook for log summary
-export function useLogsSummary(searchSpaceId: number, hours: number = 24) {
-	const [summary, setSummary] = useState<LogSummary | null>(null);
-	const [loading, setLoading] = useState(true);
-	const [error, setError] = useState<string | null>(null);
+// Separate hook for log summary with optional polling support for document processing indicator UI
+export function useLogsSummary(
+	searchSpaceId: number,
+	hours: number = 24,
+	options: { refetchInterval?: number } = {}
+) {
+	const {
+		data: summary,
+		isLoading: loading,
+		error,
+		refetch,
+	} = useQuery({
+		queryKey: cacheKeys.logs.summary(searchSpaceId),
+		queryFn: () =>
+			logsApiService.getLogSummary({
+				search_space_id: searchSpaceId,
+				hours: hours,
+			}),
+		enabled: !!searchSpaceId,
+		staleTime: 3 * 60 * 1000,
+		// Enable refetch interval for document processing indicator polling
+		refetchInterval:
+			options.refetchInterval && options.refetchInterval > 0 ? options.refetchInterval : undefined,
+	});
 
-	const fetchSummary = useCallback(async () => {
-		if (!searchSpaceId) return;
-
-		try {
-			setLoading(true);
-			const response = await authenticatedFetch(
-				`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/logs/search-space/${searchSpaceId}/summary?hours=${hours}`,
-				{ method: "GET" }
-			);
-
-			if (!response.ok) {
-				const errorData = await response.json().catch(() => ({}));
-				throw new Error(errorData.detail || "Failed to fetch logs summary");
-			}
-
-			const data = await response.json();
-			setSummary(data);
-			setError(null);
-			return data;
-		} catch (err: any) {
-			setError(err.message || "Failed to fetch logs summary");
-			console.error("Error fetching logs summary:", err);
-			throw err;
-		} finally {
-			setLoading(false);
-		}
-	}, [searchSpaceId, hours]);
-
-	useEffect(() => {
-		fetchSummary();
-	}, [fetchSummary]);
-
-	const refreshSummary = useCallback(() => {
-		return fetchSummary();
-	}, [fetchSummary]);
-
-	return { summary, loading, error, refreshSummary };
+	return { summary, loading, error, refreshSummary: refetch };
 }

surfsense_web/lib/apis/base-api.service.ts

@@ -21,18 +21,23 @@ export type RequestOptions = {
 };
 
 class BaseApiService {
-	bearerToken: string;
 	baseUrl: string;
 
 	noAuthEndpoints: string[] = ["/auth/jwt/login", "/auth/register", "/auth/refresh"]; // Add more endpoints as needed
 
-	constructor(bearerToken: string, baseUrl: string) {
-		this.bearerToken = bearerToken;
+	// Use a getter to always read fresh token from localStorage
+	// This ensures the token is always up-to-date after login/logout
+	get bearerToken(): string {
+		return typeof window !== "undefined" ? getBearerToken() || "" : "";
+	}
+
+	constructor(baseUrl: string) {
 		this.baseUrl = baseUrl;
 	}
 
-	setBearerToken(bearerToken: string) {
-		this.bearerToken = bearerToken;
+	// Keep for backward compatibility, but token is now always read from localStorage
+	setBearerToken(_bearerToken: string) {
+		// No-op: token is now always read fresh from localStorage via the getter
 	}
 
 	async request<T, R extends ResponseType = ResponseType.JSON>(
@@ -293,7 +298,4 @@ class BaseApiService {
 	}
 }
 
-export const baseApiService = new BaseApiService(
-	typeof window !== "undefined" ? getBearerToken() || "" : "",
-	process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || ""
-);
+export const baseApiService = new BaseApiService(process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || "");

surfsense_web/lib/apis/connectors-api.service.ts

@@ -0,0 +1,200 @@
+import {
+	type CreateConnectorRequest,
+	createConnectorRequest,
+	createConnectorResponse,
+	type DeleteConnectorRequest,
+	deleteConnectorRequest,
+	deleteConnectorResponse,
+	type GetConnectorRequest,
+	type GetConnectorsRequest,
+	getConnectorRequest,
+	getConnectorResponse,
+	getConnectorsRequest,
+	getConnectorsResponse,
+	type IndexConnectorRequest,
+	indexConnectorRequest,
+	indexConnectorResponse,
+	type ListGitHubRepositoriesRequest,
+	listGitHubRepositoriesRequest,
+	listGitHubRepositoriesResponse,
+	type UpdateConnectorRequest,
+	updateConnectorRequest,
+	updateConnectorResponse,
+} from "@/contracts/types/connector.types";
+import { ValidationError } from "../error";
+import { baseApiService } from "./base-api.service";
+
+class ConnectorsApiService {
+	/**
+	 * Get all connectors for a search space
+	 */
+	getConnectors = async (request: GetConnectorsRequest) => {
+		const parsedRequest = getConnectorsRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		// Transform query params to be string values
+		const transformedQueryParams = parsedRequest.data.queryParams
+			? Object.fromEntries(
+					Object.entries(parsedRequest.data.queryParams).map(([k, v]) => {
+						return [k, String(v)];
+					})
+				)
+			: undefined;
+
+		const queryParams = transformedQueryParams
+			? new URLSearchParams(transformedQueryParams).toString()
+			: "";
+
+		return baseApiService.get(
+			`/api/v1/search-source-connectors?${queryParams}`,
+			getConnectorsResponse
+		);
+	};
+
+	/**
+	 * Get a single connector by ID
+	 */
+	getConnector = async (request: GetConnectorRequest) => {
+		const parsedRequest = getConnectorRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		return baseApiService.get(
+			`/api/v1/search-source-connectors/${request.id}`,
+			getConnectorResponse
+		);
+	};
+
+	/**
+	 * Create a new connector
+	 */
+	createConnector = async (request: CreateConnectorRequest) => {
+		const parsedRequest = createConnectorRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		const { data, queryParams } = parsedRequest.data;
+
+		// Transform query params to be string values
+		const transformedQueryParams = Object.fromEntries(
+			Object.entries(queryParams).map(([k, v]) => {
+				return [k, String(v)];
+			})
+		);
+
+		const queryString = new URLSearchParams(transformedQueryParams).toString();
+
+		return baseApiService.post(
+			`/api/v1/search-source-connectors?${queryString}`,
+			createConnectorResponse,
+			{
+				body: data,
+			}
+		);
+	};
+
+	/**
+	 * Update an existing connector
+	 */
+	updateConnector = async (request: UpdateConnectorRequest) => {
+		const parsedRequest = updateConnectorRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		const { id, data } = parsedRequest.data;
+
+		return baseApiService.put(`/api/v1/search-source-connectors/${id}`, updateConnectorResponse, {
+			body: data,
+		});
+	};
+
+	/**
+	 * Delete a connector
+	 */
+	deleteConnector = async (request: DeleteConnectorRequest) => {
+		const parsedRequest = deleteConnectorRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		return baseApiService.delete(
+			`/api/v1/search-source-connectors/${request.id}`,
+			deleteConnectorResponse
+		);
+	};
+
+	/**
+	 * Index connector content
+	 */
+	indexConnector = async (request: IndexConnectorRequest) => {
+		const parsedRequest = indexConnectorRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		const { connector_id, queryParams } = parsedRequest.data;
+
+		// Transform query params to be string values
+		const transformedQueryParams = Object.fromEntries(
+			Object.entries(queryParams).map(([k, v]) => {
+				return [k, String(v)];
+			})
+		);
+
+		const queryString = new URLSearchParams(transformedQueryParams).toString();
+
+		return baseApiService.post(
+			`/api/v1/search-source-connectors/${connector_id}/index?${queryString}`,
+			indexConnectorResponse
+		);
+	};
+
+	/**
+	 * List GitHub repositories using a Personal Access Token
+	 */
+	listGitHubRepositories = async (request: ListGitHubRepositoriesRequest) => {
+		const parsedRequest = listGitHubRepositoriesRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		return baseApiService.post(`/api/v1/github/repositories`, listGitHubRepositoriesResponse, {
+			body: parsedRequest.data,
+		});
+	};
+}
+
+export const connectorsApiService = new ConnectorsApiService();

surfsense_web/lib/apis/logs-api.service.ts

@@ -0,0 +1,128 @@
+import {
+	type CreateLogRequest,
+	createLogRequest,
+	createLogResponse,
+	type DeleteLogRequest,
+	deleteLogRequest,
+	deleteLogResponse,
+	type GetLogRequest,
+	type GetLogSummaryRequest,
+	type GetLogsRequest,
+	getLogRequest,
+	getLogResponse,
+	getLogSummaryRequest,
+	getLogSummaryResponse,
+	getLogsRequest,
+	getLogsResponse,
+	type Log,
+	log,
+	type UpdateLogRequest,
+	updateLogRequest,
+	updateLogResponse,
+} from "@/contracts/types/log.types";
+import { ValidationError } from "../error";
+import { baseApiService } from "./base-api.service";
+
+class LogsApiService {
+	/**
+	 * Get a list of logs with optional filtering and pagination
+	 */
+	getLogs = async (request: GetLogsRequest) => {
+		const parsedRequest = getLogsRequest.safeParse(request);
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+		// Transform query params to be string values
+		const transformedQueryParams = parsedRequest.data.queryParams
+			? Object.fromEntries(
+					Object.entries(parsedRequest.data.queryParams).map(([k, v]) => {
+						// Handle array values (document_type)
+						if (Array.isArray(v)) {
+							return [k, v.join(",")];
+						}
+						return [k, String(v)];
+					})
+				)
+			: undefined;
+
+		const queryParams = transformedQueryParams
+			? new URLSearchParams(transformedQueryParams).toString()
+			: "";
+		return baseApiService.get(`/api/v1/logs?${queryParams}`, getLogsResponse);
+	};
+
+	/**
+	 * Get a single log by ID
+	 */
+	getLog = async (request: GetLogRequest) => {
+		const parsedRequest = getLogRequest.safeParse(request);
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+		return baseApiService.get(`/api/v1/logs/${request.id}`, getLogResponse);
+	};
+
+	/**
+	 * Create a log entry
+	 */
+	createLog = async (request: CreateLogRequest) => {
+		const parsedRequest = createLogRequest.safeParse(request);
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+		return baseApiService.post(`/api/v1/logs`, createLogResponse, {
+			body: parsedRequest.data,
+		});
+	};
+
+	/**
+	 * Update a log entry
+	 */
+	updateLog = async (logId: number, request: UpdateLogRequest) => {
+		const parsedRequest = updateLogRequest.safeParse(request);
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+		return baseApiService.put(`/api/v1/logs/${logId}`, updateLogResponse, {
+			body: parsedRequest.data,
+		});
+	};
+
+	/**
+	 * Delete a log entry
+	 */
+	deleteLog = async (request: DeleteLogRequest) => {
+		const parsedRequest = deleteLogRequest.safeParse(request);
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+		return baseApiService.delete(`/api/v1/logs/${parsedRequest.data.id}`, deleteLogResponse);
+	};
+
+	/**
+	 * Get summary for logs by search space
+	 */
+	getLogSummary = async (request: GetLogSummaryRequest) => {
+		const parsedRequest = getLogSummaryRequest.safeParse(request);
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+		const { search_space_id, hours } = parsedRequest.data;
+		const url = `/api/v1/logs/search-space/${search_space_id}/summary${hours ? `?hours=${hours}` : ""}`;
+		return baseApiService.get(url, getLogSummaryResponse);
+	};
+}
+
+export const logsApiService = new LogsApiService();

surfsense_web/lib/query-client/cache-keys.ts

@@ -1,4 +1,6 @@
+import type { GetConnectorsRequest } from "@/contracts/types/connector.types";
 import type { GetDocumentsRequest } from "@/contracts/types/document.types";
+import type { GetLogsRequest } from "@/contracts/types/log.types";
 import type { GetSearchSpacesRequest } from "@/contracts/types/search-space.types";
 
 export const cacheKeys = {
@@ -18,6 +20,13 @@ export const cacheKeys = {
 		typeCounts: (searchSpaceId?: string) => ["documents", "type-counts", searchSpaceId] as const,
 		byChunk: (chunkId: string) => ["documents", "by-chunk", chunkId] as const,
 	},
+	logs: {
+		list: (searchSpaceId?: number | string) => ["logs", "list", searchSpaceId] as const,
+		detail: (logId: number | string) => ["logs", "detail", logId] as const,
+		summary: (searchSpaceId?: number | string) => ["logs", "summary", searchSpaceId] as const,
+		withQueryParams: (queries: GetLogsRequest["queryParams"]) =>
+			["logs", "with-query-params", ...(queries ? Object.values(queries) : [])] as const,
+	},
 	newLLMConfigs: {
 		all: (searchSpaceId: number) => ["new-llm-configs", searchSpaceId] as const,
 		byId: (configId: number) => ["new-llm-configs", "detail", configId] as const,
@@ -52,4 +61,11 @@ export const cacheKeys = {
 		all: (searchSpaceId: string) => ["invites", searchSpaceId] as const,
 		info: (inviteCode: string) => ["invites", "info", inviteCode] as const,
 	},
+	connectors: {
+		all: (searchSpaceId: string) => ["connectors", searchSpaceId] as const,
+		withQueryParams: (queries: GetConnectorsRequest["queryParams"]) =>
+			["connectors", ...(queries ? Object.values(queries) : [])] as const,
+		byId: (connectorId: string) => ["connector", connectorId] as const,
+		index: () => ["connector", "index"] as const,
+	},
 };

surfsense_web/messages/en.json

@@ -267,7 +267,11 @@
 		"content_summary": "Content Summary",
 		"view_full": "View Full Content",
 		"filter_placeholder": "Filter by title...",
-		"rows_per_page": "Rows per page"
+		"rows_per_page": "Rows per page",
+		"refresh": "Refresh",
+		"refresh_success": "Documents refreshed",
+		"processing_documents": "Processing documents...",
+		"active_tasks_count": "{count} active task(s)"
 	},
 	"add_connector": {
 		"title": "Connect Your Tools",

surfsense_web/messages/zh.json

@@ -267,7 +267,11 @@
 		"content_summary": "内容摘要",
 		"view_full": "查看完整内容",
 		"filter_placeholder": "按标题筛选...",
-		"rows_per_page": "每页行数"
+		"rows_per_page": "每页行数",
+		"refresh": "刷新",
+		"refresh_success": "文档已刷新",
+		"processing_documents": "正在处理文档...",
+		"active_tasks_count": "{count} 个正在进行的工作项"
 	},
 	"add_connector": {
 		"title": "连接您的工具",

surfsense_web/package.json

@@ -30,7 +30,6 @@
 		"@blocknote/react": "^0.45.0",
 		"@blocknote/server-util": "^0.45.0",
 		"@hookform/resolvers": "^5.2.2",
-		"@next/third-parties": "^16.1.0",
 		"@number-flow/react": "^0.5.10",
 		"@posthog/react": "^1.5.2",
 		"@radix-ui/react-accordion": "^1.2.11",

surfsense_web/pnpm-lock.yaml

@@ -35,9 +35,6 @@ importers:
       '@hookform/resolvers':
         specifier: ^5.2.2
         version: 5.2.2(react-hook-form@7.69.0(react@19.2.3))
-      '@next/third-parties':
-        specifier: ^16.1.0
-        version: 16.1.0(next@16.1.0(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)
       '@number-flow/react':
         specifier: ^0.5.10
         version: 0.5.10(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
@@ -1383,12 +1380,6 @@ packages:
     cpu: [x64]
     os: [win32]
 
-  '@next/third-parties@16.1.0':
-    resolution: {integrity: sha512-VxD1UxwXNgCnDDBW+oinysZORkzir2B/MvCYF8S02r78VnGr37cbkMlM0LESrE9Nc/qlo2bLBgvpvnyOy4vleg==}
-    peerDependencies:
-      next: ^13.0.0 || ^14.0.0 || ^15.0.0 || ^16.0.0-beta.0
-      react: ^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0
-
   '@nodelib/fs.scandir@2.1.5':
     resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
     engines: {node: '>= 8'}
@@ -5882,9 +5873,6 @@ packages:
     resolution: {integrity: sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==}
     engines: {node: '>=6'}
 
-  third-party-capital@1.0.20:
-    resolution: {integrity: sha512-oB7yIimd8SuGptespDAZnNkzIz+NWaJCu2RMsbs4Wmp9zSDUM8Nhi3s2OOcqYuv3mN4hitXc8DVx+LyUmbUDiA==}
-
   throttleit@2.1.0:
     resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==}
     engines: {node: '>=18'}
@@ -7214,12 +7202,6 @@ snapshots:
   '@next/swc-win32-x64-msvc@16.1.0':
     optional: true
 
-  '@next/third-parties@16.1.0(next@16.1.0(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)':
-    dependencies:
-      next: 16.1.0(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
-      react: 19.2.3
-      third-party-capital: 1.0.20
-
   '@nodelib/fs.scandir@2.1.5':
     dependencies:
       '@nodelib/fs.stat': 2.0.5
@@ -12560,8 +12542,6 @@ snapshots:
 
   tapable@2.3.0: {}
 
-  third-party-capital@1.0.20: {}
-
   throttleit@2.1.0: {}
 
   tinyexec@1.0.2: {}