feat: refactor agent tools management and add UI integration

- Added endpoint to list agent tools with metadata, excluding hidden tools.
- Updated NewChatRequest and RegenerateRequest schemas to include disabled tools.
- Integrated disabled tools management in the NewChatPage and Composer components.
- Improved tool instructions and visibility in the system prompt.
- Refactored tool registration to support hidden tools and default enabled states.
- Enhanced document chunk creation to handle strict zip behavior.
- Cleaned up imports and formatting across various files for consistency.

surfsense_backend/app/routes/new_chat_routes.py

@@ -35,6 +35,7 @@ from app.db import (
     shielded_async_session,
 )
 from app.schemas.new_chat import (
+    AgentToolInfo,
     NewChatMessageRead,
     NewChatRequest,
     NewChatThreadCreate,
@@ -1024,6 +1025,32 @@ async def list_messages(
         ) from None
 
 
+# =============================================================================
+# Agent Tools Endpoint
+# =============================================================================
+
+
+@router.get("/agent/tools", response_model=list[AgentToolInfo])
+async def list_agent_tools(
+    _user: User = Depends(current_active_user),
+):
+    """Return the list of built-in agent tools with their metadata.
+
+    Hidden (WIP) tools are excluded from the response.
+    """
+    from app.agents.new_chat.tools.registry import BUILTIN_TOOLS
+
+    return [
+        AgentToolInfo(
+            name=t.name,
+            description=t.description,
+            enabled_by_default=t.enabled_by_default,
+        )
+        for t in BUILTIN_TOOLS
+        if not t.hidden
+    ]
+
+
 # =============================================================================
 # Chat Streaming Endpoint
 # =============================================================================
@@ -1108,6 +1135,7 @@ async def handle_new_chat(
                 needs_history_bootstrap=thread.needs_history_bootstrap,
                 thread_visibility=thread.visibility,
                 current_user_display_name=user.display_name or "A team member",
+                disabled_tools=request.disabled_tools,
             ),
             media_type="text/event-stream",
             headers={
@@ -1344,6 +1372,7 @@ async def regenerate_response(
                     needs_history_bootstrap=thread.needs_history_bootstrap,
                     thread_visibility=thread.visibility,
                     current_user_display_name=user.display_name or "A team member",
+                    disabled_tools=request.disabled_tools,
                 ):
                     yield chunk
                 streaming_completed = True

surfsense_backend/app/routes/search_source_connectors_routes.py

@@ -18,6 +18,7 @@ Note: OAuth connectors (Gmail, Drive, Slack, etc.) support multiple accounts per
 Non-OAuth connectors (BookStack, GitHub, etc.) are limited to one per search space.
 """
 
+import asyncio
 import logging
 from contextlib import suppress
 from datetime import UTC, datetime, timedelta
@@ -52,8 +53,6 @@ from app.schemas import (
     SearchSourceConnectorRead,
     SearchSourceConnectorUpdate,
 )
-import asyncio
-
 from app.services.composio_service import ComposioService, get_composio_service
 from app.services.notification_service import NotificationService
 from app.tasks.connector_indexers import (
@@ -3124,7 +3123,9 @@ async def get_drive_picker_token(
                 detail="Composio connected account not found. Please reconnect.",
             )
         service = get_composio_service()
-        access_token = await asyncio.to_thread(service.get_access_token, composio_account_id)
+        access_token = await asyncio.to_thread(
+            service.get_access_token, composio_account_id
+        )
         return {
             "access_token": access_token,
             "client_id": config.GOOGLE_OAUTH_CLIENT_ID,