refactor: remove search_surfsense_docs tool and related references

- Deleted the `search_surfsense_docs` tool and its associated files, streamlining the agent's toolset.
- Updated various components and prompts to remove references to the now-removed tool, ensuring consistency across the codebase.
- Adjusted documentation to direct users to the SurfSense documentation link for product-related queries instead.

surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/system_prompt.md

@@ -8,7 +8,6 @@ Gather and synthesize evidence using SurfSense research tools with clear citatio
 <available_tools>
 - `web_search`
 - `scrape_webpage`
-- `search_surfsense_docs`
 </available_tools>
 
 <tool_policy>

surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/tools/__init__.py

@@ -1,11 +1,9 @@
-"""Research-stage tools: web search, scrape, and in-product doc search."""
+"""Research-stage tools: web search and scrape."""
 
 from .scrape_webpage import create_scrape_webpage_tool
-from .search_surfsense_docs import create_search_surfsense_docs_tool
 from .web_search import create_web_search_tool
 
 __all__ = [
     "create_scrape_webpage_tool",
-    "create_search_surfsense_docs_tool",
     "create_web_search_tool",
 ]

surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/tools/index.py

@@ -9,7 +9,6 @@ from langchain_core.tools import BaseTool
 from app.agents.new_chat.permissions import Ruleset
 
 from .scrape_webpage import create_scrape_webpage_tool
-from .search_surfsense_docs import create_search_surfsense_docs_tool
 from .web_search import create_web_search_tool
 
 NAME = "research"
@@ -27,5 +26,4 @@ def load_tools(
             available_connectors=d.get("available_connectors"),
         ),
         create_scrape_webpage_tool(firecrawl_api_key=d.get("firecrawl_api_key")),
-        create_search_surfsense_docs_tool(db_session=d["db_session"]),
     ]

surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/tools/search_surfsense_docs.py

@@ -1,145 +0,0 @@
-"""Semantic search over pre-indexed in-app documentation chunks for user how-to questions."""
-
-import asyncio
-import json
-
-from langchain_core.tools import tool
-from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db import SurfsenseDocsChunk, SurfsenseDocsDocument
-from app.utils.document_converters import embed_text
-from app.utils.surfsense_docs import surfsense_docs_public_url
-
-
-def format_surfsense_docs_results(results: list[tuple]) -> str:
-    """Format (chunk, document) rows as XML with ``doc-`` chunk IDs for citations and UI routing."""
-    if not results:
-        return "No relevant Surfsense documentation found for your query."
-
-    # Group chunks by document
-    grouped: dict[int, dict] = {}
-    for chunk, doc in results:
-        public_url = surfsense_docs_public_url(doc.source)
-        if doc.id not in grouped:
-            grouped[doc.id] = {
-                "document_id": f"doc-{doc.id}",
-                "document_type": "SURFSENSE_DOCS",
-                "title": doc.title,
-                "url": public_url,
-                "metadata": {"source": doc.source, "public_url": public_url},
-                "chunks": [],
-            }
-        grouped[doc.id]["chunks"].append(
-            {
-                "chunk_id": f"doc-{chunk.id}",
-                "content": chunk.content,
-            }
-        )
-
-    # Render XML matching format_documents_for_context structure
-    parts: list[str] = []
-    for g in grouped.values():
-        metadata_json = json.dumps(g["metadata"], ensure_ascii=False)
-
-        parts.append("<document>")
-        parts.append("<document_metadata>")
-        parts.append(f"  <document_id>{g['document_id']}</document_id>")
-        parts.append(f"  <document_type>{g['document_type']}</document_type>")
-        parts.append(f"  <title><![CDATA[{g['title']}]]></title>")
-        parts.append(f"  <url><![CDATA[{g['url']}]]></url>")
-        parts.append(f"  <metadata_json><![CDATA[{metadata_json}]]></metadata_json>")
-        parts.append("</document_metadata>")
-        parts.append("")
-        parts.append("<document_content>")
-
-        for ch in g["chunks"]:
-            parts.append(
-                f"  <chunk id='{ch['chunk_id']}'><![CDATA[{ch['content']}]]></chunk>"
-            )
-
-        parts.append("</document_content>")
-        parts.append("</document>")
-        parts.append("")
-
-    return "\n".join(parts).strip()
-
-
-async def search_surfsense_docs_async(
-    query: str,
-    db_session: AsyncSession,
-    top_k: int = 10,
-) -> str:
-    """
-    Search Surfsense documentation using vector similarity.
-
-    Args:
-        query: The search query about Surfsense usage
-        db_session: Database session for executing queries
-        top_k: Number of results to return
-
-    Returns:
-        Formatted string with relevant documentation content
-    """
-    # Get embedding for the query
-    query_embedding = await asyncio.to_thread(embed_text, query)
-
-    # Vector similarity search on chunks, joining with documents
-    stmt = (
-        select(SurfsenseDocsChunk, SurfsenseDocsDocument)
-        .join(
-            SurfsenseDocsDocument,
-            SurfsenseDocsChunk.document_id == SurfsenseDocsDocument.id,
-        )
-        .order_by(SurfsenseDocsChunk.embedding.op("<=>")(query_embedding))
-        .limit(top_k)
-    )
-
-    result = await db_session.execute(stmt)
-    rows = result.all()
-
-    return format_surfsense_docs_results(rows)
-
-
-def create_search_surfsense_docs_tool(db_session: AsyncSession):
-    """
-    Factory function to create the search_surfsense_docs tool.
-
-    Args:
-        db_session: Database session for executing queries
-
-    Returns:
-        A configured tool function for searching Surfsense documentation
-    """
-
-    @tool
-    async def search_surfsense_docs(query: str, top_k: int = 10) -> str:
-        """
-        Search Surfsense documentation for help with using the application.
-
-        Use this tool when the user asks questions about:
-        - How to use Surfsense features
-        - Installation and setup instructions
-        - Configuration options and settings
-        - Troubleshooting common issues
-        - Available connectors and integrations
-        - Browser extension usage
-        - API documentation
-
-        This searches the official Surfsense documentation that was indexed
-        at deployment time. It does NOT search the user's personal knowledge base.
-
-        Args:
-            query: The search query about Surfsense usage or features
-            top_k: Number of documentation chunks to retrieve (default: 10)
-
-        Returns:
-            Relevant documentation content formatted with chunk IDs for citations
-        """
-        return await search_surfsense_docs_async(
-            query=query,
-            db_session=db_session,
-            top_k=top_k,
-        )
-
-    return search_surfsense_docs