feat(agents): consolidate connectors under mcp_discovery; route web search through google_search

MCP consolidation:
- Route all MCP-capable connectors (Slack, Jira, Linear, ClickUp, Airtable,
  Notion, Confluence, interim Gmail/Calendar, custom MCP) through a single
  `mcp_discovery` subagent. Drive/OneDrive/Dropbox stay native to enrich the KB.
- Deprecate Discord/Teams/Luma: no viable official MCP server.

Google-only web search:
- Remove the main-agent `web_search` tool and the SearXNG platform service;
  all public web search now flows through the `google_search` subagent via task().
- Deprecate the Tavily/SearXNG/Linkup/Baidu search connectors (HTTP 410 on
  create, "Deprecated" badge); guide heavy users to the custom MCP connector.
- Remove web search from anonymous chat (pure Q&A).
- Tear SearXNG out of docker compose + install scripts; drop tavily-python
  and linkup-sdk deps and their config/env vars.

Fix:
- metrics._package_version() now swallows any metadata lookup failure. A
  malformed editable-install distribution with no `Version` field raised
  KeyError deep in importlib.metadata, and since it runs on every
  record_subagent_invoke_duration call it was crashing every task()
  delegation. Verified end-to-end against live GPT-5.4.

Co-authored-by: Cursor <cursoragent@cursor.com>

surfsense_backend/tests/unit/agents/multi_agent_chat/shared/document_render/test_web_results.py

@@ -1,84 +0,0 @@
-"""Tests for the ``<web_results>`` wrapper around web-result excerpt documents."""
-
-from __future__ import annotations
-
-import pytest
-
-from app.agents.chat.multi_agent_chat.shared.citations import (
-    CitationRegistry,
-    CitationSourceType,
-)
-from app.agents.chat.multi_agent_chat.shared.document_render import (
-    RenderableDocument,
-    RenderablePassage,
-    render_web_results,
-)
-
-pytestmark = pytest.mark.unit
-
-
-def _web_doc(url: str, title: str, content: str) -> RenderableDocument:
-    return RenderableDocument(
-        title=title,
-        source=f"Web · {url.split('//', 1)[-1].split('/', 1)[0]}",
-        passages=[
-            RenderablePassage(
-                content=content,
-                locator={"url": url},
-                source_type=CitationSourceType.WEB_RESULT,
-            )
-        ],
-    )
-
-
-def test_returns_none_when_nothing_to_show() -> None:
-    registry = CitationRegistry()
-
-    assert render_web_results([], registry) is None
-
-
-def test_wraps_in_web_results_container() -> None:
-    registry = CitationRegistry()
-
-    block = render_web_results(
-        [_web_doc("https://example.com/a", "Example", "the answer is 42")],
-        registry,
-    )
-
-    assert block is not None
-    assert block.startswith("<web_results>")
-    assert block.endswith("</web_results>")
-    assert "cite a result with its [n]" in block
-    assert (
-        '<document title="Example" source="Web · example.com" view="excerpt">' in block
-    )
-    assert "[1] the answer is 42" in block
-
-
-def test_registers_each_result_as_web_result_with_url_locator() -> None:
-    registry = CitationRegistry()
-
-    render_web_results(
-        [
-            _web_doc("https://a.com/x", "A", "alpha"),
-            _web_doc("https://b.com/y", "B", "beta"),
-        ],
-        registry,
-    )
-
-    first = registry.resolve(1)
-    second = registry.resolve(2)
-    assert first is not None and second is not None
-    assert first.source_type is CitationSourceType.WEB_RESULT
-    assert first.locator == {"url": "https://a.com/x"}
-    assert second.locator == {"url": "https://b.com/y"}
-
-
-def test_same_url_reuses_label_across_calls() -> None:
-    registry = CitationRegistry()
-    doc = _web_doc("https://example.com/a", "Example", "stable fact")
-
-    render_web_results([doc], registry)
-    render_web_results([doc], registry)
-
-    assert registry.next_n == 2

surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py

@@ -191,25 +191,25 @@ def test_missing_user_allowlist_keeps_coded_behaviour():
 
 
 def test_user_allowlist_for_different_subagent_does_not_leak():
-    """User trust for ``linear`` must not affect a ``jira`` subagent compile."""
+    """User trust for one subagent must not affect a different subagent compile."""
     coded = Ruleset(
-        origin="jira",
+        origin="mcp_discovery",
         rules=[Rule(permission="save_issue", pattern="*", action="ask")],
     )
-    linear_allowlist = Ruleset(
-        origin="user_allowlist:linear",
+    other_allowlist = Ruleset(
+        origin="user_allowlist:knowledge_base",
         rules=[Rule(permission="save_issue", pattern="*", action="allow")],
     )
 
     result = pack_subagent(
-        name="jira",
+        name="mcp_discovery",
         description="test",
         system_prompt="x",
         tools=[],
         ruleset=coded,
         dependencies={
             "flags": AgentFeatureFlags(),
-            "user_allowlist_by_subagent": {"linear": linear_allowlist},
+            "user_allowlist_by_subagent": {"knowledge_base": other_allowlist},
         },
     )
 

surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_discovery_migration.py

@@ -0,0 +1,153 @@
+"""Guardrails for the MCP-consolidation migration.
+
+Every MCP-backed connector now routes to the single ``mcp_discovery`` subagent
+(file connectors stay native; Discord/Teams/Luma are deprecated). These tests
+pin the pieces that make that safe: connector→route mapping, any-of gating,
+legacy checkpoint aliasing, tool-name collision prefixing, the metadata-derived
+approval ruleset, and the KB indexing-deprecation sets.
+"""
+
+from __future__ import annotations
+
+import pytest
+from langchain_core.tools import StructuredTool
+
+from app.agents.chat.multi_agent_chat.constants import (
+    CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS,
+    LEGACY_SUBAGENT_ALIASES,
+    SUBAGENT_TO_REQUIRED_CONNECTOR_MAP,
+)
+from app.agents.chat.multi_agent_chat.subagents.builtins.mcp_discovery.tools.index import (
+    NAME as MCP_DISCOVERY_NAME,
+    build_ruleset,
+)
+from app.agents.chat.multi_agent_chat.subagents.mcp_tools.index import (
+    resolve_tool_name_collisions,
+)
+from app.agents.chat.multi_agent_chat.subagents.registry import (
+    SUBAGENT_BUILDERS_BY_NAME,
+    get_subagents_to_exclude,
+)
+from app.services.mcp_oauth.registry import (
+    DEPRECATED_INDEXING_CONNECTOR_TYPES,
+    LIVE_CONNECTOR_TYPES,
+)
+
+pytestmark = pytest.mark.unit
+
+# Connectors that must all funnel into ``mcp_discovery`` (not their own routes).
+_MCP_ROUTED = {
+    "SLACK_CONNECTOR",
+    "JIRA_CONNECTOR",
+    "LINEAR_CONNECTOR",
+    "CLICKUP_CONNECTOR",
+    "AIRTABLE_CONNECTOR",
+    "NOTION_CONNECTOR",
+    "CONFLUENCE_CONNECTOR",
+    "GOOGLE_GMAIL_CONNECTOR",
+    "GOOGLE_CALENDAR_CONNECTOR",
+    "MCP_CONNECTOR",
+}
+
+
+def _tool(name: str, metadata: dict) -> StructuredTool:
+    return StructuredTool.from_function(
+        func=lambda: name,
+        name=name,
+        description=name,
+        metadata=metadata,
+    )
+
+
+def test_all_mcp_connectors_route_to_discovery():
+    for connector_type in _MCP_ROUTED:
+        assert (
+            CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS[connector_type]
+            == MCP_DISCOVERY_NAME
+        ), connector_type
+
+
+def test_file_connectors_keep_native_routes():
+    assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["GOOGLE_DRIVE_CONNECTOR"] == "google_drive"
+    assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["DROPBOX_CONNECTOR"] == "dropbox"
+    assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["ONEDRIVE_CONNECTOR"] == "onedrive"
+
+
+def test_deprecated_connectors_have_no_route():
+    for connector_type in ("DISCORD_CONNECTOR", "TEAMS_CONNECTOR", "LUMA_CONNECTOR"):
+        assert connector_type not in CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS
+
+
+def test_discovery_gating_is_any_of():
+    """One connected app is enough to keep ``mcp_discovery``; none excludes it."""
+    assert MCP_DISCOVERY_NAME not in get_subagents_to_exclude(["SLACK_CONNECTOR"])
+    assert MCP_DISCOVERY_NAME in get_subagents_to_exclude([])
+    # A generic user MCP server alone still unlocks it.
+    assert MCP_DISCOVERY_NAME not in get_subagents_to_exclude(["MCP_CONNECTOR"])
+
+
+def test_discovery_gating_tokens_match_routed_connectors():
+    assert SUBAGENT_TO_REQUIRED_CONNECTOR_MAP[MCP_DISCOVERY_NAME] == frozenset(_MCP_ROUTED)
+
+
+def test_legacy_aliases_resolve_to_a_live_subagent():
+    """Old per-connector task names must alias onto a subagent that still exists."""
+    for legacy, target in LEGACY_SUBAGENT_ALIASES.items():
+        assert legacy not in SUBAGENT_BUILDERS_BY_NAME, legacy
+        assert target in SUBAGENT_BUILDERS_BY_NAME, target
+
+
+def test_collision_only_prefixes_shared_names():
+    """A name on two connectors is prefixed; a unique name is left untouched."""
+    tools = [
+        _tool("search", {"mcp_connector_id": 1, "mcp_transport": "http"}),
+        _tool("search", {"mcp_connector_id": 2, "mcp_transport": "http"}),
+        _tool("list_bases", {"mcp_connector_id": 2, "mcp_transport": "http"}),
+    ]
+    resolved = {t.name: t for t in resolve_tool_name_collisions(tools, {1: "NOTION_CONNECTOR", 2: "AIRTABLE_CONNECTOR"})}
+
+    # The unique tool keeps its bare name (trusted_tools / history stay valid).
+    assert "list_bases" in resolved
+    # The colliding name is gone; both are prefixed with service + connector id.
+    assert "search" not in resolved
+    assert "notion_1_search" in resolved
+    assert "airtable_2_search" in resolved
+    # Original name preserved for the "Always Allow" fallback key.
+    for name in ("notion_1_search", "airtable_2_search"):
+        meta = resolved[name].metadata or {}
+        assert meta["mcp_original_tool_name"] == "search"
+        assert meta["mcp_collision_prefixed"] is True
+
+
+def test_collision_noop_without_collisions():
+    tools = [
+        _tool("a", {"mcp_connector_id": 1}),
+        _tool("b", {"mcp_connector_id": 2}),
+    ]
+    assert [t.name for t in resolve_tool_name_collisions(tools, {})] == ["a", "b"]
+
+
+def test_ruleset_reads_hitl_from_metadata():
+    """Read-only MCP tools ``allow``; every other MCP tool ``ask``; natives skip."""
+    tools = [
+        _tool("readonly_search", {"mcp_transport": "http", "hitl": False}),
+        _tool("mutating_create", {"mcp_transport": "http", "hitl": True}),
+        _tool("native_helper", {}),  # no mcp_transport => no rule
+    ]
+    rules = {r.permission: r.action for r in build_ruleset(tools).rules}
+    assert rules == {"readonly_search": "allow", "mutating_create": "ask"}
+
+
+def test_indexing_deprecation_sets():
+    """Indexing-only connectors are deprecated; migrated ones are LIVE; Obsidian stays."""
+    deprecated = {t.value for t in DEPRECATED_INDEXING_CONNECTOR_TYPES}
+    assert deprecated == {
+        "GITHUB_CONNECTOR",
+        "BOOKSTACK_CONNECTOR",
+        "ELASTICSEARCH_CONNECTOR",
+        "CIRCLEBACK_CONNECTOR",
+    }
+    assert "OBSIDIAN_CONNECTOR" not in deprecated
+
+    live = {t.value for t in LIVE_CONNECTOR_TYPES}
+    assert {"NOTION_CONNECTOR", "CONFLUENCE_CONNECTOR"} <= live

surfsense_backend/tests/unit/agents/multi_agent_chat/test_prompt_resources.py

@@ -45,7 +45,7 @@ def test_every_subagent_has_description_md(name: str):
     [
         "core_behavior.md",
         "routing.md",
-        "tools/web_search/description.md",
+        "tools/scrape_webpage/description.md",
     ],
 )
 def test_main_agent_prompt_fragments_resolve(filename: str):

surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py

@@ -19,32 +19,24 @@ from app.agents.chat.multi_agent_chat.subagents.registry import (
 
 pytestmark = pytest.mark.unit
 
-# The full specialist roster the main agent composes from: 8 builtins + 15
-# connector routes. Adding/removing a specialist is a deliberate product change
-# and must be reflected here.
+# The full specialist roster the main agent composes from after the MCP
+# migration: builtins + the three file-connector routes (Drive/Dropbox/
+# OneDrive). Every MCP-backed connector (Slack/Jira/Linear/ClickUp/Airtable/
+# Notion/Confluence/Gmail/Calendar) now lives behind the single
+# ``mcp_discovery`` route; Discord/Teams/Luma were deprecated. Adding/removing a
+# specialist is a deliberate product change and must be reflected here.
 _EXPECTED_SUBAGENTS = frozenset(
     {
-        "airtable",
-        "calendar",
-        "clickup",
-        "confluence",
         "deliverables",
-        "discord",
         "dropbox",
-        "gmail",
         "google_drive",
         "google_maps",
         "google_search",
-        "jira",
         "knowledge_base",
-        "linear",
-        "luma",
+        "mcp_discovery",
         "memory",
-        "notion",
         "onedrive",
         "reddit",
-        "slack",
-        "teams",
         "web_crawler",
         "youtube",
     }

surfsense_backend/tests/unit/agents/multi_agent_chat/test_web_search_removed.py

@@ -0,0 +1,45 @@
+"""Guardrail: the multi-engine ``web_search`` tool is fully retired.
+
+Public web search now runs exclusively through the ``google_search`` subagent,
+and the four search connector types (Tavily/SearXNG/Linkup/Baidu) are soft-
+deprecated. This pins those invariants so a future change can't quietly bring
+the ``web_search`` tool back or un-deprecate a search connector.
+"""
+
+from __future__ import annotations
+
+import pytest
+from fastapi import HTTPException
+
+from app.agents.chat.multi_agent_chat.main_agent.tools.index import (
+    MAIN_AGENT_SURFSENSE_TOOL_NAMES,
+)
+from app.agents.chat.multi_agent_chat.subagents.registry import (
+    SUBAGENT_BUILDERS_BY_NAME,
+)
+from app.utils.validators import DEPRECATED_CONNECTOR_TYPES, raise_if_connector_deprecated
+
+pytestmark = pytest.mark.unit
+
+_DEPRECATED_SEARCH_TYPES = (
+    "TAVILY_API",
+    "SEARXNG_API",
+    "LINKUP_API",
+    "BAIDU_SEARCH_API",
+)
+
+
+def test_web_search_tool_removed_from_main_agent():
+    assert "web_search" not in MAIN_AGENT_SURFSENSE_TOOL_NAMES
+
+
+def test_google_search_specialist_present():
+    assert "google_search" in SUBAGENT_BUILDERS_BY_NAME
+
+
+@pytest.mark.parametrize("connector_type", _DEPRECATED_SEARCH_TYPES)
+def test_search_connectors_deprecated(connector_type):
+    assert connector_type in DEPRECATED_CONNECTOR_TYPES
+    with pytest.raises(HTTPException) as excinfo:
+        raise_if_connector_deprecated(connector_type)
+    assert excinfo.value.status_code == 410