feat(agents): consolidate connectors under mcp_discovery; route web search through google_search

MCP consolidation:
- Route all MCP-capable connectors (Slack, Jira, Linear, ClickUp, Airtable,
  Notion, Confluence, interim Gmail/Calendar, custom MCP) through a single
  `mcp_discovery` subagent. Drive/OneDrive/Dropbox stay native to enrich the KB.
- Deprecate Discord/Teams/Luma: no viable official MCP server.

Google-only web search:
- Remove the main-agent `web_search` tool and the SearXNG platform service;
  all public web search now flows through the `google_search` subagent via task().
- Deprecate the Tavily/SearXNG/Linkup/Baidu search connectors (HTTP 410 on
  create, "Deprecated" badge); guide heavy users to the custom MCP connector.
- Remove web search from anonymous chat (pure Q&A).
- Tear SearXNG out of docker compose + install scripts; drop tavily-python
  and linkup-sdk deps and their config/env vars.

Fix:
- metrics._package_version() now swallows any metadata lookup failure. A
  malformed editable-install distribution with no `Version` field raised
  KeyError deep in importlib.metadata, and since it runs on every
  record_subagent_invoke_duration call it was crashing every task()
  delegation. Verified end-to-end against live GPT-5.4.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-04 21:06:04 -07:00
parent ff2e5f390f
commit ab747e7a49
206 changed files with 1704 additions and 7223 deletions

View file

@ -1,93 +0,0 @@
"""Tests for the shared ``web_search`` tool's citable-result adaptation.
The tool's network path (SearXNG + live connectors) is out of scope here; these
cover the pure mapping from raw web results to renderable, citable documents and
the end-to-end registration of ``WEB_RESULT`` ``[n]`` labels.
"""
from __future__ import annotations
import pytest
from app.agents.chat.multi_agent_chat.shared.citations import (
CitationRegistry,
CitationSourceType,
)
from app.agents.chat.multi_agent_chat.shared.document_render import render_web_results
from app.agents.chat.shared.tools.web_search import (
_to_renderable_web_documents,
_web_source_label,
)
pytestmark = pytest.mark.unit
def _raw_result(url: str, title: str, content: str) -> dict:
return {
"document": {"title": title, "metadata": {"url": url}},
"content": content,
}
def test_web_source_label_strips_scheme_and_www() -> None:
assert _web_source_label("https://www.example.com/path") == "Web · example.com"
assert _web_source_label("http://news.site.org/a/b") == "Web · news.site.org"
assert _web_source_label("") == "Web"
def test_adapter_maps_each_result_to_one_web_passage() -> None:
docs = _to_renderable_web_documents(
[
_raw_result("https://a.com/x", "Alpha", "alpha body"),
_raw_result("https://b.com/y", "Beta", "beta body"),
]
)
assert [d.title for d in docs] == ["Alpha", "Beta"]
passages = [p for d in docs for p in d.passages]
assert all(p.source_type is CitationSourceType.WEB_RESULT for p in passages)
assert passages[0].locator == {"url": "https://a.com/x"}
assert passages[0].content == "alpha body"
def test_adapter_skips_results_without_url_or_content() -> None:
docs = _to_renderable_web_documents(
[
_raw_result("", "No URL", "has content"),
_raw_result("https://c.com/z", "Empty", " "),
_raw_result("https://d.com/w", "Good", "real content"),
]
)
assert [d.title for d in docs] == ["Good"]
def test_adapter_truncates_on_char_budget() -> None:
big = "x" * 30
docs = _to_renderable_web_documents(
[
_raw_result("https://a.com", "A", big),
_raw_result("https://b.com", "B", big),
_raw_result("https://c.com", "C", big),
],
max_chars=50,
)
# First fits (30), second crosses 50 and stops the loop.
assert [d.title for d in docs] == ["A"]
def test_end_to_end_registers_web_results_for_citation() -> None:
registry = CitationRegistry()
docs = _to_renderable_web_documents(
[_raw_result("https://example.com/a", "Example", "the answer is 42")]
)
block = render_web_results(docs, registry)
assert block is not None
assert "[1] the answer is 42" in block
entry = registry.resolve(1)
assert entry is not None
assert entry.source_type is CitationSourceType.WEB_RESULT
assert entry.locator == {"url": "https://example.com/a"}

View file

@ -1,84 +0,0 @@
"""Tests for the ``<web_results>`` wrapper around web-result excerpt documents."""
from __future__ import annotations
import pytest
from app.agents.chat.multi_agent_chat.shared.citations import (
CitationRegistry,
CitationSourceType,
)
from app.agents.chat.multi_agent_chat.shared.document_render import (
RenderableDocument,
RenderablePassage,
render_web_results,
)
pytestmark = pytest.mark.unit
def _web_doc(url: str, title: str, content: str) -> RenderableDocument:
return RenderableDocument(
title=title,
source=f"Web · {url.split('//', 1)[-1].split('/', 1)[0]}",
passages=[
RenderablePassage(
content=content,
locator={"url": url},
source_type=CitationSourceType.WEB_RESULT,
)
],
)
def test_returns_none_when_nothing_to_show() -> None:
registry = CitationRegistry()
assert render_web_results([], registry) is None
def test_wraps_in_web_results_container() -> None:
registry = CitationRegistry()
block = render_web_results(
[_web_doc("https://example.com/a", "Example", "the answer is 42")],
registry,
)
assert block is not None
assert block.startswith("<web_results>")
assert block.endswith("</web_results>")
assert "cite a result with its [n]" in block
assert (
'<document title="Example" source="Web · example.com" view="excerpt">' in block
)
assert "[1] the answer is 42" in block
def test_registers_each_result_as_web_result_with_url_locator() -> None:
registry = CitationRegistry()
render_web_results(
[
_web_doc("https://a.com/x", "A", "alpha"),
_web_doc("https://b.com/y", "B", "beta"),
],
registry,
)
first = registry.resolve(1)
second = registry.resolve(2)
assert first is not None and second is not None
assert first.source_type is CitationSourceType.WEB_RESULT
assert first.locator == {"url": "https://a.com/x"}
assert second.locator == {"url": "https://b.com/y"}
def test_same_url_reuses_label_across_calls() -> None:
registry = CitationRegistry()
doc = _web_doc("https://example.com/a", "Example", "stable fact")
render_web_results([doc], registry)
render_web_results([doc], registry)
assert registry.next_n == 2

View file

@ -191,25 +191,25 @@ def test_missing_user_allowlist_keeps_coded_behaviour():
def test_user_allowlist_for_different_subagent_does_not_leak():
"""User trust for ``linear`` must not affect a ``jira`` subagent compile."""
"""User trust for one subagent must not affect a different subagent compile."""
coded = Ruleset(
origin="jira",
origin="mcp_discovery",
rules=[Rule(permission="save_issue", pattern="*", action="ask")],
)
linear_allowlist = Ruleset(
origin="user_allowlist:linear",
other_allowlist = Ruleset(
origin="user_allowlist:knowledge_base",
rules=[Rule(permission="save_issue", pattern="*", action="allow")],
)
result = pack_subagent(
name="jira",
name="mcp_discovery",
description="test",
system_prompt="x",
tools=[],
ruleset=coded,
dependencies={
"flags": AgentFeatureFlags(),
"user_allowlist_by_subagent": {"linear": linear_allowlist},
"user_allowlist_by_subagent": {"knowledge_base": other_allowlist},
},
)

View file

@ -0,0 +1,153 @@
"""Guardrails for the MCP-consolidation migration.
Every MCP-backed connector now routes to the single ``mcp_discovery`` subagent
(file connectors stay native; Discord/Teams/Luma are deprecated). These tests
pin the pieces that make that safe: connectorroute mapping, any-of gating,
legacy checkpoint aliasing, tool-name collision prefixing, the metadata-derived
approval ruleset, and the KB indexing-deprecation sets.
"""
from __future__ import annotations
import pytest
from langchain_core.tools import StructuredTool
from app.agents.chat.multi_agent_chat.constants import (
CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS,
LEGACY_SUBAGENT_ALIASES,
SUBAGENT_TO_REQUIRED_CONNECTOR_MAP,
)
from app.agents.chat.multi_agent_chat.subagents.builtins.mcp_discovery.tools.index import (
NAME as MCP_DISCOVERY_NAME,
build_ruleset,
)
from app.agents.chat.multi_agent_chat.subagents.mcp_tools.index import (
resolve_tool_name_collisions,
)
from app.agents.chat.multi_agent_chat.subagents.registry import (
SUBAGENT_BUILDERS_BY_NAME,
get_subagents_to_exclude,
)
from app.services.mcp_oauth.registry import (
DEPRECATED_INDEXING_CONNECTOR_TYPES,
LIVE_CONNECTOR_TYPES,
)
pytestmark = pytest.mark.unit
# Connectors that must all funnel into ``mcp_discovery`` (not their own routes).
_MCP_ROUTED = {
"SLACK_CONNECTOR",
"JIRA_CONNECTOR",
"LINEAR_CONNECTOR",
"CLICKUP_CONNECTOR",
"AIRTABLE_CONNECTOR",
"NOTION_CONNECTOR",
"CONFLUENCE_CONNECTOR",
"GOOGLE_GMAIL_CONNECTOR",
"GOOGLE_CALENDAR_CONNECTOR",
"MCP_CONNECTOR",
}
def _tool(name: str, metadata: dict) -> StructuredTool:
return StructuredTool.from_function(
func=lambda: name,
name=name,
description=name,
metadata=metadata,
)
def test_all_mcp_connectors_route_to_discovery():
for connector_type in _MCP_ROUTED:
assert (
CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS[connector_type]
== MCP_DISCOVERY_NAME
), connector_type
def test_file_connectors_keep_native_routes():
assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["GOOGLE_DRIVE_CONNECTOR"] == "google_drive"
assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["DROPBOX_CONNECTOR"] == "dropbox"
assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["ONEDRIVE_CONNECTOR"] == "onedrive"
def test_deprecated_connectors_have_no_route():
for connector_type in ("DISCORD_CONNECTOR", "TEAMS_CONNECTOR", "LUMA_CONNECTOR"):
assert connector_type not in CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS
def test_discovery_gating_is_any_of():
"""One connected app is enough to keep ``mcp_discovery``; none excludes it."""
assert MCP_DISCOVERY_NAME not in get_subagents_to_exclude(["SLACK_CONNECTOR"])
assert MCP_DISCOVERY_NAME in get_subagents_to_exclude([])
# A generic user MCP server alone still unlocks it.
assert MCP_DISCOVERY_NAME not in get_subagents_to_exclude(["MCP_CONNECTOR"])
def test_discovery_gating_tokens_match_routed_connectors():
assert SUBAGENT_TO_REQUIRED_CONNECTOR_MAP[MCP_DISCOVERY_NAME] == frozenset(_MCP_ROUTED)
def test_legacy_aliases_resolve_to_a_live_subagent():
"""Old per-connector task names must alias onto a subagent that still exists."""
for legacy, target in LEGACY_SUBAGENT_ALIASES.items():
assert legacy not in SUBAGENT_BUILDERS_BY_NAME, legacy
assert target in SUBAGENT_BUILDERS_BY_NAME, target
def test_collision_only_prefixes_shared_names():
"""A name on two connectors is prefixed; a unique name is left untouched."""
tools = [
_tool("search", {"mcp_connector_id": 1, "mcp_transport": "http"}),
_tool("search", {"mcp_connector_id": 2, "mcp_transport": "http"}),
_tool("list_bases", {"mcp_connector_id": 2, "mcp_transport": "http"}),
]
resolved = {t.name: t for t in resolve_tool_name_collisions(tools, {1: "NOTION_CONNECTOR", 2: "AIRTABLE_CONNECTOR"})}
# The unique tool keeps its bare name (trusted_tools / history stay valid).
assert "list_bases" in resolved
# The colliding name is gone; both are prefixed with service + connector id.
assert "search" not in resolved
assert "notion_1_search" in resolved
assert "airtable_2_search" in resolved
# Original name preserved for the "Always Allow" fallback key.
for name in ("notion_1_search", "airtable_2_search"):
meta = resolved[name].metadata or {}
assert meta["mcp_original_tool_name"] == "search"
assert meta["mcp_collision_prefixed"] is True
def test_collision_noop_without_collisions():
tools = [
_tool("a", {"mcp_connector_id": 1}),
_tool("b", {"mcp_connector_id": 2}),
]
assert [t.name for t in resolve_tool_name_collisions(tools, {})] == ["a", "b"]
def test_ruleset_reads_hitl_from_metadata():
"""Read-only MCP tools ``allow``; every other MCP tool ``ask``; natives skip."""
tools = [
_tool("readonly_search", {"mcp_transport": "http", "hitl": False}),
_tool("mutating_create", {"mcp_transport": "http", "hitl": True}),
_tool("native_helper", {}), # no mcp_transport => no rule
]
rules = {r.permission: r.action for r in build_ruleset(tools).rules}
assert rules == {"readonly_search": "allow", "mutating_create": "ask"}
def test_indexing_deprecation_sets():
"""Indexing-only connectors are deprecated; migrated ones are LIVE; Obsidian stays."""
deprecated = {t.value for t in DEPRECATED_INDEXING_CONNECTOR_TYPES}
assert deprecated == {
"GITHUB_CONNECTOR",
"BOOKSTACK_CONNECTOR",
"ELASTICSEARCH_CONNECTOR",
"CIRCLEBACK_CONNECTOR",
}
assert "OBSIDIAN_CONNECTOR" not in deprecated
live = {t.value for t in LIVE_CONNECTOR_TYPES}
assert {"NOTION_CONNECTOR", "CONFLUENCE_CONNECTOR"} <= live

View file

@ -45,7 +45,7 @@ def test_every_subagent_has_description_md(name: str):
[
"core_behavior.md",
"routing.md",
"tools/web_search/description.md",
"tools/scrape_webpage/description.md",
],
)
def test_main_agent_prompt_fragments_resolve(filename: str):

View file

@ -19,32 +19,24 @@ from app.agents.chat.multi_agent_chat.subagents.registry import (
pytestmark = pytest.mark.unit
# The full specialist roster the main agent composes from: 8 builtins + 15
# connector routes. Adding/removing a specialist is a deliberate product change
# and must be reflected here.
# The full specialist roster the main agent composes from after the MCP
# migration: builtins + the three file-connector routes (Drive/Dropbox/
# OneDrive). Every MCP-backed connector (Slack/Jira/Linear/ClickUp/Airtable/
# Notion/Confluence/Gmail/Calendar) now lives behind the single
# ``mcp_discovery`` route; Discord/Teams/Luma were deprecated. Adding/removing a
# specialist is a deliberate product change and must be reflected here.
_EXPECTED_SUBAGENTS = frozenset(
{
"airtable",
"calendar",
"clickup",
"confluence",
"deliverables",
"discord",
"dropbox",
"gmail",
"google_drive",
"google_maps",
"google_search",
"jira",
"knowledge_base",
"linear",
"luma",
"mcp_discovery",
"memory",
"notion",
"onedrive",
"reddit",
"slack",
"teams",
"web_crawler",
"youtube",
}

View file

@ -0,0 +1,45 @@
"""Guardrail: the multi-engine ``web_search`` tool is fully retired.
Public web search now runs exclusively through the ``google_search`` subagent,
and the four search connector types (Tavily/SearXNG/Linkup/Baidu) are soft-
deprecated. This pins those invariants so a future change can't quietly bring
the ``web_search`` tool back or un-deprecate a search connector.
"""
from __future__ import annotations
import pytest
from fastapi import HTTPException
from app.agents.chat.multi_agent_chat.main_agent.tools.index import (
MAIN_AGENT_SURFSENSE_TOOL_NAMES,
)
from app.agents.chat.multi_agent_chat.subagents.registry import (
SUBAGENT_BUILDERS_BY_NAME,
)
from app.utils.validators import DEPRECATED_CONNECTOR_TYPES, raise_if_connector_deprecated
pytestmark = pytest.mark.unit
_DEPRECATED_SEARCH_TYPES = (
"TAVILY_API",
"SEARXNG_API",
"LINKUP_API",
"BAIDU_SEARCH_API",
)
def test_web_search_tool_removed_from_main_agent():
assert "web_search" not in MAIN_AGENT_SURFSENSE_TOOL_NAMES
def test_google_search_specialist_present():
assert "google_search" in SUBAGENT_BUILDERS_BY_NAME
@pytest.mark.parametrize("connector_type", _DEPRECATED_SEARCH_TYPES)
def test_search_connectors_deprecated(connector_type):
assert connector_type in DEPRECATED_CONNECTOR_TYPES
with pytest.raises(HTTPException) as excinfo:
raise_if_connector_deprecated(connector_type)
assert excinfo.value.status_code == 410

View file

@ -9,7 +9,7 @@ earliest -> latest:
3. (future) user-defined rules from the Agent Permissions UI
Without #1 every read-only built-in (``ls``, ``read_file``, ``grep``,
``glob``, ``web_search`` ) defaulted to ``ask`` because
``glob``, ``scrape_webpage`` ) defaulted to ``ask`` because
``permissions.evaluate`` returns ``ask`` when no rule matches. That
caused two production-painful behaviors:
@ -58,7 +58,6 @@ class TestReadOnlyToolsAllowed:
"read_file",
"grep",
"glob",
"web_search",
"scrape_webpage",
"get_connected_accounts",
"write_todos",

View file

@ -65,8 +65,8 @@ class TestResolveToolName:
def test_falls_back_to_tool_call_name(self) -> None:
request = MagicMock()
request.tool = None
request.tool_call = {"name": "web_search", "args": {}}
assert _resolve_tool_name(request) == "web_search"
request.tool_call = {"name": "create_automation", "args": {}}
assert _resolve_tool_name(request) == "create_automation"
def test_unknown_when_nothing_resolves(self) -> None:
request = MagicMock()
@ -278,8 +278,8 @@ class TestMiddlewareIntegration:
request = MagicMock()
request.tool = MagicMock()
request.tool.name = "web_search"
request.tool.name = "scrape_webpage"
await mw.awrap_tool_call(request, handler)
assert errors == ["web_search"]
assert errors == ["scrape_webpage"]
finally:
ot.reload_for_tests()

View file

@ -166,8 +166,8 @@ class TestMetricHelpers:
model="gpt-4o",
provider="openai",
)
metrics.record_tool_call_duration(3.0, tool_name="web_search")
metrics.record_tool_call_error(tool_name="web_search")
metrics.record_tool_call_duration(3.0, tool_name="scrape_webpage")
metrics.record_tool_call_error(tool_name="scrape_webpage")
metrics.record_kb_search_duration(
4.0,
workspace_id=1,
@ -278,3 +278,30 @@ class TestEnabledIntegration:
sp.set_attribute("tool.truncated", False)
with otel.model_call_span(model_id="m", provider="p") as sp:
sp.set_attribute("retry.count", 3)
class TestPackageVersionResilience:
"""A version-tag lookup must never crash the request path (e.g. subagents).
An editable/dynamic install can have distribution metadata with no
``Version`` field, which raises ``KeyError`` deep inside importlib.metadata.
``_package_version`` must swallow that and every other lookup failure.
"""
def test_missing_version_key_falls_back(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
def _raise_key_error(_name: str) -> str:
raise KeyError("Version")
monkeypatch.setattr(metrics.metadata, "version", _raise_key_error)
assert metrics._package_version() == "unknown"
def test_package_not_found_falls_back(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
def _raise_not_found(_name: str) -> str:
raise metrics.metadata.PackageNotFoundError("surf-new-backend")
monkeypatch.setattr(metrics.metadata, "version", _raise_not_found)
assert metrics._package_version() == "unknown"

View file

@ -28,7 +28,7 @@ def test_clear_ignored_for_non_task_tool() -> None:
open_task_span(state, run_id="run-1")
sid = state.active_span_id
clear_task_span_if_delegating_task_ended(
state, tool_name="web_search", run_id="run-1"
state, tool_name="scrape_webpage", run_id="run-1"
)
assert state.active_span_id == sid
assert state.active_task_run_id == "run-1"

View file

@ -128,14 +128,14 @@ class TestToolHeavyTurn:
b.on_tool_input_start(
ui_id="call_run123",
tool_name="web_search",
tool_name="scrape_webpage",
langchain_tool_call_id="lc_tool_abc",
)
b.on_tool_input_delta("call_run123", '{"query":')
b.on_tool_input_delta("call_run123", '"surfsense"}')
b.on_tool_input_available(
ui_id="call_run123",
tool_name="web_search",
tool_name="scrape_webpage",
args={"query": "surfsense"},
langchain_tool_call_id="lc_tool_abc",
)
@ -150,7 +150,7 @@ class TestToolHeavyTurn:
tool_part = snap[1]
assert tool_part["type"] == "tool-call"
assert tool_part["toolCallId"] == "call_run123"
assert tool_part["toolName"] == "web_search"
assert tool_part["toolName"] == "scrape_webpage"
assert tool_part["args"] == {"query": "surfsense"}
# ``argsText`` is the pretty-printed final JSON, not the raw
# streaming buffer (FE ``stream-pipeline.ts:128``).

View file

@ -4,6 +4,7 @@ import pytest
from fastapi import HTTPException
from app.utils.validators import (
raise_if_connector_deprecated,
validate_connector_config,
validate_connectors,
validate_document_ids,
@ -331,3 +332,31 @@ def test_validate_connector_config_invalid():
# Invalid URL format in SEARXNG_API
with pytest.raises(ValueError):
validate_connector_config("SEARXNG_API", {"SEARXNG_HOST": "not-a-url"})
@pytest.mark.parametrize(
"connector_type",
[
"DISCORD_CONNECTOR",
"TEAMS_CONNECTOR",
"LUMA_CONNECTOR",
"TAVILY_API",
"SEARXNG_API",
"LINKUP_API",
"BAIDU_SEARCH_API",
],
)
def test_raise_if_connector_deprecated_blocks(connector_type):
"""Deprecated connector types are refused with HTTP 410."""
with pytest.raises(HTTPException) as excinfo:
raise_if_connector_deprecated(connector_type)
assert excinfo.value.status_code == 410
@pytest.mark.parametrize(
"connector_type",
["SLACK_CONNECTOR", "NOTION_CONNECTOR", "SERPER_API", "MCP_CONNECTOR"],
)
def test_raise_if_connector_deprecated_allows_active(connector_type):
"""Active connector types pass through without raising."""
raise_if_connector_deprecated(connector_type)