apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-10 22:32:16 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(agents): consolidate connectors under mcp_discovery; route web search through google_search

Browse source 
MCP consolidation:
- Route all MCP-capable connectors (Slack, Jira, Linear, ClickUp, Airtable,
  Notion, Confluence, interim Gmail/Calendar, custom MCP) through a single
  `mcp_discovery` subagent. Drive/OneDrive/Dropbox stay native to enrich the KB.
- Deprecate Discord/Teams/Luma: no viable official MCP server.

Google-only web search:
- Remove the main-agent `web_search` tool and the SearXNG platform service;
  all public web search now flows through the `google_search` subagent via task().
- Deprecate the Tavily/SearXNG/Linkup/Baidu search connectors (HTTP 410 on
  create, "Deprecated" badge); guide heavy users to the custom MCP connector.
- Remove web search from anonymous chat (pure Q&A).
- Tear SearXNG out of docker compose + install scripts; drop tavily-python
  and linkup-sdk deps and their config/env vars.

Fix:
- metrics._package_version() now swallows any metadata lookup failure. A
  malformed editable-install distribution with no `Version` field raised
  KeyError deep in importlib.metadata, and since it runs on every
  record_subagent_invoke_duration call it was crashing every task()
  delegation. Verified end-to-end against live GPT-5.4.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-04 21:06:04 -07:00
parent ff2e5f390f
commit ab747e7a49
206 changed files with 1704 additions and 7223 deletions
Download patch file Download diff file Expand all files Collapse all files

2
surfsense_backend/tests/e2e/README.md
View file

@ -72,7 +72,7 @@ pnpm exec playwright install --with-deps chromium
### Each run
**1. Bring up Postgres + Redis** from the repo root (the other deps-only
services (SearXNG, Zero, pgAdmin) are not needed for E2E):
services (Zero, pgAdmin) are not needed for E2E):
```bash
docker compose -f docker/docker-compose.deps-only.yml up -d db redis

51
surfsense_backend/tests/e2e/fakes/chat_llm.py
View file

@ -142,6 +142,13 @@ class FakeChatLLM(BaseChatModel):
and CLICKUP_CANARY_TOKEN in latest_tool_text
):
return f"ClickUp live tool content found: {CLICKUP_CANARY_TOKEN}"
if latest_tool_name == "search" and NOTION_CANARY_TOKEN in latest_tool_text:
return f"Notion live tool content found: {NOTION_CANARY_TOKEN}"
if (
latest_tool_name == "searchConfluenceUsingCql"
and CONFLUENCE_CANARY_TOKEN in latest_tool_text
):
return f"Confluence live tool content found: {CONFLUENCE_CANARY_TOKEN}"
wants_gmail = _contains_any(
latest_human,
@ -556,7 +563,7 @@ class FakeChatLLM(BaseChatModel):
# Marker unique to a connector subagent's prompt: the main agent must
# delegate via ``task``; only the subagent has connector tools registered.
in_connector_subagent = (
"specialist for the user's connected" in _messages_to_text(messages)
"connected-apps specialist" in _messages_to_text(messages)
)
# Main agent: delegate live-tool connector work to its subagent (which
@ -579,8 +586,12 @@ class FakeChatLLM(BaseChatModel):
("linear", ("linear", "issue", LINEAR_CANARY_TITLE)),
("slack", ("slack", SLACK_CANARY_TOKEN)),
("clickup", ("clickup", CLICKUP_CANARY_TITLE)),
("notion", ("notion", NOTION_CANARY_TITLE)),
("confluence", ("confluence", CONFLUENCE_CANARY_TITLE)),
)
for subagent_type, needles in connector_delegations:
# Every MCP-backed connector is now one ``mcp_discovery`` route; the
# needle set only decides which canary the delegation targets.
for connector_key, needles in connector_delegations:
if _contains_any(latest_human, needles):
return AIMessage(
content="",
@ -588,10 +599,10 @@ class FakeChatLLM(BaseChatModel):
{
"name": "task",
"args": {
"subagent_type": subagent_type,
"subagent_type": "mcp_discovery",
"description": latest_human,
},
"id": f"call_e2e_task_{subagent_type}",
"id": f"call_e2e_task_{connector_key}",
}
],
)
@ -718,6 +729,38 @@ class FakeChatLLM(BaseChatModel):
],
)
# Confluence check precedes Notion: the Confluence prompt also contains
# the word "page", so Notion's needle omits it to avoid cross-matching.
if latest_tool is None and _contains_any(
latest_human,
("confluence", CONFLUENCE_CANARY_TITLE),
):
return AIMessage(
content="",
tool_calls=[
{
"name": "searchConfluenceUsingCql",
"args": {"cql": f'text ~ "{CONFLUENCE_CANARY_TITLE}"'},
"id": "call_e2e_search_confluence",
}
],
)
if latest_tool is None and _contains_any(
latest_human,
("notion", NOTION_CANARY_TITLE),
):
return AIMessage(
content="",
tool_calls=[
{
"name": "search",
"args": {"query": NOTION_CANARY_TITLE},
"id": "call_e2e_search_notion",
}
],
)
return None
def _generate(

75
surfsense_backend/tests/e2e/fakes/jira_module.py
View file

@ -1,4 +1,13 @@
"""Strict Jira MCP OAuth/tool fakes for Playwright E2E."""
"""Strict Atlassian (Jira + Confluence) MCP OAuth/tool fakes for Playwright E2E.
Jira and Confluence share one hosted Atlassian Rovo MCP server
(``https://mcp.atlassian.com/v1/mcp``) and one OAuth surface. Because the MCP
OAuth/runtime dispatchers are keyed by URL, a single handler here serves both
connectors; production keeps their tool sets disjoint via each service's
``allowed_tools`` curation. The OAuth ``redirect_uri`` substring is therefore
relaxed to the shared ``/api/v1/auth/mcp/`` prefix so both the
``.../mcp/jira/...`` and ``.../mcp/confluence/...`` callbacks validate.
"""
from __future__ import annotations
@ -22,6 +31,13 @@ _ACCESS_TOKEN = "fake-jira-mcp-access-token"
_REFRESH_TOKEN = "fake-jira-mcp-refresh-token"
_OAUTH_CODE = "fake-jira-oauth-code"
# Confluence canary — keep in sync with FAKE_CONFLUENCE_PAGES /
# CANARY_TOKENS.confluenceCanary in surfsense_web/tests/helpers/canary.ts.
_CONFLUENCE_TOKEN = "SURFSENSE_E2E_CANARY_TOKEN_CONFLUENCE_001"
_CONFLUENCE_PAGE_ID = "fake-confluence-page-canary-001"
_CONFLUENCE_TITLE = "E2E Canary Confluence Page"
_CONFLUENCE_SPACE_ID = "fake-confluence-space-001"
def _load_fixture() -> dict[str, Any]:
with _FIXTURE_PATH.open() as f:
@ -69,6 +85,34 @@ async def _list_tools() -> SimpleNamespace:
"required": ["jql"],
},
),
SimpleNamespace(
name="searchConfluenceUsingCql",
description="Search Confluence content using a CQL expression.",
inputSchema={
"type": "object",
"properties": {
"cql": {
"type": "string",
"description": "CQL query used to search Confluence content.",
}
},
"required": ["cql"],
},
),
SimpleNamespace(
name="getConfluencePage",
description="Fetch a Confluence page's body by id.",
inputSchema={
"type": "object",
"properties": {
"pageId": {
"type": "string",
"description": "The Confluence page id to fetch.",
}
},
"required": ["pageId"],
},
),
]
)
@ -98,11 +142,30 @@ async def _call_tool(
text = _issue_text(issue)
return SimpleNamespace(content=[SimpleNamespace(text=text)])
raise NotImplementedError(f"Unexpected Jira MCP tool call: {tool_name!r}")
if tool_name == "searchConfluenceUsingCql":
cql = str(arguments.get("cql", ""))
if _CONFLUENCE_TITLE.lower() not in cql.lower():
raise ValueError(f"Unexpected Confluence CQL query: {cql!r}")
text = (
f"{_CONFLUENCE_TITLE}\n"
f"id: {_CONFLUENCE_PAGE_ID}\n"
f"space: {_CONFLUENCE_SPACE_ID}\n"
f"excerpt: {_CONFLUENCE_TOKEN}"
)
return SimpleNamespace(content=[SimpleNamespace(text=text)])
if tool_name == "getConfluencePage":
page_id = str(arguments.get("pageId", ""))
if page_id != _CONFLUENCE_PAGE_ID:
raise ValueError(f"Unexpected Confluence page id: {page_id!r}")
text = f"{_CONFLUENCE_TITLE}\n\n{_CONFLUENCE_TOKEN}"
return SimpleNamespace(content=[SimpleNamespace(text=text)])
raise NotImplementedError(f"Unexpected Atlassian MCP tool call: {tool_name!r}")
def install(active_patches: list[Any]) -> None:
"""Register Jira MCP OAuth/tool handlers with the shared dispatchers."""
"""Register the shared Atlassian (Jira + Confluence) MCP handlers."""
del active_patches
mcp_oauth_runtime.register_service(
mcp_url=_MCP_URL,
@ -122,8 +185,10 @@ def install(active_patches: list[Any]) -> None:
oauth_code=_OAUTH_CODE,
access_token=_ACCESS_TOKEN,
refresh_token=_REFRESH_TOKEN,
scope="read:jira-work read:me write:jira-work",
redirect_uri_substring="/api/v1/auth/mcp/jira/connector/callback",
scope="read:jira-work read:confluence-content.all read:me write:jira-work",
# Shared Atlassian server serves both Jira and Confluence connectors, so
# accept either service's callback under the common MCP OAuth prefix.
redirect_uri_substring="/api/v1/auth/mcp/",
)
mcp_runtime.register(
url=_MCP_URL,

6
surfsense_backend/tests/e2e/fakes/native_google.py
View file

@ -430,15 +430,15 @@ def install(active_patches: list[Any]) -> None:
("app.connectors.google_gmail_connector.build", _fake_build),
("app.connectors.google_calendar_connector.build", _fake_build),
(
"app.agents.chat.multi_agent_chat.subagents.connectors.calendar.tools.create_event.build",
"app.agents.chat.multi_agent_chat.subagents.builtins.mcp_discovery.tools.calendar.create_event.build",
_fake_build,
),
(
"app.agents.chat.multi_agent_chat.subagents.connectors.calendar.tools.update_event.build",
"app.agents.chat.multi_agent_chat.subagents.builtins.mcp_discovery.tools.calendar.update_event.build",
_fake_build,
),
(
"app.agents.chat.multi_agent_chat.subagents.connectors.calendar.tools.delete_event.build",
"app.agents.chat.multi_agent_chat.subagents.builtins.mcp_discovery.tools.calendar.delete_event.build",
_fake_build,
),
("googleapiclient.http.MediaIoBaseDownload", _FakeMediaIoBaseDownload),

127
surfsense_backend/tests/e2e/fakes/notion_mcp_module.py Normal file
View file

@ -0,0 +1,127 @@
"""Strict Notion MCP OAuth/tool fakes for Playwright E2E.
Notion migrated from indexed OAuth to the hosted Notion MCP server
(``https://mcp.notion.com/mcp``, DCR/RFC 7591). This fake mirrors
``jira_module`` for the generic MCP OAuth + streamable-HTTP tool boundaries;
the older ``notion_module`` (a ``notion_client`` SDK stand-in) stays only to
satisfy production's import-time ``import notion_client``.
"""
from __future__ import annotations
from types import SimpleNamespace
from typing import Any
from tests.e2e.fakes import mcp_oauth_runtime, mcp_runtime
_AUTHORIZATION_URL = "https://mcp.notion.com/authorize"
_REGISTRATION_URL = "https://mcp.notion.com/register"
_TOKEN_URL = "https://mcp.notion.com/token"
_MCP_URL = "https://mcp.notion.com/mcp"
_CLIENT_ID = "fake-notion-mcp-client-id"
_CLIENT_SECRET = "fake-notion-mcp-client-secret"
_ACCESS_TOKEN = "fake-notion-mcp-access-token"
_REFRESH_TOKEN = "fake-notion-mcp-refresh-token"
_OAUTH_CODE = "fake-notion-oauth-code"
# Keep in sync with FAKE_NOTION_PAGES / CANARY_TOKENS.notionCanary in
# surfsense_web/tests/helpers/canary.ts.
_CANARY_TOKEN = "SURFSENSE_E2E_CANARY_TOKEN_NOTION_001"
_CANARY_PAGE_ID = "fake-notion-page-canary-001"
_CANARY_TITLE = "E2E Canary Notion Page"
_WORKSPACE_NAME = "SurfSense E2E Notion Workspace"
async def _list_tools() -> SimpleNamespace:
return SimpleNamespace(
tools=[
SimpleNamespace(
name="search",
description="Search the connected Notion workspace for pages and databases.",
inputSchema={
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "Text to search Notion pages for.",
}
},
"required": ["query"],
},
),
SimpleNamespace(
name="fetch",
description="Fetch the full contents of a Notion page by id.",
inputSchema={
"type": "object",
"properties": {
"id": {
"type": "string",
"description": "The Notion page id to fetch.",
}
},
"required": ["id"],
},
),
]
)
async def _call_tool(
tool_name: str, arguments: dict[str, Any] | None = None
) -> SimpleNamespace:
arguments = arguments or {}
if tool_name == "search":
query = str(arguments.get("query", ""))
if _CANARY_TITLE.lower() not in query.lower():
raise ValueError(f"Unexpected Notion search query: {query!r}")
text = (
f"{_CANARY_TITLE}\n"
f"id: {_CANARY_PAGE_ID}\n"
f"workspace: {_WORKSPACE_NAME}\n"
f"snippet: {_CANARY_TOKEN}"
)
return SimpleNamespace(content=[SimpleNamespace(text=text)])
if tool_name == "fetch":
page_id = str(arguments.get("id", ""))
if page_id != _CANARY_PAGE_ID:
raise ValueError(f"Unexpected Notion fetch id: {page_id!r}")
text = f"{_CANARY_TITLE}\n\n{_CANARY_TOKEN}"
return SimpleNamespace(content=[SimpleNamespace(text=text)])
raise NotImplementedError(f"Unexpected Notion MCP tool call: {tool_name!r}")
def install(active_patches: list[Any]) -> None:
"""Register Notion MCP OAuth/tool handlers with the shared dispatchers."""
del active_patches
mcp_oauth_runtime.register_service(
mcp_url=_MCP_URL,
discovery_metadata={
"issuer": "https://mcp.notion.com",
"authorization_endpoint": _AUTHORIZATION_URL,
"token_endpoint": _TOKEN_URL,
"registration_endpoint": _REGISTRATION_URL,
"code_challenge_methods_supported": ["S256"],
"grant_types_supported": ["authorization_code", "refresh_token"],
"response_types_supported": ["code"],
},
client_id=_CLIENT_ID,
client_secret=_CLIENT_SECRET,
token_endpoint=_TOKEN_URL,
registration_endpoint=_REGISTRATION_URL,
oauth_code=_OAUTH_CODE,
access_token=_ACCESS_TOKEN,
refresh_token=_REFRESH_TOKEN,
scope="read write",
redirect_uri_substring="/api/v1/auth/mcp/notion/connector/callback",
)
mcp_runtime.register(
url=_MCP_URL,
expected_bearer=_ACCESS_TOKEN,
list_tools=_list_tools,
call_tool=_call_tool,
)

2
surfsense_backend/tests/e2e/run_backend.py
View file

@ -282,6 +282,7 @@ def _install_runtime_fakes() -> None:
mcp_oauth_runtime as _fake_mcp_oauth_runtime,
mcp_runtime as _fake_mcp_runtime,
native_google as _fake_native_google,
notion_mcp_module as _fake_notion_mcp_module,
notion_module as _fake_notion_module,
onedrive_graph as _fake_onedrive_graph,
slack_module as _fake_slack_module,
@ -295,6 +296,7 @@ def _install_runtime_fakes() -> None:
_fake_onedrive_graph.install(_active_patches)
_fake_dropbox_api.install(_active_patches)
_fake_notion_module.install(_active_patches)
_fake_notion_mcp_module.install(_active_patches)
_fake_linear_module.install(_active_patches)
_fake_jira_module.install(_active_patches)
_fake_clickup_module.install(_active_patches)