mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
feat(agents): consolidate connectors under mcp_discovery; route web search through google_search
MCP consolidation: - Route all MCP-capable connectors (Slack, Jira, Linear, ClickUp, Airtable, Notion, Confluence, interim Gmail/Calendar, custom MCP) through a single `mcp_discovery` subagent. Drive/OneDrive/Dropbox stay native to enrich the KB. - Deprecate Discord/Teams/Luma: no viable official MCP server. Google-only web search: - Remove the main-agent `web_search` tool and the SearXNG platform service; all public web search now flows through the `google_search` subagent via task(). - Deprecate the Tavily/SearXNG/Linkup/Baidu search connectors (HTTP 410 on create, "Deprecated" badge); guide heavy users to the custom MCP connector. - Remove web search from anonymous chat (pure Q&A). - Tear SearXNG out of docker compose + install scripts; drop tavily-python and linkup-sdk deps and their config/env vars. Fix: - metrics._package_version() now swallows any metadata lookup failure. A malformed editable-install distribution with no `Version` field raised KeyError deep in importlib.metadata, and since it runs on every record_subagent_invoke_duration call it was crashing every task() delegation. Verified end-to-end against live GPT-5.4. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
ff2e5f390f
commit
ab747e7a49
206 changed files with 1704 additions and 7223 deletions
|
|
@ -1,21 +1,20 @@
|
|||
import { expect, confluenceWithChatTest as test } from "../../fixtures";
|
||||
import { streamChatToCompletion } from "../../helpers/api/chat";
|
||||
import { listConnectors, triggerIndex } from "../../helpers/api/connectors";
|
||||
import { getEditorContent, listDocuments } from "../../helpers/api/documents";
|
||||
import { listConnectors, triggerIndexExpectDisabled } from "../../helpers/api/connectors";
|
||||
import { listDocuments } from "../../helpers/api/documents";
|
||||
import { CANARY_TOKENS, FAKE_CONFLUENCE_PAGES } from "../../helpers/canary";
|
||||
import { openConnectorPopup } from "../../helpers/ui/connector-popup";
|
||||
import { waitForDocumentByTitle, waitForIndexingComplete } from "../../helpers/waits/indexing";
|
||||
|
||||
/**
|
||||
* Proves Confluence OAuth -> indexed Confluence pages -> stored source_markdown -> chat.
|
||||
* Proves Confluence MCP OAuth -> live MCP tool discovery/call -> chat.
|
||||
*
|
||||
* The external Atlassian provider is faked at the OAuth token/resource and
|
||||
* Confluence page-fetch boundaries; SurfSense's own add/callback routes,
|
||||
* encrypted config storage, indexing endpoint, indexing pipeline, and chat
|
||||
* retrieval remain real.
|
||||
* Confluence migrated to the hosted Atlassian Rovo MCP server (shared with
|
||||
* Jira), so it is live-tool only: the public indexing route returns
|
||||
* indexing_started=false and chat should call Confluence MCP tools
|
||||
* (`searchConfluenceUsingCql`) instead of retrieving from the KB.
|
||||
*/
|
||||
test.describe("Confluence connector journey", () => {
|
||||
test("user connects Confluence through OAuth, indexes a page, and chats with the canary token", async ({
|
||||
test("user connects Confluence and chats through live MCP tools with indexing disabled", async ({
|
||||
page,
|
||||
request,
|
||||
apiToken,
|
||||
|
|
@ -23,17 +22,27 @@ test.describe("Confluence connector journey", () => {
|
|||
confluenceConnector,
|
||||
chatThread,
|
||||
}) => {
|
||||
test.setTimeout(180_000); // worker cold-start + summarize + embed + chunk
|
||||
test.setTimeout(90_000); // worker cold-start + live tool chat
|
||||
|
||||
expect(confluenceConnector.connector_type).toBe("CONFLUENCE_CONNECTOR");
|
||||
expect(confluenceConnector.is_indexable).toBe(true);
|
||||
expect(confluenceConnector.is_indexable).toBe(false);
|
||||
expect(confluenceConnector.config._token_encrypted).toBe(true);
|
||||
expect(confluenceConnector.config.cloud_id).toBe(FAKE_CONFLUENCE_PAGES.canary.cloudId);
|
||||
expect(confluenceConnector.config.base_url).toBe(FAKE_CONFLUENCE_PAGES.canary.baseUrl);
|
||||
expect(confluenceConnector.config.access_token).toBeTruthy();
|
||||
expect(confluenceConnector.config.CONFLUENCE_BASE_URL).toBeUndefined();
|
||||
expect(confluenceConnector.config.CONFLUENCE_EMAIL).toBeUndefined();
|
||||
expect(confluenceConnector.config.CONFLUENCE_API_TOKEN).toBeUndefined();
|
||||
expect(confluenceConnector.config.mcp_service).toBe("confluence");
|
||||
expect(confluenceConnector.config.server_config).toMatchObject({
|
||||
transport: "streamable-http",
|
||||
url: "https://mcp.atlassian.com/v1/mcp",
|
||||
});
|
||||
// Shared Atlassian server issues the token via the same cf.mcp endpoint
|
||||
// as Jira; assert the endpoint + a live token rather than the (Jira-named)
|
||||
// shared client id.
|
||||
expect(confluenceConnector.config.mcp_oauth).toMatchObject({
|
||||
token_endpoint: "https://cf.mcp.atlassian.com/v1/token",
|
||||
});
|
||||
expect(
|
||||
(confluenceConnector.config.mcp_oauth as Record<string, unknown>).access_token
|
||||
).toBeTruthy();
|
||||
expect(confluenceConnector.config.access_token).toBeUndefined();
|
||||
expect(confluenceConnector.config.refresh_token).toBeUndefined();
|
||||
|
||||
await page.goto(`/dashboard/${searchSpace.id}/new-chat`, {
|
||||
waitUntil: "domcontentloaded",
|
||||
|
|
@ -44,45 +53,17 @@ test.describe("Confluence connector journey", () => {
|
|||
await connectorDialog.getByPlaceholder("Search").fill("Confluence");
|
||||
await expect(connectorDialog.getByText("Confluence", { exact: true })).toBeVisible();
|
||||
|
||||
await triggerIndex(request, apiToken, confluenceConnector.id, searchSpace.id, {});
|
||||
const beforeDocs = await listDocuments(request, apiToken, searchSpace.id);
|
||||
expect(beforeDocs).toHaveLength(0);
|
||||
|
||||
await waitForIndexingComplete(request, apiToken, confluenceConnector.id, searchSpace.id, {
|
||||
timeoutMs: 150_000,
|
||||
intervalMs: 1_500,
|
||||
minDocuments: 1,
|
||||
});
|
||||
|
||||
await waitForDocumentByTitle(
|
||||
const disabledIndex = await triggerIndexExpectDisabled(
|
||||
request,
|
||||
apiToken,
|
||||
searchSpace.id,
|
||||
FAKE_CONFLUENCE_PAGES.canary.title,
|
||||
{
|
||||
timeoutMs: 30_000,
|
||||
}
|
||||
confluenceConnector.id,
|
||||
searchSpace.id
|
||||
);
|
||||
|
||||
const docs = await listDocuments(request, apiToken, searchSpace.id);
|
||||
const canaryDoc = docs.find((d) => d.title === FAKE_CONFLUENCE_PAGES.canary.title);
|
||||
|
||||
expect(canaryDoc, "Confluence canary document must exist after indexing").toBeDefined();
|
||||
if (!canaryDoc) throw new Error("unreachable: canaryDoc asserted defined above");
|
||||
expect(canaryDoc.document_type).toBe("CONFLUENCE_CONNECTOR");
|
||||
|
||||
const editor = await getEditorContent(request, apiToken, searchSpace.id, canaryDoc.id);
|
||||
expect(
|
||||
editor.source_markdown,
|
||||
`canary token ${CANARY_TOKENS.confluenceCanary} should appear in editor source_markdown; ` +
|
||||
`got first 200 chars: ${editor.source_markdown.slice(0, 200)}`
|
||||
).toContain(CANARY_TOKENS.confluenceCanary);
|
||||
expect(editor.document_type).toBe("CONFLUENCE_CONNECTOR");
|
||||
expect(editor.chunk_count).toBeGreaterThan(0);
|
||||
|
||||
const refreshedConnectors = await listConnectors(request, apiToken, searchSpace.id);
|
||||
const refreshed = refreshedConnectors.find((c) => c.id === confluenceConnector.id);
|
||||
expect(refreshed?.connector_type).toBe("CONFLUENCE_CONNECTOR");
|
||||
expect(refreshed?.is_indexable).toBe(true);
|
||||
expect(refreshed?.last_indexed_at).not.toBeNull();
|
||||
expect(disabledIndex.message ?? "").toContain("real-time agent tools");
|
||||
expect(disabledIndex.message ?? "").toContain("background indexing is disabled");
|
||||
|
||||
const chat = await streamChatToCompletion(request, apiToken, {
|
||||
searchSpaceId: searchSpace.id,
|
||||
|
|
@ -91,7 +72,19 @@ test.describe("Confluence connector journey", () => {
|
|||
});
|
||||
expect(
|
||||
chat.assistantText,
|
||||
`chat agent should surface Confluence canary token after indexing; got: ${chat.assistantText.slice(0, 200)}`
|
||||
`chat agent should surface Confluence canary token from live MCP tools; got: ${chat.assistantText.slice(0, 200)}`
|
||||
).toContain(CANARY_TOKENS.confluenceCanary);
|
||||
|
||||
const eventText = JSON.stringify(chat.events);
|
||||
expect(eventText).toContain("searchConfluenceUsingCql");
|
||||
|
||||
const refreshedConnectors = await listConnectors(request, apiToken, searchSpace.id);
|
||||
const refreshed = refreshedConnectors.find((c) => c.id === confluenceConnector.id);
|
||||
expect(refreshed?.connector_type).toBe("CONFLUENCE_CONNECTOR");
|
||||
expect(refreshed?.is_indexable).toBe(false);
|
||||
expect(refreshed?.last_indexed_at).toBeNull();
|
||||
|
||||
const afterDocs = await listDocuments(request, apiToken, searchSpace.id);
|
||||
expect(afterDocs).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,20 +1,19 @@
|
|||
import { expect, notionWithChatTest as test } from "../../fixtures";
|
||||
import { streamChatToCompletion } from "../../helpers/api/chat";
|
||||
import { listConnectors, triggerIndex } from "../../helpers/api/connectors";
|
||||
import { getEditorContent, listDocuments } from "../../helpers/api/documents";
|
||||
import { listConnectors, triggerIndexExpectDisabled } from "../../helpers/api/connectors";
|
||||
import { listDocuments } from "../../helpers/api/documents";
|
||||
import { CANARY_TOKENS, FAKE_NOTION_PAGES } from "../../helpers/canary";
|
||||
import { openConnectorPopup } from "../../helpers/ui/connector-popup";
|
||||
import { waitForDocumentByTitle, waitForIndexingComplete } from "../../helpers/waits/indexing";
|
||||
|
||||
/**
|
||||
* Proves Notion OAuth -> indexed Notion API pages -> stored source_markdown -> chat.
|
||||
* Proves Notion MCP OAuth -> live MCP tool discovery/call -> chat.
|
||||
*
|
||||
* The external Notion provider is faked at the OAuth token/API boundary;
|
||||
* SurfSense's own add/callback routes, encrypted config storage, connector
|
||||
* indexing endpoint, indexing pipeline, and chat retrieval remain real.
|
||||
* Notion migrated from indexed OAuth to the hosted Notion MCP server, so it is
|
||||
* live-tool only: the public indexing route returns indexing_started=false and
|
||||
* chat should call Notion MCP tools (`search`) instead of retrieving from the KB.
|
||||
*/
|
||||
test.describe("Notion connector journey", () => {
|
||||
test("user connects Notion through OAuth, indexes a page, and chats with the canary token", async ({
|
||||
test("user connects Notion and chats through live MCP tools with indexing disabled", async ({
|
||||
page,
|
||||
request,
|
||||
apiToken,
|
||||
|
|
@ -22,16 +21,25 @@ test.describe("Notion connector journey", () => {
|
|||
notionConnector,
|
||||
chatThread,
|
||||
}) => {
|
||||
test.setTimeout(180_000); // worker cold-start + summarize + embed + chunk
|
||||
test.setTimeout(90_000); // worker cold-start + live tool chat
|
||||
|
||||
expect(notionConnector.connector_type).toBe("NOTION_CONNECTOR");
|
||||
expect(notionConnector.is_indexable).toBe(true);
|
||||
expect(notionConnector.is_indexable).toBe(false);
|
||||
expect(notionConnector.config._token_encrypted).toBe(true);
|
||||
expect(notionConnector.config.NOTION_INTEGRATION_TOKEN).toBeUndefined();
|
||||
expect(notionConnector.config.access_token).toBeTruthy();
|
||||
expect(notionConnector.config.workspace_id).toBe(FAKE_NOTION_PAGES.canary.workspaceId);
|
||||
expect(notionConnector.config.workspace_name).toBe(FAKE_NOTION_PAGES.canary.workspaceName);
|
||||
expect(notionConnector.config.bot_id).toBe(FAKE_NOTION_PAGES.canary.botId);
|
||||
expect(notionConnector.config.mcp_service).toBe("notion");
|
||||
expect(notionConnector.config.server_config).toMatchObject({
|
||||
transport: "streamable-http",
|
||||
url: "https://mcp.notion.com/mcp",
|
||||
});
|
||||
expect(notionConnector.config.mcp_oauth).toMatchObject({
|
||||
client_id: "fake-notion-mcp-client-id",
|
||||
token_endpoint: "https://mcp.notion.com/token",
|
||||
});
|
||||
expect(
|
||||
(notionConnector.config.mcp_oauth as Record<string, unknown>).access_token
|
||||
).toBeTruthy();
|
||||
expect(notionConnector.config.access_token).toBeUndefined();
|
||||
expect(notionConnector.config.refresh_token).toBeUndefined();
|
||||
|
||||
await page.goto(`/dashboard/${searchSpace.id}/new-chat`, {
|
||||
waitUntil: "domcontentloaded",
|
||||
|
|
@ -39,46 +47,20 @@ test.describe("Notion connector journey", () => {
|
|||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
await connectorDialog.getByPlaceholder("Search").fill("Notion");
|
||||
await expect(connectorDialog.getByText("Notion", { exact: true })).toBeVisible();
|
||||
|
||||
await triggerIndex(request, apiToken, notionConnector.id, searchSpace.id, {});
|
||||
const beforeDocs = await listDocuments(request, apiToken, searchSpace.id);
|
||||
expect(beforeDocs).toHaveLength(0);
|
||||
|
||||
await waitForIndexingComplete(request, apiToken, notionConnector.id, searchSpace.id, {
|
||||
timeoutMs: 150_000,
|
||||
intervalMs: 1_500,
|
||||
minDocuments: 1,
|
||||
});
|
||||
|
||||
await waitForDocumentByTitle(
|
||||
const disabledIndex = await triggerIndexExpectDisabled(
|
||||
request,
|
||||
apiToken,
|
||||
searchSpace.id,
|
||||
FAKE_NOTION_PAGES.canary.title,
|
||||
{
|
||||
timeoutMs: 30_000,
|
||||
}
|
||||
notionConnector.id,
|
||||
searchSpace.id
|
||||
);
|
||||
|
||||
const docs = await listDocuments(request, apiToken, searchSpace.id);
|
||||
const canaryDoc = docs.find((d) => d.title === FAKE_NOTION_PAGES.canary.title);
|
||||
|
||||
expect(canaryDoc, "Notion canary document must exist after indexing").toBeDefined();
|
||||
if (!canaryDoc) throw new Error("unreachable: canaryDoc asserted defined above");
|
||||
expect(canaryDoc.document_type).toBe("NOTION_CONNECTOR");
|
||||
|
||||
const editor = await getEditorContent(request, apiToken, searchSpace.id, canaryDoc.id);
|
||||
expect(
|
||||
editor.source_markdown,
|
||||
`canary token ${CANARY_TOKENS.notionCanary} should appear in editor source_markdown; ` +
|
||||
`got first 200 chars: ${editor.source_markdown.slice(0, 200)}`
|
||||
).toContain(CANARY_TOKENS.notionCanary);
|
||||
expect(editor.document_type).toBe("NOTION_CONNECTOR");
|
||||
expect(editor.chunk_count).toBeGreaterThan(0);
|
||||
|
||||
const refreshedConnectors = await listConnectors(request, apiToken, searchSpace.id);
|
||||
const refreshed = refreshedConnectors.find((c) => c.id === notionConnector.id);
|
||||
expect(refreshed?.connector_type).toBe("NOTION_CONNECTOR");
|
||||
expect(refreshed?.is_indexable).toBe(true);
|
||||
expect(refreshed?.last_indexed_at).not.toBeNull();
|
||||
expect(disabledIndex.message ?? "").toContain("real-time agent tools");
|
||||
expect(disabledIndex.message ?? "").toContain("background indexing is disabled");
|
||||
|
||||
const chat = await streamChatToCompletion(request, apiToken, {
|
||||
searchSpaceId: searchSpace.id,
|
||||
|
|
@ -87,7 +69,19 @@ test.describe("Notion connector journey", () => {
|
|||
});
|
||||
expect(
|
||||
chat.assistantText,
|
||||
`chat agent should surface Notion canary token after indexing; got: ${chat.assistantText.slice(0, 200)}`
|
||||
`chat agent should surface Notion canary token from live MCP tools; got: ${chat.assistantText.slice(0, 200)}`
|
||||
).toContain(CANARY_TOKENS.notionCanary);
|
||||
|
||||
const eventText = JSON.stringify(chat.events);
|
||||
expect(eventText).toContain(FAKE_NOTION_PAGES.canary.id);
|
||||
|
||||
const refreshedConnectors = await listConnectors(request, apiToken, searchSpace.id);
|
||||
const refreshed = refreshedConnectors.find((c) => c.id === notionConnector.id);
|
||||
expect(refreshed?.connector_type).toBe("NOTION_CONNECTOR");
|
||||
expect(refreshed?.is_indexable).toBe(false);
|
||||
expect(refreshed?.last_indexed_at).toBeNull();
|
||||
|
||||
const afterDocs = await listDocuments(request, apiToken, searchSpace.id);
|
||||
expect(afterDocs).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -454,12 +454,12 @@ export async function runNativeGoogleCalendarOAuth(
|
|||
}
|
||||
|
||||
/**
|
||||
* Drives the Notion OAuth flow programmatically.
|
||||
* Drives the Notion MCP OAuth flow programmatically.
|
||||
*
|
||||
* The E2E backend keeps SurfSense's OAuth add/callback routes real and
|
||||
* patches only Notion's external token endpoint. Notion's authorization
|
||||
* URL stays off-origin, so this helper extracts the signed state and calls
|
||||
* the backend callback directly with the deterministic fake code.
|
||||
* Notion migrated from indexed OAuth to the hosted Notion MCP server, so this
|
||||
* uses SurfSense's generic MCP OAuth routes. The E2E backend keeps those routes
|
||||
* real and patches Notion's external discovery/DCR/token/MCP tool boundaries
|
||||
* (see surfsense_backend/tests/e2e/fakes/notion_mcp_module.py).
|
||||
*/
|
||||
export async function runNotionOAuth(
|
||||
request: APIRequestContext,
|
||||
|
|
@ -471,26 +471,26 @@ export async function runNotionOAuth(
|
|||
connector: ConnectorRow | null;
|
||||
}> {
|
||||
const initiateResp = await request.get(
|
||||
`${BACKEND_URL}/api/v1/auth/notion/connector/add?space_id=${searchSpaceId}`,
|
||||
`${BACKEND_URL}/api/v1/auth/mcp/notion/connector/add?space_id=${searchSpaceId}`,
|
||||
{ headers: authHeaders(token) }
|
||||
);
|
||||
if (!initiateResp.ok()) {
|
||||
throw new Error(
|
||||
`Notion initiate failed (${initiateResp.status()}): ${await initiateResp.text()}`
|
||||
`Notion MCP initiate failed (${initiateResp.status()}): ${await initiateResp.text()}`
|
||||
);
|
||||
}
|
||||
const { auth_url } = (await initiateResp.json()) as { auth_url: string };
|
||||
if (!auth_url) {
|
||||
throw new Error("Notion initiate response missing auth_url");
|
||||
throw new Error("Notion MCP initiate response missing auth_url");
|
||||
}
|
||||
|
||||
const state = new URL(auth_url).searchParams.get("state");
|
||||
if (!state) {
|
||||
throw new Error(`Notion auth_url missing state: ${auth_url}`);
|
||||
throw new Error(`Notion MCP auth_url missing state: ${auth_url}`);
|
||||
}
|
||||
|
||||
const callbackResp = await request.get(
|
||||
`${BACKEND_URL}/api/v1/auth/notion/connector/callback?code=fake-notion-oauth-code&state=${encodeURIComponent(state)}`,
|
||||
`${BACKEND_URL}/api/v1/auth/mcp/notion/connector/callback?code=fake-notion-oauth-code&state=${encodeURIComponent(state)}`,
|
||||
{
|
||||
headers: authHeaders(token),
|
||||
maxRedirects: 0,
|
||||
|
|
@ -506,10 +506,14 @@ export async function runNotionOAuth(
|
|||
}
|
||||
|
||||
/**
|
||||
* Drives the Confluence OAuth flow programmatically.
|
||||
* Drives the Confluence MCP OAuth flow programmatically.
|
||||
*
|
||||
* The E2E backend keeps SurfSense's OAuth add/callback routes real and
|
||||
* patches only Atlassian's external token/resource endpoints.
|
||||
* Confluence migrated to the hosted Atlassian Rovo MCP server (shared with
|
||||
* Jira), so this uses SurfSense's generic MCP OAuth routes. The E2E backend
|
||||
* keeps those routes real and patches the shared Atlassian
|
||||
* discovery/DCR/token/MCP tool boundaries (see
|
||||
* surfsense_backend/tests/e2e/fakes/jira_module.py). The OAuth code is the
|
||||
* shared Atlassian one because both connectors hit the same token endpoint.
|
||||
*/
|
||||
export async function runConfluenceOAuth(
|
||||
request: APIRequestContext,
|
||||
|
|
@ -521,26 +525,26 @@ export async function runConfluenceOAuth(
|
|||
connector: ConnectorRow | null;
|
||||
}> {
|
||||
const initiateResp = await request.get(
|
||||
`${BACKEND_URL}/api/v1/auth/confluence/connector/add?space_id=${searchSpaceId}`,
|
||||
`${BACKEND_URL}/api/v1/auth/mcp/confluence/connector/add?space_id=${searchSpaceId}`,
|
||||
{ headers: authHeaders(token) }
|
||||
);
|
||||
if (!initiateResp.ok()) {
|
||||
throw new Error(
|
||||
`Confluence initiate failed (${initiateResp.status()}): ${await initiateResp.text()}`
|
||||
`Confluence MCP initiate failed (${initiateResp.status()}): ${await initiateResp.text()}`
|
||||
);
|
||||
}
|
||||
const { auth_url } = (await initiateResp.json()) as { auth_url: string };
|
||||
if (!auth_url) {
|
||||
throw new Error("Confluence initiate response missing auth_url");
|
||||
throw new Error("Confluence MCP initiate response missing auth_url");
|
||||
}
|
||||
|
||||
const state = new URL(auth_url).searchParams.get("state");
|
||||
if (!state) {
|
||||
throw new Error(`Confluence auth_url missing state: ${auth_url}`);
|
||||
throw new Error(`Confluence MCP auth_url missing state: ${auth_url}`);
|
||||
}
|
||||
|
||||
const callbackResp = await request.get(
|
||||
`${BACKEND_URL}/api/v1/auth/confluence/connector/callback?code=fake-confluence-oauth-code&state=${encodeURIComponent(state)}`,
|
||||
`${BACKEND_URL}/api/v1/auth/mcp/confluence/connector/callback?code=fake-jira-oauth-code&state=${encodeURIComponent(state)}`,
|
||||
{
|
||||
headers: authHeaders(token),
|
||||
maxRedirects: 0,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue