apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-08 22:22:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(agents): consolidate connectors under mcp_discovery; route web search through google_search

Browse source 
MCP consolidation:
- Route all MCP-capable connectors (Slack, Jira, Linear, ClickUp, Airtable,
  Notion, Confluence, interim Gmail/Calendar, custom MCP) through a single
  `mcp_discovery` subagent. Drive/OneDrive/Dropbox stay native to enrich the KB.
- Deprecate Discord/Teams/Luma: no viable official MCP server.

Google-only web search:
- Remove the main-agent `web_search` tool and the SearXNG platform service;
  all public web search now flows through the `google_search` subagent via task().
- Deprecate the Tavily/SearXNG/Linkup/Baidu search connectors (HTTP 410 on
  create, "Deprecated" badge); guide heavy users to the custom MCP connector.
- Remove web search from anonymous chat (pure Q&A).
- Tear SearXNG out of docker compose + install scripts; drop tavily-python
  and linkup-sdk deps and their config/env vars.

Fix:
- metrics._package_version() now swallows any metadata lookup failure. A
  malformed editable-install distribution with no `Version` field raised
  KeyError deep in importlib.metadata, and since it runs on every
  record_subagent_invoke_duration call it was crashing every task()
  delegation. Verified end-to-end against live GPT-5.4.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-04 21:06:04 -07:00
parent ff2e5f390f
commit ab747e7a49
206 changed files with 1704 additions and 7223 deletions
Download patch file Download diff file Expand all files Collapse all files

49
surfsense_web/components/assistant-ui/citation-metadata-context.tsx
View file

@ -1,7 +1,6 @@
"use client";
import { useAuiState } from "@assistant-ui/react";
import { createContext, type FC, type ReactNode, useContext, useMemo } from "react";
import { createContext, type FC, type ReactNode, useContext } from "react";
export interface CitationMeta {
title: string;
@ -10,50 +9,16 @@ export interface CitationMeta {
type CitationMetadataMap = ReadonlyMap<string, CitationMeta>;
const CitationMetadataContext = createContext<CitationMetadataMap>(new Map());
const EMPTY_CITATION_METADATA: CitationMetadataMap = new Map();
interface ToolCallResult {
status?: string;
citations?: Record<string, { title: string; snippet?: string }>;
}
interface MessageContent {
type: string;
toolName?: string;
result?: unknown;
}
const CitationMetadataContext = createContext<CitationMetadataMap>(EMPTY_CITATION_METADATA);
// The previous web-search citation spine (WEB_RESULT hover cards) was removed
// with the multi-engine web_search tool. Agent citation logic is being reworked
// wholesale, so this provider currently yields no web citation metadata.
export const CitationMetadataProvider: FC<{ children: ReactNode }> = ({ children }) => {
const content = useAuiState(
({ message }) => (message as { content?: MessageContent[] })?.content
);
const metadataMap = useMemo<CitationMetadataMap>(() => {
if (!content || !Array.isArray(content)) return new Map();
const merged = new Map<string, CitationMeta>();
for (const part of content) {
if (part.type !== "tool-call" || part.toolName !== "web_search" || !part.result) {
continue;
}
const result = part.result as ToolCallResult;
const citations = result.citations;
if (!citations || typeof citations !== "object") continue;
for (const [url, meta] of Object.entries(citations)) {
if (url.startsWith("http") && meta.title && !merged.has(url)) {
merged.set(url, { title: meta.title, snippet: meta.snippet });
}
}
}
return merged;
}, [content]);
return (
<CitationMetadataContext.Provider value={metadataMap}>
<CitationMetadataContext.Provider value={EMPTY_CITATION_METADATA}>
{children}
</CitationMetadataContext.Provider>
);

50
surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx
View file

@ -23,6 +23,7 @@ interface ConnectorCardProps {
accountCount?: number;
connectorCount?: number;
isIndexing?: boolean;
deprecated?: boolean;
onConnect?: () => void;
onManage?: () => void;
}
@ -52,11 +53,15 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
accountCount,
connectorCount,
isIndexing = false,
deprecated = false,
onConnect,
onManage,
}) => {
const isMCP = connectorType === EnumConnectorName.MCP_CONNECTOR;
const isLive = !!connectorType && LIVE_CONNECTOR_TYPES.has(connectorType);
// Deprecated connectors can no longer be connected, but existing rows stay
// manageable (so users can disconnect them).
const isDeprecatedForConnect = deprecated && !isConnected;
// Get connector status
const { getConnectorStatus, isConnectorEnabled, getConnectorStatusMessage, shouldShowWarnings } =
useConnectorStatus();
@ -73,6 +78,10 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
return null;
}
if (isDeprecatedForConnect) {
return "Deprecated — no longer available to connect.";
}
return description;
};
@ -104,12 +113,19 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
<div className="flex-1 min-w-0">
<div className="flex items-center gap-1.5">
<span className="text-[14px] font-semibold leading-tight truncate">{title}</span>
{showWarnings && status.status !== "active" && (
<ConnectorStatusBadge
status={status.status}
statusMessage={statusMessage}
className="flex-shrink-0"
/>
{deprecated ? (
<span className="shrink-0 rounded-full bg-muted px-1.5 py-0.5 text-[9px] font-medium uppercase tracking-wide text-muted-foreground">
Deprecated
</span>
) : (
showWarnings &&
status.status !== "active" && (
<ConnectorStatusBadge
status={status.status}
statusMessage={statusMessage}
className="flex-shrink-0"
/>
)
)}
</div>
{isIndexing ? (
@ -151,18 +167,20 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
!isConnected && "shadow-xs"
)}
onClick={isConnected ? onManage : onConnect}
disabled={isConnecting || !isEnabled}
disabled={isConnecting || !isEnabled || isDeprecatedForConnect}
>
<span className={isConnecting ? "opacity-0" : ""}>
{!isEnabled
? "Unavailable"
: isConnected
? "Manage"
: id === "youtube-crawler"
? "Add"
: connectorType
? "Connect"
: "Add"}
{isDeprecatedForConnect
? "Deprecated"
: !isEnabled
? "Unavailable"
: isConnected
? "Manage"
: id === "youtube-crawler"
? "Add"
: connectorType
? "Connect"
: "Add"}
</span>
{isConnecting && <Spinner size="xs" className="absolute" />}
</Button>

33
surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts
View file

@ -1,5 +1,25 @@
import { EnumConnectorName } from "@/contracts/enums/connector";
/**
* Connectors retired during the MCP migration (no viable official MCP server).
* The catalog card is shown disabled with a "Deprecated" badge so existing
* users understand why; the backend `/add` routes also refuse with HTTP 410.
* Reinstate by removing the type here and in the backend
* `DEPRECATED_CONNECTOR_TYPES` if demand returns.
*/
export const DEPRECATED_CONNECTOR_TYPES = new Set<string>([
EnumConnectorName.DISCORD_CONNECTOR,
EnumConnectorName.TEAMS_CONNECTOR,
EnumConnectorName.LUMA_CONNECTOR,
// Search APIs retired by the Google-only web-search consolidation. Public
// web search now runs through the google_search subagent; Tavily/Linkup can
// still be added via the generic Custom MCP connector (API-key headers).
EnumConnectorName.TAVILY_API,
EnumConnectorName.SEARXNG_API,
EnumConnectorName.LINKUP_API,
EnumConnectorName.BAIDU_SEARCH_API,
]);
/**
* Connectors that operate in real time (no background indexing).
* Used to adjust UI: hide sync controls, show "Connected" instead of doc counts.
@ -17,6 +37,9 @@ export const LIVE_CONNECTOR_TYPES = new Set<string>([
EnumConnectorName.GOOGLE_GMAIL_CONNECTOR,
EnumConnectorName.COMPOSIO_GMAIL_CONNECTOR,
EnumConnectorName.LUMA_CONNECTOR,
// Migrated to hosted MCP: real-time agent tools, no background indexing.
EnumConnectorName.NOTION_CONNECTOR,
EnumConnectorName.CONFLUENCE_CONNECTOR,
]);
// OAuth Connectors (Quick Connect)
@ -55,9 +78,9 @@ export const OAUTH_CONNECTORS = [
{
id: "notion-connector",
title: "Notion",
description: "Search your Notion pages",
description: "Search, read, and create pages",
connectorType: EnumConnectorName.NOTION_CONNECTOR,
authEndpoint: "/api/v1/auth/notion/connector/add",
authEndpoint: "/api/v1/auth/mcp/notion/connector/add/",
},
{
id: "linear-connector",
@ -104,16 +127,16 @@ export const OAUTH_CONNECTORS = [
{
id: "jira-connector",
title: "Jira",
description: "Rework in progress.",
description: "Search, read, and manage issues",
connectorType: EnumConnectorName.JIRA_CONNECTOR,
authEndpoint: "/api/v1/auth/mcp/jira/connector/add/",
},
{
id: "confluence-connector",
title: "Confluence",
description: "Rework in progress.",
description: "Search, read, and create pages",
connectorType: EnumConnectorName.CONFLUENCE_CONNECTOR,
authEndpoint: "/api/v1/auth/confluence/connector/add/",
authEndpoint: "/api/v1/auth/mcp/confluence/connector/add/",
},
{
id: "clickup-connector",

3
surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx
View file

@ -12,6 +12,7 @@ import {
CONNECTOR_CATEGORY_LABELS,
type ConnectorCategory,
CRAWLERS,
DEPRECATED_CONNECTOR_TYPES,
getConnectorCategory,
OAUTH_CONNECTORS,
OTHER_CONNECTORS,
@ -146,6 +147,7 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
documentCount={documentCount}
accountCount={accountCount}
isIndexing={isIndexing}
deprecated={DEPRECATED_CONNECTOR_TYPES.has(connector.connectorType)}
onConnect={() => onConnectOAuth(connector)}
onManage={
isConnected && onViewAccountsList
@ -194,6 +196,7 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
documentCount={documentCount}
connectorCount={mcpConnectorCount}
isIndexing={isIndexing}
deprecated={DEPRECATED_CONNECTOR_TYPES.has(connector.connectorType)}
onConnect={handleConnect}
onManage={actualConnector && onManage ? () => onManage(actualConnector) : undefined}
/>

18
surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx
View file

@ -11,7 +11,7 @@ import { EnumConnectorName } from "@/contracts/enums/connector";
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import { authenticatedFetch } from "@/lib/auth-fetch";
import { getReauthEndpoint } from "@/lib/connector-telemetry";
import { getReauthEndpoint, needsMcpReconnect } from "@/lib/connector-telemetry";
import { buildBackendUrl } from "@/lib/env-config";
import { formatRelativeDate } from "@/lib/format-date";
import { cn } from "@/lib/utils";
@ -192,6 +192,9 @@ export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
const connectorReauthEndpoint = getReauthEndpoint(connector);
const isAuthExpired =
!!connectorReauthEndpoint && connector.config?.auth_expired === true;
// Migrated type still on legacy native config: reconnect via MCP to
// start producing agent tools again.
const needsReconnect = !isAuthExpired && needsMcpReconnect(connector);
const isLive =
LIVE_CONNECTOR_TYPES.has(connector.connector_type) ||
Boolean(connector.config?.server_config);
@ -245,6 +248,19 @@ export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
/>
Re-authenticate
</Button>
) : needsReconnect ? (
<Button
size="sm"
className="h-8 text-[11px] px-3 font-medium bg-amber-600 hover:bg-amber-700 text-white border-0 shadow-xs shrink-0"
onClick={() => handleReauth(connector)}
disabled={reauthingId === connector.id}
title="This connector moved to MCP. Reconnect to use it with the agent."
>
<RefreshCw
className={cn("size-3.5", reauthingId === connector.id && "animate-spin")}
/>
Reconnect via MCP
</Button>
) : isLive && onDisconnect ? (
confirmDisconnectId === connector.id ? (
<div className="flex items-center gap-1.5 shrink-0">

4
surfsense_web/components/assistant-ui/inline-citation.tsx
View file

@ -115,9 +115,9 @@ interface UrlCitationProps {
}
/**
* Inline citation for live web search results (URL-based chunk IDs).
* Inline citation for URL-based chunk IDs (e.g. scraped/linked web pages).
* Renders a compact chip with favicon + domain and a hover popover showing the
* page title and snippet (extracted deterministically from web_search tool results).
* page title and snippet when citation metadata is available.
*/
export const UrlCitation: FC<UrlCitationProps> = ({ url }) => {
const reactId = useId();

44
surfsense_web/components/assistant-ui/thread.tsx
View file

@ -14,7 +14,6 @@ import {
ChevronDown,
ChevronRight,
Clipboard,
Globe,
Plus,
Settings2,
SquareIcon,
@ -1057,12 +1056,7 @@ const ComposerAction: FC<ComposerActionProps> = ({
});
}, []);
const hasWebSearchTool = agentTools?.some((t) => t.name === "web_search") ?? false;
const isWebSearchEnabled = hasWebSearchTool && !disabledToolsSet.has("web_search");
const filteredTools = useMemo(
() => agentTools?.filter((t) => t.name !== "web_search"),
[agentTools]
);
const filteredTools = agentTools;
const groupedTools = useMemo(() => {
if (!filteredTools) return [];
const toolsByName = new Map(filteredTools.map((t) => [t.name, t]));
@ -1143,22 +1137,6 @@ const ComposerAction: FC<ComposerActionProps> = ({
<Upload className="size-4" />
Upload Files
</DropdownMenuItem>
{hasWebSearchTool && (
<DropdownMenuItem
onSelect={(event) => {
event.preventDefault();
toggleTool("web_search");
}}
>
<Globe className="size-4" />
<span className="flex-1">Web Search</span>
<Switch
checked={isWebSearchEnabled}
tabIndex={-1}
className="pointer-events-none shrink-0 origin-right scale-[0.6]"
/>
</DropdownMenuItem>
)}
<DropdownMenuItem onSelect={() => setConnectorDialogOpen(true)}>
<Unplug className="size-4" />
Manage Connectors
@ -1379,26 +1357,6 @@ const ComposerAction: FC<ComposerActionProps> = ({
<Camera className="h-4 w-4" />
Take a screenshot
</DropdownMenuItem>
{hasWebSearchTool && (
<DropdownMenuItem
onSelect={(event) => {
event.preventDefault();
toggleTool("web_search");
}}
className={cn(
"hover:bg-accent hover:text-accent-foreground",
isWebSearchEnabled && "text-primary"
)}
>
<Globe className="h-4 w-4" />
<span className="flex-1 min-w-0 truncate">Web Search</span>
<Switch
checked={isWebSearchEnabled}
tabIndex={-1}
className="pointer-events-none shrink-0 origin-right scale-[0.6]"
/>
</DropdownMenuItem>
)}
<DropdownMenuSub
open={toolsPopoverOpen}
onOpenChange={(open) => {