feat(agents): consolidate connectors under mcp_discovery; route web search through google_search

MCP consolidation:
- Route all MCP-capable connectors (Slack, Jira, Linear, ClickUp, Airtable,
  Notion, Confluence, interim Gmail/Calendar, custom MCP) through a single
  `mcp_discovery` subagent. Drive/OneDrive/Dropbox stay native to enrich the KB.
- Deprecate Discord/Teams/Luma: no viable official MCP server.

Google-only web search:
- Remove the main-agent `web_search` tool and the SearXNG platform service;
  all public web search now flows through the `google_search` subagent via task().
- Deprecate the Tavily/SearXNG/Linkup/Baidu search connectors (HTTP 410 on
  create, "Deprecated" badge); guide heavy users to the custom MCP connector.
- Remove web search from anonymous chat (pure Q&A).
- Tear SearXNG out of docker compose + install scripts; drop tavily-python
  and linkup-sdk deps and their config/env vars.

Fix:
- metrics._package_version() now swallows any metadata lookup failure. A
  malformed editable-install distribution with no `Version` field raised
  KeyError deep in importlib.metadata, and since it runs on every
  record_subagent_invoke_duration call it was crashing every task()
  delegation. Verified end-to-end against live GPT-5.4.

Co-authored-by: Cursor <cursoragent@cursor.com>

surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py

@@ -64,12 +64,20 @@ async def _check_and_trigger_schedules():
                 SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR: index_google_drive_files_task,
             }
 
-            from app.services.mcp_oauth.registry import LIVE_CONNECTOR_TYPES
+            from app.services.mcp_oauth.registry import (
+                DEPRECATED_INDEXING_CONNECTOR_TYPES,
+                LIVE_CONNECTOR_TYPES,
+            )
 
-            # Disable obsolete periodic indexing for live connectors in one batch.
+            # Disable obsolete periodic indexing in one batch: live connectors
+            # (now real-time agent tools) and deprecated-indexing connectors
+            # (KB is files/notes/uploads only) no longer index on a schedule.
             live_disabled = []
             for connector in due_connectors:
-                if connector.connector_type in LIVE_CONNECTOR_TYPES:
+                if (
+                    connector.connector_type in LIVE_CONNECTOR_TYPES
+                    or connector.connector_type in DEPRECATED_INDEXING_CONNECTOR_TYPES
+                ):
                     connector.periodic_indexing_enabled = False
                     connector.next_scheduled_at = None
                     live_disabled.append(connector)
@@ -77,7 +85,7 @@ async def _check_and_trigger_schedules():
                 await session.commit()
                 for c in live_disabled:
                     logger.info(
-                        "Disabled obsolete periodic indexing for live connector %s (%s)",
+                        "Disabled obsolete periodic indexing for connector %s (%s)",
                         c.id,
                         c.connector_type.value,
                     )

surfsense_backend/app/tasks/chat/streaming/flows/shared/assistant_finalize.py

@@ -54,7 +54,7 @@ def _resolve_citations(
 ) -> list[dict[str, Any]]:
     """Rewrite ``[n]`` -> ``[citation:<payload>]`` in each text part before persisting.
 
-    No-op when the turn registered no citable sources; ``web_search``'s existing
+    No-op when the turn registered no citable sources; any pre-existing
     ``[citation:url]`` markers pass through untouched (the regex matches bare ``[n]``).
     """
     registry = _as_registry(raw_registry)

surfsense_backend/app/tasks/chat/streaming/handlers/tools/web_search/emission.py

@@ -1,37 +0,0 @@
-"""web_search: citations parsed from provider XML."""
-
-from __future__ import annotations
-
-import re
-from collections.abc import Iterator
-
-from app.tasks.chat.streaming.handlers.tools.emission_context import (
-    ToolCompletionEmissionContext,
-)
-
-
-def iter_completion_emission_frames(
-    ctx: ToolCompletionEmissionContext,
-) -> Iterator[str]:
-    out = ctx.tool_output
-    xml = out.get("result", str(out)) if isinstance(out, dict) else str(out)
-    citations: dict[str, dict[str, str]] = {}
-    for m in re.finditer(
-        r"<title><!\[CDATA\[(.*?)\]\]></title>\s*<url><!\[CDATA\[(.*?)\]\]></url>",
-        xml,
-    ):
-        title, url = m.group(1).strip(), m.group(2).strip()
-        if url.startswith("http") and url not in citations:
-            citations[url] = {"title": title}
-    for m in re.finditer(
-        r"<chunk\s+id='([^']*)'><!\[CDATA\[([\s\S]*?)\]\]></chunk>",
-        xml,
-    ):
-        chunk_url, content = m.group(1).strip(), m.group(2).strip()
-        if chunk_url.startswith("http") and chunk_url in citations and content:
-            citations[chunk_url]["snippet"] = (
-                content[:200] + "…" if len(content) > 200 else content
-            )
-    yield ctx.emit_tool_output_card(
-        {"status": "completed", "citations": citations},
-    )