feat(agents): consolidate connectors under mcp_discovery; route web search through google_search

MCP consolidation:
- Route all MCP-capable connectors (Slack, Jira, Linear, ClickUp, Airtable,
  Notion, Confluence, interim Gmail/Calendar, custom MCP) through a single
  `mcp_discovery` subagent. Drive/OneDrive/Dropbox stay native to enrich the KB.
- Deprecate Discord/Teams/Luma: no viable official MCP server.

Google-only web search:
- Remove the main-agent `web_search` tool and the SearXNG platform service;
  all public web search now flows through the `google_search` subagent via task().
- Deprecate the Tavily/SearXNG/Linkup/Baidu search connectors (HTTP 410 on
  create, "Deprecated" badge); guide heavy users to the custom MCP connector.
- Remove web search from anonymous chat (pure Q&A).
- Tear SearXNG out of docker compose + install scripts; drop tavily-python
  and linkup-sdk deps and their config/env vars.

Fix:
- metrics._package_version() now swallows any metadata lookup failure. A
  malformed editable-install distribution with no `Version` field raised
  KeyError deep in importlib.metadata, and since it runs on every
  record_subagent_invoke_duration call it was crashing every task()
  delegation. Verified end-to-end against live GPT-5.4.

Co-authored-by: Cursor <cursoragent@cursor.com>

docker/.env.example

@@ -333,16 +333,6 @@ STT_SERVICE=local/base
 # GATEWAY_DISCORD_ENABLED=FALSE
 # GATEWAY_DISCORD_REDIRECT_URI=http://localhost:3929/api/v1/gateway/discord/callback
 
-# ------------------------------------------------------------------------------
-# SearXNG (bundled web search, works out of the box with no config needed)
-# ------------------------------------------------------------------------------
-# SearXNG provides web search to all search spaces automatically.
-# To access the SearXNG UI directly in dev/deps-only compose: http://localhost:8888
-# To disable the service entirely: docker compose up --scale searxng=0
-# To point at your own SearXNG instance instead of the bundled one:
-# SEARXNG_DEFAULT_HOST=http://your-searxng:8080
-# SEARXNG_SECRET=surfsense-searxng-secret
-
 # ------------------------------------------------------------------------------
 # Daytona Sandbox (optional cloud code execution for the deep agent)
 # ------------------------------------------------------------------------------
@@ -492,9 +482,6 @@ NOLOGIN_MODE_ENABLED=FALSE
 # -- Redis exposed port (dev/deps-only only; Redis is internal-only in prod) --
 # REDIS_PORT=6379
 
-# -- SearXNG exposed port (dev/deps-only only; internal-only in prod) --
-# SEARXNG_PORT=8888
-
 # -- WhatsApp bridge exposed port (dev/hybrid only; prod keeps it Docker-internal) --
 # WHATSAPP_BRIDGE_PORT=9929
 

docker/docker-compose.deps-only.yml

@@ -1,7 +1,7 @@
 # =============================================================================
 # SurfSense — Dependencies only (no backend / frontend / Celery images)
 # =============================================================================
-# Postgres, Redis, SearXNG, pgAdmin, Zero — run API + Next + Celery on the host.
+# Postgres, Redis, pgAdmin, Zero — run API + Next + Celery on the host.
 # Celery is not Dockerized here: use `uv run` from surfsense_backend/ (no extra
 # backend image build just for workers).
 #
@@ -9,7 +9,7 @@
 #   docker compose -f docker/docker-compose.deps-only.yml up -d
 #
 # Compose variable substitution uses `docker/.env` (copy from .env.example).
-# Bind mounts use ./postgresql.conf and ./searxng in this directory.
+# Bind mounts use ./postgresql.conf in this directory.
 #
 # Local Celery (from surfsense_backend/, after Redis is up):
 #   uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues=surfsense,surfsense.connectors
@@ -17,7 +17,6 @@
 #
 # Host setup:
 #   - Backend .env: DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense
-#   - Backend .env: SEARXNG_DEFAULT_HOST=http://localhost:${SEARXNG_PORT:-8888}
 #   - Backend .env: CELERY_BROKER_URL / REDIS_APP_URL → redis://localhost:6379/0
 #   - Web .env: NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:${ZERO_CACHE_PORT:-4848}
 #
@@ -80,20 +79,6 @@ services:
       timeout: 5s
       retries: 5
 
-  searxng:
-    image: searxng/searxng:2026.3.13-3c1f68c59
-    ports:
-      - "${SEARXNG_PORT:-8888}:8080"
-    volumes:
-      - ./searxng:/etc/searxng
-    environment:
-      - SEARXNG_SECRET=${SEARXNG_SECRET:-surfsense-searxng-secret}
-    healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
   # NOTE: zero-cache requires the `zero_publication` Postgres publication to
   # exist before it starts. In this deps-only stack there is no backend
   # container to run migrations, so you must run `uv run alembic upgrade head`

docker/docker-compose.dev.yml

@@ -85,20 +85,6 @@ services:
       - "${OTEL_TEMPO_PORT:-3200}:3200"
     restart: unless-stopped
 
-  searxng:
-    image: searxng/searxng:2026.3.13-3c1f68c59
-    ports:
-      - "${SEARXNG_PORT:-8888}:8080"
-    volumes:
-      - ./searxng:/etc/searxng
-    environment:
-      - SEARXNG_SECRET=${SEARXNG_SECRET:-surfsense-searxng-secret}
-    healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
   backend:
     build: *backend-build
     ports:
@@ -125,7 +111,6 @@ services:
       - LANGSMITH_TRACING=false
       - AUTH_TYPE=${AUTH_TYPE:-LOCAL}
       - NEXT_FRONTEND_URL=${NEXT_FRONTEND_URL:-http://localhost:3000}
-      - SEARXNG_DEFAULT_HOST=${SEARXNG_DEFAULT_HOST:-http://searxng:8080}
       - WHATSAPP_BRIDGE_URL=${WHATSAPP_BRIDGE_URL:-http://whatsapp-bridge:9929}
       # Daytona Sandbox – uncomment and set credentials to enable cloud code execution
       # - DAYTONA_SANDBOX_ENABLED=TRUE
@@ -138,8 +123,6 @@ services:
         condition: service_healthy
       redis:
         condition: service_healthy
-      searxng:
-        condition: service_healthy
       migrations:
         condition: service_completed_successfully
     healthcheck:
@@ -186,7 +169,6 @@ services:
       - CELERY_TASK_DEFAULT_QUEUE=surfsense
       - PYTHONPATH=/app
       - FILE_STORAGE_LOCAL_PATH=/app/.local_object_store
-      - SEARXNG_DEFAULT_HOST=${SEARXNG_DEFAULT_HOST:-http://searxng:8080}
       - SERVICE_ROLE=worker
     depends_on:
       db:

docker/docker-compose.yml

@@ -81,19 +81,6 @@ services:
   #     timeout: 5s
   #     retries: 3
 
-  searxng:
-    image: searxng/searxng:2026.3.13-3c1f68c59
-    volumes:
-      - ./searxng:/etc/searxng
-    environment:
-      SEARXNG_SECRET: ${SEARXNG_SECRET:-surfsense-searxng-secret}
-    restart: unless-stopped
-    healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
   # Single public entry point for the Docker stack. Comment this service out
   # only if you front SurfSense with your own reverse proxy.
   proxy:
@@ -146,7 +133,6 @@ services:
       FILE_STORAGE_LOCAL_PATH: /app/.local_object_store
       NEXT_FRONTEND_URL: ${NEXT_FRONTEND_URL:-${SURFSENSE_PUBLIC_URL:-http://localhost:${LISTEN_HTTP_PORT:-3929}}}
       BACKEND_URL: ${BACKEND_URL:-${SURFSENSE_PUBLIC_URL:-http://localhost:${LISTEN_HTTP_PORT:-3929}}}
-      SEARXNG_DEFAULT_HOST: ${SEARXNG_DEFAULT_HOST:-http://searxng:8080}
       WHATSAPP_BRIDGE_URL: ${WHATSAPP_BRIDGE_URL:-http://whatsapp-bridge:9929}
       # Daytona Sandbox – uncomment and set credentials to enable cloud code execution
       # DAYTONA_SANDBOX_ENABLED: "TRUE"
@@ -161,8 +147,6 @@ services:
         condition: service_healthy
       redis:
         condition: service_healthy
-      searxng:
-        condition: service_healthy
       migrations:
         condition: service_completed_successfully
     restart: unless-stopped
@@ -210,7 +194,6 @@ services:
       CELERY_TASK_DEFAULT_QUEUE: surfsense
       PYTHONPATH: /app
       FILE_STORAGE_LOCAL_PATH: /app/.local_object_store
-      SEARXNG_DEFAULT_HOST: ${SEARXNG_DEFAULT_HOST:-http://searxng:8080}
       SERVICE_ROLE: worker
     depends_on:
       db:

docker/scripts/install.ps1

@@ -329,7 +329,6 @@ Write-Step "Downloading SurfSense files"
 Write-Info "Installation directory: $InstallDir"
 
 New-Item -ItemType Directory -Path "$InstallDir\scripts" -Force | Out-Null
-New-Item -ItemType Directory -Path "$InstallDir\searxng" -Force | Out-Null
 New-Item -ItemType Directory -Path "$InstallDir\proxy" -Force | Out-Null
 
 $Files = @(
@@ -339,8 +338,6 @@ $Files = @(
     @{ Src = "docker/proxy/Caddyfile";                   Dest = "proxy/Caddyfile" }
     @{ Src = "docker/postgresql.conf";                   Dest = "postgresql.conf" }
     @{ Src = "docker/scripts/migrate-database.ps1";      Dest = "scripts/migrate-database.ps1" }
-    @{ Src = "docker/searxng/settings.yml";              Dest = "searxng/settings.yml" }
-    @{ Src = "docker/searxng/limiter.toml";              Dest = "searxng/limiter.toml" }
 )
 
 foreach ($f in $Files) {

docker/scripts/install.sh

@@ -332,7 +332,6 @@ SELECTED_VARIANT=$(resolve_variant)
 step "Downloading SurfSense files"
 info "Installation directory: ${INSTALL_DIR}"
 mkdir -p "${INSTALL_DIR}/scripts"
-mkdir -p "${INSTALL_DIR}/searxng"
 mkdir -p "${INSTALL_DIR}/proxy"
 
 FILES=(
@@ -342,8 +341,6 @@ FILES=(
     "docker/proxy/Caddyfile:proxy/Caddyfile"
     "docker/postgresql.conf:postgresql.conf"
     "docker/scripts/migrate-database.sh:scripts/migrate-database.sh"
-    "docker/searxng/settings.yml:searxng/settings.yml"
-    "docker/searxng/limiter.toml:searxng/limiter.toml"
 )
 
 for entry in "${FILES[@]}"; do

docker/searxng/limiter.toml

@@ -1,5 +0,0 @@
-[botdetection.ip_limit]
-link_token = false
-
-[botdetection.ip_lists]
-pass_ip = ["0.0.0.0/0"]

docker/searxng/settings.yml

@@ -1,90 +0,0 @@
-use_default_settings:
-  engines:
-    remove:
-      - ahmia
-      - torch
-      - qwant
-      - qwant news
-      - qwant images
-      - qwant videos
-      - mojeek
-      - mojeek images
-      - mojeek news
-
-server:
-  secret_key: "override-me-via-env"
-  limiter: false
-  image_proxy: false
-  method: "GET"
-  default_http_headers:
-    X-Robots-Tag: "noindex, nofollow"
-
-search:
-  formats:
-    - html
-    - json
-  default_lang: "auto"
-  autocomplete: ""
-  safe_search: 0
-  ban_time_on_fail: 5
-  max_ban_time_on_fail: 120
-  suspended_times:
-    SearxEngineAccessDenied: 3600
-    SearxEngineCaptcha: 3600
-    SearxEngineTooManyRequests: 600
-    cf_SearxEngineCaptcha: 7200
-    cf_SearxEngineAccessDenied: 3600
-    recaptcha_SearxEngineCaptcha: 7200
-
-ui:
-  static_use_hash: true
-
-outgoing:
-  request_timeout: 12.0
-  max_request_timeout: 20.0
-  pool_connections: 100
-  pool_maxsize: 20
-  enable_http2: true
-  extra_proxy_timeout: 10
-  retries: 1
-  # Uncomment and set your residential proxy URL to route search engine requests through it.
-  # Format: http://<username>:<base64_password>@<hostname>:<port>/
-  #
-  # proxies:
-  #   all://:
-  #     - http://user:pass@proxy-host:port/
-
-engines:
-  - name: google
-    disabled: false
-    weight: 1.2
-    retry_on_http_error: [429, 503]
-  - name: duckduckgo
-    disabled: false
-    weight: 1.1
-    retry_on_http_error: [429, 503]
-  - name: brave
-    disabled: false
-    weight: 1.0
-    retry_on_http_error: [429, 503]
-  - name: bing
-    disabled: false
-    weight: 0.9
-    retry_on_http_error: [429, 503]
-  - name: wikipedia
-    disabled: false
-    weight: 0.8
-  - name: stackoverflow
-    disabled: false
-    weight: 0.7
-  - name: yahoo
-    disabled: false
-    weight: 0.7
-    retry_on_http_error: [429, 503]
-  - name: wikidata
-    disabled: false
-    weight: 0.6
-  - name: currency
-    disabled: false
-  - name: ddg definitions
-    disabled: false