diff --git a/docker/.env.example b/docker/.env.example index fe57698ab..a191c27c1 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -333,16 +333,6 @@ STT_SERVICE=local/base # GATEWAY_DISCORD_ENABLED=FALSE # GATEWAY_DISCORD_REDIRECT_URI=http://localhost:3929/api/v1/gateway/discord/callback -# ------------------------------------------------------------------------------ -# SearXNG (bundled web search, works out of the box with no config needed) -# ------------------------------------------------------------------------------ -# SearXNG provides web search to all search spaces automatically. -# To access the SearXNG UI directly in dev/deps-only compose: http://localhost:8888 -# To disable the service entirely: docker compose up --scale searxng=0 -# To point at your own SearXNG instance instead of the bundled one: -# SEARXNG_DEFAULT_HOST=http://your-searxng:8080 -# SEARXNG_SECRET=surfsense-searxng-secret - # ------------------------------------------------------------------------------ # Daytona Sandbox (optional cloud code execution for the deep agent) # ------------------------------------------------------------------------------ @@ -492,9 +482,6 @@ NOLOGIN_MODE_ENABLED=FALSE # -- Redis exposed port (dev/deps-only only; Redis is internal-only in prod) -- # REDIS_PORT=6379 -# -- SearXNG exposed port (dev/deps-only only; internal-only in prod) -- -# SEARXNG_PORT=8888 - # -- WhatsApp bridge exposed port (dev/hybrid only; prod keeps it Docker-internal) -- # WHATSAPP_BRIDGE_PORT=9929 diff --git a/docker/docker-compose.deps-only.yml b/docker/docker-compose.deps-only.yml index e70e126bb..6bd21eed3 100644 --- a/docker/docker-compose.deps-only.yml +++ b/docker/docker-compose.deps-only.yml @@ -1,7 +1,7 @@ # ============================================================================= # SurfSense — Dependencies only (no backend / frontend / Celery images) # ============================================================================= -# Postgres, Redis, SearXNG, pgAdmin, Zero — run API + Next + Celery on the host. +# Postgres, Redis, pgAdmin, Zero — run API + Next + Celery on the host. # Celery is not Dockerized here: use `uv run` from surfsense_backend/ (no extra # backend image build just for workers). # @@ -9,7 +9,7 @@ # docker compose -f docker/docker-compose.deps-only.yml up -d # # Compose variable substitution uses `docker/.env` (copy from .env.example). -# Bind mounts use ./postgresql.conf and ./searxng in this directory. +# Bind mounts use ./postgresql.conf in this directory. # # Local Celery (from surfsense_backend/, after Redis is up): # uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues=surfsense,surfsense.connectors @@ -17,7 +17,6 @@ # # Host setup: # - Backend .env: DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense -# - Backend .env: SEARXNG_DEFAULT_HOST=http://localhost:${SEARXNG_PORT:-8888} # - Backend .env: CELERY_BROKER_URL / REDIS_APP_URL → redis://localhost:6379/0 # - Web .env: NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:${ZERO_CACHE_PORT:-4848} # @@ -80,20 +79,6 @@ services: timeout: 5s retries: 5 - searxng: - image: searxng/searxng:2026.3.13-3c1f68c59 - ports: - - "${SEARXNG_PORT:-8888}:8080" - volumes: - - ./searxng:/etc/searxng - environment: - - SEARXNG_SECRET=${SEARXNG_SECRET:-surfsense-searxng-secret} - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"] - interval: 10s - timeout: 5s - retries: 5 - # NOTE: zero-cache requires the `zero_publication` Postgres publication to # exist before it starts. In this deps-only stack there is no backend # container to run migrations, so you must run `uv run alembic upgrade head` diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 9660690ea..d3c3bbdae 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -85,20 +85,6 @@ services: - "${OTEL_TEMPO_PORT:-3200}:3200" restart: unless-stopped - searxng: - image: searxng/searxng:2026.3.13-3c1f68c59 - ports: - - "${SEARXNG_PORT:-8888}:8080" - volumes: - - ./searxng:/etc/searxng - environment: - - SEARXNG_SECRET=${SEARXNG_SECRET:-surfsense-searxng-secret} - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"] - interval: 10s - timeout: 5s - retries: 5 - backend: build: *backend-build ports: @@ -125,7 +111,6 @@ services: - LANGSMITH_TRACING=false - AUTH_TYPE=${AUTH_TYPE:-LOCAL} - NEXT_FRONTEND_URL=${NEXT_FRONTEND_URL:-http://localhost:3000} - - SEARXNG_DEFAULT_HOST=${SEARXNG_DEFAULT_HOST:-http://searxng:8080} - WHATSAPP_BRIDGE_URL=${WHATSAPP_BRIDGE_URL:-http://whatsapp-bridge:9929} # Daytona Sandbox – uncomment and set credentials to enable cloud code execution # - DAYTONA_SANDBOX_ENABLED=TRUE @@ -138,8 +123,6 @@ services: condition: service_healthy redis: condition: service_healthy - searxng: - condition: service_healthy migrations: condition: service_completed_successfully healthcheck: @@ -186,7 +169,6 @@ services: - CELERY_TASK_DEFAULT_QUEUE=surfsense - PYTHONPATH=/app - FILE_STORAGE_LOCAL_PATH=/app/.local_object_store - - SEARXNG_DEFAULT_HOST=${SEARXNG_DEFAULT_HOST:-http://searxng:8080} - SERVICE_ROLE=worker depends_on: db: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 3b47d6670..fe878858a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -81,19 +81,6 @@ services: # timeout: 5s # retries: 3 - searxng: - image: searxng/searxng:2026.3.13-3c1f68c59 - volumes: - - ./searxng:/etc/searxng - environment: - SEARXNG_SECRET: ${SEARXNG_SECRET:-surfsense-searxng-secret} - restart: unless-stopped - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"] - interval: 10s - timeout: 5s - retries: 5 - # Single public entry point for the Docker stack. Comment this service out # only if you front SurfSense with your own reverse proxy. proxy: @@ -146,7 +133,6 @@ services: FILE_STORAGE_LOCAL_PATH: /app/.local_object_store NEXT_FRONTEND_URL: ${NEXT_FRONTEND_URL:-${SURFSENSE_PUBLIC_URL:-http://localhost:${LISTEN_HTTP_PORT:-3929}}} BACKEND_URL: ${BACKEND_URL:-${SURFSENSE_PUBLIC_URL:-http://localhost:${LISTEN_HTTP_PORT:-3929}}} - SEARXNG_DEFAULT_HOST: ${SEARXNG_DEFAULT_HOST:-http://searxng:8080} WHATSAPP_BRIDGE_URL: ${WHATSAPP_BRIDGE_URL:-http://whatsapp-bridge:9929} # Daytona Sandbox – uncomment and set credentials to enable cloud code execution # DAYTONA_SANDBOX_ENABLED: "TRUE" @@ -161,8 +147,6 @@ services: condition: service_healthy redis: condition: service_healthy - searxng: - condition: service_healthy migrations: condition: service_completed_successfully restart: unless-stopped @@ -210,7 +194,6 @@ services: CELERY_TASK_DEFAULT_QUEUE: surfsense PYTHONPATH: /app FILE_STORAGE_LOCAL_PATH: /app/.local_object_store - SEARXNG_DEFAULT_HOST: ${SEARXNG_DEFAULT_HOST:-http://searxng:8080} SERVICE_ROLE: worker depends_on: db: diff --git a/docker/scripts/install.ps1 b/docker/scripts/install.ps1 index 761fef80e..144944fe7 100644 --- a/docker/scripts/install.ps1 +++ b/docker/scripts/install.ps1 @@ -329,7 +329,6 @@ Write-Step "Downloading SurfSense files" Write-Info "Installation directory: $InstallDir" New-Item -ItemType Directory -Path "$InstallDir\scripts" -Force | Out-Null -New-Item -ItemType Directory -Path "$InstallDir\searxng" -Force | Out-Null New-Item -ItemType Directory -Path "$InstallDir\proxy" -Force | Out-Null $Files = @( @@ -339,8 +338,6 @@ $Files = @( @{ Src = "docker/proxy/Caddyfile"; Dest = "proxy/Caddyfile" } @{ Src = "docker/postgresql.conf"; Dest = "postgresql.conf" } @{ Src = "docker/scripts/migrate-database.ps1"; Dest = "scripts/migrate-database.ps1" } - @{ Src = "docker/searxng/settings.yml"; Dest = "searxng/settings.yml" } - @{ Src = "docker/searxng/limiter.toml"; Dest = "searxng/limiter.toml" } ) foreach ($f in $Files) { diff --git a/docker/scripts/install.sh b/docker/scripts/install.sh index f9660b132..03f38ce38 100644 --- a/docker/scripts/install.sh +++ b/docker/scripts/install.sh @@ -332,7 +332,6 @@ SELECTED_VARIANT=$(resolve_variant) step "Downloading SurfSense files" info "Installation directory: ${INSTALL_DIR}" mkdir -p "${INSTALL_DIR}/scripts" -mkdir -p "${INSTALL_DIR}/searxng" mkdir -p "${INSTALL_DIR}/proxy" FILES=( @@ -342,8 +341,6 @@ FILES=( "docker/proxy/Caddyfile:proxy/Caddyfile" "docker/postgresql.conf:postgresql.conf" "docker/scripts/migrate-database.sh:scripts/migrate-database.sh" - "docker/searxng/settings.yml:searxng/settings.yml" - "docker/searxng/limiter.toml:searxng/limiter.toml" ) for entry in "${FILES[@]}"; do diff --git a/docker/searxng/limiter.toml b/docker/searxng/limiter.toml deleted file mode 100644 index dce84146f..000000000 --- a/docker/searxng/limiter.toml +++ /dev/null @@ -1,5 +0,0 @@ -[botdetection.ip_limit] -link_token = false - -[botdetection.ip_lists] -pass_ip = ["0.0.0.0/0"] diff --git a/docker/searxng/settings.yml b/docker/searxng/settings.yml deleted file mode 100644 index 0b805b6aa..000000000 --- a/docker/searxng/settings.yml +++ /dev/null @@ -1,90 +0,0 @@ -use_default_settings: - engines: - remove: - - ahmia - - torch - - qwant - - qwant news - - qwant images - - qwant videos - - mojeek - - mojeek images - - mojeek news - -server: - secret_key: "override-me-via-env" - limiter: false - image_proxy: false - method: "GET" - default_http_headers: - X-Robots-Tag: "noindex, nofollow" - -search: - formats: - - html - - json - default_lang: "auto" - autocomplete: "" - safe_search: 0 - ban_time_on_fail: 5 - max_ban_time_on_fail: 120 - suspended_times: - SearxEngineAccessDenied: 3600 - SearxEngineCaptcha: 3600 - SearxEngineTooManyRequests: 600 - cf_SearxEngineCaptcha: 7200 - cf_SearxEngineAccessDenied: 3600 - recaptcha_SearxEngineCaptcha: 7200 - -ui: - static_use_hash: true - -outgoing: - request_timeout: 12.0 - max_request_timeout: 20.0 - pool_connections: 100 - pool_maxsize: 20 - enable_http2: true - extra_proxy_timeout: 10 - retries: 1 - # Uncomment and set your residential proxy URL to route search engine requests through it. - # Format: http://:@:/ - # - # proxies: - # all://: - # - http://user:pass@proxy-host:port/ - -engines: - - name: google - disabled: false - weight: 1.2 - retry_on_http_error: [429, 503] - - name: duckduckgo - disabled: false - weight: 1.1 - retry_on_http_error: [429, 503] - - name: brave - disabled: false - weight: 1.0 - retry_on_http_error: [429, 503] - - name: bing - disabled: false - weight: 0.9 - retry_on_http_error: [429, 503] - - name: wikipedia - disabled: false - weight: 0.8 - - name: stackoverflow - disabled: false - weight: 0.7 - - name: yahoo - disabled: false - weight: 0.7 - retry_on_http_error: [429, 503] - - name: wikidata - disabled: false - weight: 0.6 - - name: currency - disabled: false - - name: ddg definitions - disabled: false diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index f2b3f73b1..65b60bdd1 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -55,18 +55,6 @@ WHATSAPP_WEBHOOK_VERIFY_TOKEN= WHATSAPP_WEBHOOK_APP_SECRET= WHATSAPP_BRIDGE_URL=http://whatsapp-bridge:9929 -# Platform Web Search (SearXNG) -# Set this to enable built-in web search. Docker Compose sets it automatically. -# Only uncomment if running the backend outside Docker (e.g. uvicorn on host). -# SEARXNG_DEFAULT_HOST=http://localhost:8888 - -# web.discover fallback providers (env-keyed). SearXNG above is preferred; these -# are used when it is not configured. web.discover self-disables if none are set. -# LINKUP_API_KEY= -# BAIDU_API_KEY= -# BAIDU_MODEL=ernie-3.5-8k -# BAIDU_SEARCH_SOURCE=baidu_search_v2 - # Periodic task interval # # Run every minute (default) # SCHEDULE_CHECKER_INTERVAL=1m diff --git a/surfsense_backend/app/agents/chat/anonymous_chat/agent.py b/surfsense_backend/app/agents/chat/anonymous_chat/agent.py index eb0f46037..168eaaff0 100644 --- a/surfsense_backend/app/agents/chat/anonymous_chat/agent.py +++ b/surfsense_backend/app/agents/chat/anonymous_chat/agent.py @@ -1,16 +1,16 @@ """Minimal anonymous / free-chat agent. The no-login chat experience must stay dead simple: the user asks a question -and the model answers, optionally using ``web_search`` and an optionally -uploaded **read-only** document. We deliberately bypass the full SurfSense deep -agent stack (filesystem, file-intent, knowledge-base persistence, subagents, -skills, memory) because those middlewares stage or persist "documents" that an -anonymous session can never see again -- which produced phantom -"I saved it to a file" answers for free users. +and the model answers over an optionally uploaded **read-only** document. We +deliberately bypass the full SurfSense deep agent stack (filesystem, +file-intent, knowledge-base persistence, subagents, skills, memory) because +those middlewares stage or persist "documents" that an anonymous session can +never see again -- which produced phantom "I saved it to a file" answers for +free users. -For any other SurfSense capability the model is instructed (via the system -prompt built here) to tell the user to create a free account instead of -pretending to perform the action. +For any other SurfSense capability (including web search) the model is +instructed (via the system prompt built here) to tell the user to create a +free account instead of pretending to perform the action. """ from __future__ import annotations @@ -22,7 +22,6 @@ from deepagents.backends import StateBackend from langchain.agents import create_agent from langchain.agents.middleware import ( ModelCallLimitMiddleware, - ToolCallLimitMiddleware, ) from langchain_core.language_models import BaseChatModel from langgraph.types import Checkpointer @@ -32,7 +31,6 @@ from app.agents.chat.shared.middleware import ( RetryAfterMiddleware, create_surfsense_compaction_middleware, ) -from app.agents.chat.shared.tools.web_search import create_web_search_tool # Cap how much of an uploaded document we inline into the system prompt. The # upload endpoint allows files up to several MB, but the doc is re-sent on @@ -77,25 +75,26 @@ def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str "## How to help\n" "- Answer the user's questions directly and conversationally. You are " "a straightforward question-and-answer assistant.\n" - "- When a question needs current, real-time, or factual information " - "from the internet (news, prices, weather, recent events, live data), " - "use the `web_search` tool. Otherwise, answer directly from your own " - "knowledge.\n" + "- Answer from your own knowledge. You do NOT have web access here, so " + "for current, real-time, or fast-changing facts (news, prices, " + "weather, recent events, live data) say you can't look them up in the " + "free experience and may be out of date, and invite the user to create " + "a free account for live web search.\n" "- Be concise, accurate, and helpful. Use Markdown formatting when it " "improves readability." f"{doc_section}\n\n" "## What is not available here\n" - "This is the free, no-login experience. You CANNOT save files or " - "notes, generate reports, podcasts, resumes, presentations, or images, " - "search or build a knowledge base, connect to apps (Gmail, Google " - "Drive, Notion, Slack, Calendar, Discord, and similar), set up " + "This is the free, no-login experience. You CANNOT search the web, save " + "files or notes, generate reports, podcasts, resumes, presentations, or " + "images, search or build a knowledge base, connect to apps (Gmail, " + "Google Drive, Notion, Slack, Calendar, Discord, and similar), set up " "automations, or remember anything across sessions.\n\n" "If the user asks for any of these, do NOT pretend to do them and " "never claim you saved, created, or stored anything. Instead, briefly " "let them know the feature requires a free SurfSense account and " "invite them to create one at https://www.surfsense.com. Then offer to " - "help with what you can do here (answering questions and searching the " - "web)." + "help with what you can do here (answering questions from your own " + "knowledge and about any uploaded document)." ) @@ -105,14 +104,13 @@ async def create_anonymous_chat_agent( checkpointer: Checkpointer, anon_session_id: str | None = None, anon_doc: dict[str, Any] | None = None, - enable_web_search: bool = True, ): """Create a minimal Q/A agent for anonymous / free chat. Unlike :func:`create_surfsense_deep_agent`, this agent has no filesystem, file-intent, knowledge-base persistence, subagent, skills, or memory - middleware. Its only tool is ``web_search`` (when ``enable_web_search`` is - True), and any uploaded document is injected into the system prompt as + middleware -- and no tools at all. It answers purely from the model's own + knowledge; any uploaded document is injected into the system prompt as read-only context. Args: @@ -120,36 +118,23 @@ async def create_anonymous_chat_agent( checkpointer: LangGraph checkpointer for the ephemeral anon thread. anon_session_id: Anonymous session id (used only for telemetry/metadata). anon_doc: Optional ``{"title", "content"}`` for an uploaded document. - enable_web_search: When False, the agent runs as a pure LLM with no - tools (used when the user toggles web search off). """ - tools = ( - [create_web_search_tool(workspace_id=None, available_connectors=None)] - if enable_web_search - else [] - ) - # Reliability-only middleware. Nothing here touches the database or - # filesystem: call limits guard against loops, compaction summarises long - # histories into in-graph state, and retry handles provider rate limits. + # filesystem: the call limit guards against loops, compaction summarises + # long histories into in-graph state, and retry handles provider rate + # limits. middleware: list[Any] = [ ModelCallLimitMiddleware(thread_limit=120, run_limit=80, exit_behavior="end"), + create_surfsense_compaction_middleware(llm, StateBackend), + RetryAfterMiddleware(max_retries=3), ] - if tools: - middleware.append( - ToolCallLimitMiddleware( - thread_limit=300, run_limit=80, exit_behavior="continue" - ) - ) - middleware.append(create_surfsense_compaction_middleware(llm, StateBackend)) - middleware.append(RetryAfterMiddleware(max_retries=3)) system_prompt = build_anonymous_system_prompt(anon_doc) agent = create_agent( llm, system_prompt=system_prompt, - tools=tools, + tools=[], middleware=middleware, context_schema=SurfSenseContextSchema, checkpointer=checkpointer, diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py index 0605de008..b132618d7 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py @@ -2,27 +2,32 @@ from __future__ import annotations +# Connected apps (hosted MCP + interim-native Gmail/Calendar) all route to the +# single ``mcp_discovery`` subagent. File connectors stay native (they enrich +# the knowledge base). Deprecated connectors (Discord/Teams/Luma) are omitted: +# they have no agent subagent, so their rows produce no tools. CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS: dict[str, str] = { - "GOOGLE_GMAIL_CONNECTOR": "gmail", - "COMPOSIO_GMAIL_CONNECTOR": "gmail", - "GOOGLE_CALENDAR_CONNECTOR": "calendar", - "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR": "calendar", - "DISCORD_CONNECTOR": "discord", - "TEAMS_CONNECTOR": "teams", - "LUMA_CONNECTOR": "luma", - "LINEAR_CONNECTOR": "linear", - "JIRA_CONNECTOR": "jira", - "CLICKUP_CONNECTOR": "clickup", - "SLACK_CONNECTOR": "slack", - "AIRTABLE_CONNECTOR": "airtable", - "NOTION_CONNECTOR": "notion", - "CONFLUENCE_CONNECTOR": "confluence", + "GOOGLE_GMAIL_CONNECTOR": "mcp_discovery", + "COMPOSIO_GMAIL_CONNECTOR": "mcp_discovery", + "GOOGLE_CALENDAR_CONNECTOR": "mcp_discovery", + "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR": "mcp_discovery", + "LINEAR_CONNECTOR": "mcp_discovery", + "JIRA_CONNECTOR": "mcp_discovery", + "CLICKUP_CONNECTOR": "mcp_discovery", + "SLACK_CONNECTOR": "mcp_discovery", + "AIRTABLE_CONNECTOR": "mcp_discovery", + "NOTION_CONNECTOR": "mcp_discovery", + "CONFLUENCE_CONNECTOR": "mcp_discovery", + "MCP_CONNECTOR": "mcp_discovery", "GOOGLE_DRIVE_CONNECTOR": "google_drive", "COMPOSIO_GOOGLE_DRIVE_CONNECTOR": "google_drive", "DROPBOX_CONNECTOR": "dropbox", "ONEDRIVE_CONNECTOR": "onedrive", } +# ``mcp_discovery`` is gated any-of: present iff the workspace has at least one +# connected app. Tokens are searchable-type strings (Composio Gmail/Calendar +# map to the GOOGLE_* tokens in connector_searchable_types). SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = { "deliverables": frozenset(), "knowledge_base": frozenset(), @@ -31,19 +36,40 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = { "google_maps": frozenset(), "google_search": frozenset(), "reddit": frozenset(), - "airtable": frozenset({"AIRTABLE_CONNECTOR"}), - "calendar": frozenset({"GOOGLE_CALENDAR_CONNECTOR"}), - "clickup": frozenset({"CLICKUP_CONNECTOR"}), - "confluence": frozenset({"CONFLUENCE_CONNECTOR"}), - "discord": frozenset({"DISCORD_CONNECTOR"}), + "mcp_discovery": frozenset( + { + "SLACK_CONNECTOR", + "JIRA_CONNECTOR", + "LINEAR_CONNECTOR", + "CLICKUP_CONNECTOR", + "AIRTABLE_CONNECTOR", + "NOTION_CONNECTOR", + "CONFLUENCE_CONNECTOR", + "GOOGLE_GMAIL_CONNECTOR", + "GOOGLE_CALENDAR_CONNECTOR", + "MCP_CONNECTOR", + } + ), "dropbox": frozenset({"DROPBOX_FILE"}), - "gmail": frozenset({"GOOGLE_GMAIL_CONNECTOR"}), "google_drive": frozenset({"GOOGLE_DRIVE_FILE"}), - "jira": frozenset({"JIRA_CONNECTOR"}), - "linear": frozenset({"LINEAR_CONNECTOR"}), - "luma": frozenset({"LUMA_CONNECTOR"}), - "notion": frozenset({"NOTION_CONNECTOR"}), "onedrive": frozenset({"ONEDRIVE_FILE"}), - "slack": frozenset({"SLACK_CONNECTOR"}), - "teams": frozenset({"TEAMS_CONNECTOR"}), +} + +# Old per-connector subagent names, kept working for checkpoint resume: a +# ``task(subagent_type="gmail")`` paused before the MCP consolidation resolves +# to the consolidated ``mcp_discovery`` subagent instead of hard-failing +# "subagent does not exist". New routing never emits these names. +LEGACY_SUBAGENT_ALIASES: dict[str, str] = { + "gmail": "mcp_discovery", + "calendar": "mcp_discovery", + "linear": "mcp_discovery", + "jira": "mcp_discovery", + "clickup": "mcp_discovery", + "slack": "mcp_discovery", + "airtable": "mcp_discovery", + "notion": "mcp_discovery", + "confluence": "mcp_discovery", + "discord": "mcp_discovery", + "teams": "mcp_discovery", + "luma": "mcp_discovery", } diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/task_tool.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/task_tool.py index a08e6cda1..cc5ebaa98 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/task_tool.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/task_tool.py @@ -23,6 +23,7 @@ from langchain_core.tools import StructuredTool from langgraph.errors import GraphInterrupt from langgraph.types import Command, Interrupt +from app.agents.chat.multi_agent_chat.constants import LEGACY_SUBAGENT_ALIASES from app.agents.chat.multi_agent_chat.subagents.shared.invocation import ( EXCLUDED_STATE_KEYS, subagent_invoke_config, @@ -157,6 +158,26 @@ def build_task_tool_with_parent_config( case a trivial dict-backed resolver is used. """ subagent_names: set[str] = {spec["name"] for spec in subagents} + + def _canonical_subagent_type(subagent_type: str) -> str: + """Resolve a legacy connector subagent name to its consolidated route. + + Only rewrites when the requested name is unavailable but its alias is + (checkpoint resume of a pre-consolidation ``task(...)`` call). Current + routing never emits legacy names, so live traffic is untouched. + """ + if subagent_type in subagent_names: + return subagent_type + alias = LEGACY_SUBAGENT_ALIASES.get(subagent_type) + if alias is not None and alias in subagent_names: + logger.info( + "[hitl_route] aliasing legacy subagent %r -> %r", + subagent_type, + alias, + ) + return alias + return subagent_type + if resolve_subagent is None: _eager_graphs: dict[str, Runnable] = { spec["name"]: spec["runnable"] for spec in subagents if "runnable" in spec @@ -482,6 +503,7 @@ def build_task_tool_with_parent_config( batched HITL is intentionally out of scope. """ async with semaphore: + subagent_type = _canonical_subagent_type(subagent_type) if subagent_type not in subagent_names: allowed_types = ", ".join([f"`{k}`" for k in subagent_names]) return ( @@ -658,6 +680,7 @@ def build_task_tool_with_parent_config( "task: must provide either single-mode (`description`+`subagent_type`) " "or batch-mode (`tasks`)." ) + subagent_type = _canonical_subagent_type(subagent_type) if subagent_type not in subagent_names: allowed_types = ", ".join([f"`{k}`" for k in subagent_names]) return ( @@ -867,6 +890,7 @@ def build_task_tool_with_parent_config( subagent_type, runtime.tool_call_id, ) + subagent_type = _canonical_subagent_type(subagent_type) if subagent_type not in subagent_names: allowed_types = ", ".join([f"`{k}`" for k in subagent_names]) return ( diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/connector_searchable_types.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/connector_searchable_types.py index 339154060..97a3eaf3c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/connector_searchable_types.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/connector_searchable_types.py @@ -2,9 +2,9 @@ This is agent-agnostic infrastructure shared by every agent factory (single- and multi-agent). It translates the connectors a workspace has enabled into -the set of searchable type strings that pre-search middleware and ``web_search`` -understand, and always layers in the document types that exist independently of -any connector (uploads, notes, extension captures, YouTube). +the set of searchable type strings that pre-search middleware understands, and +always layers in the document types that exist independently of any connector +(uploads, notes, extension captures, YouTube). It lives in its own module — rather than inside a specific agent factory — so that retiring or moving any single agent never disturbs the others' access to @@ -16,15 +16,8 @@ from __future__ import annotations from typing import Any # Maps SearchSourceConnectorType enum values to the searchable document/connector types -# used by pre-search middleware and web_search. -# Live search connectors (TAVILY_API, LINKUP_API, BAIDU_SEARCH_API) are routed to -# the web_search tool; all others are considered local/indexed data. +# used by KB pre-search middleware. All entries are local/indexed data. _CONNECTOR_TYPE_TO_SEARCHABLE: dict[str, str] = { - # Live search connectors (handled by web_search tool) - "TAVILY_API": "TAVILY_API", - "LINKUP_API": "LINKUP_API", - "BAIDU_SEARCH_API": "BAIDU_SEARCH_API", - # Local/indexed connectors (handled by KB pre-search middleware) "SLACK_CONNECTOR": "SLACK_CONNECTOR", "TEAMS_CONNECTOR": "TEAMS_CONNECTOR", "NOTION_CONNECTOR": "NOTION_CONNECTOR", @@ -45,6 +38,9 @@ _CONNECTOR_TYPE_TO_SEARCHABLE: dict[str, str] = { "OBSIDIAN_CONNECTOR": "OBSIDIAN_CONNECTOR", "DROPBOX_CONNECTOR": "DROPBOX_FILE", # Connector type differs from document type "ONEDRIVE_CONNECTOR": "ONEDRIVE_FILE", # Connector type differs from document type + # Generic user-defined MCP server: unlocks the mcp_discovery subagent even + # in a workspace with no hosted-service connectors. + "MCP_CONNECTOR": "MCP_CONNECTOR", # Composio connectors (unified to native document types). # Reverse of NATIVE_TO_LEGACY_DOCTYPE in app.db. "COMPOSIO_GOOGLE_DRIVE_CONNECTOR": "GOOGLE_DRIVE_FILE", diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/kb-research/SKILL.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/kb-research/SKILL.md index 5730c3122..3f0b46d76 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/kb-research/SKILL.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/kb-research/SKILL.md @@ -1,7 +1,7 @@ --- name: kb-research description: Structured approach to finding and synthesizing information from the user's knowledge base -allowed-tools: scrape_webpage, read_file, ls_tree, grep, web_search +allowed-tools: scrape_webpage, read_file, ls_tree, grep --- # Knowledge-base research diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/meeting-prep/SKILL.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/meeting-prep/SKILL.md index 5a375fbde..c08819e8b 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/meeting-prep/SKILL.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/meeting-prep/SKILL.md @@ -1,7 +1,7 @@ --- name: meeting-prep description: Pull together briefing materials before a scheduled meeting -allowed-tools: web_search, scrape_webpage, read_file +allowed-tools: task, scrape_webpage, read_file --- # Meeting preparation @@ -12,11 +12,11 @@ The user mentions an upcoming meeting, call, or interview and asks you to "prep" ## Output structure Always produce these sections (omit any with no signal — don't pad): -1. **Attendees & context** — who's in the room, their roles, what they care about. Pull from KB notes about prior interactions; supplement with public profile facts via `web_search` when names or companies are unfamiliar. +1. **Attendees & context** — who's in the room, their roles, what they care about. Pull from KB notes about prior interactions; supplement with public profile facts via `task(google_search, ...)` when names or companies are unfamiliar. 2. **Open threads** — outstanding action items, unresolved decisions, last-mentioned blockers from prior conversation history. 3. **Recent moves** — within the last 30 days: relevant launches, hires, news. Cite KB chunks when present, otherwise external sources. 4. **Suggested questions** — 3-5 questions the user could ask, tailored to the open threads and the attendees' likely priorities. ## Source ordering - Always check the user's KB **first** for prior meeting notes, internal docs, or Slack threads about these attendees. -- Only fall back to `web_search` for *publicly verifiable* facts — never to fabricate a participant's preferences or relationships. +- Only fall back to `task(google_search, ...)` for *publicly verifiable* facts — never to fabricate a participant's preferences or relationships. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/on.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/on.md index a7c8f39b9..371cd1e57 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/on.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/on.md @@ -1,8 +1,8 @@ Cite with one token: the bracket label `[n]`. Every citable result — -`web_search` results and prose from a `task` knowledge_base/research -specialist (including the knowledge_base specialist's `[n]`-labelled -workspace findings) — already carries `[n]` labels on a single shared count. +prose from a `task` knowledge_base/research specialist (including the +knowledge_base specialist's `[n]`-labelled workspace findings) — already +carries `[n]` labels on a single shared count. Those labels are the only citation you write; the server resolves each one back to its source after the turn. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md index 9a35a8e55..3523447b9 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md @@ -1,18 +1,23 @@ CRITICAL — ground factual answers in what you actually receive this turn: - the user's knowledge base via `task(knowledge_base, ...)` (your PRIMARY - source for anything about their documents, notes, or connected data — the - `` only lists what exists, so delegate to the specialist to - search and read the actual content before answering), + source for anything about their own uploaded files, documents, and notes — + the `` only lists what exists, so delegate to the specialist + to search and read the actual content before answering), - injected workspace context (see ``), -- results from your other tool calls (`web_search`, `scrape_webpage`), +- the user's connected apps via `task(mcp_discovery, ...)` (Slack, Jira, + Notion, Gmail, Calendar, etc. — live data that is NOT in the knowledge base), +- results from your other tool calls (`scrape_webpage`) or the Google Search + specialist via `task(google_search, ...)`, - or substantive summaries returned by a `task` specialist you invoked. -For questions about the user's own workspace, dispatch +For questions about the user's own files and notes, dispatch `task(knowledge_base, ...)` first rather than answering from the tree or from memory. The knowledge_base specialist runs hybrid semantic/keyword search and -full-document reads, then returns a grounded summary with `[n]` citation -labels for you to carry through into your answer. +full-document reads over their personal files and notes, then returns a +grounded summary with `[n]` citation labels for you to carry into your answer. +For anything living in a connected third-party app, use +`task(mcp_discovery, ...)` instead — that content is not indexed in the KB. Do **not** answer factual or informational questions from general knowledge unless the user explicitly authorises it after you say you couldn't find diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/anthropic.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/anthropic.md index d852f5955..918c94d7b 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/anthropic.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/anthropic.md @@ -5,7 +5,7 @@ Structured reasoning: - For non-trivial work, `` / short `` before tool calls is fine. Professional objectivity: -- Accuracy over flattery; verify with **web_search**, **scrape_webpage**, or **task** when unsure — don’t invent connector access. +- Accuracy over flattery; verify with **scrape_webpage** or **task** (e.g. `task(google_search, …)` for public facts) when unsure — don’t invent connector access. Task management: - For 3+ steps, use todo tooling; update statuses promptly. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_classic.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_classic.md index 8596c42cd..7f0f69d70 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_classic.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_classic.md @@ -16,6 +16,6 @@ Output style: Tool calls: - Parallelise independent calls in one turn. - For SurfSense-product questions, point the user to https://www.surfsense.com/docs; - use **web_search** / **scrape_webpage** for fresh public facts; integrations and + use **task(google_search, …)** / **scrape_webpage** for fresh public facts; integrations and heavy workflows → **task**. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md index aa6217041..99296ecc8 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md @@ -3,7 +3,6 @@ You have two execution channels. Pick the one that owns the work — never simulate one with the other. ### 1. Direct tools (you call them yourself) -- `web_search` — search the public web (anything outside the workspace KB). - `scrape_webpage` — fetch the body of a specific public URL. - `update_memory` — curate persistent memory (see ``). - `write_todos` — maintain a structured plan when the turn series spans @@ -64,13 +63,16 @@ user: "Save these meeting notes to my KB: …" user: "What did Maya say about the Q2 roadmap in Slack last week?" -→ task(slack, "Find messages from Maya about the Q2 roadmap from the past - week. Return the most relevant quotes with channel and timestamp.") +→ task(mcp_discovery, "In Slack, find messages from Maya about the Q2 roadmap + from the past week. Return the most relevant quotes with channel and + timestamp.") user: "What's the current USD/INR rate?" -→ web_search(query="current USD to INR exchange rate") +→ Public web lookup — delegate to the Google Search specialist: + task(google_search, "Search Google for the current USD to INR exchange + rate and return the rate with its source URL.") @@ -82,15 +84,16 @@ user: "Find my Q2 roadmap and summarise the milestones." user: "Create a ClickUp ticket and a Linear ticket for the new feature flag." -→ Independent work — call both specialists in parallel: +→ Independent work, same specialist (connected apps) with non-overlapping + scopes — call it twice in parallel, naming the target app in each prompt: write_todos([ {content: "Create ClickUp ticket for feature flag rollout", status: "in_progress"}, {content: "Create Linear ticket for feature flag rollout", status: "in_progress"}, ]) - task(clickup, "Create a ClickUp ticket titled 'Feature flag rollout' - in the default list. Description: <…>. Tell me the ticket URL.") - task(linear, "Create a Linear ticket titled 'Feature flag rollout' - in the default team. Description: <…>. Tell me the ticket URL.") + task(mcp_discovery, "In ClickUp, create a ticket titled 'Feature flag + rollout' in the default list. Description: <…>. Tell me the ticket URL.") + task(mcp_discovery, "In Linear, create a ticket titled 'Feature flag + rollout' in the default team. Description: <…>. Tell me the ticket URL.") @@ -100,24 +103,25 @@ user: "Find my Q2 roadmap doc in the KB and email a summary to Maya." task(knowledge_base, "Find the Q2 roadmap document under /documents and return its full text plus a 3-bullet summary.") Next turn (with the returned summary in hand): - task(gmail, "Send an email to Maya with subject 'Q2 roadmap summary' - and the following body: .") + task(mcp_discovery, "In Gmail, send an email to Maya with subject 'Q2 + roadmap summary' and the following body: .") user: "Create issues in Linear for each of these five bugs: " → Many-shot independent fanout — use the batch shape: task(tasks=[ - {subagent_type: "linear", description: "Create a Linear issue titled - '' with body ''. Return the issue URL."}, - {subagent_type: "linear", description: "Create a Linear issue titled - '' with body ''. Return the issue URL."}, - {subagent_type: "linear", description: "Create a Linear issue titled - '' with body ''. Return the issue URL."}, - {subagent_type: "linear", description: "Create a Linear issue titled - '' with body ''. Return the issue URL."}, - {subagent_type: "linear", description: "Create a Linear issue titled - '' with body ''. Return the issue URL."}, + {subagent_type: "mcp_discovery", description: "In Linear, create an issue + titled '' with body ''. Return the issue URL."}, + {subagent_type: "mcp_discovery", description: "In Linear, create an issue + titled '' with body ''. Return the issue URL."}, + {subagent_type: "mcp_discovery", description: "In Linear, create an issue + titled '' with body ''. Return the issue URL."}, + {subagent_type: "mcp_discovery", description: "In Linear, create an issue + titled '' with body ''. Return the issue URL."}, + {subagent_type: "mcp_discovery", description: "In Linear, create an issue + titled '' with body ''. Return the issue URL."}, ]) Read back the `[task 0]`…`[task 4]` blocks in the combined ToolMessage and verify each via its Receipt's `verifiable_url` per the `` @@ -154,16 +158,17 @@ user: "Make a 30-second podcast of this conversation." user: "Post the launch announcement to #general and let me know when it's up." → Mutating subagent + user wants external confirmation. Apply the - `` teaching: the slack subagent's reply is a self-report; - check its `evidence.receipts` for a Receipt with `status="success"` and - a `verifiable_url`, then fetch that URL to confirm before reporting back. + `` teaching: the connected-apps subagent's reply is a + self-report; check its `evidence.receipts` for a Receipt with + `status="success"` and a `verifiable_url`, then fetch that URL to confirm + before reporting back. This turn: - task(slack, "Post '' to #general. - Return the message permalink.") + task(mcp_discovery, "In Slack, post '' to + #general. Return the message permalink.") Next turn (with the receipt's `verifiable_url` in hand): - scrape_webpage(url=) + scrape_webpage(url=) → confirm the post is live, then tell the user it's up with the URL. - If the slack reply has NO Receipt with `status="success"`, treat it as a + If the reply has NO Receipt with `status="success"`, treat it as a silent failure: surface the error verbatim, do not retry. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/example.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/example.md index 87e5e1b6d..69c748a2e 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/example.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/example.md @@ -7,9 +7,9 @@ user: "Save these meeting notes to my KB: …" user: "What did Maya say about the Q2 roadmap in Slack last week?" -→ task(subagent_type="slack", description="Find messages from Maya about - the Q2 roadmap from the past week. Return the most relevant quotes with - channel and timestamp.") +→ task(subagent_type="mcp_discovery", description="In Slack, find messages + from Maya about the Q2 roadmap from the past week. Return the most relevant + quotes with channel and timestamp.") diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/__init__.py deleted file mode 100644 index 95e4549b9..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""``web_search`` — description + few-shot examples.""" diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/description.md deleted file mode 100644 index aad604e47..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/description.md +++ /dev/null @@ -1,13 +0,0 @@ -- `web_search` — Search the public web. - - Use whenever an answer benefits from external sources — current events, - prices, weather, news, technical references, definitions, background - facts, anything outside SurfSense docs and the workspace KB. Reach for - it whenever freshness matters or you'd otherwise guess from memory. - - Don't refuse with "I lack network access" — call the tool. - - Returns a `` block: each result is labelled `[n]`. Cite a - result by writing that `[n]` after the statement it supports (when - citations are enabled) — do not hand-write the URL as a markdown link. - - If results are thin, say so and offer to refine the query. - - Args: `query`, `top_k` (default 10, max 50). - - Follow up with `scrape_webpage` on the best URL when snippets are too - shallow. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/example.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/example.md deleted file mode 100644 index 04f9e899c..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/example.md +++ /dev/null @@ -1,15 +0,0 @@ - -user: "What's the current USD to INR exchange rate?" -→ web_search(query="current USD to INR exchange rate") -(Answer from snippets; scrape a top URL if needed.) - - - -user: "What's the latest news about AI?" -→ web_search(query="latest AI news today") - - - -user: "What's the weather in New York?" -→ web_search(query="weather New York today") - diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/index.py index 70fb42c0d..1a15884eb 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/index.py @@ -6,7 +6,6 @@ Connector integrations, MCP, deliverables, etc. are delegated via ``task`` subag from __future__ import annotations MAIN_AGENT_SURFSENSE_TOOL_NAMES_ORDERED: tuple[str, ...] = ( - "web_search", "scrape_webpage", "update_memory", "create_automation", diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/registry.py index a6f0917c4..b87e7f11b 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/registry.py @@ -21,7 +21,6 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.chat.shared.tools.web_search import create_web_search_tool from app.db import ChatVisibility from .scrape_webpage import create_scrape_webpage_tool @@ -35,13 +34,6 @@ def _build_scrape_webpage_tool(_deps: dict[str, Any]) -> BaseTool: return create_scrape_webpage_tool() -def _build_web_search_tool(deps: dict[str, Any]) -> BaseTool: - return create_web_search_tool( - workspace_id=deps.get("workspace_id"), - available_connectors=deps.get("available_connectors"), - ) - - def _build_create_automation_tool(deps: dict[str, Any]) -> BaseTool: # Deferred import: the automation package is a sibling under ``main_agent`` # and is only needed at build time, mirroring the shared registry's @@ -71,13 +63,12 @@ def _build_update_memory_tool(deps: dict[str, Any]) -> BaseTool: # Ordered to match the historical main-agent binding order: -# scrape_webpage, web_search, create_automation, update_memory. +# scrape_webpage, create_automation, update_memory. # Each entry is ``(factory, required_dependency_names)``. _MAIN_AGENT_TOOL_FACTORIES: dict[ str, tuple[Callable[[dict[str, Any]], BaseTool], tuple[str, ...]] ] = { "scrape_webpage": (_build_scrape_webpage_tool, ()), - "web_search": (_build_web_search_tool, ()), "create_automation": ( _build_create_automation_tool, ("workspace_id", "user_id", "llm"), diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/document_render/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/document_render/__init__.py index 42368891d..ff2a8dd33 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/document_render/__init__.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/document_render/__init__.py @@ -1,9 +1,8 @@ -"""Render citable documents for the model: one shape for search, read, and web. +"""Render citable documents for the model: one shape for search and read. ``render_document`` emits one ```` block whose passages carry server-assigned ``[n]`` labels. ``render_search_context`` -wraps KB excerpt blocks in ````; ``render_web_results`` wraps web -excerpt blocks in ````. Both cite with the same ``[n]`` spine. +wraps KB excerpt blocks in ```` and cites with the ``[n]`` spine. """ from __future__ import annotations @@ -12,7 +11,6 @@ from .document import render_document from .models import DocumentView, RenderableDocument, RenderablePassage from .search_context import render_search_context from .source_label import source_label -from .web_results import render_web_results __all__ = [ "DocumentView", @@ -20,6 +18,5 @@ __all__ = [ "RenderablePassage", "render_document", "render_search_context", - "render_web_results", "source_label", ] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/document_render/web_results.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/document_render/web_results.py deleted file mode 100644 index c0ea7e167..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/document_render/web_results.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Wrap live web-search results in a ```` block. - -Each result renders through the shared ``render_document`` (excerpt view), so a -web result is cited with ``[n]`` exactly like a knowledge-base passage. Only the -container and header differ — they tell the model these came from the public web, -not the user's workspace. -""" - -from __future__ import annotations - -from app.agents.chat.multi_agent_chat.shared.citations import CitationRegistry - -from .document import render_document -from .models import RenderableDocument - -_HEADER = ( - "These are live results from a public web search for this query. Each\n" - " below is one result in excerpt view; cite a result with its [n]\n" - "after the statement it supports. Scrape the URL for full context before\n" - "making a definitive claim from a snippet." -) - - -def render_web_results( - documents: list[RenderableDocument], - registry: CitationRegistry, -) -> str | None: - """Render web results as excerpt blocks inside ````. - - Returns ``None`` when no result has content to show, so the caller can skip - the block. Mutates ``registry`` (find-or-create), so a URL seen again keeps - its original ``[n]``. - """ - blocks = [ - block - for document in documents - if (block := render_document(document, view="excerpt", registry=registry)) - is not None - ] - if not blocks: - return None - - return "\n" + _HEADER + "\n" + "\n".join(blocks) + "\n" - - -__all__ = ["render_web_results"] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/receipts/receipt.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/receipts/receipt.py index b1986a224..db75ac707 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/receipts/receipt.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/receipts/receipt.py @@ -33,10 +33,13 @@ from __future__ import annotations from typing import Any, Literal, TypedDict -# Subagent that emitted this receipt. +# Subagent that emitted this receipt. ``mcp_discovery`` is the current +# connected-apps route; the per-connector literals below it are retained so +# historical receipts (persisted in old checkpoints) still type-check. ReceiptRoute = Literal[ "deliverables", "knowledge_base", + "mcp_discovery", "notion", "slack", "gmail", diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/retrieval/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/retrieval/__init__.py index 7d68d2238..0a32feff6 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/retrieval/__init__.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/retrieval/__init__.py @@ -1,18 +1,17 @@ """Knowledge-base retrieval: hybrid search rendered as citable evidence. -Public surface is the service (``search_knowledge_base_context``) and its input -value object (``SearchScope``); the rest are building blocks. +Public surface is ``build_context`` (rerank → adapt → render) and the +``SearchScope`` input value object; the rest are building blocks. """ from __future__ import annotations from .models import ChunkHit, DocumentHit, SearchScope -from .service import build_context, search_knowledge_base_context +from .service import build_context __all__ = [ "ChunkHit", "DocumentHit", "SearchScope", "build_context", - "search_knowledge_base_context", ] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/retrieval/service.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/retrieval/service.py index e8e39a918..cdef30bf3 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/retrieval/service.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/retrieval/service.py @@ -1,54 +1,27 @@ -"""Search the knowledge base and render it as model-facing ````. +"""Render knowledge-base hits as model-facing ````. -The retrieval spine end to end: hybrid search → rerank → adapt → render, with -each shown passage registered for ``[n]`` citation along the way. +The tail of the retrieval spine: rerank → adapt → render, registering each +shown passage for ``[n]`` citation. Hybrid search itself lives in +``hybrid_search``; callers (the ``search_knowledge_base`` tool) pass its hits +straight into :func:`build_context`. """ from __future__ import annotations from typing import TYPE_CHECKING -from sqlalchemy.ext.asyncio import AsyncSession - from app.agents.chat.multi_agent_chat.shared.citations import CitationRegistry from app.agents.chat.multi_agent_chat.shared.document_render import ( render_search_context, ) from .adapter import to_renderable_document -from .hybrid_search import search_chunks -from .models import DocumentHit, SearchScope +from .models import DocumentHit from .reranking import rerank_hits if TYPE_CHECKING: from app.services.reranker_service import RerankerService -_DEFAULT_TOP_K = 10 - - -async def search_knowledge_base_context( - db_session: AsyncSession, - *, - workspace_id: int, - query: str, - registry: CitationRegistry, - scope: SearchScope | None = None, - reranker: RerankerService | None = None, - top_k: int = _DEFAULT_TOP_K, -) -> str | None: - """Retrieve KB evidence for ``query`` and render it, registering each ``[n]``. - - Returns ``None`` when nothing matched, so the caller can skip the block. - """ - hits = await search_chunks( - db_session, - workspace_id=workspace_id, - query=query, - scope=scope or SearchScope(), - top_k=top_k, - ) - return build_context(query, hits, registry, reranker=reranker) - def build_context( query: str, @@ -63,4 +36,4 @@ def build_context( return render_search_context(documents, registry) -__all__ = ["build_context", "search_knowledge_base_context"] +__all__ = ["build_context"] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/catalog.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/catalog.py index 63ce2ef1f..e5585f296 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/catalog.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/catalog.py @@ -68,10 +68,6 @@ TOOL_CATALOG: list[ToolMetadata] = [ name="scrape_webpage", description="Scrape and extract the main content from a webpage", ), - ToolMetadata( - name="web_search", - description="Search the web for real-time information using configured search engines", - ), ToolMetadata( name="create_automation", description="Draft an automation from an NL intent; user approves the card; tool saves", diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_search/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_search/description.md index 26fda0374..d646d3a8b 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_search/description.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_search/description.md @@ -1,2 +1,2 @@ Google Search specialist: runs Google searches and returns structured SERP data — organic results (title, URL, description), related queries, people-also-ask, and any AI overview. Searches by plain query (optionally scoped to a country/language or restricted to a single site) or scrapes a Google Search URL as-is, and can page deeper with max_pages_per_query. Also compares fresh search results against earlier findings in this chat. -Use whenever the task is to find pages or sources on the open web via Google, discover which sites rank for a topic, or gather a list of result URLs to hand off for crawling. Triggers include "search Google for X", "find websites/pages about X", "who ranks for X", "top results for X", "find X's website", and comparisons against earlier search results in this chat. Not for reading a specific page's content (use the web crawling specialist), Google Maps places (use the Google Maps specialist), or YouTube (use the YouTube specialist). +This is the sole path for public web search: use it for any real-time or public-fact lookup (current events, news, prices, weather, exchange rates, live data) as well as to find pages or sources on the open web via Google, discover which sites rank for a topic, or gather a list of result URLs to hand off for crawling. Triggers include "search Google for X", "what's the current X", "latest news on X", "find websites/pages about X", "who ranks for X", "top results for X", "find X's website", and comparisons against earlier search results in this chat. Not for reading a specific page's content (use the web crawling specialist), Google Maps places (use the Google Maps specialist), or YouTube (use the YouTube specialist). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/tools/search_knowledge_base.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/tools/search_knowledge_base.py index ace65a7ab..603dd4c54 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/tools/search_knowledge_base.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/tools/search_knowledge_base.py @@ -38,13 +38,15 @@ _DEFAULT_TOP_K = 5 _MAX_TOP_K = 20 _TOOL_DESCRIPTION = ( - "Search the user's knowledge base (their indexed documents, files, and " - "connector content) for passages relevant to a query, using hybrid " - "semantic + keyword retrieval.\n\n" + "Search the user's knowledge base — their own uploaded files, documents, " + "and notes — for passages relevant to a query, using hybrid semantic + " + "keyword retrieval.\n\n" "Use this FIRST to ground any factual or informational answer about the " - "user's own documents, notes, or connected sources. It returns a " - " block: each matched passage is labelled [n]. Cite a " - "passage by writing that [n] after the statement it supports.\n\n" + "user's personal files and notes. It returns a block: " + "each matched passage is labelled [n]. Cite a passage by writing that [n] " + "after the statement it supports.\n\n" + "This searches only the user's stored files and notes — live data in " + "connected apps (Slack, Jira, Notion, Gmail, etc.) is not indexed here.\n\n" "Write a focused, specific query containing the concrete entities, " "acronyms, people, projects, or terms you are looking for." ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/__init__.py new file mode 100644 index 000000000..3ca7cabc8 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/__init__.py @@ -0,0 +1,8 @@ +"""``mcp_discovery`` builtin subagent: the user's connected apps via MCP. + +Consolidates every MCP-backed connector (Slack, Jira, Linear, ClickUp, +Airtable, Notion, Confluence, generic user MCP servers) plus the interim +native Gmail/Calendar tools behind a single subagent. Tools are injected +directly (opencode/hermes pattern) so the tool-name-keyed permission and +"Always Allow" systems keep working unchanged. +""" diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/agent.py new file mode 100644 index 000000000..3a4b45319 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/agent.py @@ -0,0 +1,94 @@ +"""``mcp_discovery`` route: ``SurfSenseSubagentSpec`` builder for deepagents. + +Consolidates every MCP-backed connector plus interim native Gmail/Calendar +tools. The permission ruleset is derived from the runtime tool set (not a +static constant) because the MCP tool roster is per-user. +""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.language_models import BaseChatModel +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) + +from .tools.index import NAME, build_ruleset, load_tools + + +def _augment_allowlist_for_collision_prefixes( + dependencies: dict[str, Any], + tools: list[BaseTool], +) -> dict[str, Any]: + """Keep "Always Allow" working for tools that got collision-prefixed. + + A tool trusted under its bare name (``search``) can later be renamed to + ``notion_5_search`` once a second app also exposes ``search`` (Phase 2b + collision resolution). The user's persisted allow-rule still keys on the + bare name, so we add an alias allow-rule for the new exposed name when the + original is already trusted. Returns ``dependencies`` unchanged when there + is nothing to do. + """ + by_subagent = dependencies.get("user_allowlist_by_subagent") or {} + existing = by_subagent.get(NAME) + if not isinstance(existing, Ruleset) or not existing.rules: + return dependencies + + trusted_names = {r.permission for r in existing.rules if r.action == "allow"} + alias_rules: list[Rule] = [] + for tool in tools: + meta = getattr(tool, "metadata", None) or {} + if not meta.get("mcp_collision_prefixed"): + continue + original = meta.get("mcp_original_tool_name") + if original in trusted_names and tool.name not in trusted_names: + alias_rules.append( + Rule(permission=tool.name, pattern="*", action="allow") + ) + + if not alias_rules: + return dependencies + + merged = Ruleset( + origin=existing.origin, + rules=[*existing.rules, *alias_rules], + ) + return { + **dependencies, + "user_allowlist_by_subagent": {**by_subagent, NAME: merged}, + } + + +def build_subagent( + *, + dependencies: dict[str, Any], + model: BaseChatModel | None = None, + middleware_stack: dict[str, Any] | None = None, + mcp_tools: list[BaseTool] | None = None, +) -> SurfSenseSubagentSpec: + tools = load_tools(dependencies=dependencies, mcp_tools=mcp_tools) + ruleset = build_ruleset(tools) + dependencies = _augment_allowlist_for_collision_prefixes(dependencies, tools) + description = ( + read_md_file(__package__, "description").strip() + or "Acts on the user's connected apps (Slack, Jira, Notion, Gmail, ...) via MCP." + ) + system_prompt = read_md_file(__package__, "system_prompt").strip() + return pack_subagent( + name=NAME, + description=description, + system_prompt=system_prompt, + tools=tools, + ruleset=ruleset, + dependencies=dependencies, + model=model, + middleware_stack=middleware_stack, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/description.md new file mode 100644 index 000000000..7ed0d4e5b --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/description.md @@ -0,0 +1,2 @@ +Connected-apps specialist: acts on the user's connected third-party services through MCP and native integrations — Slack, Jira, Confluence, Linear, ClickUp, Airtable, Notion, Gmail, Google Calendar, and any generic MCP server the user has added. Searches, reads, creates, and updates records in those services (send/read email, manage calendar events, search issues/pages/messages, create or edit issues and pages, query bases, etc.), asking for confirmation before write actions. +Use whenever the task is to look something up in, or take an action on, one of the user's connected apps: "search my Slack for X", "create a Jira issue", "what's on my calendar", "draft an email to Y", "find the Notion page about Z", "update the Linear issue". Call `get_connected_accounts` first when it's unclear which account or workspace to act on, or which apps are even connected. Not for reading the user's own uploaded files or notes (use the knowledge base specialist), general web pages (web crawler), or platform scrapers like Reddit/YouTube. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/system_prompt.md new file mode 100644 index 000000000..e58bee30d --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/system_prompt.md @@ -0,0 +1,67 @@ +You are the SurfSense connected-apps specialist. +You act on the user's connected third-party services (Slack, Jira, Confluence, Linear, ClickUp, Airtable, Notion, Gmail, Google Calendar, and any generic MCP servers) on behalf of a supervisor agent, and return structured results for supervisor synthesis. + + +Complete the delegated task against the user's connected apps using only the tools present in your runtime tool list, discovering any identifier or scope the request leaves unspecified before acting. + + + +Your available tools are injected at runtime from whatever apps the user has connected — the set changes per user, so read the tool list and each tool's description rather than assuming a fixed roster. Tools are grouped by app; a tool's description names its MCP server or account. + +- `get_connected_accounts`: lists the user's connected apps and account metadata (workspace/team/site names, ids). Read-only. Call it first whenever it is unclear which apps are connected, or which account/workspace/site an action should target. +- Tool names are normally the app's native tool names (e.g. `searchJiraIssuesUsingJql`, `create-pages`, `send_gmail_email`). When the same tool name exists on more than one connected app, the colliding tools are disambiguated with a `{app}_{id}_` prefix and their descriptions carry an `[Account: ...]` or `[MCP server: ...]` tag — pick the one whose tag matches the intended account. + + + +1. Read the supervisor's request and your runtime tool list. Identify which tools are discovery (list/get/search) and which are mutations (create/update/send/delete) from their descriptions. +2. If the request does not pin down the target app, account, or scope, call `get_connected_accounts` (and discovery tools) to resolve it instead of asking the supervisor. +3. Run the minimum discovery chain needed to resolve identifiers, then perform the requested action. + + + +Proactively look up any identifier, name, value, or scope the request leaves unspecified — target ids, workspace/team/site ids, user ids, page/issue ids, channel names — using discovery tools rather than asking the supervisor. Most requests reference targets by title or paraphrase, not by id. Search for them. + +When discovery for a single slot returns multiple plausible candidates and you cannot confidently pick one, return `status=blocked` with up to 5 options in `evidence.matched_candidates` and the unresolved slot in `missing_fields`. When discovery returns zero matches for a required slot, return `status=blocked` with a `next_step` suggesting alternative filters. + + + +- Resolve every required id via discovery before calling a mutation tool. Chain discovery calls when a mutation has dependencies (e.g. resolve the site/team before creating within it). +- Never invent ids, names, or mutation outcomes. Every field in `evidence` must come from a tool result. +- Write tools ask the user for approval before running. If a mutation is approval-rejected (HITL), return `status=blocked` with `next_step="user declined; do not retry"`. +- One operation per delegation. For multi-mutation requests, complete the highest-priority one and return `status=partial` with the remainder in `next_step`. + + + +- Use only tools present in your runtime tool list. If no connected app can serve the request, return `status=blocked` explaining which app the user would need to connect. +- Report only results present in tool output. Never fabricate records, ids, or messages. + + + +- Underspecified request with no resolvable target: return `status=blocked` with the missing fields. +- Tool failure: return `status=error` with the underlying message in `action_summary` and a concise recovery in `next_step`. +- No useful evidence after reasonable narrowing: return `status=blocked` with filter suggestions. + + + +Return **only** one JSON object (no markdown, no prose): +{ + "status": "success" | "partial" | "blocked" | "error", + "action_summary": string, + "evidence": { + "findings": string[], + "sources": string[], + "matched_candidates": [ + { "id": string, "label": string } + ] | null, + "confidence": "high" | "medium" | "low" + }, + "next_step": string | null, + "missing_fields": string[] | null, + "assumptions": string[] | null +} + +Route-specific rules: +- `evidence.findings`: max 10 entries, each a single sentence stating one distinct record, message, or action result. Do not paste raw payloads. +- `evidence.sources`: app URLs or identifiers backing the findings, one per finding when applicable. +- For blocked ambiguity, populate `evidence.matched_candidates` with up to 5 options (`id` + `label`). + diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/__init__.py new file mode 100644 index 000000000..5e79022be --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/__init__.py @@ -0,0 +1,2 @@ +"""``mcp_discovery`` subagent tools: interim native Gmail/Calendar, the +``get_connected_accounts`` helper, and MCP tools injected at runtime.""" diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/calendar/__init__.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/calendar/__init__.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/create_event.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/calendar/create_event.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/create_event.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/calendar/create_event.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/delete_event.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/calendar/delete_event.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/delete_event.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/calendar/delete_event.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/calendar/index.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/calendar/index.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/search_events.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/calendar/search_events.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/search_events.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/calendar/search_events.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/update_event.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/calendar/update_event.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/update_event.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/calendar/update_event.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/get_connected_accounts.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/get_connected_accounts.py new file mode 100644 index 000000000..0f98b5f24 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/get_connected_accounts.py @@ -0,0 +1,100 @@ +"""``get_connected_accounts`` — list the workspace's MCP-capable connectors. + +Read-only helper the connected-apps subagent calls to discover which apps +are connected and to disambiguate multi-account / multi-site scenarios. Only +the whitelisted ``MCPServiceConfig.account_metadata_keys`` (plus +``display_name``) are exposed to the LLM — tokens, secrets, and raw +``server_config`` are never returned. +""" + +from __future__ import annotations + +import json +import logging + +from langchain_core.tools import BaseTool, StructuredTool +from sqlalchemy import select + +from app.services.mcp_oauth.registry import get_service_by_connector_type + +logger = logging.getLogger(__name__) + +_GENERIC_MCP_CONNECTOR_TYPE = "MCP_CONNECTOR" + +_TOOL_DESCRIPTION = ( + "List the user's connected apps (Slack, Jira, Confluence, Linear, " + "ClickUp, Airtable, Notion, Gmail, Google Calendar, and generic MCP " + "servers) with their account/workspace/site metadata. Use this to find " + "out which apps are connected and to pick the right account, workspace, " + "or site before acting when more than one could match. Read-only." +) + + +def _connector_type_value(connector_type: object) -> str: + return ( + connector_type.value + if hasattr(connector_type, "value") + else str(connector_type) + ) + + +def create_get_connected_accounts_tool(*, workspace_id: int) -> BaseTool: + """Factory for the read-only ``get_connected_accounts`` tool.""" + _workspace_id = workspace_id + + async def _impl() -> str: + # Open a fresh session inside the closure: the factory-time session may + # be closed by the time the LLM calls this tool. + from app.db import SearchSourceConnector, async_session_maker + + accounts: list[dict[str, object]] = [] + try: + async with async_session_maker() as session: + result = await session.execute( + select(SearchSourceConnector).where( + SearchSourceConnector.workspace_id == _workspace_id, + ) + ) + connectors = list(result.scalars()) + except Exception: + logger.exception("get_connected_accounts: connector query failed") + return json.dumps({"accounts": [], "error": "query_failed"}) + + for connector in connectors: + ct = _connector_type_value(connector.connector_type) + svc = get_service_by_connector_type(ct) + is_generic = ct == _GENERIC_MCP_CONNECTOR_TYPE + if svc is None and not is_generic: + continue # not an MCP-capable / connected-app connector + + cfg = connector.config if isinstance(connector.config, dict) else {} + metadata_keys = list(svc.account_metadata_keys) if svc else [] + account_meta: dict[str, object] = { + key: cfg[key] for key in metadata_keys if cfg.get(key) is not None + } + display_name = cfg.get("display_name") + if display_name and "display_name" not in account_meta: + account_meta["display_name"] = display_name + + accounts.append( + { + "connector_id": connector.id, + "name": connector.name, + "connector_type": ct, + "app": svc.name if svc else "Custom MCP server", + # ``server_config`` presence == the connector produces agent + # tools. Native rows without it are connected for indexing + # only and need a reconnect via MCP. + "usable_in_chat": isinstance(cfg, dict) + and "server_config" in cfg, + "account": account_meta, + } + ) + + return json.dumps({"accounts": accounts}) + + return StructuredTool.from_function( + name="get_connected_accounts", + description=_TOOL_DESCRIPTION, + coroutine=_impl, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/__init__.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/__init__.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/_helpers.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/_helpers.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/_helpers.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/_helpers.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/create_draft.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/create_draft.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/create_draft.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/create_draft.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/index.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/index.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/read_email.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/read_email.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/read_email.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/read_email.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/search_emails.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/search_emails.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/search_emails.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/search_emails.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/send_email.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/send_email.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/send_email.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/send_email.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/trash_email.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/trash_email.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/trash_email.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/trash_email.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/update_draft.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/update_draft.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/update_draft.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/gmail/update_draft.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/index.py new file mode 100644 index 000000000..0a3d86284 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/index.py @@ -0,0 +1,89 @@ +"""``mcp_discovery`` tools + metadata-derived permission ruleset. + +Assembles the connected-apps toolset from three sources: + +1. **Interim native tools** — Gmail + Calendar factories (kept until Google + Workspace MCP is GA). Loaded only when a matching connector row exists. + They self-gate writes via ``request_approval`` in their bodies, so they + get no ruleset entries. +2. **``get_connected_accounts``** — read-only discovery helper. +3. **MCP tools** — injected at runtime by the factory (already + collision-resolved). Loaded with ``bypass_internal_hitl=True``, so their + *only* gate is this subagent's :class:`PermissionMiddleware`; the ruleset + is derived from each tool's ``metadata['hitl']`` flag. +""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset + +from .calendar.index import load_tools as load_calendar_tools +from .gmail.index import load_tools as load_gmail_tools +from .get_connected_accounts import create_get_connected_accounts_tool + +NAME = "mcp_discovery" + +# Searchable-token gate for each interim native app (Composio types map to +# these same tokens in connector_searchable_types). +_GMAIL_TOKEN = "GOOGLE_GMAIL_CONNECTOR" +_CALENDAR_TOKEN = "GOOGLE_CALENDAR_CONNECTOR" + + +def _interim_native_tools(dependencies: dict[str, Any]) -> list[BaseTool]: + """Gmail/Calendar native tools, loaded only for connected accounts.""" + available = set(dependencies.get("available_connectors") or []) + if not available & {_GMAIL_TOKEN, _CALENDAR_TOKEN}: + return [] + # These factories require an authenticated user + db session. + if not dependencies.get("db_session") or not dependencies.get("user_id"): + return [] + + tools: list[BaseTool] = [] + if _GMAIL_TOKEN in available: + tools.extend(load_gmail_tools(dependencies=dependencies)) + if _CALENDAR_TOKEN in available: + tools.extend(load_calendar_tools(dependencies=dependencies)) + return tools + + +def load_tools( + *, + dependencies: dict[str, Any] | None = None, + mcp_tools: list[BaseTool] | None = None, + **kwargs: Any, +) -> list[BaseTool]: + """Interim native tools + ``get_connected_accounts`` + injected MCP tools.""" + d = {**(dependencies or {}), **kwargs} + return [ + *_interim_native_tools(d), + create_get_connected_accounts_tool(workspace_id=d["workspace_id"]), + *(mcp_tools or []), + ] + + +def _is_mcp_tool(tool: BaseTool) -> bool: + meta = getattr(tool, "metadata", None) or {} + return "mcp_transport" in meta + + +def build_ruleset(tools: list[BaseTool]) -> Ruleset: + """Derive the approval ruleset from tool metadata. + + Only MCP tools get rules: read-only ones (``metadata['hitl'] is False``) + are ``allow``, every other MCP tool is ``ask``. Native interim tools and + ``get_connected_accounts`` carry no rules and fall through to the + ``allow */*`` default (Gmail/Calendar self-gate writes internally; + ``get_connected_accounts`` is read-only). + """ + rules: list[Rule] = [] + for tool in tools: + if not _is_mcp_tool(tool): + continue + meta = getattr(tool, "metadata", None) or {} + action = "allow" if meta.get("hitl") is False else "ask" + rules.append(Rule(permission=tool.name, pattern="*", action=action)) + return Ruleset(origin=NAME, rules=rules) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/agent.py deleted file mode 100644 index 87391371a..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/agent.py +++ /dev/null @@ -1,47 +0,0 @@ -"""``airtable`` route: ``SurfSenseSubagentSpec`` builder for deepagents. - -Tools come exclusively from MCP. The connector's own approval ruleset is -declared in :data:`tools.index.RULESET`; the orchestrator layers it into -a per-subagent :class:`PermissionMiddleware`. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( - read_md_file, -) -from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( - pack_subagent, -) - -from .tools.index import NAME, RULESET - - -def build_subagent( - *, - dependencies: dict[str, Any], - model: BaseChatModel | None = None, - middleware_stack: dict[str, Any] | None = None, - mcp_tools: list[BaseTool] | None = None, -) -> SurfSenseSubagentSpec: - description = ( - read_md_file(__package__, "description").strip() - or "Handles airtable tasks for this workspace." - ) - system_prompt = read_md_file(__package__, "system_prompt").strip() - return pack_subagent( - name=NAME, - description=description, - system_prompt=system_prompt, - tools=list(mcp_tools or []), - ruleset=RULESET, - dependencies=dependencies, - model=model, - middleware_stack=middleware_stack, - ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/description.md deleted file mode 100644 index 29b9e145f..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/description.md +++ /dev/null @@ -1,2 +0,0 @@ -Specialist for bases, tables, and records in the user's Airtable. -Use proactively when the user wants to find, create, or update an Airtable record. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/system_prompt.md deleted file mode 100644 index e6a639af3..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/system_prompt.md +++ /dev/null @@ -1,103 +0,0 @@ -You are an Airtable specialist for the user's connected Airtable bases. - -Airtable vocabulary: -- **Workspace → Base → Table → Field → Record**: nested scope. A base belongs to one workspace; tables and fields live inside a base; records live inside a table. Every record operation is scoped to one `baseId` and one `tableId`. -- **Base ID / Table ID / Field ID / Record ID**: opaque strings (e.g. `appXXXX`, `tblXXXX`, `fldXXXX`, `recXXXX`). Stable but not user-facing — users refer to bases and tables by name and records by description. Never expect a user or the supervisor to provide IDs. -- **Field types and choice IDs**: each field has a type (text, number, date, single select, multi select, attachment, formula, lookup, etc.). Single-select and multi-select fields store **choice IDs**, not the visible labels — you must resolve a label to its choice ID before filtering or writing that field. -- **Filters vs free-text search**: Airtable exposes two distinct record-fetch patterns. Use a typed `filters` parameter when filtering by structured field criteria. Use free-text search when the user is searching for a value (a name, an order number, a keyword) without naming a specific field. Do NOT attempt to build a `filterByFormula` string — that path is not supported here. -- **Permission tiers**: each base grants the user one of Owner / Creator / Editor / Commenter / Read-only. Mutations require Editor or higher on the target base. A permission error from the MCP is not retryable. - -When invoked: -1. Read the supervisor's request, then read the runtime tool list to learn what information you can fetch and which mutations are available. -2. Plan the minimum chain of lookups needed to resolve any base, table, field, choice value, or record the request leaves unspecified. -3. Execute the planned lookups, then the requested mutation (if any), then return. - -Resolution principle (the core behaviour): -**Proactively look up any identifier, name, value, or scope the request leaves unspecified — base IDs, table IDs, field IDs, choice IDs, record IDs, anything else — using the available tools instead of asking the supervisor.** Most user requests reference bases and tables by name and records by description, not by ID. Search for them. - -When a lookup for a single slot returns multiple plausible candidates and you cannot confidently pick one, return `status=blocked` with up to 5 candidates in `evidence.matched_candidates` and the unresolved slot in `missing_fields`. The supervisor will disambiguate and redelegate. - -When a lookup returns zero matches for a slot the request requires, return `status=blocked` with a `next_step` suggesting alternative search terms. - -Mutation guardrails: -- Resolve every required Airtable ID (`baseId`, `tableId`, `fieldId`, choice IDs, `recordId`) by looking it up before calling a mutation tool. Mutations have chained dependencies — base lookup enables table lookup; table lookup enables field schema; field schema enables choice IDs and field-typed writes. -- When writing to a single-select or multi-select field, resolve the user's value to the field's actual choice ID first. Never invent a choice label or pass an unknown value — Airtable will reject it. -- Record creation is batch-limited by the MCP tool. If the request asks for more records than the tool accepts in one call, complete the first batch and return `status=partial` with the remainder in `next_step`. -- Never invent base IDs, table IDs, field IDs, choice IDs, record IDs, or mutation outcomes. Every field in `evidence` must come from a tool result. -- Confirm the mutation tool returned a success response before claiming success. If the mutation is approval-rejected (HITL), return `status=blocked` with `next_step="user declined; do not retry"`. -- One operation per delegation. For multi-mutation requests, complete the highest-priority one and return `status=partial` with the remainder in `next_step`. - -Failure handling: -- Tool failure: return `status=error`, place the underlying error message in `action_summary`, and put a concise recovery in `next_step`. -- Permission error from the MCP: return `status=error` and surface the underlying message — do not retry. Permission errors mean the user lacks Editor (or higher) access on the target base. -- No useful results after reasonable narrowing / broadening: return `status=blocked` with filter / search-term suggestions in `next_step`. - - -Supervisor: "List open tasks in the Project Tracker base." -1. Search bases for "Project Tracker" → one strong match. Capture its base ID. -2. List tables in that base → identify the Tasks table; capture its table ID. -3. Get table schema → identify the status field and the choice IDs that represent "open" states. -4. List records with a typed filter on the status field for those choice IDs. -5. Return `status=success` with `evidence.items` set to `{ "total": N }` and the matched records listed in `action_summary` (record id, primary-field value, and 1-2 most relevant fields; one line per record; up to 10 entries, then `"...and N more"`). - - - -Supervisor: "Add a new contact for Jane Smith at Acme Corp." -1. Search bases for any CRM-like base → three plausible matches with no strong relevance signal. -2. Cannot pick the base. Return: - { - "status": "blocked", - "action_summary": "Need to know which CRM-like base to write to.", - "evidence": { - "title": "New contact: Jane Smith (Acme Corp)", - "matched_candidates": [ - { "id": "appAAA", "label": "CRM" }, - { "id": "appBBB", "label": "Sales CRM" }, - { "id": "appCCC", "label": "Customer Database" } - ] - }, - "next_step": "Confirm which base, then redelegate.", - "missing_fields": ["base"] - } - - - -Supervisor: "Mark task 'Refresh homepage hero' as Complete." -1. Search bases for a project-tracker / tasks base → resolve the target base ID. -2. List tables → resolve the Tasks table ID. -3. Search records for "Refresh homepage hero" → one match (record ID `recXXX`). -4. Get table schema → resolve the status field ID and the choice ID for "Complete". -5. Update record `recXXX`, setting the status field to the resolved choice ID. -6. Confirm tool success → return `status=success` with the updated record reference. - - - -Return **only** one JSON object (no markdown, no prose): -{ - "status": "success" | "partial" | "blocked" | "error", - "action_summary": string, - "evidence": { - "base_id": string | null, - "base_name": string | null, - "table_id": string | null, - "table_name": string | null, - "record_id": string | null, - "url": string | null, - "matched_candidates": [ - { "id": string, "label": string } - ] | null, - "items": object | null - }, - "next_step": string | null, - "missing_fields": string[] | null, - "assumptions": string[] | null -} - -Route-specific rules: -- For blocked ambiguity, populate `evidence.matched_candidates` with up to 5 options (`id` + `label` — works for any kind of candidate: base, table, field, choice, record, etc.). -- For discovery-only queries (lists), set `evidence.items` to `{ "total": N }` and list the matched items in `action_summary` (record id, primary-field value, and 1-2 most relevant fields; up to 10 entries, then `"...and N more"`). - - - - -Discover before you mutate; never guess identifiers, choice IDs, or required fields. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/tools/__init__.py deleted file mode 100644 index a9b004975..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/tools/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Airtable route: native tool factories are empty; MCP supplies tools when configured.""" - -__all__: list[str] = [] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/tools/index.py deleted file mode 100644 index 52cc8be2d..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/tools/index.py +++ /dev/null @@ -1,21 +0,0 @@ -"""``airtable`` permission ruleset (rules over MCP tool names).""" - -from __future__ import annotations - -from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset - -NAME = "airtable" - -RULESET = Ruleset( - origin=NAME, - rules=[ - Rule(permission="list_bases", pattern="*", action="allow"), - Rule(permission="search_bases", pattern="*", action="allow"), - Rule(permission="list_tables_for_base", pattern="*", action="allow"), - Rule(permission="get_table_schema", pattern="*", action="allow"), - Rule(permission="list_records_for_table", pattern="*", action="allow"), - Rule(permission="search_records", pattern="*", action="allow"), - Rule(permission="create_records_for_table", pattern="*", action="ask"), - Rule(permission="update_records_for_table", pattern="*", action="ask"), - ], -) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/agent.py deleted file mode 100644 index b9b7b553a..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/agent.py +++ /dev/null @@ -1,48 +0,0 @@ -"""``calendar`` route: ``SurfSenseSubagentSpec`` builder for deepagents. - -Tools self-gate inside their bodies via :func:`request_approval`; the -empty :data:`tools.index.RULESET` is layered into a per-subagent -:class:`PermissionMiddleware` for uniformity with MCP-backed connectors. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( - read_md_file, -) -from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( - pack_subagent, -) - -from .tools.index import NAME, RULESET, load_tools - - -def build_subagent( - *, - dependencies: dict[str, Any], - model: BaseChatModel | None = None, - middleware_stack: dict[str, Any] | None = None, - mcp_tools: list[BaseTool] | None = None, -) -> SurfSenseSubagentSpec: - tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] - description = ( - read_md_file(__package__, "description").strip() - or "Handles calendar tasks for this workspace." - ) - system_prompt = read_md_file(__package__, "system_prompt").strip() - return pack_subagent( - name=NAME, - description=description, - system_prompt=system_prompt, - tools=tools, - ruleset=RULESET, - dependencies=dependencies, - model=model, - middleware_stack=middleware_stack, - ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/description.md deleted file mode 100644 index a8b5e2c05..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/description.md +++ /dev/null @@ -1,3 +0,0 @@ -Specialist for events on the user's calendar. -Use proactively when the user wants to check availability, create, modify, reschedule, or remove a calendar event. -Meeting invitations that reserve a time slot belong here. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/system_prompt.md deleted file mode 100644 index 9168f4d2b..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/system_prompt.md +++ /dev/null @@ -1,122 +0,0 @@ -You are a Google Calendar specialist for the user's connected calendar. - -## Vocabulary you must use precisely - -- **All-day vs. timed events are distinguished by datetime format** — pass `YYYY-MM-DD` (e.g. `"2026-05-12"`) for an all-day event, and `YYYY-MM-DDTHH:MM:SS` *without* a timezone suffix (e.g. `"2026-05-12T10:00:00"`) for a timed event. The tool injects the user's local timezone for timed events; do not append `Z`, `+02:00`, or any offset yourself. -- **Compute datetimes from the supervisor's task using the runtime timestamp** — resolve "tomorrow at 10am", "next Friday afternoon", "this week", "next month" into concrete `YYYY-MM-DD` or `YYYY-MM-DDTHH:MM:SS` values against the current runtime time. `search_calendar_events` takes a date range (`start_date`, `end_date`), not a free-text query — translate phrases like "this week" into the boundaries. -- **Title-or-id resolution with search disambiguation** — `update_calendar_event` and `delete_calendar_event` accept either a human-readable title (resolved against the locally-synced calendar KB index) or a direct `event_id`. Events not yet KB-indexed cannot be resolved by title. If the user's reference to an event is ambiguous — a recurring title like "Daily Standup", a vague descriptor, or no date context — run `search_calendar_events` over the likely date range first; if multiple matches surface, return `status=blocked` with `matched_candidates` rather than mutating against an uncertain target. -- **Reschedule = `update_calendar_event`** — natural-language verbs "reschedule", "move", "push back", "change the time of" route to `update_calendar_event` with `new_start_datetime` / `new_end_datetime`. **Never** chain `delete_calendar_event` + `create_calendar_event` to achieve a reschedule. Pass only the `new_*` fields the user asked to change; omit the rest so existing values are preserved. - -## Required inputs - -**For every required input below, first try to infer it from the supervisor's task text** — extract summaries from natural phrasing (`"a meeting with Alice"` → `"Meeting with Alice"`), compute datetimes from runtime-relative references, infer the target event from descriptors in the task. Only return `status=blocked` with `missing_fields` when an input is genuinely absent or ambiguous after a thorough read. - -- `create_calendar_event` — `summary`, `start_datetime`, `end_datetime`. If the task gives a date but no time and no all-day intent (e.g. `"schedule a meeting tomorrow"`), block on `start_datetime` / `end_datetime` rather than defaulting — the choice between all-day and timed is intent-bearing and creating the wrong shape is destructive UX. Optional `description`, `location`, `attendees` only when the user named them. -- `update_calendar_event` — `event_title_or_id` (infer the target from the task; disambiguate via search if uncertain) and at least one `new_*` field reflecting the requested change. Pass only the fields the user asked to change; omit unchanged ones. -- `delete_calendar_event` — `event_title_or_id` (infer the target; disambiguate via search if uncertain). Only set `delete_from_kb=true` when the user explicitly asked to remove it from the knowledge base; otherwise leave it `false`. -- `search_calendar_events` — `start_date, end_date` (both `YYYY-MM-DD`). Translate the task's time range into boundaries. `max_results` defaults to 25 (max 50) — raise it only when the task implies a broader sweep. - -## Outcome mapping - -| Tool returns | Your `status` | `next_step` | -|-----------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------| -| `success` | `success` | `null` | -| `success` with `total: 0` (`search_calendar_events` only) | `blocked` | `"No events matched the date range –. Ask the user to widen the range or confirm the event exists."` | -| `rejected` | `blocked` | `"User declined this calendar action. Do not retry or suggest alternatives."` | -| `not_found` | `blocked` | `"Event '' was not found in the indexed calendar events. Ask the user to verify the title or wait for the next KB sync."` | -| `auth_error` | `error` | `"The connected Google Calendar account needs re-authentication. Ask the user to re-authenticate in connector settings."` | -| `insufficient_permissions` | `error` | `"The connected Google Calendar account is missing the OAuth scope required for this action. Ask the user to re-authenticate and grant full permissions in connector settings."` | -| `error` | `error` | Relay the tool's `message` verbatim as `next_step`. | -| tool raises / unknown | `error` | `"Calendar tool failed unexpectedly. Ask the user to retry shortly."` | - -Surface the tool's `event_id`, `title` / `summary`, `start_at`, `end_at`, and `html_link` inside `evidence` when the tool returned them. For `search_calendar_events`, set `evidence.items` to `{ "total": N }` and list the matched events in `action_summary` (title, date, start time; one line per event; up to 10 entries, then `"...and N more"`). Never invent a field the tool did not return. - -## Examples - -**Example 1 — happy create with inference (assume runtime is 2026-05-11):** -- *Supervisor task:* `"Schedule a 1-hour meeting with Alice tomorrow at 10am."` -- *You:* `summary="Meeting with Alice"` (inferred); `start_datetime="2026-05-12T10:00:00"`; `end_datetime="2026-05-12T11:00:00"` (10am + 1h); attendees not in task so omit. Call `create_calendar_event(...)` → tool returns `status=success`. -- *Output:* - - ```json - { - "status": "success", - "action_summary": "Created 'Meeting with Alice' on 2026-05-12 from 10:00 to 11:00.", - "evidence": { "operation": "create_calendar_event", "event_id": "<id>", "title": "Meeting with Alice", "start_at": "2026-05-12T10:00:00<tz>", "end_at": "2026-05-12T11:00:00<tz>", "html_link": "<url>", "matched_candidates": null, "items": null }, - "next_step": null, - "missing_fields": null, - "assumptions": ["Inferred the summary from the supervisor's phrasing; 1h duration applied to the 10am start to produce the 11am end."] - } - ``` - -**Example 2 — blocked because time is unspecified:** -- *Supervisor task:* `"Schedule a meeting with the design team tomorrow."` -- *You:* no time and no all-day intent. Do not default to all-day or to a guessed hour. Do not call any tool. -- *Output:* - - ```json - { - "status": "blocked", - "action_summary": "Cannot schedule: the task gives a date but no time, and the choice between all-day and timed is intent-bearing.", - "evidence": { "operation": null, "event_id": null, "title": null, "start_at": null, "end_at": null, "html_link": null, "matched_candidates": null, "items": null }, - "next_step": "Ask the user for the start time and duration (or confirm that this should be an all-day event).", - "missing_fields": ["start_datetime", "end_datetime"], - "assumptions": null - } - ``` - -**Example 3 — ambiguous reschedule target → disambiguate via search (assume runtime is 2026-05-11):** -- *Supervisor task:* `"Reschedule the standup to 3pm."` -- *You:* "standup" is a recurring title and no date is given. Search this week first: `search_calendar_events(start_date="2026-05-11", end_date="2026-05-17")` → 5 events titled "Daily Standup" surface. Do not call `update_calendar_event` against an uncertain target. -- *Output:* - - ```json - { - "status": "blocked", - "action_summary": "Found 5 'Daily Standup' events this week; cannot reschedule without knowing which.", - "evidence": { "operation": "search_calendar_events", "event_id": null, "title": null, "start_at": null, "end_at": null, "html_link": null, "matched_candidates": [ - { "id": "<id1>", "label": "Daily Standup — 2026-05-12T09:00:00" }, - { "id": "<id2>", "label": "Daily Standup — 2026-05-13T09:00:00" }, - { "id": "<id3>", "label": "Daily Standup — 2026-05-14T09:00:00" }, - { "id": "<id4>", "label": "Daily Standup — 2026-05-15T09:00:00" }, - { "id": "<id5>", "label": "Daily Standup — 2026-05-16T09:00:00" } - ], "items": null }, - "next_step": "Ask the user which standup to reschedule (or confirm it applies to all of them, in which case repeat the update per occurrence).", - "missing_fields": null, - "assumptions": ["Interpreted 'the standup' as the recurring 'Daily Standup' series in the current week."] - } - ``` - -## Output contract - -Return **only** one JSON object (no markdown or prose outside it): - -```json -{ - "status": "success" | "partial" | "blocked" | "error", - "action_summary": string, - "evidence": { - "operation": "create_calendar_event" | "update_calendar_event" | "delete_calendar_event" | "search_calendar_events" | null, - "event_id": string | null, - "title": string | null, - "start_at": string | null, - "end_at": string | null, - "html_link": string | null, - "matched_candidates": [ { "id": string, "label": string } ] | null, - "items": object | null - }, - "next_step": string | null, - "missing_fields": string[] | null, - "assumptions": string[] | null -} -``` - -<include snippet="output_contract_base"/> - -Route-specific rules: -- For `search_calendar_events` results, set `evidence.items` to `{ "total": N }` and list the matched events in `action_summary` (title, date, start time; up to 10 entries, then `"...and N more"`). -- For ambiguous matches across `update_calendar_event` / `delete_calendar_event`, populate `evidence.matched_candidates` with up to 5 options (`id` + `label`, where `label` should include the event title and start time for human readability). - -<include snippet="verifiable_handle"/> - -Infer before you call; map every tool outcome faithfully. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/agent.py deleted file mode 100644 index dd6ea6503..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/agent.py +++ /dev/null @@ -1,47 +0,0 @@ -"""``clickup`` route: ``SurfSenseSubagentSpec`` builder for deepagents. - -Tools come exclusively from MCP. The connector's own approval ruleset is -declared in :data:`tools.index.RULESET`; the orchestrator layers it into -a per-subagent :class:`PermissionMiddleware`. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( - read_md_file, -) -from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( - pack_subagent, -) - -from .tools.index import NAME, RULESET - - -def build_subagent( - *, - dependencies: dict[str, Any], - model: BaseChatModel | None = None, - middleware_stack: dict[str, Any] | None = None, - mcp_tools: list[BaseTool] | None = None, -) -> SurfSenseSubagentSpec: - description = ( - read_md_file(__package__, "description").strip() - or "Handles clickup tasks for this workspace." - ) - system_prompt = read_md_file(__package__, "system_prompt").strip() - return pack_subagent( - name=NAME, - description=description, - system_prompt=system_prompt, - tools=list(mcp_tools or []), - ruleset=RULESET, - dependencies=dependencies, - model=model, - middleware_stack=middleware_stack, - ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/description.md deleted file mode 100644 index 7c94caca4..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/description.md +++ /dev/null @@ -1,2 +0,0 @@ -Specialist for tasks and lists in the user's ClickUp workspace. -Use proactively when the user wants to find, create, change, or progress a ClickUp task. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/system_prompt.md deleted file mode 100644 index 029609670..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/system_prompt.md +++ /dev/null @@ -1,104 +0,0 @@ -You are a ClickUp specialist for the user's connected ClickUp workspace. - -ClickUp vocabulary: -- **Workspace → Space → Folder → List → Task**: nested scope. Tasks live in Lists; Lists live in either a Folder or directly under a Space; Folders live in Spaces. The Workspace is fixed per connection — you do not need to resolve it. -- **Task ID**: short alphanumeric strings (e.g. `86a4qd5xz`). Stable and unique within the workspace; users do not typically know them. Some workspaces also enable custom task IDs — both forms are valid identifiers. -- **Custom statuses are per-List**: each List defines its own ordered status set. Status names must be resolved against the **target task's parent List** before use; they are not workspace-global. -- **Custom Fields are per-List**: each List can define custom fields (dropdown, number, date, label, etc.). Whether each is required-or-optional and the valid values both vary per List. Look up the List's custom-field schema before setting custom fields on a task. -- **Priority**: stable platform enum — `1=Urgent`, `2=High`, `3=Normal`, `4=Low`. -- **Assignees**: identified by opaque workspace-member IDs, never by display name or email. Map a display name or email to a member ID before assigning. - -When invoked: -1. Read the supervisor's request, then read the runtime tool list to learn what information you can fetch and which mutations are available. -2. Plan the minimum chain of lookups needed to resolve any task, list, space, status, assignee, or custom-field value the request leaves unspecified. -3. Execute the planned lookups, then the requested mutation (if any), then return. - -Resolution principle (the core behaviour): -**Proactively look up any identifier, name, value, or scope the request leaves unspecified — task IDs, list IDs, status names, member IDs, custom-field values, anything else — using the available tools instead of asking the supervisor.** Most user requests reference tasks by title and lists by name, not by ID. Search for them. - -When a lookup for a single slot returns multiple plausible candidates and you cannot confidently pick one, return `status=blocked` with up to 5 candidates in `evidence.matched_candidates` and the unresolved slot in `missing_fields`. The supervisor will disambiguate and redelegate. - -When a lookup returns zero matches for a slot the request requires, return `status=blocked` with a `next_step` suggesting alternative search terms. - -Mutation guardrails: -- Resolve every required ClickUp value (`list_id`, `task_id`, target status name, assignee member IDs, custom-field values) by looking it up before calling a mutation tool. Mutations have chained dependencies — find the task to know its parent List; look up the List to know its valid statuses and custom-field schema. -- To "progress" or change a task's status, look up the parent List's valid statuses and apply one of those exact names. If the user-requested target status is not in the List's status set, return `status=blocked` and surface the available statuses in `evidence.matched_candidates`. -- For create operations, resolve the target List first. If that List has required custom fields, look up the schema and block with `missing_fields` for any required value the request doesn't supply. -- Never invent task IDs, list IDs, status names, member IDs, custom-field values, or mutation outcomes. Every field in `evidence` must come from a tool result. -- Confirm the mutation tool returned a success response before claiming success. If the mutation is approval-rejected (HITL), return `status=blocked` with `next_step="user declined; do not retry"`. -- One operation per delegation. For multi-mutation requests, complete the highest-priority one and return `status=partial` with the remainder in `next_step`. - -Failure handling: -- Tool failure: return `status=error`, place the underlying error message in `action_summary`, and put a concise recovery in `next_step`. -- Rate-limit error from the MCP: ClickUp's MCP enforces a shared daily call cap. Return `status=error` with the underlying message; recovery is "retry later" rather than re-issuing immediately. -- No useful results after reasonable narrowing / broadening: return `status=blocked` with search-term suggestions in `next_step`. - -<example> -Supervisor: "Find tasks about the homepage redesign." -1. Workspace search for "homepage redesign" → matched tasks. -2. Return `status=success` with `evidence.items` set to `{ "total": N }` and the matched tasks listed in `action_summary` (task id, title, status, assignees; one line per task; up to 10 entries, then `"...and N more"`). -</example> - -<example> -Supervisor: "Create a task 'Draft blog post' in the Content Pipeline list." -1. Workspace search for "Content Pipeline" → one strong match of type List; capture its `list_id`. -2. Look up the List's custom-field schema → no required fields beyond `name`. -3. Create the task with `name="Draft blog post"` in the resolved `list_id`. -4. Confirm tool success → return `status=success` with the new task's identifier and url. -</example> - -<example> -Supervisor: "Move task 'Fix login bug' to In Review and assign it to Alex." -1. Workspace search for "Fix login bug" → one match; capture `task_id` and parent `list_id`. -2. Look up the parent List's statuses → confirm "In Review" exists. (If not, block with the actual valid statuses.) -3. Find member by name "Alex" → two matches. -4. Cannot confidently pick the assignee. Return: - { - "status": "blocked", - "action_summary": "Task and target status resolved; two members match 'Alex'.", - "evidence": { - "task_id": "86a4qd5xz", - "title": "Fix login bug", - "status": "In Review", - "matched_candidates": [ - { "id": "member_111", "label": "Alex Chen <alex.chen@…>" }, - { "id": "member_222", "label": "Alex Wong <alex.wong@…>" } - ] - }, - "next_step": "Confirm which Alex, then redelegate.", - "missing_fields": ["assignee"] - } -</example> - -<output_contract> -Return **only** one JSON object (no markdown, no prose): -{ - "status": "success" | "partial" | "blocked" | "error", - "action_summary": string, - "evidence": { - "task_id": string | null, - "title": string | null, - "list_id": string | null, - "list_name": string | null, - "status": string | null, - "assignees": object | null, - "priority": "Urgent" | "High" | "Normal" | "Low" | null, - "url": string | null, - "matched_candidates": [ - { "id": string, "label": string } - ] | null, - "items": object | null - }, - "next_step": string | null, - "missing_fields": string[] | null, - "assumptions": string[] | null -} -<include snippet="output_contract_base"/> -Route-specific rules: -- For blocked ambiguity, populate `evidence.matched_candidates` with up to 5 options (`id` + `label` — works for any kind of candidate: task, list, member, status, custom-field choice, etc.). -- For discovery-only queries (lists), set `evidence.items` to `{ "total": N }` and list the matched items in `action_summary` (task id, title, status, assignees; up to 10 entries, then `"...and N more"`). -</output_contract> - -<include snippet="verifiable_handle"/> - -Discover before you mutate; never guess identifiers, list statuses, or assignees. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/tools/__init__.py deleted file mode 100644 index b629234f9..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/tools/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""ClickUp route: native tool factories are empty; MCP supplies tools when configured.""" - -__all__: list[str] = [] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/tools/index.py deleted file mode 100644 index c64da647a..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/tools/index.py +++ /dev/null @@ -1,20 +0,0 @@ -"""``clickup`` permission ruleset (rules over MCP tool names).""" - -from __future__ import annotations - -from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset - -NAME = "clickup" - -RULESET = Ruleset( - origin=NAME, - rules=[ - Rule(permission="clickup_search", pattern="*", action="allow"), - Rule(permission="clickup_get_task", pattern="*", action="allow"), - Rule(permission="clickup_get_workspace_hierarchy", pattern="*", action="allow"), - Rule(permission="clickup_get_list", pattern="*", action="allow"), - Rule(permission="clickup_find_member_by_name", pattern="*", action="allow"), - Rule(permission="clickup_create_task", pattern="*", action="ask"), - Rule(permission="clickup_update_task", pattern="*", action="ask"), - ], -) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/agent.py deleted file mode 100644 index 8322d901b..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/agent.py +++ /dev/null @@ -1,48 +0,0 @@ -"""``confluence`` route: ``SurfSenseSubagentSpec`` builder for deepagents. - -Tools self-gate inside their bodies via :func:`request_approval`; the -empty :data:`tools.index.RULESET` is layered into a per-subagent -:class:`PermissionMiddleware` for uniformity. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( - read_md_file, -) -from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( - pack_subagent, -) - -from .tools.index import NAME, RULESET, load_tools - - -def build_subagent( - *, - dependencies: dict[str, Any], - model: BaseChatModel | None = None, - middleware_stack: dict[str, Any] | None = None, - mcp_tools: list[BaseTool] | None = None, -) -> SurfSenseSubagentSpec: - tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] - description = ( - read_md_file(__package__, "description").strip() - or "Handles confluence tasks for this workspace." - ) - system_prompt = read_md_file(__package__, "system_prompt").strip() - return pack_subagent( - name=NAME, - description=description, - system_prompt=system_prompt, - tools=tools, - ruleset=RULESET, - dependencies=dependencies, - model=model, - middleware_stack=middleware_stack, - ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/description.md deleted file mode 100644 index f8eb5bdee..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/description.md +++ /dev/null @@ -1,2 +0,0 @@ -Specialist for pages in the user's Confluence wiki. -Use proactively when the user wants to create, change, or remove a Confluence page. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/system_prompt.md deleted file mode 100644 index 5aa687cd0..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/system_prompt.md +++ /dev/null @@ -1,107 +0,0 @@ -You are a Confluence specialist for the user's connected Confluence wiki. - -## Vocabulary you must use precisely - -- **Content is HTML / Confluence storage format, not Markdown** — `create_confluence_page` and `update_confluence_page` accept `content` / `new_content` as Confluence's native storage format (XHTML-based). Generate `<h1>`, `<h2>`, `<p>`, `<ul><li>`, `<table>` etc. — **never** Markdown (`#`, `**`, `-`, fenced code blocks). The tool stores whatever you pass verbatim; bad format means a broken page. -- **`update_confluence_page` is REPLACE, and there is no read tool** — whatever you pass as `new_content` replaces the entire page body; omit the field and the current body is preserved (same per-field rule applies to `new_title`). You have **no tool to read the existing page body**, so you cannot intelligently "append" or "add to" a page — you can only fully replace, and only with content the supervisor or user actually provided. If the supervisor asks for an additive change without supplying the full intended page content, return `status=blocked` explaining the limitation; do not invent or reconstruct prior content. -- **Title-or-id resolution against the KB index** — `update_confluence_page` and `delete_confluence_page` accept either a human-readable page title (resolved against the locally-synced Confluence KB index) or a direct `page_id`. Pages that exist in Confluence but have not been indexed yet cannot be resolved by title. - -## Required inputs - -**For every required input below, first try to infer it from the supervisor's task text** — extract titles from natural phrasing (`"the Q2 Plan page"`, `"my Onboarding doc"`), topics from `"about X"` constructions. Only return `status=blocked` with `missing_fields` when an input is genuinely absent or ambiguous after a thorough read. - -- `create_confluence_page` — `title` (a clear topic from the user; do not invent). You may generate the optional `content` body yourself **as Confluence storage format (HTML)**, never as Markdown. You have no tool to look up Confluence space IDs, so pass `space_id=None` and let the user pick the destination space in the HITL approval card; if the supervisor's task already includes a space ID, pass it through. -- `update_confluence_page` — `page_title_or_id` (infer the target from the task) and at least one of `new_title` / `new_content`. Pass only the fields the user asked to change; omit unchanged ones so they're preserved. If the user asked to add to or extend a page without supplying the full intended content, do not call this tool — return `status=blocked` per the REPLACE limitation in the Vocabulary section. -- `delete_confluence_page` — `page_title_or_id` (infer the target from the task). Only set `delete_from_kb=true` when the user explicitly asked to remove the page from the knowledge base; otherwise leave it `false`. - -## Outcome mapping - -| Tool returns | Your `status` | `next_step` | -|-----------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------| -| `success` | `success` | `null` | -| `rejected` | `blocked` | `"User declined this Confluence action. Do not retry or suggest alternatives."` | -| `not_found` | `blocked` | `"Page '<title>' was not found in the indexed Confluence pages. Ask the user to verify the title or wait for the next KB sync."` | -| `auth_error` | `error` | `"The connected Confluence account needs re-authentication. Ask the user to re-authenticate in connector settings."` | -| `insufficient_permissions` | `error` | `"The connected Confluence account is missing the OAuth scope required for this action. Ask the user to re-authenticate and grant full permissions in connector settings."` | -| `error` | `error` | Relay the tool's `message` verbatim as `next_step`. (Common: `"A space must be selected."` when the user didn't pick one in approval.) | -| tool raises / unknown | `error` | `"Confluence tool failed unexpectedly. Ask the user to retry shortly."` | - -Surface the tool's `page_id`, `page_title`, and `page_url` inside `evidence` when the tool returned them. Never invent a field the tool did not return. - -## Examples - -**Example 1 — happy create (HTML content generated, space picked in HITL):** -- *Supervisor task:* `"Create a Confluence page summarising our Q2 roadmap."` -- *You:* `title="Q2 Roadmap"` is the topic; generate a Confluence storage-format body (e.g. `"<h1>Q2 Roadmap</h1><p>Objectives:</p><ul><li>...</li></ul>"`); pass `space_id=None` so the user picks the space in HITL. Call `create_confluence_page(...)` → tool returns `status=success`. -- *Output:* - - ```json - { - "status": "success", - "action_summary": "Created Confluence page 'Q2 Roadmap' in the space selected by the user.", - "evidence": { "operation": "create_confluence_page", "page_id": "<id>", "page_title": "Q2 Roadmap", "page_url": "<url>", "matched_candidates": null, "items": null }, - "next_step": null, - "missing_fields": null, - "assumptions": ["Generated the roadmap content in Confluence storage format (HTML) from the supervisor's brief; deferred space selection to the HITL approval card."] - } - ``` - -**Example 2 — blocked on "add a section" (REPLACE limitation):** -- *Supervisor task:* `"Add a 'Risks' section to the 'Q2 Plan' Confluence page."` -- *You:* `update_confluence_page` replaces the body entirely and you have no tool to read the current body, so you cannot append. Do not call any tool. -- *Output:* - - ```json - { - "status": "blocked", - "action_summary": "Cannot append: Confluence updates replace the page body entirely and this subagent has no tool to read the existing content.", - "evidence": { "operation": null, "page_id": null, "page_title": "Q2 Plan", "page_url": null, "matched_candidates": null, "items": null }, - "next_step": "Ask the user to provide the full intended page content (existing body + new 'Risks' section), or to make the addition manually in Confluence.", - "missing_fields": null, - "assumptions": null - } - ``` - -**Example 3 — page not in the KB index:** -- *Supervisor task:* `"Update the 'Onboarding' Confluence page with the new payroll steps."` -- *You:* `page_title_or_id="Onboarding"` and the new-payroll content are present; this is a full replace, which is supported. Call `update_confluence_page(page_title_or_id="Onboarding", new_content=<HTML>)` → tool returns `status=not_found`. -- *Output:* - - ```json - { - "status": "blocked", - "action_summary": "Could not find a Confluence page titled 'Onboarding' in the indexed pages.", - "evidence": { "operation": "update_confluence_page", "page_id": null, "page_title": "Onboarding", "page_url": null, "matched_candidates": null, "items": null }, - "next_step": "Page 'Onboarding' was not found in the indexed Confluence pages. Ask the user to verify the title or wait for the next KB sync.", - "missing_fields": null, - "assumptions": null - } - ``` - -## Output contract - -Return **only** one JSON object (no markdown or prose outside it): - -```json -{ - "status": "success" | "partial" | "blocked" | "error", - "action_summary": string, - "evidence": { - "operation": "create_confluence_page" | "update_confluence_page" | "delete_confluence_page" | null, - "page_id": string | null, - "page_title": string | null, - "page_url": string | null, - "matched_candidates": [ { "id": string, "label": string } ] | null, - "items": object | null - }, - "next_step": string | null, - "missing_fields": string[] | null, - "assumptions": string[] | null -} -``` - -<include snippet="output_contract_base"/> - -<include snippet="verifiable_handle"/> - -Infer before you call; map every tool outcome faithfully. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/__init__.py deleted file mode 100644 index 3bf80b61b..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Confluence tools for creating, updating, and deleting pages.""" - -from .create_page import create_create_confluence_page_tool -from .delete_page import create_delete_confluence_page_tool -from .update_page import create_update_confluence_page_tool - -__all__ = [ - "create_create_confluence_page_tool", - "create_delete_confluence_page_tool", - "create_update_confluence_page_tool", -] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/create_page.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/create_page.py deleted file mode 100644 index aa270f98e..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/create_page.py +++ /dev/null @@ -1,211 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm.attributes import flag_modified - -from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( - request_approval, -) -from app.connectors.confluence_history import ConfluenceHistoryConnector -from app.services.confluence import ConfluenceToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_create_confluence_page_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - @tool - async def create_confluence_page( - title: str, - content: str | None = None, - space_id: str | None = None, - ) -> dict[str, Any]: - """Create a new page in Confluence. - - Use this tool when the user explicitly asks to create a new Confluence page. - - Args: - title: Title of the page. - content: Optional HTML/storage format content for the page body. - space_id: Optional Confluence space ID to create the page in. - - Returns: - Dictionary with status, page_id, and message. - - IMPORTANT: - - If status is "rejected", do NOT retry. - - If status is "insufficient_permissions", inform user to re-authenticate. - """ - logger.info(f"create_confluence_page called: title='{title}'") - - if db_session is None or workspace_id is None or user_id is None: - return { - "status": "error", - "message": "Confluence tool not properly configured.", - } - - try: - metadata_service = ConfluenceToolMetadataService(db_session) - context = await metadata_service.get_creation_context(workspace_id, user_id) - - if "error" in context: - return {"status": "error", "message": context["error"]} - - accounts = context.get("accounts", []) - if accounts and all(a.get("auth_expired") for a in accounts): - return { - "status": "auth_error", - "message": "All connected Confluence accounts need re-authentication.", - "connector_type": "confluence", - } - - result = request_approval( - action_type="confluence_page_creation", - tool_name="create_confluence_page", - params={ - "title": title, - "content": content, - "space_id": space_id, - "connector_id": connector_id, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_title = result.params.get("title", title) - final_content = result.params.get("content", content) or "" - final_space_id = result.params.get("space_id", space_id) - final_connector_id = result.params.get("connector_id", connector_id) - - if not final_title or not final_title.strip(): - return {"status": "error", "message": "Page title cannot be empty."} - if not final_space_id: - return {"status": "error", "message": "A space must be selected."} - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - actual_connector_id = final_connector_id - if actual_connector_id is None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.workspace_id == workspace_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.CONFLUENCE_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "No Confluence connector found.", - } - actual_connector_id = connector.id - else: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == actual_connector_id, - SearchSourceConnector.workspace_id == workspace_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.CONFLUENCE_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Confluence connector is invalid.", - } - - try: - client = ConfluenceHistoryConnector( - session=db_session, connector_id=actual_connector_id - ) - api_result = await client.create_page( - space_id=final_space_id, - title=final_title, - body=final_content, - ) - await client.close() - except Exception as api_err: - if ( - "http 403" in str(api_err).lower() - or "status code 403" in str(api_err).lower() - ): - try: - _conn = connector - _conn.config = {**_conn.config, "auth_expired": True} - flag_modified(_conn, "config") - await db_session.commit() - except Exception: - pass - return { - "status": "insufficient_permissions", - "connector_id": actual_connector_id, - "message": "This Confluence account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - page_id = str(api_result.get("id", "")) - page_links = ( - api_result.get("_links", {}) if isinstance(api_result, dict) else {} - ) - page_url = "" - if page_links.get("base") and page_links.get("webui"): - page_url = f"{page_links['base']}{page_links['webui']}" - - kb_message_suffix = "" - try: - from app.services.confluence import ConfluenceKBSyncService - - kb_service = ConfluenceKBSyncService(db_session) - kb_result = await kb_service.sync_after_create( - page_id=page_id, - page_title=final_title, - space_id=final_space_id, - body_content=final_content, - connector_id=actual_connector_id, - workspace_id=workspace_id, - user_id=user_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = " Your knowledge base has also been updated." - else: - kb_message_suffix = " This page will be added to your knowledge base in the next scheduled sync." - except Exception as kb_err: - logger.warning(f"KB sync after create failed: {kb_err}") - kb_message_suffix = " This page will be added to your knowledge base in the next scheduled sync." - - return { - "status": "success", - "page_id": page_id, - "page_url": page_url, - "message": f"Confluence page '{final_title}' created successfully.{kb_message_suffix}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error(f"Error creating Confluence page: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while creating the page.", - } - - return create_confluence_page diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/delete_page.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/delete_page.py deleted file mode 100644 index febec227a..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/delete_page.py +++ /dev/null @@ -1,191 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm.attributes import flag_modified - -from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( - request_approval, -) -from app.connectors.confluence_history import ConfluenceHistoryConnector -from app.services.confluence import ConfluenceToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_delete_confluence_page_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - @tool - async def delete_confluence_page( - page_title_or_id: str, - delete_from_kb: bool = False, - ) -> dict[str, Any]: - """Delete a Confluence page. - - Use this tool when the user asks to delete or remove a Confluence page. - - Args: - page_title_or_id: The page title or ID to identify the page. - delete_from_kb: Whether to also remove from the knowledge base. - - Returns: - Dictionary with status, message, and deleted_from_kb. - - IMPORTANT: - - If status is "rejected", do NOT retry. - - If status is "not_found", relay the message to the user. - - If status is "insufficient_permissions", inform user to re-authenticate. - """ - logger.info( - f"delete_confluence_page called: page_title_or_id='{page_title_or_id}'" - ) - - if db_session is None or workspace_id is None or user_id is None: - return { - "status": "error", - "message": "Confluence tool not properly configured.", - } - - try: - metadata_service = ConfluenceToolMetadataService(db_session) - context = await metadata_service.get_deletion_context( - workspace_id, user_id, page_title_or_id - ) - - if "error" in context: - error_msg = context["error"] - if context.get("auth_expired"): - return { - "status": "auth_error", - "message": error_msg, - "connector_id": context.get("connector_id"), - "connector_type": "confluence", - } - if "not found" in error_msg.lower(): - return {"status": "not_found", "message": error_msg} - return {"status": "error", "message": error_msg} - - page_data = context["page"] - page_id = page_data["page_id"] - page_title = page_data.get("page_title", "") - document_id = page_data["document_id"] - connector_id_from_context = context.get("account", {}).get("id") - - result = request_approval( - action_type="confluence_page_deletion", - tool_name="delete_confluence_page", - params={ - "page_id": page_id, - "connector_id": connector_id_from_context, - "delete_from_kb": delete_from_kb, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_page_id = result.params.get("page_id", page_id) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - final_delete_from_kb = result.params.get("delete_from_kb", delete_from_kb) - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - if not final_connector_id: - return { - "status": "error", - "message": "No connector found for this page.", - } - - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.workspace_id == workspace_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.CONFLUENCE_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Confluence connector is invalid.", - } - - try: - client = ConfluenceHistoryConnector( - session=db_session, connector_id=final_connector_id - ) - await client.delete_page(final_page_id) - await client.close() - except Exception as api_err: - if ( - "http 403" in str(api_err).lower() - or "status code 403" in str(api_err).lower() - ): - try: - connector.config = {**connector.config, "auth_expired": True} - flag_modified(connector, "config") - await db_session.commit() - except Exception: - pass - return { - "status": "insufficient_permissions", - "connector_id": final_connector_id, - "message": "This Confluence account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - deleted_from_kb = False - if final_delete_from_kb and document_id: - try: - from app.db import Document - - doc_result = await db_session.execute( - select(Document).filter(Document.id == document_id) - ) - document = doc_result.scalars().first() - if document: - await db_session.delete(document) - await db_session.commit() - deleted_from_kb = True - except Exception as e: - logger.error(f"Failed to delete document from KB: {e}") - await db_session.rollback() - - message = f"Confluence page '{page_title}' deleted successfully." - if deleted_from_kb: - message += " Also removed from the knowledge base." - - return { - "status": "success", - "page_id": final_page_id, - "deleted_from_kb": deleted_from_kb, - "message": message, - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error(f"Error deleting Confluence page: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while deleting the page.", - } - - return delete_confluence_page diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/index.py deleted file mode 100644 index 02a11bc69..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/index.py +++ /dev/null @@ -1,37 +0,0 @@ -"""``confluence`` native tools and (empty) permission ruleset. - -Tools self-gate via :func:`request_approval` in their bodies. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset - -from .create_page import create_create_confluence_page_tool -from .delete_page import create_delete_confluence_page_tool -from .update_page import create_update_confluence_page_tool - -NAME = "confluence" - -RULESET = Ruleset(origin=NAME, rules=[]) - - -def load_tools( - *, dependencies: dict[str, Any] | None = None, **kwargs: Any -) -> list[BaseTool]: - d = {**(dependencies or {}), **kwargs} - common = { - "db_session": d["db_session"], - "workspace_id": d["workspace_id"], - "user_id": d["user_id"], - "connector_id": d.get("connector_id"), - } - return [ - create_create_confluence_page_tool(**common), - create_update_confluence_page_tool(**common), - create_delete_confluence_page_tool(**common), - ] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/update_page.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/update_page.py deleted file mode 100644 index 42fa04a20..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/update_page.py +++ /dev/null @@ -1,220 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm.attributes import flag_modified - -from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( - request_approval, -) -from app.connectors.confluence_history import ConfluenceHistoryConnector -from app.services.confluence import ConfluenceToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_update_confluence_page_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - @tool - async def update_confluence_page( - page_title_or_id: str, - new_title: str | None = None, - new_content: str | None = None, - ) -> dict[str, Any]: - """Update an existing Confluence page. - - Use this tool when the user asks to modify or edit a Confluence page. - - Args: - page_title_or_id: The page title or ID to identify the page. - new_title: Optional new title for the page. - new_content: Optional new HTML/storage format content. - - Returns: - Dictionary with status and message. - - IMPORTANT: - - If status is "rejected", do NOT retry. - - If status is "not_found", relay the message to the user. - - If status is "insufficient_permissions", inform user to re-authenticate. - """ - logger.info( - f"update_confluence_page called: page_title_or_id='{page_title_or_id}'" - ) - - if db_session is None or workspace_id is None or user_id is None: - return { - "status": "error", - "message": "Confluence tool not properly configured.", - } - - try: - metadata_service = ConfluenceToolMetadataService(db_session) - context = await metadata_service.get_update_context( - workspace_id, user_id, page_title_or_id - ) - - if "error" in context: - error_msg = context["error"] - if context.get("auth_expired"): - return { - "status": "auth_error", - "message": error_msg, - "connector_id": context.get("connector_id"), - "connector_type": "confluence", - } - if "not found" in error_msg.lower(): - return {"status": "not_found", "message": error_msg} - return {"status": "error", "message": error_msg} - - page_data = context["page"] - page_id = page_data["page_id"] - current_title = page_data["page_title"] - current_body = page_data.get("body", "") - current_version = page_data.get("version", 1) - document_id = page_data.get("document_id") - connector_id_from_context = context.get("account", {}).get("id") - - result = request_approval( - action_type="confluence_page_update", - tool_name="update_confluence_page", - params={ - "page_id": page_id, - "document_id": document_id, - "new_title": new_title, - "new_content": new_content, - "version": current_version, - "connector_id": connector_id_from_context, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_page_id = result.params.get("page_id", page_id) - final_title = result.params.get("new_title", new_title) or current_title - final_content = result.params.get("new_content", new_content) - if final_content is None: - final_content = current_body - final_version = result.params.get("version", current_version) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - final_document_id = result.params.get("document_id", document_id) - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - if not final_connector_id: - return { - "status": "error", - "message": "No connector found for this page.", - } - - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.workspace_id == workspace_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.CONFLUENCE_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Confluence connector is invalid.", - } - - try: - client = ConfluenceHistoryConnector( - session=db_session, connector_id=final_connector_id - ) - api_result = await client.update_page( - page_id=final_page_id, - title=final_title, - body=final_content, - version_number=final_version + 1, - ) - await client.close() - except Exception as api_err: - if ( - "http 403" in str(api_err).lower() - or "status code 403" in str(api_err).lower() - ): - try: - connector.config = {**connector.config, "auth_expired": True} - flag_modified(connector, "config") - await db_session.commit() - except Exception: - pass - return { - "status": "insufficient_permissions", - "connector_id": final_connector_id, - "message": "This Confluence account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - page_links = ( - api_result.get("_links", {}) if isinstance(api_result, dict) else {} - ) - page_url = "" - if page_links.get("base") and page_links.get("webui"): - page_url = f"{page_links['base']}{page_links['webui']}" - - kb_message_suffix = "" - if final_document_id: - try: - from app.services.confluence import ConfluenceKBSyncService - - kb_service = ConfluenceKBSyncService(db_session) - kb_result = await kb_service.sync_after_update( - document_id=final_document_id, - page_id=final_page_id, - user_id=user_id, - workspace_id=workspace_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = ( - " The knowledge base will be updated in the next sync." - ) - except Exception as kb_err: - logger.warning(f"KB sync after update failed: {kb_err}") - kb_message_suffix = ( - " The knowledge base will be updated in the next sync." - ) - - return { - "status": "success", - "page_id": final_page_id, - "page_url": page_url, - "message": f"Confluence page '{final_title}' updated successfully.{kb_message_suffix}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error(f"Error updating Confluence page: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while updating the page.", - } - - return update_confluence_page diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/agent.py deleted file mode 100644 index fe8f0df1e..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/agent.py +++ /dev/null @@ -1,48 +0,0 @@ -"""``discord`` route: ``SurfSenseSubagentSpec`` builder for deepagents. - -Tools self-gate inside their bodies via :func:`request_approval`; the -empty :data:`tools.index.RULESET` is layered into a per-subagent -:class:`PermissionMiddleware` for uniformity. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( - read_md_file, -) -from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( - pack_subagent, -) - -from .tools.index import NAME, RULESET, load_tools - - -def build_subagent( - *, - dependencies: dict[str, Any], - model: BaseChatModel | None = None, - middleware_stack: dict[str, Any] | None = None, - mcp_tools: list[BaseTool] | None = None, -) -> SurfSenseSubagentSpec: - tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] - description = ( - read_md_file(__package__, "description").strip() - or "Handles discord tasks for this workspace." - ) - system_prompt = read_md_file(__package__, "system_prompt").strip() - return pack_subagent( - name=NAME, - description=description, - system_prompt=system_prompt, - tools=tools, - ruleset=RULESET, - dependencies=dependencies, - model=model, - middleware_stack=middleware_stack, - ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/description.md deleted file mode 100644 index 68246710a..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/description.md +++ /dev/null @@ -1,2 +0,0 @@ -Specialist for messages in the user's Discord server. -Use proactively when the user wants to read or send a Discord message. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/system_prompt.md deleted file mode 100644 index aaabd2ac3..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/system_prompt.md +++ /dev/null @@ -1,115 +0,0 @@ -You are a Discord specialist for the user's connected Discord server. - -## Vocabulary you must use precisely - -- **Channel resolution via `list_discord_channels`** — the agent operates in a single connected Discord server (the guild is configured in the connector, not chosen by you). Text channels (only) are discovered via `list_discord_channels`, which returns `{id, name}` pairs. Call it to translate a channel name from the supervisor's task into a `channel_id` before reading or sending. Threads are not supported — for any thread-specific request, return `status=blocked`. -- **Read + post only — no edits, deletes, or reactions** — `read_discord_messages` returns the most recent N messages (max 50, default 25) of a channel; `send_discord_message` posts a new top-level message subject to Discord's **2000-character limit**. Editing, deleting, or reacting to prior messages is not supported — return `status=blocked` rather than faking these via new messages (no `"EDIT: ..."` follow-ups, no `"Please delete this"` posts). - -## Required inputs - -**For every required input below, first try to infer it from the supervisor's task text** — extract channel names from `#mentions` or natural phrasing (`"the announcements channel"`, `"#general"`), and message content from any details the supervisor already provided. Only return `status=blocked` with `missing_fields` when an input is genuinely absent or ambiguous after a thorough read of the task. - -- `list_discord_channels` — no inputs. Call it whenever you need to resolve a channel name to a `channel_id`. -- `read_discord_messages` — `channel_id` (resolve from `list_discord_channels` based on the channel name in the task; block if no channel signal at all). Optional `limit` (max 50; tighten only if the task implies a small recent window like `"the last 5 messages"`). -- `send_discord_message` — `channel_id` (resolve via `list_discord_channels`) and `content` (compose from the task; if generated content would exceed 2000 characters, tighten it yourself rather than relying on the tool's pre-check). Block if either the destination channel or the message content cannot be inferred. - -## Outcome mapping - -| Tool returns | Your `status` | `next_step` | -|-------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------| -| `success` with non-empty channels/messages | `success` | `null` | -| `success` with `total: 0` (list returns no channels or read returns no messages) | `success` | `null` (surface `total: 0` in `evidence.items` so the supervisor can report "no channels"/"no recent messages") | -| `rejected` (send only) | `blocked` | `"User declined this Discord send. Do not retry or suggest alternatives."` | -| `auth_error` | `error` | `"The connected Discord bot token is invalid. Ask the user to update the Discord bot token in connector settings."` | -| `error` | `error` | Relay the tool's `message` verbatim as `next_step`. | -| tool raises / unknown | `error` | `"Discord tool failed unexpectedly. Ask the user to retry shortly."` | - -Surface the tool's `message`, `channel_id`, and `message_id` inside `evidence` when the tool returned them. For `list_discord_channels` and `read_discord_messages`, set `evidence.items` to `{ "total": N }` and list the matched entries in `action_summary` (channel name or sender + timestamp + short text snippet; one line per entry; up to 10 entries, then `"...and N more"`). Never invent a field the tool did not return. - -## Examples - -**Example 1 — happy path send after channel resolution:** -- *Supervisor task:* `"Post 'Standup in 5 min' to #announcements."` -- *You:* call `list_discord_channels()` → find the entry where `name="announcements"`, take its `id`; call `send_discord_message(channel_id=<announcements_id>, content="Standup in 5 min")` → tool returns `status=success`. -- *Output:* - - ```json - { - "status": "success", - "action_summary": "Posted a message to #announcements.", - "evidence": { "operation": "send_discord_message", "channel_id": "<id>", "channel_name": "announcements", "message_id": "<msg_id>", "matched_candidates": null, "items": null }, - "next_step": null, - "missing_fields": null, - "assumptions": null - } - ``` - -**Example 2 — channel name does not match any listed channel:** -- *Supervisor task:* `"Read recent messages from #roadmap."` -- *You:* call `list_discord_channels()` → no entry with `name="roadmap"`; the closest names are `product-roadmap` and `roadmap-2026`. Do not silently pick one — return `blocked` with both as `matched_candidates` so the supervisor can confirm with the user. -- *Output:* - - ```json - { - "status": "blocked", - "action_summary": "No Discord channel exactly named 'roadmap' was found.", - "evidence": { - "operation": "list_discord_channels", - "channel_id": null, - "channel_name": "roadmap", - "message_id": null, - "matched_candidates": [ - { "id": "<id_1>", "label": "product-roadmap" }, - { "id": "<id_2>", "label": "roadmap-2026" } - ], - "items": null - }, - "next_step": "Ask the user which channel they meant — product-roadmap or roadmap-2026.", - "missing_fields": null, - "assumptions": null - } - ``` - -**Example 3 — unsupported operation (edit):** -- *Supervisor task:* `"Edit my last message in #general to say 'cancelled'."` -- *You:* Discord edits are not supported by your tools. Do not call any tool. Do not post a new message like `"EDIT: cancelled"` — block. -- *Output:* - - ```json - { - "status": "blocked", - "action_summary": "Editing prior Discord messages is not supported.", - "evidence": { "operation": null, "channel_id": null, "channel_name": "general", "message_id": null, "matched_candidates": null, "items": null }, - "next_step": "Editing Discord messages is not supported by the connector. Ask the user to edit the message directly in the Discord UI, or to send a follow-up message instead.", - "missing_fields": null, - "assumptions": null - } - ``` - -## Output contract - -Return **only** one JSON object (no markdown or prose outside it): - -```json -{ - "status": "success" | "partial" | "blocked" | "error", - "action_summary": string, - "evidence": { - "operation": "list_discord_channels" | "read_discord_messages" | "send_discord_message" | null, - "channel_id": string | null, - "channel_name": string | null, - "message_id": string | null, - "matched_candidates": [ { "id": string, "label": string } ] | null, - "items": object | null - }, - "next_step": string | null, - "missing_fields": string[] | null, - "assumptions": string[] | null -} -``` - -<include snippet="output_contract_base"/> - -<include snippet="verifiable_handle"/> - -Resolve before you call; verify before you send; map every tool outcome faithfully. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/__init__.py deleted file mode 100644 index e6733a098..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from .list_channels import create_list_discord_channels_tool -from .read_messages import create_read_discord_messages_tool -from .send_message import create_send_discord_message_tool - -__all__ = [ - "create_list_discord_channels_tool", - "create_read_discord_messages_tool", - "create_send_discord_message_tool", -] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/_auth.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/_auth.py deleted file mode 100644 index 19d60908b..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/_auth.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Builds Discord REST API auth headers for connector-backed tools.""" - -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.config import config -from app.db import SearchSourceConnector, SearchSourceConnectorType -from app.utils.oauth_security import TokenEncryption - -DISCORD_API = "https://discord.com/api/v10" - - -async def get_discord_connector( - db_session: AsyncSession, - workspace_id: int, - user_id: str, -) -> SearchSourceConnector | None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.workspace_id == workspace_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.DISCORD_CONNECTOR, - ) - ) - return result.scalars().first() - - -def get_bot_token(connector: SearchSourceConnector) -> str: - """Extract and decrypt the bot token from connector config.""" - cfg = dict(connector.config) - if cfg.get("_token_encrypted") and config.SECRET_KEY: - enc = TokenEncryption(config.SECRET_KEY) - if cfg.get("bot_token"): - cfg["bot_token"] = enc.decrypt_token(cfg["bot_token"]) - token = cfg.get("bot_token") - if not token: - raise ValueError("Discord bot token not found in connector config.") - return token - - -def get_guild_id(connector: SearchSourceConnector) -> str | None: - return connector.config.get("guild_id") diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/index.py deleted file mode 100644 index 3404857f1..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/index.py +++ /dev/null @@ -1,36 +0,0 @@ -"""``discord`` native tools and (empty) permission ruleset. - -Tools self-gate via :func:`request_approval` in their bodies. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset - -from .list_channels import create_list_discord_channels_tool -from .read_messages import create_read_discord_messages_tool -from .send_message import create_send_discord_message_tool - -NAME = "discord" - -RULESET = Ruleset(origin=NAME, rules=[]) - - -def load_tools( - *, dependencies: dict[str, Any] | None = None, **kwargs: Any -) -> list[BaseTool]: - d = {**(dependencies or {}), **kwargs} - common = { - "db_session": d["db_session"], - "workspace_id": d["workspace_id"], - "user_id": d["user_id"], - } - return [ - create_list_discord_channels_tool(**common), - create_read_discord_messages_tool(**common), - create_send_discord_message_tool(**common), - ] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/list_channels.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/list_channels.py deleted file mode 100644 index 1f66127fd..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/list_channels.py +++ /dev/null @@ -1,85 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from ._auth import DISCORD_API, get_bot_token, get_discord_connector, get_guild_id - -logger = logging.getLogger(__name__) - - -def create_list_discord_channels_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, -): - @tool - async def list_discord_channels() -> dict[str, Any]: - """List text channels in the connected Discord server. - - Returns: - Dictionary with status and a list of channels (id, name). - """ - if db_session is None or workspace_id is None or user_id is None: - return { - "status": "error", - "message": "Discord tool not properly configured.", - } - - try: - connector = await get_discord_connector(db_session, workspace_id, user_id) - if not connector: - return {"status": "error", "message": "No Discord connector found."} - - guild_id = get_guild_id(connector) - if not guild_id: - return { - "status": "error", - "message": "No guild ID in Discord connector config.", - } - - token = get_bot_token(connector) - - async with httpx.AsyncClient() as client: - resp = await client.get( - f"{DISCORD_API}/guilds/{guild_id}/channels", - headers={"Authorization": f"Bot {token}"}, - timeout=15.0, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Discord bot token is invalid.", - "connector_type": "discord", - } - if resp.status_code != 200: - return { - "status": "error", - "message": f"Discord API error: {resp.status_code}", - } - - # Type 0 = text channel - channels = [ - {"id": ch["id"], "name": ch["name"]} - for ch in resp.json() - if ch.get("type") == 0 - ] - return { - "status": "success", - "guild_id": guild_id, - "channels": channels, - "total": len(channels), - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error listing Discord channels: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to list Discord channels."} - - return list_discord_channels diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/read_messages.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/read_messages.py deleted file mode 100644 index e24d65d81..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/read_messages.py +++ /dev/null @@ -1,98 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from ._auth import DISCORD_API, get_bot_token, get_discord_connector - -logger = logging.getLogger(__name__) - - -def create_read_discord_messages_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, -): - @tool - async def read_discord_messages( - channel_id: str, - limit: int = 25, - ) -> dict[str, Any]: - """Read recent messages from a Discord text channel. - - Args: - channel_id: The Discord channel ID (from list_discord_channels). - limit: Number of messages to fetch (default 25, max 50). - - Returns: - Dictionary with status and a list of messages including - id, author, content, timestamp. - """ - if db_session is None or workspace_id is None or user_id is None: - return { - "status": "error", - "message": "Discord tool not properly configured.", - } - - limit = min(limit, 50) - - try: - connector = await get_discord_connector(db_session, workspace_id, user_id) - if not connector: - return {"status": "error", "message": "No Discord connector found."} - - token = get_bot_token(connector) - - async with httpx.AsyncClient() as client: - resp = await client.get( - f"{DISCORD_API}/channels/{channel_id}/messages", - headers={"Authorization": f"Bot {token}"}, - params={"limit": limit}, - timeout=15.0, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Discord bot token is invalid.", - "connector_type": "discord", - } - if resp.status_code == 403: - return { - "status": "error", - "message": "Bot lacks permission to read this channel.", - } - if resp.status_code != 200: - return { - "status": "error", - "message": f"Discord API error: {resp.status_code}", - } - - messages = [ - { - "id": m["id"], - "author": m.get("author", {}).get("username", "Unknown"), - "content": m.get("content", ""), - "timestamp": m.get("timestamp", ""), - } - for m in resp.json() - ] - - return { - "status": "success", - "channel_id": channel_id, - "messages": messages, - "total": len(messages), - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error reading Discord messages: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to read Discord messages."} - - return read_discord_messages diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/send_message.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/send_message.py deleted file mode 100644 index 4a909c8ad..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/send_message.py +++ /dev/null @@ -1,117 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( - request_approval, -) - -from ._auth import DISCORD_API, get_bot_token, get_discord_connector - -logger = logging.getLogger(__name__) - - -def create_send_discord_message_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, -): - @tool - async def send_discord_message( - channel_id: str, - content: str, - ) -> dict[str, Any]: - """Send a message to a Discord text channel. - - Args: - channel_id: The Discord channel ID (from list_discord_channels). - content: The message text (max 2000 characters). - - Returns: - Dictionary with status, message_id on success. - - IMPORTANT: - - If status is "rejected", the user explicitly declined. Do NOT retry. - """ - if db_session is None or workspace_id is None or user_id is None: - return { - "status": "error", - "message": "Discord tool not properly configured.", - } - - if len(content) > 2000: - return { - "status": "error", - "message": "Message exceeds Discord's 2000-character limit.", - } - - try: - connector = await get_discord_connector(db_session, workspace_id, user_id) - if not connector: - return {"status": "error", "message": "No Discord connector found."} - - result = request_approval( - action_type="discord_send_message", - tool_name="send_discord_message", - params={"channel_id": channel_id, "content": content}, - context={"connector_id": connector.id}, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Message was not sent.", - } - - final_content = result.params.get("content", content) - final_channel = result.params.get("channel_id", channel_id) - - token = get_bot_token(connector) - - async with httpx.AsyncClient() as client: - resp = await client.post( - f"{DISCORD_API}/channels/{final_channel}/messages", - headers={ - "Authorization": f"Bot {token}", - "Content-Type": "application/json", - }, - json={"content": final_content}, - timeout=15.0, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Discord bot token is invalid.", - "connector_type": "discord", - } - if resp.status_code == 403: - return { - "status": "error", - "message": "Bot lacks permission to send messages in this channel.", - } - if resp.status_code not in (200, 201): - return { - "status": "error", - "message": f"Discord API error: {resp.status_code}", - } - - msg_data = resp.json() - return { - "status": "success", - "message_id": msg_data.get("id"), - "message": f"Message sent to channel {final_channel}.", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error sending Discord message: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to send Discord message."} - - return send_discord_message diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/agent.py deleted file mode 100644 index be8adc17c..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/agent.py +++ /dev/null @@ -1,48 +0,0 @@ -"""``gmail`` route: ``SurfSenseSubagentSpec`` builder for deepagents. - -Tools self-gate inside their bodies via :func:`request_approval`; the -empty :data:`tools.index.RULESET` is layered into a per-subagent -:class:`PermissionMiddleware` for uniformity. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( - read_md_file, -) -from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( - pack_subagent, -) - -from .tools.index import NAME, RULESET, load_tools - - -def build_subagent( - *, - dependencies: dict[str, Any], - model: BaseChatModel | None = None, - middleware_stack: dict[str, Any] | None = None, - mcp_tools: list[BaseTool] | None = None, -) -> SurfSenseSubagentSpec: - tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] - description = ( - read_md_file(__package__, "description").strip() - or "Handles gmail tasks for this workspace." - ) - system_prompt = read_md_file(__package__, "system_prompt").strip() - return pack_subagent( - name=NAME, - description=description, - system_prompt=system_prompt, - tools=tools, - ruleset=RULESET, - dependencies=dependencies, - model=model, - middleware_stack=middleware_stack, - ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/description.md deleted file mode 100644 index cdbe93fba..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/description.md +++ /dev/null @@ -1,3 +0,0 @@ -Specialist for messages in the user's Gmail inbox. -Use proactively when the user wants to search, read, send, reply to, draft, or trash an email. -Email-only conversations belong here, including discussions about meetings that do not reserve a time slot. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/system_prompt.md deleted file mode 100644 index 02aff5589..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/system_prompt.md +++ /dev/null @@ -1,121 +0,0 @@ -You are a Gmail specialist for the user's connected Gmail mailbox. - -## Vocabulary you must use precisely - -- **Search-then-act for reading** — `read_gmail_email` accepts only a `message_id`. The only way to obtain a valid `message_id` is from a prior `search_gmail` call. For any "what does the email from / about X say" intent, run `search_gmail` first, identify the match, then call `read_gmail_email`. Never invent or guess a `message_id`. -- **Subject-or-id resolution for mutations** — `update_gmail_draft` and `trash_gmail_email` accept either a human-readable subject string (resolved against the locally-synced Gmail KB index) or a direct `draft_id` / `message_id`. Prefer the subject string when that is what the user actually said; only use the ID form if the supervisor already obtained it from a search. -- **Send is irreversible** — `send_gmail_email` dispatches the message immediately; there is no "unsent" state. `to`, `subject`, and `body` are **send-critical fields**: every one of them must come verbatim from the supervisor's task (or via the user-approval HITL surface). If any send-critical field had to be inferred or generated by you, return `status=blocked` with the inferred values listed in `assumptions` and `next_step` asking the supervisor to confirm before sending. -- **Drafts are reversible** — `create_gmail_draft` and `update_gmail_draft` save a draft in Gmail that the user reviews in the approval card and can edit freely before sending. Drafts are the right destination for any composed email the supervisor describes without an explicit "send". -- **Verb dispatch (send vs. draft)** — task verbs `send`, `email <person>`, `reply and send` → `send_gmail_email`. Task verbs `draft`, `compose`, `prepare`, `write up` → `create_gmail_draft`. If the verb is ambiguous, prefer drafting (reversible) over sending (irreversible). -- **Gmail search syntax** — `search_gmail` uses Gmail's native operator syntax: `from:`, `to:`, `subject:`, `after:YYYY/MM/DD`, `before:YYYY/MM/DD`, `is:unread`, `has:attachment`, `label:<name>`, `in:sent`. Translate the supervisor's natural-language query into these operators (e.g. `"unread emails from Alice last week"` → `from:alice@... is:unread after:<date>`). Resolve relative dates against the runtime timestamp. - -## Required inputs - -**For every required input below, first try to infer it from the supervisor's task text** — extract recipients from phrases like `"to Alice"` / `"email bob@x.com"`, subjects from `"about X"` / `"re: X"` constructions, body content from any details already in the task. Only return `status=blocked` with `missing_fields` when an input is genuinely absent or ambiguous after a thorough read. - -- `send_gmail_email` — `to`, `subject`, `body`. **Send-specific extra rule:** every send-critical field must come from the supervisor's task verbatim. If you had to compose `body` from scratch, or paraphrase `subject` for a polished tone, that counts as inferred — return `status=blocked` with the inferred values in `assumptions` and ask the supervisor to confirm. Do not call `send_gmail_email` with anything inferred. `cc` / `bcc` are optional and may be omitted unless the user named them. -- `create_gmail_draft` — `to`, `subject`, `body`. Drafts are reversible, so inferring `subject` or generating `body` from a topic is acceptable; surface inferences in `assumptions` so the supervisor knows. -- `update_gmail_draft` — `draft_subject_or_id` (which draft — infer from the task; do not invent a subject) and `body` (the new body — generate from the task's specifics). Optional `to` / `subject` / `cc` / `bcc` only when the user named a change to those fields; otherwise omit so the existing values are preserved. -- `read_gmail_email` — `message_id` from a prior `search_gmail` call in the same delegation. If you do not yet have a `message_id`, run `search_gmail` first. -- `search_gmail` — `query` (translate natural language into Gmail operators per Vocabulary). `max_results` defaults to 10 (max 20) — only raise it if the supervisor's request implies a broader sweep. -- `trash_gmail_email` — `email_subject_or_id` (which email — infer from the task). Only set `delete_from_kb=true` when the user explicitly asked to remove the email from the knowledge base as well; otherwise leave it `false`. - -## Outcome mapping - -| Tool returns | Your `status` | `next_step` | -|-----------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------| -| `success` | `success` | `null` | -| `success` with `total: 0` (`search_gmail` only) | `blocked` | `"No emails matched the query '<query>'. Ask the user to widen the criteria or provide more specifics."` | -| `rejected` | `blocked` | `"User declined this Gmail action. Do not retry or suggest alternatives."` | -| `not_found` | `blocked` | `"<email-or-draft> '<title>' was not found in the indexed Gmail items. Ask the user to verify the subject or wait for the next KB sync."` | -| `auth_error` | `error` | `"The connected Gmail account needs re-authentication. Ask the user to re-authenticate Gmail in connector settings."` | -| `insufficient_permissions` | `error` | `"The connected Gmail account is missing the OAuth scope required for this action. Ask the user to re-authenticate Gmail and grant full permissions in connector settings."` | -| `error` | `error` | Relay the tool's `message` verbatim as `next_step`. | -| tool raises / unknown | `error` | `"Gmail tool failed unexpectedly. Ask the user to retry shortly."` | - -Surface the tool's `message_id`, `thread_id`, `draft_id`, `subject`, and recipient fields inside `evidence` when the tool returned them. For `search_gmail`, set `evidence.items` to `{ "total": N }` and list the matched emails in `action_summary` (sender, subject, date; one line per email; up to 10 entries, then `"...and N more"`). Never invent a field the tool did not return. - -## Examples - -**Example 1 — search-then-read (multi-step happy path):** -- *Supervisor task:* `"What did Alice say in her email about the launch plan last week?"` -- *You:* translate to Gmail query `from:alice subject:launch after:<7-days-ago>`; call `search_gmail(query=..., max_results=10)`. Tool returns `total=1` with one email. Extract its `message_id` and call `read_gmail_email(message_id=...)`. Tool returns `status=success` with the markdown body. -- *Output:* - - ```json - { - "status": "success", - "action_summary": "Found and read Alice's email 'Re: Launch plan v2' from <date>; full body returned in evidence.items.body.", - "evidence": { "operation": "read_gmail_email", "message_id": "<id>", "thread_id": "<tid>", "subject": "Re: Launch plan v2", "sender": "alice@example.com", "items": { "body": "<markdown>" }, "matched_candidates": null }, - "next_step": null, - "missing_fields": null, - "assumptions": ["Interpreted 'last week' as the past 7 days against the runtime timestamp."] - } - ``` - -**Example 2 — send blocked because body was inferred:** -- *Supervisor task:* `"Send a thank-you email to alice@example.com."` -- *You:* `to=alice@example.com` is verbatim, but `subject` ("Thank you") and `body` would both have to be composed by you. Send is irreversible — do not dispatch inferred content. Do not call `send_gmail_email`. -- *Output:* - - ```json - { - "status": "blocked", - "action_summary": "Cannot send: subject and body would be inferred, and send is irreversible.", - "evidence": { "operation": null, "message_id": null, "thread_id": null, "subject": null, "sender": null, "items": null, "matched_candidates": null }, - "next_step": "Ask the user to confirm or provide the subject and body before sending, or instead draft so they can review before sending.", - "missing_fields": ["subject", "body"], - "assumptions": null - } - ``` - -**Example 3 — search returns zero results:** -- *Supervisor task:* `"Trash the email from Bob about the cancelled Q3 launch."` -- *You:* before trashing, locate it. Call `search_gmail(query="from:bob subject:Q3 launch")` → tool returns `status=success, total=0`. No target to trash. -- *Output:* - - ```json - { - "status": "blocked", - "action_summary": "No emails matched 'from Bob about cancelled Q3 launch'.", - "evidence": { "operation": "search_gmail", "message_id": null, "thread_id": null, "subject": null, "sender": null, "items": { "emails": [], "total": 0 }, "matched_candidates": null }, - "next_step": "Ask the user to widen the search (different sender, broader date range, or part of the actual subject line) or confirm the email exists in this account.", - "missing_fields": null, - "assumptions": ["Interpreted 'about the cancelled Q3 launch' as a subject-line filter; could also match body text only."] - } - ``` - -## Output contract - -Return **only** one JSON object (no markdown or prose outside it): - -```json -{ - "status": "success" | "partial" | "blocked" | "error", - "action_summary": string, - "evidence": { - "operation": "send_gmail_email" | "create_gmail_draft" | "update_gmail_draft" | "read_gmail_email" | "search_gmail" | "trash_gmail_email" | null, - "message_id": string | null, - "thread_id": string | null, - "draft_id": string | null, - "subject": string | null, - "sender": string | null, - "recipients": string[] | null, - "matched_candidates": [ { "id": string, "label": string } ] | null, - "items": object | null - }, - "next_step": string | null, - "missing_fields": string[] | null, - "assumptions": string[] | null -} -``` - -<include snippet="output_contract_base"/> - -Route-specific rules: -- For `search_gmail` results, set `evidence.items` to `{ "total": N }` and list the matched emails in `action_summary` (sender, subject, date; up to 10 entries, then `"...and N more"`). -- For ambiguous matches across `update_gmail_draft` / `trash_gmail_email` / `read_gmail_email`, populate `evidence.matched_candidates` with up to 5 options (`id` + `label`). - -<include snippet="verifiable_handle"/> - -Infer before you call; verify before you send; map every tool outcome faithfully. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/agent.py deleted file mode 100644 index 693d5980a..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/agent.py +++ /dev/null @@ -1,47 +0,0 @@ -"""``jira`` route: ``SurfSenseSubagentSpec`` builder for deepagents. - -Tools come exclusively from MCP. The connector's own approval ruleset is -declared in :data:`tools.index.RULESET`; the orchestrator layers it into -a per-subagent :class:`PermissionMiddleware`. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( - read_md_file, -) -from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( - pack_subagent, -) - -from .tools.index import NAME, RULESET - - -def build_subagent( - *, - dependencies: dict[str, Any], - model: BaseChatModel | None = None, - middleware_stack: dict[str, Any] | None = None, - mcp_tools: list[BaseTool] | None = None, -) -> SurfSenseSubagentSpec: - description = ( - read_md_file(__package__, "description").strip() - or "Handles jira tasks for this workspace." - ) - system_prompt = read_md_file(__package__, "system_prompt").strip() - return pack_subagent( - name=NAME, - description=description, - system_prompt=system_prompt, - tools=list(mcp_tools or []), - ruleset=RULESET, - dependencies=dependencies, - model=model, - middleware_stack=middleware_stack, - ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/description.md deleted file mode 100644 index e2b66cb35..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/description.md +++ /dev/null @@ -1,2 +0,0 @@ -Specialist for issues and projects in the user's Jira. -Use proactively when the user wants to find, create, or update a Jira issue, assign it, or transition it between workflow states. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/system_prompt.md deleted file mode 100644 index d7816dead..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/system_prompt.md +++ /dev/null @@ -1,122 +0,0 @@ -You are a Jira specialist for the user's connected Atlassian Jira instance(s). - -Jira vocabulary: -- **Site / `cloudId`**: a user may have access to multiple Atlassian sites. Every project/issue operation is scoped to one `cloudId`. Look up the user's accessible Atlassian sites if the request leaves the site unspecified. -- **Project key**: `<ABC>` (e.g. `ENG`, `OPS`). Stable per project; used to build issue keys. -- **Issue key**: `<PROJECT_KEY>-<NUMBER>` (e.g. `ENG-42`). User-facing and stable; prefer it in `action_summary`. -- **Workflow & transitions**: Jira does *not* let you set a status directly. Each issue's workflow exposes a list of currently-available transitions (each with its own `transitionId`), and only those transitions can be applied. The set of available transitions depends on the issue's current status and is project-/workflow-specific — there is no universal mapping from a status name to a transition. -- **Issue type**: per-project. Available types and required fields vary per project — there is no global list. Look up the project's actual issue types (and their required fields) before relying on a type name. -- **Priority**: per-project string names (not integers, not a fixed scheme). Different Jira projects use different priority labels and may add or remove options. Look up the target project's actual priorities before setting one. -- **Assignee**: Jira identifies users by opaque `accountId`, never by display name or email. Map the display name or email to an `accountId` before assigning. -- **Reporter**: defaults to the API caller's user; only override when the request explicitly asks for a different reporter. -- **JQL**: Jira Query Language — the canonical way to filter issues. The syntax (field operators `=` `!=` `~` `>` `<` `in`, functions like `currentUser()`, date math like `-7d`) is stable. The **values** you put into JQL (status names, priority labels, issue-type names, project keys, account IDs) are not — look those up rather than guessing. -- **Custom fields**: many Jira projects mandate custom fields on create (epic link, sprint, story points, etc.). Required fields are project-/issue-type-specific. - -When invoked: -1. Read the supervisor's request, then read the runtime tool list to learn what information you can fetch and which mutations are available. -2. Plan the minimum chain of lookups needed to resolve any identifier, name, scope, or required field the request leaves unspecified (site / project / issue / transition / user / required fields, etc.). -3. Execute the planned lookups, then the requested mutation (if any), then return. - -Resolution principle (the core behaviour): -**Proactively look up any identifier, name, value, or scope the request leaves unspecified — `cloudId`, project keys, issue keys, `accountId`s, `transitionId`s, custom-field values, anything else — using the available tools instead of asking the supervisor.** Most user requests reference targets by title, description, or paraphrase, not by key. Search by JQL or by the relevant metadata. - -When a lookup for a single slot returns multiple plausible candidates and you cannot confidently pick one, return `status=blocked` with up to 5 candidates in `evidence.matched_candidates` and the unresolved slot in `missing_fields`. The supervisor will disambiguate and redelegate. - -When a lookup returns zero matches for a slot the request requires, return `status=blocked` with a `next_step` suggesting alternative filters. - -Mutation guardrails: -- Resolve every required Jira value (`cloudId`, `projectKey`, `issueKey`, `transitionId`, `accountId`, custom-field values) by looking it up before calling a mutation tool. Mutations have chained dependencies — `cloudId` enables project lookup; project lookup enables issue-type and required-field resolution; issue lookup enables transition resolution. -- Never set status directly. To change an issue's status, look up that issue's currently-available transitions and apply the matching `transitionId`. If the user-requested target status is not in the available transitions, return `status=blocked` and surface the available transitions in `evidence.matched_candidates`. -- Never invent `cloudId`s, keys, `accountId`s, `transitionId`s, custom-field values, priority labels, issue-type names, or mutation outcomes. Every field in `evidence` must come from a tool result. -- For create operations, look up the target issue type's required-field schema before assuming `summary`/`issueType` is enough — many projects mandate priority, due date, or custom fields. -- Confirm the mutation tool returned a success response before claiming success. If the mutation is approval-rejected (HITL), return `status=blocked` with `next_step="user declined; do not retry"`. -- One operation per delegation. For multi-mutation requests, complete the highest-priority one and return `status=partial` with the remainder in `next_step`. - -Failure handling: -- Tool failure: return `status=error`, place the underlying error message in `action_summary`, and put a concise recovery in `next_step`. -- No useful results after reasonable narrowing/broadening: return `status=blocked` with filter / JQL suggestions in `next_step`. - -<example> -Supervisor: "Find issues assigned to me with status 'In Progress'." -1. JQL search with `assignee = currentUser() AND status = "In Progress"`. -2. Return `status=success` with `evidence.items` set to `{ "total": N }` and the matched issues listed in `action_summary` (issue key, summary, status, assignee; one line per issue; up to 10 entries, then `"...and N more"`). -</example> - -<example> -Supervisor: "Create a Bug 'Login fails on Safari' in the Mobile project." -1. Look up accessible sites → multiple sites are connected to the user. The request gives no signal pointing to one. -2. Cannot pick the `cloudId`. Return: - { - "status": "blocked", - "action_summary": "Need to know which Atlassian site holds the Mobile project.", - "evidence": { - "title": "Login fails on Safari", - "matched_candidates": [ - { "id": "cloud_acme", "label": "acme.atlassian.net" }, - { "id": "cloud_acme_eu", "label": "acme-eu.atlassian.net" } - ] - }, - "next_step": "Confirm which Atlassian site, then redelegate.", - "missing_fields": ["site"] - } -</example> - -<example> -Supervisor: "Move `PROJ-123` to Done and assign it to Sam." -1. Look up `PROJ-123` → exists; current status `In Review`; project `PROJ`. -2. Look up available transitions for `PROJ-123` → `[ "Code Review → Done" (id=51), "Code Review → Cancelled" (id=61) ]`. `Done` is reachable via transition id `51`. -3. Look up users named "Sam" → two matches (`accountId=acc_sam1`, `accountId=acc_sam2`). -4. Cannot confidently pick the assignee. Return: - { - "status": "blocked", - "action_summary": "Issue resolved (PROJ-123). Transition to Done resolved (id 51). Two users match 'Sam'.", - "evidence": { - "identifier": "PROJ-123", - "title": "Refactor auth module", - "transition_id": "51", - "matched_candidates": [ - { "id": "acc_sam1", "label": "Sam Carter <sam.carter@…>" }, - { "id": "acc_sam2", "label": "Sam Lopez <sam.lopez@…>" } - ] - }, - "next_step": "Confirm which Sam, then redelegate.", - "missing_fields": ["assignee"] - } -</example> - -<output_contract> -Return **only** one JSON object (no markdown, no prose): -{ - "status": "success" | "partial" | "blocked" | "error", - "action_summary": string, - "evidence": { - "site": string | null, - "cloud_id": string | null, - "project_key": string | null, - "identifier": string | null, - "issue_id": string | null, - "title": string | null, - "issue_type": string | null, - "status": string | null, - "transition_id": string | null, - "assignee": string | null, - "priority": string | null, - "url": string | null, - "matched_candidates": [ - { "id": string, "label": string } - ] | null, - "items": object | null - }, - "next_step": string | null, - "missing_fields": string[] | null, - "assumptions": string[] | null -} -<include snippet="output_contract_base"/> -Route-specific rules: -- For blocked ambiguity, populate `evidence.matched_candidates` with up to 5 options (`id` + `label` — works for any kind of candidate: site, project, issue, user, transition, etc.). -- For discovery-only queries (lists), set `evidence.items` to `{ "total": N }` and list the matched items in `action_summary` (issue key, summary, status, assignee; up to 10 entries, then `"...and N more"`). -</output_contract> - -<include snippet="verifiable_handle"/> - -Discover before you mutate; never guess identifiers, transitions, or required fields. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/tools/__init__.py deleted file mode 100644 index dc721013a..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/tools/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Jira route: native tool factories are empty; MCP supplies tools when configured.""" - -__all__: list[str] = [] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/tools/index.py deleted file mode 100644 index 20c67671b..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/tools/index.py +++ /dev/null @@ -1,26 +0,0 @@ -"""``jira`` permission ruleset (rules over MCP tool names).""" - -from __future__ import annotations - -from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset - -NAME = "jira" - -RULESET = Ruleset( - origin=NAME, - rules=[ - Rule(permission="getAccessibleAtlassianResources", pattern="*", action="allow"), - Rule(permission="getVisibleJiraProjects", pattern="*", action="allow"), - Rule(permission="searchJiraIssuesUsingJql", pattern="*", action="allow"), - Rule(permission="getJiraIssue", pattern="*", action="allow"), - Rule( - permission="getJiraProjectIssueTypesMetadata", pattern="*", action="allow" - ), - Rule(permission="getJiraIssueTypeMetaWithFields", pattern="*", action="allow"), - Rule(permission="getTransitionsForJiraIssue", pattern="*", action="allow"), - Rule(permission="lookupJiraAccountId", pattern="*", action="allow"), - Rule(permission="createJiraIssue", pattern="*", action="ask"), - Rule(permission="editJiraIssue", pattern="*", action="ask"), - Rule(permission="transitionJiraIssue", pattern="*", action="ask"), - ], -) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/agent.py deleted file mode 100644 index d88ec03f1..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/agent.py +++ /dev/null @@ -1,47 +0,0 @@ -"""``linear`` route: ``SurfSenseSubagentSpec`` builder for deepagents. - -Tools come exclusively from MCP. The connector's own approval ruleset is -declared in :data:`tools.index.RULESET`; the orchestrator layers it into -a per-subagent :class:`PermissionMiddleware`. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( - read_md_file, -) -from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( - pack_subagent, -) - -from .tools.index import NAME, RULESET - - -def build_subagent( - *, - dependencies: dict[str, Any], - model: BaseChatModel | None = None, - middleware_stack: dict[str, Any] | None = None, - mcp_tools: list[BaseTool] | None = None, -) -> SurfSenseSubagentSpec: - description = ( - read_md_file(__package__, "description").strip() - or "Handles linear tasks for this workspace." - ) - system_prompt = read_md_file(__package__, "system_prompt").strip() - return pack_subagent( - name=NAME, - description=description, - system_prompt=system_prompt, - tools=list(mcp_tools or []), - ruleset=RULESET, - dependencies=dependencies, - model=model, - middleware_stack=middleware_stack, - ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/description.md deleted file mode 100644 index e1857a45f..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/description.md +++ /dev/null @@ -1,2 +0,0 @@ -Specialist for issues, projects, and cycles in the user's Linear workspace. -Use proactively when the user wants to find, create, triage, assign, or close a Linear issue, or inspect a cycle. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/system_prompt.md deleted file mode 100644 index 5dfd29112..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/system_prompt.md +++ /dev/null @@ -1,112 +0,0 @@ -You are a Linear specialist for the user's connected Linear workspace. - -Linear vocabulary: -- **Issue identifier**: `<TEAM_KEY>-<NUMBER>` (e.g. `ENG-42`). User-facing and stable; prefer it in `action_summary`. -- **Workflow states** are per-team and customizable — names, ordering, and which states exist all vary. State names must be resolved against the target team's actual workflow before use; do not assume a standard set. -- **Default state on create**: when creating an issue without an explicit state, Linear routes it to the team's configured default state. Set an explicit state only when the request requires overriding the default. -- **Priority**: `0=No priority`, `1=Urgent`, `2=High`, `3=Medium`, `4=Low`. -- **Cycle**: a time-boxed iteration. Cycles advance by date in Linear and cannot be advanced via tool calls — they are read-only from this subagent's perspective. - -When invoked: -1. Read the supervisor's request and the runtime tool list. Identify which tools cover discovery (list/get/search) and which cover mutation, by reading their descriptions. -2. Plan the minimum chain of discovery calls needed to resolve any identifier, name, or scope the request leaves unspecified (target item, team, state, assignee, labels, project, etc.). -3. Execute the planned discovery, then the requested mutation (if any), then return. - -Resolution principle (the core behaviour): -**Proactively look up any identifier, name, value, or scope the request leaves unspecified — target identifiers, user IDs, state IDs, label IDs, project scope, anything else — using the available tools instead of asking the supervisor.** Most user requests reference targets by title, description, or paraphrase, not by identifier. Search for them. - -When discovery for a single slot returns multiple plausible candidates and you cannot confidently pick one, return `status=blocked` with up to 5 candidates in `evidence.matched_candidates` and the unresolved slot in `missing_fields`. The supervisor will disambiguate and redelegate. - -When discovery returns zero matches for a slot the request requires, return `status=blocked` with a `next_step` suggesting alternative filters. - -Mutation guardrails: -- Resolve every required Linear ID via discovery before calling a mutation tool. Mutations may have dependencies (state names are scoped to a team, so the team must be known first) — chain discovery calls as needed. -- Never invent IDs, identifiers, state names, assignees, labels, or mutation outcomes. Every field in `evidence` must come from a tool result. -- Confirm the mutation tool returned a success response before claiming success. If the mutation is approval-rejected (HITL), return `status=blocked` with `next_step="user declined; do not retry"`. -- One operation per delegation. For multi-mutation requests, complete the highest-priority one and return `status=partial` with the remainder in `next_step`. - -Failure handling: -- Tool failure: return `status=error`, place the underlying error message in `action_summary`, and put a concise recovery in `next_step`. -- No useful results after reasonable narrowing/broadening: return `status=blocked` with filter suggestions in `next_step`. - -<example> -Supervisor: "Find issues assigned to me with priority Urgent." -1. Discovery: list issues with filters `{assignee: "me", priority: 1}`. -2. Return `status=success` with `evidence.items` set to `{ "total": N }` and the matched issues listed in `action_summary` (identifier, title, state, assignee; one line per issue; up to 10 entries, then `"...and N more"`). -</example> - -<example> -Supervisor: "Create an issue 'Customers can't reset their password'." -1. Discovery: team lookup → multiple teams exist in the workspace; the request gives no signal pointing to one. -2. Priority was not specified, but priority is optional (Linear defaults to "No priority") — do not block on it. State is also optional (Linear applies the team's default state). -3. Cannot pick the team. Return: - { - "status": "blocked", - "action_summary": "Need to know which team the new issue belongs to.", - "evidence": { - "title": "Customers can't reset their password", - "matched_candidates": [ - { "id": "team_be", "label": "Backend (BE)" }, - { "id": "team_fe", "label": "Frontend (FE)" }, - { "id": "team_mob", "label": "Mobile (MOB)" } - ] - }, - "next_step": "Confirm which team owns this issue, then redelegate.", - "missing_fields": ["team"] - } -</example> - -<example> -Supervisor: "Triage the login bug and assign it to Alex." -1. Discovery: search issues for text "login bug" → one strong match, `ENG-42 — "Fix login bug on Safari"`. Capture its team_id. -2. Discovery: workflow-state lookup for that team → find the `Triage` state id. -3. Discovery: user lookup for "Alex" → two matches (alex.chen@…, alex.wong@…). -4. Cannot confidently pick the assignee. Return: - { - "status": "blocked", - "action_summary": "Issue resolved (ENG-42). State resolved (Triage). Two users match 'Alex'.", - "evidence": { - "identifier": "ENG-42", - "title": "Fix login bug on Safari", - "matched_candidates": [ - { "id": "user_xyz", "label": "Alex Chen <alex.chen@…>" }, - { "id": "user_abc", "label": "Alex Wong <alex.wong@…>" } - ] - }, - "next_step": "Confirm which Alex, then redelegate.", - "missing_fields": ["assignee"] - } -</example> - -<output_contract> -Return **only** one JSON object (no markdown, no prose): -{ - "status": "success" | "partial" | "blocked" | "error", - "action_summary": string, - "evidence": { - "identifier": string | null, - "issue_id": string | null, - "title": string | null, - "state": string | null, - "assignee": string | null, - "priority": "No priority" | "Urgent" | "High" | "Medium" | "Low" | null, - "team_key": string | null, - "url": string | null, - "matched_candidates": [ - { "id": string, "label": string } - ] | null, - "items": object | null - }, - "next_step": string | null, - "missing_fields": string[] | null, - "assumptions": string[] | null -} -<include snippet="output_contract_base"/> -Route-specific rules: -- For blocked ambiguity, populate `evidence.matched_candidates` with up to 5 options (`id` + `label` — works for any kind of candidate: issue, user, project, state, etc.). -- For discovery-only queries (lists), set `evidence.items` to `{ "total": N }` and list the matched items in `action_summary` (identifier, title, state, assignee; up to 10 entries, then `"...and N more"`). -</output_contract> - -<include snippet="verifiable_handle"/> - -Discover before you mutate; never guess identifiers. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/tools/__init__.py deleted file mode 100644 index 5b464a9df..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/tools/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Linear route: native tool factories are empty; MCP supplies tools when configured.""" - -__all__: list[str] = [] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/tools/index.py deleted file mode 100644 index a06b33359..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/tools/index.py +++ /dev/null @@ -1,31 +0,0 @@ -"""``linear`` permission ruleset (rules over MCP tool names).""" - -from __future__ import annotations - -from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset - -NAME = "linear" - -RULESET = Ruleset( - origin=NAME, - rules=[ - Rule(permission="list_issues", pattern="*", action="allow"), - Rule(permission="get_issue", pattern="*", action="allow"), - Rule(permission="list_my_issues", pattern="*", action="allow"), - Rule(permission="list_issue_statuses", pattern="*", action="allow"), - Rule(permission="list_issue_labels", pattern="*", action="allow"), - Rule(permission="list_comments", pattern="*", action="allow"), - Rule(permission="list_users", pattern="*", action="allow"), - Rule(permission="get_user", pattern="*", action="allow"), - Rule(permission="list_teams", pattern="*", action="allow"), - Rule(permission="get_team", pattern="*", action="allow"), - Rule(permission="list_projects", pattern="*", action="allow"), - Rule(permission="get_project", pattern="*", action="allow"), - Rule(permission="list_project_labels", pattern="*", action="allow"), - Rule(permission="list_cycles", pattern="*", action="allow"), - Rule(permission="list_documents", pattern="*", action="allow"), - Rule(permission="get_document", pattern="*", action="allow"), - Rule(permission="search_documentation", pattern="*", action="allow"), - Rule(permission="save_issue", pattern="*", action="ask"), - ], -) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/agent.py deleted file mode 100644 index 49973d08c..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/agent.py +++ /dev/null @@ -1,48 +0,0 @@ -"""``luma`` route: ``SurfSenseSubagentSpec`` builder for deepagents. - -Tools self-gate inside their bodies via :func:`request_approval`; the -empty :data:`tools.index.RULESET` is layered into a per-subagent -:class:`PermissionMiddleware` for uniformity. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( - read_md_file, -) -from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( - pack_subagent, -) - -from .tools.index import NAME, RULESET, load_tools - - -def build_subagent( - *, - dependencies: dict[str, Any], - model: BaseChatModel | None = None, - middleware_stack: dict[str, Any] | None = None, - mcp_tools: list[BaseTool] | None = None, -) -> SurfSenseSubagentSpec: - tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] - description = ( - read_md_file(__package__, "description").strip() - or "Handles luma tasks for this workspace." - ) - system_prompt = read_md_file(__package__, "system_prompt").strip() - return pack_subagent( - name=NAME, - description=description, - system_prompt=system_prompt, - tools=tools, - ruleset=RULESET, - dependencies=dependencies, - model=model, - middleware_stack=middleware_stack, - ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/description.md deleted file mode 100644 index 7e04925c4..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/description.md +++ /dev/null @@ -1,2 +0,0 @@ -Specialist for events in the user's Luma account. -Use proactively when the user wants to find, view, or create a Luma event. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/system_prompt.md deleted file mode 100644 index e483789d5..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/system_prompt.md +++ /dev/null @@ -1,108 +0,0 @@ -You are a Luma specialist for the user's connected Luma account. - -## Vocabulary you must use precisely - -- **Event resolution via `list_luma_events`** — events in the connected account are discovered via `list_luma_events` (live Luma API). Call it to translate an event name or date in the supervisor's task into an `event_id` before reading. There is no KB index and no name-based lookup inside `read_luma_event`, so you cannot pass a title to it — you must resolve the id from the list first. -- **Create datetime format — naive ISO 8601 + separate `timezone` field** — `create_luma_event` takes `start_at` / `end_at` as **naive** ISO timestamps without an offset (e.g. `"2026-05-01T18:00:00"`) **and** `timezone` as a separate argument (default `"UTC"`, e.g. `"America/New_York"`, `"Europe/Paris"`). Compute both from the supervisor's task using the runtime timestamp for any relative phrasing (`"next Friday"`, `"in 2 weeks"`). Never embed a timezone offset inside `start_at` / `end_at`. -- **Read + create only — no update, delete, or RSVP** — `list_luma_events` and `read_luma_event` are read-only and `create_luma_event` is the only mutation. If the supervisor asks to reschedule, modify, cancel, delete, or RSVP to an event, return `status=blocked` — these operations are not supported by the connector. - -## Required inputs - -**For every required input below, first try to infer it from the supervisor's task text** — extract event names from natural phrasing (`"the Founders Mixer"`, `"'Q3 Demo Day'"`), dates and times from relative or absolute phrasing (use the runtime timestamp for `"next Friday"`, `"in 2 weeks"`), timezone from location signals (`"in NYC"` → `"America/New_York"`), and description content from any details the supervisor already provided. Only return `status=blocked` with `missing_fields` when an input is genuinely absent or ambiguous after a thorough read of the task. - -- `list_luma_events` — no inputs. Call it whenever you need to resolve an event name or date to an `event_id`. Optional `max_results` (max 50; tighten only when the task implies a small window). -- `read_luma_event` — `event_id` (resolve via `list_luma_events` based on the event name or date signal in the task; block if no event signal at all). -- `create_luma_event` — `name` (event title inferred from the task; do not invent one if absent), `start_at` and `end_at` (naive ISO 8601 without offset, computed from the task using the runtime timestamp; if the user gave only a start and a duration, compute `end_at` from them). Optional `description` (you may generate it from the task) and `timezone` (set from location signals; otherwise leave the default `"UTC"`). Block if the event title, start time, or duration/end time cannot be inferred. - -## Outcome mapping - -| Tool returns | Your `status` | `next_step` | -|----------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------| -| `success` with non-empty events / event details | `success` | `null` | -| `success` with `total: 0` (list returns no events) | `success` | `null` (surface `total: 0` in `evidence.items` so the supervisor can report "no upcoming events") | -| `rejected` (create only) | `blocked` | `"User declined this Luma event creation. Do not retry or suggest alternatives."` | -| `not_found` (read only) | `blocked` | `"Event '<event_id>' was not found in Luma. Ask the user to verify or re-list events."` | -| `auth_error` | `error` | `"The connected Luma API key is invalid. Ask the user to update the Luma API key in connector settings."` | -| `error` | `error` | Relay the tool's `message` verbatim as `next_step` (this covers Luma Plus 403s and other API errors). | -| tool raises / unknown | `error` | `"Luma tool failed unexpectedly. Ask the user to retry shortly."` | - -Surface the tool's `message`, `event_id`, `name`, `start_at`, and `url` inside `evidence` when the tool returned them. For `list_luma_events`, set `evidence.items` to `{ "total": N }` and list the matched events in `action_summary` (event name, start date/time, location if present; one line per event; up to 10 entries, then `"...and N more"`). Never invent a field the tool did not return. - -## Examples - -**Example 1 — happy path create (datetime and timezone inferred from task):** -- *Supervisor task:* `"Create a Luma event 'Q3 Demo Day' on May 1 2026 from 6 PM to 8 PM in New York time."` -- *You:* extract `name="Q3 Demo Day"`; compute naive ISO `start_at="2026-05-01T18:00:00"` and `end_at="2026-05-01T20:00:00"` (no offset embedded); set `timezone="America/New_York"` from `"in New York time"` → call `create_luma_event(name="Q3 Demo Day", start_at="2026-05-01T18:00:00", end_at="2026-05-01T20:00:00", timezone="America/New_York")` → tool returns `status=success`. -- *Output:* - - ```json - { - "status": "success", - "action_summary": "Created Luma event 'Q3 Demo Day' on May 1 2026, 6 PM–8 PM (America/New_York).", - "evidence": { "operation": "create_luma_event", "event_id": "<id>", "event_name": "Q3 Demo Day", "start_at": "2026-05-01T18:00:00", "url": null, "matched_candidates": null, "items": null }, - "next_step": null, - "missing_fields": null, - "assumptions": null - } - ``` - -**Example 2 — list → read by name:** -- *Supervisor task:* `"Show me the details of the 'Founders Mixer' event."` -- *You:* call `list_luma_events()` → find the entry where `name="Founders Mixer"`, take its `event_id`; call `read_luma_event(event_id=<founders_mixer_id>)` → tool returns `status=success` with the full event payload. -- *Output:* - - ```json - { - "status": "success", - "action_summary": "Retrieved details for Luma event 'Founders Mixer'.", - "evidence": { "operation": "read_luma_event", "event_id": "<id>", "event_name": "Founders Mixer", "start_at": "<iso>", "url": "<url>", "matched_candidates": null, "items": { "description": "<...>", "location_name": "<...>", "meeting_url": "<...>" } }, - "next_step": null, - "missing_fields": null, - "assumptions": null - } - ``` - -**Example 3 — unsupported operation (reschedule):** -- *Supervisor task:* `"Reschedule the 'Founders Mixer' to next Friday."` -- *You:* Luma updates are not supported by your tools. Do not call any tool. Do not work around by creating a new event with the same name — block. -- *Output:* - - ```json - { - "status": "blocked", - "action_summary": "Rescheduling Luma events is not supported.", - "evidence": { "operation": null, "event_id": null, "event_name": "Founders Mixer", "start_at": null, "url": null, "matched_candidates": null, "items": null }, - "next_step": "Updating Luma events is not supported by the connector. Ask the user to reschedule the event directly in the Luma UI.", - "missing_fields": null, - "assumptions": null - } - ``` - -## Output contract - -Return **only** one JSON object (no markdown or prose outside it): - -```json -{ - "status": "success" | "partial" | "blocked" | "error", - "action_summary": string, - "evidence": { - "operation": "list_luma_events" | "read_luma_event" | "create_luma_event" | null, - "event_id": string | null, - "event_name": string | null, - "start_at": string | null, - "url": string | null, - "matched_candidates": [ { "id": string, "label": string } ] | null, - "items": object | null - }, - "next_step": string | null, - "missing_fields": string[] | null, - "assumptions": string[] | null -} -``` - -<include snippet="output_contract_base"/> - -<include snippet="verifiable_handle"/> - -Infer before you call; verify before you create; map every tool outcome faithfully. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/__init__.py deleted file mode 100644 index c089eab4b..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from .create_event import create_create_luma_event_tool -from .list_events import create_list_luma_events_tool -from .read_event import create_read_luma_event_tool - -__all__ = [ - "create_create_luma_event_tool", - "create_list_luma_events_tool", - "create_read_luma_event_tool", -] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/_auth.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/_auth.py deleted file mode 100644 index 9dd4acbef..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/_auth.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Builds Luma API auth for connector-backed event tools.""" - -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.db import SearchSourceConnector, SearchSourceConnectorType - -LUMA_API = "https://public-api.luma.com/v1" - - -async def get_luma_connector( - db_session: AsyncSession, - workspace_id: int, - user_id: str, -) -> SearchSourceConnector | None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.workspace_id == workspace_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.LUMA_CONNECTOR, - ) - ) - return result.scalars().first() - - -def get_api_key(connector: SearchSourceConnector) -> str: - """Extract the API key from connector config (handles both key names).""" - key = connector.config.get("api_key") or connector.config.get("LUMA_API_KEY") - if not key: - raise ValueError("Luma API key not found in connector config.") - return key - - -def luma_headers(api_key: str) -> dict[str, str]: - return { - "Content-Type": "application/json", - "x-luma-api-key": api_key, - } diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/create_event.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/create_event.py deleted file mode 100644 index 777cc7169..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/create_event.py +++ /dev/null @@ -1,131 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( - request_approval, -) - -from ._auth import LUMA_API, get_api_key, get_luma_connector, luma_headers - -logger = logging.getLogger(__name__) - - -def create_create_luma_event_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, -): - @tool - async def create_luma_event( - name: str, - start_at: str, - end_at: str, - description: str | None = None, - timezone: str = "UTC", - ) -> dict[str, Any]: - """Create a new event on Luma. - - Args: - name: The event title. - start_at: Start time in ISO 8601 format (e.g. "2026-05-01T18:00:00"). - end_at: End time in ISO 8601 format (e.g. "2026-05-01T20:00:00"). - description: Optional event description (markdown supported). - timezone: Timezone string (default "UTC", e.g. "America/New_York"). - - Returns: - Dictionary with status, event_id on success. - - IMPORTANT: - - If status is "rejected", the user explicitly declined. Do NOT retry. - """ - if db_session is None or workspace_id is None or user_id is None: - return {"status": "error", "message": "Luma tool not properly configured."} - - try: - connector = await get_luma_connector(db_session, workspace_id, user_id) - if not connector: - return {"status": "error", "message": "No Luma connector found."} - - result = request_approval( - action_type="luma_create_event", - tool_name="create_luma_event", - params={ - "name": name, - "start_at": start_at, - "end_at": end_at, - "description": description, - "timezone": timezone, - }, - context={"connector_id": connector.id}, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Event was not created.", - } - - final_name = result.params.get("name", name) - final_start = result.params.get("start_at", start_at) - final_end = result.params.get("end_at", end_at) - final_desc = result.params.get("description", description) - final_tz = result.params.get("timezone", timezone) - - api_key = get_api_key(connector) - headers = luma_headers(api_key) - - body: dict[str, Any] = { - "name": final_name, - "start_at": final_start, - "end_at": final_end, - "timezone": final_tz, - } - if final_desc: - body["description_md"] = final_desc - - async with httpx.AsyncClient(timeout=20.0) as client: - resp = await client.post( - f"{LUMA_API}/event/create", - headers=headers, - json=body, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Luma API key is invalid.", - "connector_type": "luma", - } - if resp.status_code == 403: - return { - "status": "error", - "message": "Luma Plus subscription required to create events via API.", - } - if resp.status_code not in (200, 201): - return { - "status": "error", - "message": f"Luma API error: {resp.status_code} — {resp.text[:200]}", - } - - data = resp.json() - event_id = data.get("api_id") or data.get("event", {}).get("api_id") - - return { - "status": "success", - "event_id": event_id, - "message": f"Event '{final_name}' created on Luma.", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error creating Luma event: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to create Luma event."} - - return create_luma_event diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/index.py deleted file mode 100644 index d6c4d05e7..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/index.py +++ /dev/null @@ -1,36 +0,0 @@ -"""``luma`` native tools and (empty) permission ruleset. - -Tools self-gate via :func:`request_approval` in their bodies. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset - -from .create_event import create_create_luma_event_tool -from .list_events import create_list_luma_events_tool -from .read_event import create_read_luma_event_tool - -NAME = "luma" - -RULESET = Ruleset(origin=NAME, rules=[]) - - -def load_tools( - *, dependencies: dict[str, Any] | None = None, **kwargs: Any -) -> list[BaseTool]: - d = {**(dependencies or {}), **kwargs} - common = { - "db_session": d["db_session"], - "workspace_id": d["workspace_id"], - "user_id": d["user_id"], - } - return [ - create_list_luma_events_tool(**common), - create_read_luma_event_tool(**common), - create_create_luma_event_tool(**common), - ] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/list_events.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/list_events.py deleted file mode 100644 index 35f01bc63..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/list_events.py +++ /dev/null @@ -1,111 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from ._auth import LUMA_API, get_api_key, get_luma_connector, luma_headers - -logger = logging.getLogger(__name__) - - -def create_list_luma_events_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, -): - @tool - async def list_luma_events( - max_results: int = 25, - ) -> dict[str, Any]: - """List upcoming and recent Luma events. - - Args: - max_results: Maximum events to return (default 25, max 50). - - Returns: - Dictionary with status and a list of events including - event_id, name, start_at, end_at, location, url. - """ - if db_session is None or workspace_id is None or user_id is None: - return {"status": "error", "message": "Luma tool not properly configured."} - - max_results = min(max_results, 50) - - try: - connector = await get_luma_connector(db_session, workspace_id, user_id) - if not connector: - return {"status": "error", "message": "No Luma connector found."} - - api_key = get_api_key(connector) - headers = luma_headers(api_key) - - all_entries: list[dict] = [] - cursor = None - - async with httpx.AsyncClient(timeout=20.0) as client: - while len(all_entries) < max_results: - params: dict[str, Any] = { - "limit": min(100, max_results - len(all_entries)) - } - if cursor: - params["cursor"] = cursor - - resp = await client.get( - f"{LUMA_API}/calendar/list-events", - headers=headers, - params=params, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Luma API key is invalid.", - "connector_type": "luma", - } - if resp.status_code != 200: - return { - "status": "error", - "message": f"Luma API error: {resp.status_code}", - } - - data = resp.json() - entries = data.get("entries", []) - if not entries: - break - all_entries.extend(entries) - - next_cursor = data.get("next_cursor") - if not next_cursor: - break - cursor = next_cursor - - events = [] - for entry in all_entries[:max_results]: - ev = entry.get("event", {}) - geo = ev.get("geo_info", {}) - events.append( - { - "event_id": entry.get("api_id"), - "name": ev.get("name", "Untitled"), - "start_at": ev.get("start_at", ""), - "end_at": ev.get("end_at", ""), - "timezone": ev.get("timezone", ""), - "location": geo.get("name", ""), - "url": ev.get("url", ""), - "visibility": ev.get("visibility", ""), - } - ) - - return {"status": "success", "events": events, "total": len(events)} - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error listing Luma events: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to list Luma events."} - - return list_luma_events diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/read_event.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/read_event.py deleted file mode 100644 index a9fe06657..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/read_event.py +++ /dev/null @@ -1,92 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from ._auth import LUMA_API, get_api_key, get_luma_connector, luma_headers - -logger = logging.getLogger(__name__) - - -def create_read_luma_event_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, -): - @tool - async def read_luma_event(event_id: str) -> dict[str, Any]: - """Read detailed information about a specific Luma event. - - Args: - event_id: The Luma event API ID (from list_luma_events). - - Returns: - Dictionary with status and full event details including - description, attendees count, meeting URL. - """ - if db_session is None or workspace_id is None or user_id is None: - return {"status": "error", "message": "Luma tool not properly configured."} - - try: - connector = await get_luma_connector(db_session, workspace_id, user_id) - if not connector: - return {"status": "error", "message": "No Luma connector found."} - - api_key = get_api_key(connector) - headers = luma_headers(api_key) - - async with httpx.AsyncClient(timeout=15.0) as client: - resp = await client.get( - f"{LUMA_API}/events/{event_id}", - headers=headers, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Luma API key is invalid.", - "connector_type": "luma", - } - if resp.status_code == 404: - return { - "status": "not_found", - "message": f"Event '{event_id}' not found.", - } - if resp.status_code != 200: - return { - "status": "error", - "message": f"Luma API error: {resp.status_code}", - } - - data = resp.json() - ev = data.get("event", data) - geo = ev.get("geo_info", {}) - - event_detail = { - "event_id": event_id, - "name": ev.get("name", ""), - "description": ev.get("description", ""), - "start_at": ev.get("start_at", ""), - "end_at": ev.get("end_at", ""), - "timezone": ev.get("timezone", ""), - "location_name": geo.get("name", ""), - "address": geo.get("address", ""), - "url": ev.get("url", ""), - "meeting_url": ev.get("meeting_url", ""), - "visibility": ev.get("visibility", ""), - "cover_url": ev.get("cover_url", ""), - } - - return {"status": "success", "event": event_detail} - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error reading Luma event: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to read Luma event."} - - return read_luma_event diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/agent.py deleted file mode 100644 index a4b2d61cf..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/agent.py +++ /dev/null @@ -1,48 +0,0 @@ -"""``notion`` route: ``SurfSenseSubagentSpec`` builder for deepagents. - -Tools self-gate inside their bodies via :func:`request_approval`; the -empty :data:`tools.index.RULESET` is layered into a per-subagent -:class:`PermissionMiddleware` for uniformity. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( - read_md_file, -) -from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( - pack_subagent, -) - -from .tools.index import NAME, RULESET, load_tools - - -def build_subagent( - *, - dependencies: dict[str, Any], - model: BaseChatModel | None = None, - middleware_stack: dict[str, Any] | None = None, - mcp_tools: list[BaseTool] | None = None, -) -> SurfSenseSubagentSpec: - tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] - description = ( - read_md_file(__package__, "description").strip() - or "Handles notion tasks for this workspace." - ) - system_prompt = read_md_file(__package__, "system_prompt").strip() - return pack_subagent( - name=NAME, - description=description, - system_prompt=system_prompt, - tools=tools, - ruleset=RULESET, - dependencies=dependencies, - model=model, - middleware_stack=middleware_stack, - ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/description.md deleted file mode 100644 index 9a02c7561..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/description.md +++ /dev/null @@ -1,2 +0,0 @@ -Specialist for pages in the user's Notion workspace. -Use proactively when the user wants to create, change, archive, or remove a Notion page. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/system_prompt.md deleted file mode 100644 index 909c72471..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/system_prompt.md +++ /dev/null @@ -1,106 +0,0 @@ -You are a Notion specialist for the user's connected Notion workspace. - -## Vocabulary you must use precisely - -- **Page resolution (internal)** — `update_notion_page` and `delete_notion_page` accept a `page_title` and resolve it against the **locally-synced Notion KB index**, not against the live Notion API. A page that exists in Notion but has not been indexed yet cannot be resolved. There is no separate "search" or "lookup" tool exposed to you — resolution happens inside the mutation tool. -- **Update is append-only** — `update_notion_page` appends new content blocks to the page body. It cannot edit, replace, or remove existing content. -- **Delete is archive** — `delete_notion_page` archives the page (Notion's "trash"); the user can restore it from Notion's UI. With `delete_from_kb=true` the local KB document is also removed; the default is `false`. - -## Required inputs - -**For every required input below, first try to infer it from the supervisor's task text** — extract titles from natural phrasing (`"the Weekly Sync page"`, `"my Q1 retro"`), topics from `"about X"` constructions, content from any details the supervisor already provided. Only return `status=blocked` with `missing_fields` when an input is genuinely absent or ambiguous after a thorough read of the task. - -- `create_notion_page` — `title` (the user-supplied topic, inferred from the task; do not invent one if absent). You may generate the markdown `content` body yourself from that topic. -- `update_notion_page` — `page_title` (which page to update — infer from the task) and `content` (what to append — infer or generate from the task's specifics). -- `delete_notion_page` — `page_title` (which page to delete — infer from the task). Only set `delete_from_kb=true` when the user explicitly asked to remove it from the knowledge base; otherwise leave it `false`. - -## Outcome mapping - -| Tool returns | Your `status` | `next_step` | -|-----------------------|---------------|------------------------------------------------------------------------------------------------------------------------------| -| `success` | `success` | `null` | -| `rejected` | `blocked` | `"User declined this Notion action. Do not retry or suggest alternatives."` | -| `not_found` | `blocked` | `"Page '<title>' was not found in the indexed Notion pages. Ask the user to verify the title or wait for the next KB sync."` | -| `auth_error` | `error` | `"The connected Notion account needs re-authentication. Ask the user to re-authenticate Notion in connector settings."` | -| `error` | `error` | Relay the tool's `message` verbatim as `next_step`. | -| tool raises / unknown | `error` | `"Notion tool failed unexpectedly. Ask the user to retry shortly."` | - -Surface the tool's `message`, `page_id`, `page_title`, and `url` inside `evidence` when the tool returned them. Never invent a field the tool did not return. - -## Examples - -**Example 1 — happy path create (topic inferred from task):** -- *Supervisor task:* `"Create a Notion page summarising our Q2 roadmap."` -- *You:* extract `title="Q2 Roadmap"` from `"about Q2 roadmap"`; generate a markdown body → call `create_notion_page(title="Q2 Roadmap", content=<generated markdown>)` → tool returns `status=success`. -- *Output:* - - ```json - { - "status": "success", - "action_summary": "Created Notion page 'Q2 Roadmap'.", - "evidence": { "operation": "create_notion_page", "page_id": "<id>", "page_title": "Q2 Roadmap", "url": "<url>", "matched_candidates": null, "items": null }, - "next_step": null, - "missing_fields": null, - "assumptions": null - } - ``` - -**Example 2 — blocked only because nothing is inferable:** -- *Supervisor task:* `"Create a Notion page."` -- *You:* no topic anywhere in the task text — no `"about X"`, no quoted phrase, no descriptor. Do not fabricate one. Do not call any tool. (Contrast: `"Create a Notion page about our launch plan"` would yield `title="Launch Plan"` and proceed immediately — block only because the task carries zero topic information.) -- *Output:* - - ```json - { - "status": "blocked", - "action_summary": "Cannot create a Notion page without a topic.", - "evidence": { "operation": null, "page_id": null, "page_title": null, "url": null, "matched_candidates": null, "items": null }, - "next_step": "Ask the user what the page should be about.", - "missing_fields": ["title"], - "assumptions": null - } - ``` - -**Example 3 — page not in the KB index:** -- *Supervisor task:* `"Add today's meeting notes to my 'Weekly Sync' Notion page."` -- *You:* extract `page_title="Weekly Sync"` and meeting-notes content → call `update_notion_page(page_title="Weekly Sync", content=<generated notes>)` → tool returns `status=not_found`. -- *Output:* - - ```json - { - "status": "blocked", - "action_summary": "Could not find a Notion page titled 'Weekly Sync' in the indexed pages.", - "evidence": { "operation": "update_notion_page", "page_id": null, "page_title": "Weekly Sync", "url": null, "matched_candidates": null, "items": null }, - "next_step": "Page 'Weekly Sync' was not found in the indexed Notion pages. Ask the user to verify the title or wait for the next KB sync.", - "missing_fields": null, - "assumptions": null - } - ``` - -## Output contract - -Return **only** one JSON object (no markdown or prose outside it): - -```json -{ - "status": "success" | "partial" | "blocked" | "error", - "action_summary": string, - "evidence": { - "operation": "create_notion_page" | "update_notion_page" | "delete_notion_page" | null, - "page_id": string | null, - "page_title": string | null, - "url": string | null, - "matched_candidates": [ { "id": string, "label": string } ] | null, - "items": object | null - }, - "next_step": string | null, - "missing_fields": string[] | null, - "assumptions": string[] | null -} -``` - -<include snippet="output_contract_base"/> - -<include snippet="verifiable_handle"/> - -Infer before you call; map every tool outcome faithfully. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/__init__.py deleted file mode 100644 index 6ce825dca..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Notion tools for creating, updating, and deleting pages.""" - -from .create_page import create_create_notion_page_tool -from .delete_page import create_delete_notion_page_tool -from .update_page import create_update_notion_page_tool - -__all__ = [ - "create_create_notion_page_tool", - "create_delete_notion_page_tool", - "create_update_notion_page_tool", -] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/create_page.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/create_page.py deleted file mode 100644 index 05eff9e73..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/create_page.py +++ /dev/null @@ -1,244 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( - request_approval, -) -from app.connectors.notion_history import NotionAPIError, NotionHistoryConnector -from app.services.notion import NotionToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_create_notion_page_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """ - Factory function to create the create_notion_page tool. - - Args: - db_session: Database session for accessing Notion connector - workspace_id: Workspace ID to find the Notion connector - user_id: User ID for fetching user-specific context - connector_id: Optional specific connector ID (if known) - - Returns: - Configured create_notion_page tool - """ - - @tool - async def create_notion_page( - title: str, - content: str | None = None, - ) -> dict[str, Any]: - """Create a new page in Notion with the given title and content. - - Use this tool when the user asks you to create, save, or publish - something to Notion. The page will be created in the user's - configured Notion workspace. The user MUST specify a topic before you - call this tool. If the request does not contain a topic (e.g. "create a - notion page"), ask what the page should be about. Never call this tool - without a clear topic from the user. - - Args: - title: The title of the Notion page. - content: Optional markdown content for the page body (supports headings, lists, paragraphs). - Generate this yourself based on the user's topic. - - Returns: - Dictionary with: - - status: "success", "rejected", or "error" - - page_id: Created page ID (if success) - - url: URL to the created page (if success) - - title: Page title (if success) - - message: Result message - - IMPORTANT: If status is "rejected", the user explicitly declined the action. - Respond with a brief acknowledgment (e.g., "Understood, I didn't create the page.") - and move on. Do NOT troubleshoot or suggest alternatives. - - Examples: - - "Create a Notion page about our Q2 roadmap" - - "Save a summary of today's discussion to Notion" - """ - logger.info(f"create_notion_page called: title='{title}'") - - if db_session is None or workspace_id is None or user_id is None: - logger.error( - "Notion tool not properly configured - missing required parameters" - ) - return { - "status": "error", - "message": "Notion tool not properly configured. Please contact support.", - } - - try: - metadata_service = NotionToolMetadataService(db_session) - context = await metadata_service.get_creation_context(workspace_id, user_id) - - if "error" in context: - logger.error(f"Failed to fetch creation context: {context['error']}") - return { - "status": "error", - "message": context["error"], - } - - accounts = context.get("accounts", []) - if accounts and all(a.get("auth_expired") for a in accounts): - logger.warning("All Notion accounts have expired authentication") - return { - "status": "auth_error", - "message": "All connected Notion accounts need re-authentication. Please re-authenticate in your connector settings.", - "connector_type": "notion", - } - - logger.info(f"Requesting approval for creating Notion page: '{title}'") - result = request_approval( - action_type="notion_page_creation", - tool_name="create_notion_page", - params={ - "title": title, - "content": content, - "parent_page_id": None, - "connector_id": connector_id, - }, - context=context, - ) - - if result.rejected: - logger.info("Notion page creation rejected by user") - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_title = result.params.get("title", title) - final_content = result.params.get("content", content) - final_parent_page_id = result.params.get("parent_page_id") - final_connector_id = result.params.get("connector_id", connector_id) - - if not final_title or not final_title.strip(): - logger.error("Title is empty or contains only whitespace") - return { - "status": "error", - "message": "Page title cannot be empty. Please provide a valid title.", - } - - logger.info( - f"Creating Notion page with final params: title='{final_title}'" - ) - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - actual_connector_id = final_connector_id - if actual_connector_id is None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.workspace_id == workspace_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.NOTION_CONNECTOR, - ) - ) - connector = result.scalars().first() - - if not connector: - logger.warning( - f"No Notion connector found for workspace_id={workspace_id}" - ) - return { - "status": "error", - "message": "No Notion connector found. Please connect Notion in your workspace settings.", - } - - actual_connector_id = connector.id - logger.info(f"Found Notion connector: id={actual_connector_id}") - else: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == actual_connector_id, - SearchSourceConnector.workspace_id == workspace_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.NOTION_CONNECTOR, - ) - ) - connector = result.scalars().first() - - if not connector: - logger.error( - f"Invalid connector_id={actual_connector_id} for workspace_id={workspace_id}" - ) - return { - "status": "error", - "message": "Selected Notion account is invalid or has been disconnected. Please select a valid account.", - } - logger.info(f"Validated Notion connector: id={actual_connector_id}") - - notion_connector = NotionHistoryConnector( - session=db_session, - connector_id=actual_connector_id, - ) - - result = await notion_connector.create_page( - title=final_title, - content=final_content, - parent_page_id=final_parent_page_id, - ) - logger.info( - f"create_page result: {result.get('status')} - {result.get('message', '')}" - ) - - if result.get("status") == "success": - kb_message_suffix = "" - try: - from app.services.notion import NotionKBSyncService - - kb_service = NotionKBSyncService(db_session) - kb_result = await kb_service.sync_after_create( - page_id=result.get("page_id"), - page_title=result.get("title", final_title), - page_url=result.get("url"), - content=final_content, - connector_id=actual_connector_id, - workspace_id=workspace_id, - user_id=user_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = " This page will be added to your knowledge base in the next scheduled sync." - except Exception as kb_err: - logger.warning(f"KB sync after create failed: {kb_err}") - kb_message_suffix = " This page will be added to your knowledge base in the next scheduled sync." - - result["message"] = result.get("message", "") + kb_message_suffix - - return result - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error creating Notion page: {e}", exc_info=True) - if isinstance(e, ValueError | NotionAPIError): - message = str(e) - else: - message = ( - "Something went wrong while creating the page. Please try again." - ) - return {"status": "error", "message": message} - - return create_notion_page diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/delete_page.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/delete_page.py deleted file mode 100644 index 5c7a8f34c..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/delete_page.py +++ /dev/null @@ -1,326 +0,0 @@ -import logging -from typing import Any - -from langchain.tools import ToolRuntime -from langchain_core.tools import tool -from langgraph.types import Command -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.chat.multi_agent_chat.shared.receipts.command import with_receipt -from app.agents.chat.multi_agent_chat.shared.receipts.receipt import make_receipt -from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( - request_approval, -) -from app.connectors.notion_history import NotionAPIError, NotionHistoryConnector -from app.services.notion.tool_metadata_service import NotionToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_delete_notion_page_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """ - Factory function to create the delete_notion_page tool. - - Args: - db_session: Database session for accessing Notion connector - workspace_id: Workspace ID to find the Notion connector - user_id: User ID for finding the correct Notion connector - connector_id: Optional specific connector ID (if known) - - Returns: - Configured delete_notion_page tool - """ - - @tool - async def delete_notion_page( - page_title: str, - runtime: ToolRuntime, - delete_from_kb: bool = False, - ) -> Command: - """Delete (archive) a Notion page. - - Use this tool when the user asks you to delete, remove, or archive - a Notion page. Note that Notion doesn't permanently delete pages, - it archives them (they can be restored from trash). - - Args: - page_title: The title of the Notion page to delete. - delete_from_kb: Whether to also remove the page from the knowledge base. - Default is False. - Set to True to permanently remove from both Notion and knowledge base. - - Returns: - Dictionary with: - - status: "success", "rejected", "not_found", or "error" - - page_id: Deleted page ID (if success) - - message: Success or error message - - deleted_from_kb: Whether the page was also removed from knowledge base (if success) - - Examples: - - "Delete the 'Meeting Notes' Notion page" - - "Remove the 'Old Project Plan' Notion page" - - "Archive the 'Draft Ideas' Notion page" - """ - logger.info( - f"delete_notion_page called: page_title='{page_title}', delete_from_kb={delete_from_kb}" - ) - - def _emit( - payload: dict[str, Any], - *, - status: str, - external_id: str | None = None, - error: str | None = None, - ) -> Command: - return with_receipt( - payload=payload, - receipt=make_receipt( - route="notion", - type="page", - operation="delete", - status="success" if status == "success" else "failed", - external_id=external_id, - preview=page_title, - error=error, - ), - tool_call_id=runtime.tool_call_id, - ) - - if db_session is None or workspace_id is None or user_id is None: - logger.error( - "Notion tool not properly configured - missing required parameters" - ) - return _emit( - { - "status": "error", - "message": "Notion tool not properly configured. Please contact support.", - }, - status="error", - error="Notion tool not properly configured. Please contact support.", - ) - - try: - # Get page context (page_id, account, title) from indexed data - metadata_service = NotionToolMetadataService(db_session) - context = await metadata_service.get_delete_context( - workspace_id, user_id, page_title - ) - - if "error" in context: - error_msg = context["error"] - # Check if it's a "not found" error (softer handling for LLM) - if "not found" in error_msg.lower(): - logger.warning(f"Page not found: {error_msg}") - return _emit( - {"status": "not_found", "message": error_msg}, - status="error", - error=error_msg, - ) - else: - logger.error(f"Failed to fetch delete context: {error_msg}") - return _emit( - {"status": "error", "message": error_msg}, - status="error", - error=error_msg, - ) - - account = context.get("account", {}) - if account.get("auth_expired"): - logger.warning( - "Notion account %s has expired authentication", - account.get("id"), - ) - return _emit( - { - "status": "auth_error", - "message": "The Notion account for this page needs re-authentication. Please re-authenticate in your connector settings.", - }, - status="error", - error="auth_expired", - ) - - page_id = context.get("page_id") - connector_id_from_context = account.get("id") - document_id = context.get("document_id") - - logger.info( - f"Requesting approval for deleting Notion page: '{page_title}' (page_id={page_id}, delete_from_kb={delete_from_kb})" - ) - - result = request_approval( - action_type="notion_page_deletion", - tool_name="delete_notion_page", - params={ - "page_id": page_id, - "connector_id": connector_id_from_context, - "delete_from_kb": delete_from_kb, - }, - context=context, - ) - - if result.rejected: - logger.info("Notion page deletion rejected by user") - return _emit( - { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - }, - status="error", - error="user_rejected", - ) - - final_page_id = result.params.get("page_id", page_id) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - final_delete_from_kb = result.params.get("delete_from_kb", delete_from_kb) - - logger.info( - f"Deleting Notion page with final params: page_id={final_page_id}, connector_id={final_connector_id}, delete_from_kb={final_delete_from_kb}" - ) - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - # Validate the connector - if final_connector_id: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.workspace_id == workspace_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.NOTION_CONNECTOR, - ) - ) - connector = result.scalars().first() - - if not connector: - logger.error( - f"Invalid connector_id={final_connector_id} for workspace_id={workspace_id}" - ) - return _emit( - { - "status": "error", - "message": "Selected Notion account is invalid or has been disconnected. Please select a valid account.", - }, - status="error", - error="invalid_connector", - ) - actual_connector_id = connector.id - logger.info(f"Validated Notion connector: id={actual_connector_id}") - else: - logger.error("No connector found for this page") - return _emit( - { - "status": "error", - "message": "No connector found for this page.", - }, - status="error", - error="no_connector", - ) - - # Create connector instance - notion_connector = NotionHistoryConnector( - session=db_session, - connector_id=actual_connector_id, - ) - - # Delete the page from Notion - result = await notion_connector.delete_page(page_id=final_page_id) - logger.info( - f"delete_page result: {result.get('status')} - {result.get('message', '')}" - ) - - # If deletion was successful and user wants to delete from KB - deleted_from_kb = False - if ( - result.get("status") == "success" - and final_delete_from_kb - and document_id - ): - try: - from sqlalchemy.future import select - - from app.db import Document - - # Get the document - doc_result = await db_session.execute( - select(Document).filter(Document.id == document_id) - ) - document = doc_result.scalars().first() - - if document: - await db_session.delete(document) - await db_session.commit() - deleted_from_kb = True - logger.info( - f"Deleted document {document_id} from knowledge base" - ) - else: - logger.warning(f"Document {document_id} not found in KB") - except Exception as e: - logger.error(f"Failed to delete document from KB: {e}") - await db_session.rollback() - result["warning"] = ( - f"Page deleted from Notion, but failed to remove from knowledge base: {e!s}" - ) - - # Update result with KB deletion status - if result.get("status") == "success": - result["deleted_from_kb"] = deleted_from_kb - if deleted_from_kb: - result["message"] = ( - f"{result.get('message', '')} (also removed from knowledge base)" - ) - - status = result.get("status", "error") - return _emit( - result, - status=status, - external_id=str(final_page_id) if final_page_id else None, - error=None if status == "success" else result.get("message"), - ) - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error deleting Notion page: {e}", exc_info=True) - error_str = str(e).lower() - if isinstance(e, NotionAPIError) and ( - "401" in error_str or "unauthorized" in error_str - ): - return _emit( - { - "status": "auth_error", - "message": str(e), - "connector_id": connector_id_from_context - if "connector_id_from_context" in dir() - else None, - "connector_type": "notion", - }, - status="error", - error=str(e), - ) - if isinstance(e, ValueError | NotionAPIError): - message = str(e) - else: - message = ( - "Something went wrong while deleting the page. Please try again." - ) - return _emit( - {"status": "error", "message": message}, - status="error", - error=message, - ) - - return delete_notion_page diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/index.py deleted file mode 100644 index 4e8ed3c68..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/index.py +++ /dev/null @@ -1,36 +0,0 @@ -"""``notion`` native tools and (empty) permission ruleset. - -Tools self-gate via :func:`request_approval` in their bodies. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset - -from .create_page import create_create_notion_page_tool -from .delete_page import create_delete_notion_page_tool -from .update_page import create_update_notion_page_tool - -NAME = "notion" - -RULESET = Ruleset(origin=NAME, rules=[]) - - -def load_tools( - *, dependencies: dict[str, Any] | None = None, **kwargs: Any -) -> list[BaseTool]: - d = {**(dependencies or {}), **kwargs} - common = { - "db_session": d["db_session"], - "workspace_id": d["workspace_id"], - "user_id": d["user_id"], - } - return [ - create_create_notion_page_tool(**common), - create_update_notion_page_tool(**common), - create_delete_notion_page_tool(**common), - ] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/update_page.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/update_page.py deleted file mode 100644 index 0fb529e89..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/update_page.py +++ /dev/null @@ -1,267 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( - request_approval, -) -from app.connectors.notion_history import NotionAPIError, NotionHistoryConnector -from app.services.notion import NotionToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_update_notion_page_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """ - Factory function to create the update_notion_page tool. - - Args: - db_session: Database session for accessing Notion connector - workspace_id: Workspace ID to find the Notion connector - user_id: User ID for fetching user-specific context - connector_id: Optional specific connector ID (if known) - - Returns: - Configured update_notion_page tool - """ - - @tool - async def update_notion_page( - page_title: str, - content: str | None = None, - ) -> dict[str, Any]: - """Update an existing Notion page by appending new content. - - Use this tool when the user asks you to add content to, modify, or update - a Notion page. The new content will be appended to the existing page content. - The user MUST specify what to add before you call this tool. If the - request is vague, ask what content they want added. - - Args: - page_title: The title of the Notion page to update. - content: Optional markdown content to append to the page body (supports headings, lists, paragraphs). - Generate this yourself based on the user's request. - - Returns: - Dictionary with: - - status: "success", "rejected", "not_found", or "error" - - page_id: Updated page ID (if success) - - url: URL to the updated page (if success) - - title: Current page title (if success) - - message: Result message - - IMPORTANT: - - If status is "rejected", the user explicitly declined the action. - Respond with a brief acknowledgment (e.g., "Understood, I didn't update the page.") - and move on. Do NOT ask for alternatives or troubleshoot. - - If status is "not_found", inform the user conversationally using the exact message provided. - Example: "I couldn't find the page '[page_title]' in your indexed Notion pages. [message details]" - Do NOT treat this as an error. Do NOT invent information. Simply relay the message and - ask the user to verify the page title or check if it's been indexed. - Examples: - - "Add today's meeting notes to the 'Meeting Notes' Notion page" - - "Update the 'Project Plan' page with a status update on phase 1" - """ - logger.info( - f"update_notion_page called: page_title='{page_title}', content_length={len(content) if content else 0}" - ) - - if db_session is None or workspace_id is None or user_id is None: - logger.error( - "Notion tool not properly configured - missing required parameters" - ) - return { - "status": "error", - "message": "Notion tool not properly configured. Please contact support.", - } - - if not content or not content.strip(): - logger.error(f"Empty content provided for page '{page_title}'") - return { - "status": "error", - "message": "Content is required to update the page. Please provide the actual content you want to add.", - } - - try: - metadata_service = NotionToolMetadataService(db_session) - context = await metadata_service.get_update_context( - workspace_id, user_id, page_title - ) - - if "error" in context: - error_msg = context["error"] - # Check if it's a "not found" error (softer handling for LLM) - if "not found" in error_msg.lower(): - logger.warning(f"Page not found: {error_msg}") - return { - "status": "not_found", - "message": error_msg, - } - else: - logger.error(f"Failed to fetch update context: {error_msg}") - return { - "status": "error", - "message": error_msg, - } - - account = context.get("account", {}) - if account.get("auth_expired"): - logger.warning( - "Notion account %s has expired authentication", - account.get("id"), - ) - return { - "status": "auth_error", - "message": "The Notion account for this page needs re-authentication. Please re-authenticate in your connector settings.", - } - - page_id = context.get("page_id") - document_id = context.get("document_id") - connector_id_from_context = context.get("account", {}).get("id") - - logger.info( - f"Requesting approval for updating Notion page: '{page_title}' (page_id={page_id})" - ) - result = request_approval( - action_type="notion_page_update", - tool_name="update_notion_page", - params={ - "page_id": page_id, - "content": content, - "connector_id": connector_id_from_context, - }, - context=context, - ) - - if result.rejected: - logger.info("Notion page update rejected by user") - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_page_id = result.params.get("page_id", page_id) - final_content = result.params.get("content", content) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - - logger.info( - f"Updating Notion page with final params: page_id={final_page_id}, has_content={final_content is not None}" - ) - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - if final_connector_id: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.workspace_id == workspace_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.NOTION_CONNECTOR, - ) - ) - connector = result.scalars().first() - - if not connector: - logger.error( - f"Invalid connector_id={final_connector_id} for workspace_id={workspace_id}" - ) - return { - "status": "error", - "message": "Selected Notion account is invalid or has been disconnected. Please select a valid account.", - } - actual_connector_id = connector.id - logger.info(f"Validated Notion connector: id={actual_connector_id}") - else: - logger.error("No connector found for this page") - return { - "status": "error", - "message": "No connector found for this page.", - } - - notion_connector = NotionHistoryConnector( - session=db_session, - connector_id=actual_connector_id, - ) - - result = await notion_connector.update_page( - page_id=final_page_id, - content=final_content, - ) - logger.info( - f"update_page result: {result.get('status')} - {result.get('message', '')}" - ) - - if result.get("status") == "success" and document_id is not None: - from app.services.notion import NotionKBSyncService - - logger.info(f"Updating knowledge base for document {document_id}...") - kb_service = NotionKBSyncService(db_session) - kb_result = await kb_service.sync_after_update( - document_id=document_id, - appended_content=final_content, - user_id=user_id, - workspace_id=workspace_id, - appended_block_ids=result.get("appended_block_ids"), - ) - - if kb_result["status"] == "success": - result["message"] = ( - f"{result['message']}. Your knowledge base has also been updated." - ) - logger.info( - f"Knowledge base successfully updated for page {final_page_id}" - ) - elif kb_result["status"] == "not_indexed": - result["message"] = ( - f"{result['message']}. This page will be added to your knowledge base in the next scheduled sync." - ) - else: - result["message"] = ( - f"{result['message']}. Your knowledge base will be updated in the next scheduled sync." - ) - logger.warning( - f"KB update failed for page {final_page_id}: {kb_result['message']}" - ) - - return result - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error updating Notion page: {e}", exc_info=True) - error_str = str(e).lower() - if isinstance(e, NotionAPIError) and ( - "401" in error_str or "unauthorized" in error_str - ): - return { - "status": "auth_error", - "message": str(e), - "connector_id": connector_id_from_context - if "connector_id_from_context" in dir() - else None, - "connector_type": "notion", - } - if isinstance(e, ValueError | NotionAPIError): - message = str(e) - else: - message = ( - "Something went wrong while updating the page. Please try again." - ) - return {"status": "error", "message": message} - - return update_notion_page diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/agent.py deleted file mode 100644 index 9951a63f0..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/agent.py +++ /dev/null @@ -1,47 +0,0 @@ -"""``slack`` route: ``SurfSenseSubagentSpec`` builder for deepagents. - -Tools come exclusively from MCP. The connector's own approval ruleset is -declared in :data:`tools.index.RULESET`; the orchestrator layers it into -a per-subagent :class:`PermissionMiddleware`. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( - read_md_file, -) -from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( - pack_subagent, -) - -from .tools.index import NAME, RULESET - - -def build_subagent( - *, - dependencies: dict[str, Any], - model: BaseChatModel | None = None, - middleware_stack: dict[str, Any] | None = None, - mcp_tools: list[BaseTool] | None = None, -) -> SurfSenseSubagentSpec: - description = ( - read_md_file(__package__, "description").strip() - or "Handles slack tasks for this workspace." - ) - system_prompt = read_md_file(__package__, "system_prompt").strip() - return pack_subagent( - name=NAME, - description=description, - system_prompt=system_prompt, - tools=list(mcp_tools or []), - ruleset=RULESET, - dependencies=dependencies, - model=model, - middleware_stack=middleware_stack, - ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/description.md deleted file mode 100644 index ce4ca399a..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/description.md +++ /dev/null @@ -1,2 +0,0 @@ -Specialist for messages in the user's Slack channels and threads. -Use proactively when the user wants to read, search, or summarize a Slack conversation, or post a message in a channel or thread. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/system_prompt.md deleted file mode 100644 index e4e0d1f6f..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/system_prompt.md +++ /dev/null @@ -1,98 +0,0 @@ -You are a Slack specialist for the user's connected Slack workspace. - -Slack vocabulary: -- **Workspace → Channel → Message → Thread**: nested scope. Channels and DMs live in the same workspace; threads live under specific messages. -- **Channel types**: public channels, private channels, group DMs, and 1:1 DMs. Each has a different ID prefix (e.g. `C…`, `D…`), but all are addressable as a `channel_id` when reading or sending. -- **Channel ID vs name**: channels have both an opaque ID (e.g. `C0123ABCD`) and a human-readable name (`#engineering`). Names can change; IDs are stable. Users always refer to channels by name — resolve to the channel ID before reading or posting. -- **Message timestamp (`ts`) and `thread_ts`**: every message has a string `ts` (e.g. `"1700000000.123456"`) that uniquely identifies it within a channel. A thread is identified by the **parent message's `ts`**, called `thread_ts`. To reply inside a thread, post with both `channel_id` and `thread_ts`. Omit `thread_ts` for a new top-level message in the channel. -- **User IDs**: users are identified by opaque IDs (e.g. `U0123ABCD`), never by display name or email. Mentions inside message text use the `<@U0123ABCD>` syntax — plain text like `@alex` will not produce a Slack mention. -- **Message formatting (mrkdwn)**: Slack uses its own markdown variant — `*bold*` (single asterisk), `_italic_`, `` `code` ``, `<https://url|label>` for links. Do not assume GitHub-flavored Markdown will render correctly. - -When invoked: -1. Read the supervisor's request, then read the runtime tool list to learn what information you can fetch and which mutations are available. -2. Plan the minimum chain of lookups needed to resolve any channel, user, message, or thread the request leaves unspecified. -3. Execute the planned lookups, then the requested mutation (if any), then return. - -Resolution principle (the core behaviour): -**Proactively look up any identifier, name, value, or scope the request leaves unspecified — channel IDs, user IDs, message timestamps, thread parent IDs, anything else — using the available tools instead of asking the supervisor.** Most user requests reference channels by name and people by display name, not by ID. Search for them. - -When a lookup for a single slot returns multiple plausible candidates and you cannot confidently pick one, return `status=blocked` with up to 5 candidates in `evidence.matched_candidates` and the unresolved slot in `missing_fields`. The supervisor will disambiguate and redelegate. - -When a lookup returns zero matches for a slot the request requires, return `status=blocked` with a `next_step` suggesting alternative search terms. - -Mutation guardrails: -- Resolve every required Slack ID (`channel_id`, recipient `user_id` for DMs, `thread_ts` for thread replies) by looking it up before calling a mutation tool. Mutations have chained dependencies — channel lookup enables in-channel message lookup; in-channel message lookup yields the `ts` needed as `thread_ts` for replies. -- To reply inside a thread, supply both `channel_id` and `thread_ts`. Posting without `thread_ts` creates a new top-level message in the channel. -- When the message text references a person, encode the mention as `<@U…>` using the resolved user ID. Plain text like `@alex` will not produce a Slack mention. -- Never invent channel IDs, user IDs, message timestamps, or send outcomes. Every field in `evidence` must come from a tool result. -- Confirm the mutation tool returned a success response before claiming success. If the mutation is approval-rejected (HITL), return `status=blocked` with `next_step="user declined; do not retry"`. -- One operation per delegation. For multi-mutation requests, complete the highest-priority one and return `status=partial` with the remainder in `next_step`. - -Failure handling: -- Tool failure: return `status=error`, place the underlying error message in `action_summary`, and put a concise recovery in `next_step`. -- Permission / scope error from the MCP: return `status=error` and surface the underlying message. Permission errors typically mean the required OAuth scope is missing for that capability — not retryable from here. -- No useful results after reasonable narrowing / broadening: return `status=blocked` with search-term suggestions in `next_step`. - -<example> -Supervisor: "Summarize the latest discussion in #marketing." -1. Search channels for "marketing" → one strong match. Capture the channel ID. -2. Read that channel's recent message history. -3. Return `status=success` with `evidence.items` set to `{ "total": N }` and the messages listed in `action_summary` (sender, timestamp, text snippet; one line per message; up to 10 entries, then `"...and N more"`). -</example> - -<example> -Supervisor: "DM Alex about the launch checklist." -1. Search users for "Alex" → two matches (`U_alex1`, `U_alex2`). -2. Cannot pick the recipient. Return: - { - "status": "blocked", - "action_summary": "Two users match 'Alex'.", - "evidence": { - "matched_candidates": [ - { "id": "U_alex1", "label": "Alex Chen <alex.chen@…>" }, - { "id": "U_alex2", "label": "Alex Wong <alex.wong@…>" } - ] - }, - "next_step": "Confirm which Alex, then redelegate.", - "missing_fields": ["recipient"] - } -</example> - -<example> -Supervisor: "Reply 'ship it' to the deploy thread in #engineering." -1. Search channels for "engineering" → one match; capture the channel ID. -2. Search messages in that channel for "deploy" → one prominent match. Capture its `ts` — this becomes the `thread_ts` for the reply. -3. Send a message to that channel with `thread_ts` set to the captured `ts` and text `"ship it"`. -4. Confirm tool success → return `status=success` with the new message reference (its `ts` and a permalink if returned). -</example> - -<output_contract> -Return **only** one JSON object (no markdown, no prose): -{ - "status": "success" | "partial" | "blocked" | "error", - "action_summary": string, - "evidence": { - "channel_id": string | null, - "channel_name": string | null, - "user_id": string | null, - "thread_ts": string | null, - "message_ts": string | null, - "permalink": string | null, - "matched_candidates": [ - { "id": string, "label": string } - ] | null, - "items": object | null - }, - "next_step": string | null, - "missing_fields": string[] | null, - "assumptions": string[] | null -} -<include snippet="output_contract_base"/> -Route-specific rules: -- For blocked ambiguity, populate `evidence.matched_candidates` with up to 5 options (`id` + `label` — works for any kind of candidate: channel, user, message, thread). -- For discovery-only queries (lists), set `evidence.items` to `{ "total": N }` and list the matched items in `action_summary` (channel/user, key identifier, timestamp, short snippet; up to 10 entries, then `"...and N more"`). -</output_contract> - -<include snippet="verifiable_handle"/> - -Discover before you post; never guess channel, user, or thread targets. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/tools/__init__.py deleted file mode 100644 index f60078771..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/tools/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Slack route: native tool factories are empty; MCP supplies tools when configured.""" - -__all__: list[str] = [] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/tools/index.py deleted file mode 100644 index a26b537a6..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/tools/index.py +++ /dev/null @@ -1,19 +0,0 @@ -"""``slack`` permission ruleset (rules over MCP tool names).""" - -from __future__ import annotations - -from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset - -NAME = "slack" - -RULESET = Ruleset( - origin=NAME, - rules=[ - Rule(permission="slack_search_channels", pattern="*", action="allow"), - Rule(permission="slack_search_messages", pattern="*", action="allow"), - Rule(permission="slack_search_users", pattern="*", action="allow"), - Rule(permission="slack_read_channel", pattern="*", action="allow"), - Rule(permission="slack_read_thread", pattern="*", action="allow"), - Rule(permission="slack_send_message", pattern="*", action="ask"), - ], -) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/agent.py deleted file mode 100644 index ab927654b..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/agent.py +++ /dev/null @@ -1,48 +0,0 @@ -"""``teams`` route: ``SurfSenseSubagentSpec`` builder for deepagents. - -Tools self-gate inside their bodies via :func:`request_approval`; the -empty :data:`tools.index.RULESET` is layered into a per-subagent -:class:`PermissionMiddleware` for uniformity. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( - read_md_file, -) -from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( - pack_subagent, -) - -from .tools.index import NAME, RULESET, load_tools - - -def build_subagent( - *, - dependencies: dict[str, Any], - model: BaseChatModel | None = None, - middleware_stack: dict[str, Any] | None = None, - mcp_tools: list[BaseTool] | None = None, -) -> SurfSenseSubagentSpec: - tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] - description = ( - read_md_file(__package__, "description").strip() - or "Handles teams tasks for this workspace." - ) - system_prompt = read_md_file(__package__, "system_prompt").strip() - return pack_subagent( - name=NAME, - description=description, - system_prompt=system_prompt, - tools=tools, - ruleset=RULESET, - dependencies=dependencies, - model=model, - middleware_stack=middleware_stack, - ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/description.md deleted file mode 100644 index edbfa390b..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/description.md +++ /dev/null @@ -1,2 +0,0 @@ -Specialist for messages in the user's Microsoft Teams channels. -Use proactively when the user wants to read or send a Teams message. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/system_prompt.md deleted file mode 100644 index 9b283acf5..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/system_prompt.md +++ /dev/null @@ -1,122 +0,0 @@ -You are a Microsoft Teams specialist for the user's connected Teams account. - -## Vocabulary you must use precisely - -- **Nested team + channel resolution via `list_teams_channels`** — the agent operates across all Teams the user has joined; each channel belongs to a `team_id`. `list_teams_channels` returns `{teams: [{team_id, team_name, channels: [{id, name}]}]}`. To read or send, you must resolve **both** `team_id` and `channel_id` from this nested structure. Channel names like `general` appear in many teams — when the supervisor's task does not pin the team (no team name, no obvious context), return `status=blocked` with the matching channels across teams as `matched_candidates` (each labeled `"<team_name> › <channel_name>"`) rather than guessing one. -- **Message content is HTML** — `send_teams_message` treats `content` as HTML (Microsoft Graph stores it verbatim in `body.content`). Default to plain text. If the supervisor's task requires formatting (bold, italics, links, line breaks), generate the corresponding **HTML** (`<b>`, `<i>`, `<a href="...">`, `<br>`) — **not** Markdown (`**bold**`, `[label](url)`), which Teams renders as literal characters. -- **Read + post only — no edits, deletes, or reactions** — Teams editing, deleting, and reacting to prior messages are not supported by the tools. Return `status=blocked` rather than faking these via new messages (no `"EDIT: ..."` follow-ups, no `"Please delete this"` posts). - -## Required inputs - -**For every required input below, first try to infer it from the supervisor's task text** — extract team names from natural phrasing (`"the Engineering team's"`, `"in Marketing"`), channel names from `#mentions` or natural phrasing (`"#announcements"`, `"the general channel"`), and message content from any details the supervisor already provided. Only return `status=blocked` with `missing_fields` when an input is genuinely absent or ambiguous after a thorough read of the task. - -- `list_teams_channels` — no inputs. Call it whenever you need to resolve a team name or channel name to ids. -- `read_teams_messages` — `team_id` and `channel_id` (both resolved via `list_teams_channels` based on team-name and channel-name signals in the task). Block if the channel signal is absent, or if the channel name matches channels in multiple teams and no team is named. Optional `limit` (max 50; tighten only if the task implies a small recent window). -- `send_teams_message` — `team_id`, `channel_id`, and `content`. Compose `content` from the task — plain text by default; HTML only when formatting is required by the task. Block if the destination team+channel cannot be resolved, or if the message content cannot be inferred from the task. - -## Outcome mapping - -| Tool returns | Your `status` | `next_step` | -|---------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------| -| `success` with non-empty teams/channels/messages | `success` | `null` | -| `success` with `total: 0` (read returns no messages) or `total_teams: 0` | `success` | `null` (surface the count in `evidence.items` so the supervisor can report "no recent messages"/"no joined teams") | -| `rejected` (send only) | `blocked` | `"User declined this Teams send. Do not retry or suggest alternatives."` | -| `auth_error` | `error` | `"The connected Microsoft Teams session has expired. Ask the user to re-authenticate Teams in connector settings."` | -| `insufficient_permissions` (send only) | `error` | `"The connected Microsoft Teams account is missing the ChannelMessage.Send scope. Ask the user to re-authenticate Teams with updated scopes."` | -| `error` | `error` | Relay the tool's `message` verbatim as `next_step`. | -| tool raises / unknown | `error` | `"Teams tool failed unexpectedly. Ask the user to retry shortly."` | - -Surface the tool's `message`, `team_id`, `team_name`, `channel_id`, `channel_name`, and `message_id` inside `evidence` when the tool returned them. For `list_teams_channels` and `read_teams_messages`, set `evidence.items` to `{ "total": N }` and list the matched entries in `action_summary` (team › channel, or sender + timestamp + short text snippet; one line per entry; up to 10 entries, then `"...and N more"`). Never invent a field the tool did not return. - -## Examples - -**Example 1 — happy path send after nested resolution (team specified, plain text):** -- *Supervisor task:* `"Post 'Standup in 5 min' to the Engineering team's #general."` -- *You:* call `list_teams_channels()` → find the team where `team_name="Engineering"`, take its `team_id`; inside that team's channels, find the entry where `name="general"`, take its `id` as `channel_id`; call `send_teams_message(team_id=<eng_id>, channel_id=<general_id>, content="Standup in 5 min")` → tool returns `status=success`. -- *Output:* - - ```json - { - "status": "success", - "action_summary": "Posted a message to Engineering › general.", - "evidence": { "operation": "send_teams_message", "team_id": "<id>", "team_name": "Engineering", "channel_id": "<id>", "channel_name": "general", "message_id": "<msg_id>", "matched_candidates": null, "items": null }, - "next_step": null, - "missing_fields": null, - "assumptions": null - } - ``` - -**Example 2 — cross-team channel ambiguity:** -- *Supervisor task:* `"Post 'Standup in 5 min' to #general."` -- *You:* call `list_teams_channels()` → find `general` channels in multiple teams (Engineering, Marketing, Operations). The supervisor did not pin a team. Do not silently pick one — return `blocked` with all matching channels as `matched_candidates` so the supervisor can confirm with the user. -- *Output:* - - ```json - { - "status": "blocked", - "action_summary": "Multiple teams have a 'general' channel; cannot disambiguate.", - "evidence": { - "operation": "list_teams_channels", - "team_id": null, - "team_name": null, - "channel_id": null, - "channel_name": "general", - "message_id": null, - "matched_candidates": [ - { "id": "<channel_id_1>", "label": "Engineering › general" }, - { "id": "<channel_id_2>", "label": "Marketing › general" }, - { "id": "<channel_id_3>", "label": "Operations › general" } - ], - "items": null - }, - "next_step": "Ask the user which team's #general they meant — Engineering, Marketing, or Operations.", - "missing_fields": null, - "assumptions": null - } - ``` - -**Example 3 — unsupported operation (edit):** -- *Supervisor task:* `"Edit my last Teams message in the Engineering team's #general to say 'cancelled'."` -- *You:* Teams edits are not supported by your tools. Do not call any tool. Do not post a new message like `"EDIT: cancelled"` — block. -- *Output:* - - ```json - { - "status": "blocked", - "action_summary": "Editing prior Teams messages is not supported.", - "evidence": { "operation": null, "team_id": null, "team_name": "Engineering", "channel_id": null, "channel_name": "general", "message_id": null, "matched_candidates": null, "items": null }, - "next_step": "Editing Teams messages is not supported by the connector. Ask the user to edit the message directly in the Teams UI, or to send a follow-up message instead.", - "missing_fields": null, - "assumptions": null - } - ``` - -## Output contract - -Return **only** one JSON object (no markdown or prose outside it): - -```json -{ - "status": "success" | "partial" | "blocked" | "error", - "action_summary": string, - "evidence": { - "operation": "list_teams_channels" | "read_teams_messages" | "send_teams_message" | null, - "team_id": string | null, - "team_name": string | null, - "channel_id": string | null, - "channel_name": string | null, - "message_id": string | null, - "matched_candidates": [ { "id": string, "label": string } ] | null, - "items": object | null - }, - "next_step": string | null, - "missing_fields": string[] | null, - "assumptions": string[] | null -} -``` - -<include snippet="output_contract_base"/> - -<include snippet="verifiable_handle"/> - -Resolve before you call; verify before you send; map every tool outcome faithfully. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/__init__.py deleted file mode 100644 index dbf966307..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from .list_channels import create_list_teams_channels_tool -from .read_messages import create_read_teams_messages_tool -from .send_message import create_send_teams_message_tool - -__all__ = [ - "create_list_teams_channels_tool", - "create_read_teams_messages_tool", - "create_send_teams_message_tool", -] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/_auth.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/_auth.py deleted file mode 100644 index e6600e828..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/_auth.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Builds Microsoft Graph auth headers for Teams connector tools.""" - -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.db import SearchSourceConnector, SearchSourceConnectorType - -GRAPH_API = "https://graph.microsoft.com/v1.0" - - -async def get_teams_connector( - db_session: AsyncSession, - workspace_id: int, - user_id: str, -) -> SearchSourceConnector | None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.workspace_id == workspace_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.TEAMS_CONNECTOR, - ) - ) - return result.scalars().first() - - -async def get_access_token( - db_session: AsyncSession, - connector: SearchSourceConnector, -) -> str: - """Get a valid Microsoft Graph access token, refreshing if expired.""" - from app.connectors.teams_connector import TeamsConnector - - tc = TeamsConnector( - session=db_session, - connector_id=connector.id, - ) - return await tc._get_valid_token() diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/index.py deleted file mode 100644 index ae4959d5b..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/index.py +++ /dev/null @@ -1,36 +0,0 @@ -"""``teams`` native tools and (empty) permission ruleset. - -Tools self-gate via :func:`request_approval` in their bodies. -""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset - -from .list_channels import create_list_teams_channels_tool -from .read_messages import create_read_teams_messages_tool -from .send_message import create_send_teams_message_tool - -NAME = "teams" - -RULESET = Ruleset(origin=NAME, rules=[]) - - -def load_tools( - *, dependencies: dict[str, Any] | None = None, **kwargs: Any -) -> list[BaseTool]: - d = {**(dependencies or {}), **kwargs} - common = { - "db_session": d["db_session"], - "workspace_id": d["workspace_id"], - "user_id": d["user_id"], - } - return [ - create_list_teams_channels_tool(**common), - create_read_teams_messages_tool(**common), - create_send_teams_message_tool(**common), - ] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/list_channels.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/list_channels.py deleted file mode 100644 index ca031b35b..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/list_channels.py +++ /dev/null @@ -1,92 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from ._auth import GRAPH_API, get_access_token, get_teams_connector - -logger = logging.getLogger(__name__) - - -def create_list_teams_channels_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, -): - @tool - async def list_teams_channels() -> dict[str, Any]: - """List all Microsoft Teams and their channels the user has access to. - - Returns: - Dictionary with status and a list of teams, each containing - team_id, team_name, and a list of channels (id, name). - """ - if db_session is None or workspace_id is None or user_id is None: - return {"status": "error", "message": "Teams tool not properly configured."} - - try: - connector = await get_teams_connector(db_session, workspace_id, user_id) - if not connector: - return {"status": "error", "message": "No Teams connector found."} - - token = await get_access_token(db_session, connector) - headers = {"Authorization": f"Bearer {token}"} - - async with httpx.AsyncClient(timeout=20.0) as client: - teams_resp = await client.get( - f"{GRAPH_API}/me/joinedTeams", headers=headers - ) - - if teams_resp.status_code == 401: - return { - "status": "auth_error", - "message": "Teams token expired. Please re-authenticate.", - "connector_type": "teams", - } - if teams_resp.status_code != 200: - return { - "status": "error", - "message": f"Graph API error: {teams_resp.status_code}", - } - - teams_data = teams_resp.json().get("value", []) - result_teams = [] - - async with httpx.AsyncClient(timeout=20.0) as client: - for team in teams_data: - team_id = team["id"] - ch_resp = await client.get( - f"{GRAPH_API}/teams/{team_id}/channels", - headers=headers, - ) - channels = [] - if ch_resp.status_code == 200: - channels = [ - {"id": ch["id"], "name": ch.get("displayName", "")} - for ch in ch_resp.json().get("value", []) - ] - result_teams.append( - { - "team_id": team_id, - "team_name": team.get("displayName", ""), - "channels": channels, - } - ) - - return { - "status": "success", - "teams": result_teams, - "total_teams": len(result_teams), - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error listing Teams channels: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to list Teams channels."} - - return list_teams_channels diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/read_messages.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/read_messages.py deleted file mode 100644 index ff3b7c75a..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/read_messages.py +++ /dev/null @@ -1,103 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from ._auth import GRAPH_API, get_access_token, get_teams_connector - -logger = logging.getLogger(__name__) - - -def create_read_teams_messages_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, -): - @tool - async def read_teams_messages( - team_id: str, - channel_id: str, - limit: int = 25, - ) -> dict[str, Any]: - """Read recent messages from a Microsoft Teams channel. - - Args: - team_id: The team ID (from list_teams_channels). - channel_id: The channel ID (from list_teams_channels). - limit: Number of messages to fetch (default 25, max 50). - - Returns: - Dictionary with status and a list of messages including - id, sender, content, timestamp. - """ - if db_session is None or workspace_id is None or user_id is None: - return {"status": "error", "message": "Teams tool not properly configured."} - - limit = min(limit, 50) - - try: - connector = await get_teams_connector(db_session, workspace_id, user_id) - if not connector: - return {"status": "error", "message": "No Teams connector found."} - - token = await get_access_token(db_session, connector) - - async with httpx.AsyncClient(timeout=20.0) as client: - resp = await client.get( - f"{GRAPH_API}/teams/{team_id}/channels/{channel_id}/messages", - headers={"Authorization": f"Bearer {token}"}, - params={"$top": limit}, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Teams token expired. Please re-authenticate.", - "connector_type": "teams", - } - if resp.status_code == 403: - return { - "status": "error", - "message": "Insufficient permissions to read this channel.", - } - if resp.status_code != 200: - return { - "status": "error", - "message": f"Graph API error: {resp.status_code}", - } - - raw_msgs = resp.json().get("value", []) - messages = [] - for m in raw_msgs: - sender = m.get("from", {}) - user_info = sender.get("user", {}) if sender else {} - body = m.get("body", {}) - messages.append( - { - "id": m.get("id"), - "sender": user_info.get("displayName", "Unknown"), - "content": body.get("content", ""), - "content_type": body.get("contentType", "text"), - "timestamp": m.get("createdDateTime", ""), - } - ) - - return { - "status": "success", - "team_id": team_id, - "channel_id": channel_id, - "messages": messages, - "total": len(messages), - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error reading Teams messages: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to read Teams messages."} - - return read_teams_messages diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/send_message.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/send_message.py deleted file mode 100644 index 47c0fb700..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/send_message.py +++ /dev/null @@ -1,117 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( - request_approval, -) - -from ._auth import GRAPH_API, get_access_token, get_teams_connector - -logger = logging.getLogger(__name__) - - -def create_send_teams_message_tool( - db_session: AsyncSession | None = None, - workspace_id: int | None = None, - user_id: str | None = None, -): - @tool - async def send_teams_message( - team_id: str, - channel_id: str, - content: str, - ) -> dict[str, Any]: - """Send a message to a Microsoft Teams channel. - - Requires the ChannelMessage.Send OAuth scope. If the user gets a - permission error, they may need to re-authenticate with updated scopes. - - Args: - team_id: The team ID (from list_teams_channels). - channel_id: The channel ID (from list_teams_channels). - content: The message text (HTML supported). - - Returns: - Dictionary with status, message_id on success. - - IMPORTANT: - - If status is "rejected", the user explicitly declined. Do NOT retry. - """ - if db_session is None or workspace_id is None or user_id is None: - return {"status": "error", "message": "Teams tool not properly configured."} - - try: - connector = await get_teams_connector(db_session, workspace_id, user_id) - if not connector: - return {"status": "error", "message": "No Teams connector found."} - - result = request_approval( - action_type="teams_send_message", - tool_name="send_teams_message", - params={ - "team_id": team_id, - "channel_id": channel_id, - "content": content, - }, - context={"connector_id": connector.id}, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Message was not sent.", - } - - final_content = result.params.get("content", content) - final_team = result.params.get("team_id", team_id) - final_channel = result.params.get("channel_id", channel_id) - - token = await get_access_token(db_session, connector) - - async with httpx.AsyncClient(timeout=20.0) as client: - resp = await client.post( - f"{GRAPH_API}/teams/{final_team}/channels/{final_channel}/messages", - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - }, - json={"body": {"content": final_content}}, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Teams token expired. Please re-authenticate.", - "connector_type": "teams", - } - if resp.status_code == 403: - return { - "status": "insufficient_permissions", - "message": "Missing ChannelMessage.Send permission. Please re-authenticate with updated scopes.", - } - if resp.status_code not in (200, 201): - return { - "status": "error", - "message": f"Graph API error: {resp.status_code} — {resp.text[:200]}", - } - - msg_data = resp.json() - return { - "status": "success", - "message_id": msg_data.get("id"), - "message": "Message sent to Teams channel.", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error sending Teams message: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to send Teams message."} - - return send_teams_message diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/index.py index 334f2130a..eae0105ac 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/index.py @@ -23,10 +23,70 @@ from app.agents.chat.multi_agent_chat.constants import ( ) from app.agents.chat.multi_agent_chat.shared.tools.mcp.tool import load_mcp_tools from app.db import SearchSourceConnector +from app.services.mcp_oauth.registry import MCP_SERVICES, get_service_by_connector_type logger = logging.getLogger(__name__) +def _service_key_for_type(connector_type: str | None) -> str | None: + """Return the ``MCP_SERVICES`` key for a connector type, if any.""" + if connector_type is None: + return None + svc = get_service_by_connector_type(connector_type) + if svc is None: + return None + return next((k for k, v in MCP_SERVICES.items() if v is svc), None) + + +def resolve_tool_name_collisions( + tools: Sequence[BaseTool], + connector_id_to_type: dict[int, str], +) -> list[BaseTool]: + """Prefix only the tools whose exposed name collides across connectors. + + All MCP tools now merge into a single ``mcp_discovery`` subagent, so two + apps advertising the same tool name (e.g. Jira and Confluence both expose + ``getAccessibleAtlassianResources``) would otherwise shadow each other. + We detect names carried by more than one distinct connector and rebuild + just those with a ``{service_key_or_mcp}_{connector_id}_`` prefix — the + same convention as the existing multi-account prefixing. Non-colliding + tools keep their names, so stored ``trusted_tools`` and HITL history stay + valid in the common case; for the prefixed ones, + ``metadata['mcp_original_tool_name']`` is preserved as the "Always Allow" + fallback key. + """ + names_to_connectors: dict[str, set[int]] = defaultdict(set) + for tool in tools: + meta = getattr(tool, "metadata", None) or {} + cid = meta.get("mcp_connector_id") + if isinstance(cid, int): + names_to_connectors[tool.name].add(cid) + + colliding = {n for n, cids in names_to_connectors.items() if len(cids) > 1} + if not colliding: + return list(tools) + + resolved: list[BaseTool] = [] + for tool in tools: + meta = getattr(tool, "metadata", None) or {} + cid = meta.get("mcp_connector_id") + if tool.name not in colliding or not isinstance(cid, int): + resolved.append(tool) + continue + + original_name = tool.name + prefix = _service_key_for_type(connector_id_to_type.get(cid)) or "mcp" + new_name = f"{prefix}_{cid}_{original_name}" + new_meta = { + **meta, + "mcp_original_tool_name": meta.get("mcp_original_tool_name") + or original_name, + "mcp_collision_prefixed": True, + } + resolved.append(tool.model_copy(update={"name": new_name, "metadata": new_meta})) + return resolved + + async def fetch_mcp_connector_metadata_maps( session: AsyncSession, workspace_id: int, @@ -110,4 +170,5 @@ async def load_mcp_tools_by_connector( """ flat = await load_mcp_tools(session, workspace_id, bypass_internal_hitl=True) id_map, name_map = await fetch_mcp_connector_metadata_maps(session, workspace_id) + flat = resolve_tool_name_collisions(flat, id_map) return partition_mcp_tools_by_connector(flat, id_map, name_map) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index a8fdbb45a..b490bfcac 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -24,6 +24,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.google_search.agent imp from app.agents.chat.multi_agent_chat.subagents.builtins.knowledge_base.agent import ( build_subagent as build_knowledge_base_subagent, ) +from app.agents.chat.multi_agent_chat.subagents.builtins.mcp_discovery.agent import ( + build_subagent as build_mcp_discovery_subagent, +) from app.agents.chat.multi_agent_chat.subagents.builtins.memory.agent import ( build_subagent as build_memory_subagent, ) @@ -36,51 +39,19 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent impor from app.agents.chat.multi_agent_chat.subagents.builtins.youtube.agent import ( build_subagent as build_youtube_subagent, ) -from app.agents.chat.multi_agent_chat.subagents.connectors.airtable.agent import ( - build_subagent as build_airtable_subagent, -) -from app.agents.chat.multi_agent_chat.subagents.connectors.calendar.agent import ( - build_subagent as build_calendar_subagent, -) -from app.agents.chat.multi_agent_chat.subagents.connectors.clickup.agent import ( - build_subagent as build_clickup_subagent, -) -from app.agents.chat.multi_agent_chat.subagents.connectors.confluence.agent import ( - build_subagent as build_confluence_subagent, -) -from app.agents.chat.multi_agent_chat.subagents.connectors.discord.agent import ( - build_subagent as build_discord_subagent, -) +# File connectors stay native — they enrich the knowledge base. Every other +# connector (Slack/Jira/Linear/ClickUp/Airtable/Notion/Confluence/Gmail/ +# Calendar) migrated to hosted MCP under ``mcp_discovery``; Discord/Teams/Luma +# were deprecated (no viable official MCP server). Their old packages are gone. from app.agents.chat.multi_agent_chat.subagents.connectors.dropbox.agent import ( build_subagent as build_dropbox_subagent, ) -from app.agents.chat.multi_agent_chat.subagents.connectors.gmail.agent import ( - build_subagent as build_gmail_subagent, -) from app.agents.chat.multi_agent_chat.subagents.connectors.google_drive.agent import ( build_subagent as build_google_drive_subagent, ) -from app.agents.chat.multi_agent_chat.subagents.connectors.jira.agent import ( - build_subagent as build_jira_subagent, -) -from app.agents.chat.multi_agent_chat.subagents.connectors.linear.agent import ( - build_subagent as build_linear_subagent, -) -from app.agents.chat.multi_agent_chat.subagents.connectors.luma.agent import ( - build_subagent as build_luma_subagent, -) -from app.agents.chat.multi_agent_chat.subagents.connectors.notion.agent import ( - build_subagent as build_notion_subagent, -) from app.agents.chat.multi_agent_chat.subagents.connectors.onedrive.agent import ( build_subagent as build_onedrive_subagent, ) -from app.agents.chat.multi_agent_chat.subagents.connectors.slack.agent import ( - build_subagent as build_slack_subagent, -) -from app.agents.chat.multi_agent_chat.subagents.connectors.teams.agent import ( - build_subagent as build_teams_subagent, -) from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( read_md_file, ) @@ -102,27 +73,16 @@ class SubagentBuilder(Protocol): SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { - "airtable": build_airtable_subagent, - "calendar": build_calendar_subagent, - "clickup": build_clickup_subagent, - "confluence": build_confluence_subagent, "deliverables": build_deliverables_subagent, - "discord": build_discord_subagent, "dropbox": build_dropbox_subagent, - "gmail": build_gmail_subagent, "google_drive": build_google_drive_subagent, "google_maps": build_google_maps_subagent, "google_search": build_google_search_subagent, - "jira": build_jira_subagent, "knowledge_base": build_knowledge_base_subagent, - "linear": build_linear_subagent, - "luma": build_luma_subagent, + "mcp_discovery": build_mcp_discovery_subagent, "memory": build_memory_subagent, - "notion": build_notion_subagent, "onedrive": build_onedrive_subagent, "reddit": build_reddit_subagent, - "slack": build_slack_subagent, - "teams": build_teams_subagent, "web_crawler": build_web_crawler_subagent, "youtube": build_youtube_subagent, } diff --git a/surfsense_backend/app/agents/chat/shared/tools/__init__.py b/surfsense_backend/app/agents/chat/shared/tools/__init__.py index 342fe9169..0c3bc6e57 100644 --- a/surfsense_backend/app/agents/chat/shared/tools/__init__.py +++ b/surfsense_backend/app/agents/chat/shared/tools/__init__.py @@ -1,5 +1,5 @@ """Cross-agent shared tools. -Only genuinely cross-agent tool code lives here (currently web_search, imported -directly from its module). +Only genuinely cross-agent tool code lives here, imported directly from its +module. """ diff --git a/surfsense_backend/app/agents/chat/shared/tools/web_search.py b/surfsense_backend/app/agents/chat/shared/tools/web_search.py deleted file mode 100644 index 78f3c6ae2..000000000 --- a/surfsense_backend/app/agents/chat/shared/tools/web_search.py +++ /dev/null @@ -1,287 +0,0 @@ -""" -Web search tool for the SurfSense agent. - -Provides a unified tool for real-time web searches that dispatches to all -configured search engines: the platform SearXNG instance (always available) -plus any user-configured live-search connectors (Tavily, Linkup, Baidu). - -Each result is registered into the conversation citation registry as a -``WEB_RESULT`` and rendered with a server-assigned ``[n]`` label, so the model -cites the web exactly like the knowledge base — one ``[n]`` spine, no special -web citation form. -""" - -from __future__ import annotations - -import asyncio -import time -from typing import TYPE_CHECKING, Annotated, Any -from urllib.parse import urlparse - -from langchain.tools import ToolRuntime -from langchain_core.messages import ToolMessage -from langchain_core.tools import BaseTool, StructuredTool -from langgraph.types import Command - -from app.db import shielded_async_session -from app.services.connector_service import ConnectorService -from app.utils.perf import get_perf_logger - -if TYPE_CHECKING: - from app.agents.chat.multi_agent_chat.shared.document_render import ( - RenderableDocument, - ) - -# NOTE: imports from ``app.agents.chat.multi_agent_chat`` are done lazily inside -# the functions below. This module lives under ``app.agents.chat.shared`` but is -# imported during the ``multi_agent_chat`` package's own init cascade (via the -# research subagent); importing that package at module load would re-enter a -# partially-initialized module. Lazy imports break that cycle. - -_LIVE_SEARCH_CONNECTORS: set[str] = { - "TAVILY_API", - "LINKUP_API", - "BAIDU_SEARCH_API", -} - -_LIVE_CONNECTOR_SPECS: dict[str, tuple[str, bool, bool, dict[str, Any]]] = { - "TAVILY_API": ("search_tavily", False, True, {}), - "LINKUP_API": ("search_linkup", False, False, {"mode": "standard"}), - "BAIDU_SEARCH_API": ("search_baidu", False, True, {}), -} - -_CONNECTOR_LABELS: dict[str, str] = { - "TAVILY_API": "Tavily", - "LINKUP_API": "Linkup", - "BAIDU_SEARCH_API": "Baidu", -} - - -def _web_source_label(url: str) -> str: - """A compact, human-readable source for the ``<document source=…>`` attr.""" - domain = urlparse(url).netloc.removeprefix("www.") if url else "" - return f"Web · {domain}" if domain else "Web" - - -def _to_renderable_web_documents( - documents: list[dict[str, Any]], - *, - max_chars: int = 50_000, -) -> list[RenderableDocument]: - """Map raw web results to renderable documents, one passage (the snippet) each. - - A result with no URL is skipped: ``url`` is the citation locator, so without - it the result cannot be registered or resolved. - """ - from app.agents.chat.multi_agent_chat.shared.citations import CitationSourceType - from app.agents.chat.multi_agent_chat.shared.document_render import ( - RenderableDocument, - RenderablePassage, - ) - - renderables: list[RenderableDocument] = [] - total_chars = 0 - - for doc in documents: - doc_info = doc.get("document") or {} - metadata = doc_info.get("metadata") or {} - title = doc_info.get("title") or "Web Result" - url = metadata.get("url") or "" - content = (doc.get("content") or "").strip() - if not content or not url: - continue - - total_chars += len(content) - if total_chars > max_chars: - break - - renderables.append( - RenderableDocument( - title=title, - source=_web_source_label(url), - passages=[ - RenderablePassage( - content=content, - locator={"url": url}, - source_type=CitationSourceType.WEB_RESULT, - ) - ], - ) - ) - - return renderables - - -async def _search_live_connector( - connector: str, - query: str, - workspace_id: int, - top_k: int, - semaphore: asyncio.Semaphore, -) -> list[dict[str, Any]]: - """Dispatch a single live-search connector (Tavily / Linkup / Baidu).""" - perf = get_perf_logger() - spec = _LIVE_CONNECTOR_SPECS.get(connector) - if spec is None: - return [] - - method_name, _includes_date_range, includes_top_k, extra_kwargs = spec - kwargs: dict[str, Any] = { - "user_query": query, - "workspace_id": workspace_id, - **extra_kwargs, - } - if includes_top_k: - kwargs["top_k"] = top_k - - try: - t0 = time.perf_counter() - async with semaphore, shielded_async_session() as session: - svc = ConnectorService(session, workspace_id) - _, chunks = await getattr(svc, method_name)(**kwargs) - perf.info( - "[web_search] connector=%s results=%d in %.3fs", - connector, - len(chunks), - time.perf_counter() - t0, - ) - return chunks - except Exception as e: - perf.warning("[web_search] connector=%s FAILED: %s", connector, e) - return [] - - -def create_web_search_tool( - workspace_id: int | None = None, - available_connectors: list[str] | None = None, -) -> BaseTool: - """Factory for the ``web_search`` tool. - - Dispatches in parallel to the platform SearXNG instance and any - user-configured live-search connectors (Tavily, Linkup, Baidu). - """ - active_live_connectors: list[str] = [] - if available_connectors: - active_live_connectors = [ - c for c in available_connectors if c in _LIVE_SEARCH_CONNECTORS - ] - - engine_names = ["SearXNG (platform default)"] - engine_names.extend(_CONNECTOR_LABELS.get(c, c) for c in active_live_connectors) - engines_summary = ", ".join(engine_names) - - description = ( - "Search the web for real-time information. " - "Use this for current events, news, prices, weather, public facts, or any " - "question that requires up-to-date information from the internet.\n\n" - f"Active search engines: {engines_summary}.\n" - "All configured engines are queried in parallel and results are merged." - ) - - _workspace_id = workspace_id - _active_live = active_live_connectors - - async def _web_search_impl( - query: Annotated[ - str, - "The search query to look up on the web. Use specific, descriptive terms.", - ], - runtime: ToolRuntime, - top_k: Annotated[ - int, - "Number of results to retrieve (default: 10, max: 50).", - ] = 10, - ) -> Command | str: - from app.services import web_search_service - - perf = get_perf_logger() - t0 = time.perf_counter() - clamped_top_k = min(max(1, top_k), 50) - - semaphore = asyncio.Semaphore(4) - tasks: list[asyncio.Task[list[dict[str, Any]]]] = [] - - if web_search_service.is_available(): - - async def _searxng() -> list[dict[str, Any]]: - async with semaphore: - _result_obj, docs = await web_search_service.search( - query=query, - top_k=clamped_top_k, - ) - return docs - - tasks.append(asyncio.ensure_future(_searxng())) - - if _workspace_id is not None: - for connector in _active_live: - tasks.append( - asyncio.ensure_future( - _search_live_connector( - connector=connector, - query=query, - workspace_id=_workspace_id, - top_k=clamped_top_k, - semaphore=semaphore, - ) - ) - ) - - if not tasks: - return "Web search is not available — no search engines are configured." - - results_lists = await asyncio.gather(*tasks, return_exceptions=True) - - all_documents: list[dict[str, Any]] = [] - for result in results_lists: - if isinstance(result, BaseException): - perf.warning("[web_search] a search engine failed: %s", result) - continue - all_documents.extend(result) - - seen_urls: set[str] = set() - deduplicated: list[dict[str, Any]] = [] - for doc in all_documents: - url = ((doc.get("document") or {}).get("metadata") or {}).get("url", "") - if url and url in seen_urls: - continue - if url: - seen_urls.add(url) - deduplicated.append(doc) - - from app.agents.chat.multi_agent_chat.shared.citations import load_registry - from app.agents.chat.multi_agent_chat.shared.document_render import ( - render_web_results, - ) - - registry = load_registry(getattr(runtime, "state", None)) - renderables = _to_renderable_web_documents(deduplicated) - rendered = render_web_results(renderables, registry) - - perf.info( - "[web_search] query=%r engines=%d results=%d deduped=%d renderable=%d in %.3fs", - query[:60], - len(tasks), - len(all_documents), - len(deduplicated), - len(renderables), - time.perf_counter() - t0, - ) - - if rendered is None: - return "No web search results found." - - return Command( - update={ - "messages": [ - ToolMessage(content=rendered, tool_call_id=runtime.tool_call_id) - ], - "citation_registry": registry, - } - ) - - return StructuredTool.from_function( - name="web_search", - description=description, - coroutine=_web_search_impl, - ) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 89790c952..7305f6dff 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -554,13 +554,6 @@ class Config: os.getenv("SURFSENSE_CONNECTOR_DISCOVERY_TTL_SECONDS", "30") ) - # Platform web search providers for the web.discover capability (env-keyed). - SEARXNG_DEFAULT_HOST = os.getenv("SEARXNG_DEFAULT_HOST") - LINKUP_API_KEY = os.getenv("LINKUP_API_KEY") - BAIDU_API_KEY = os.getenv("BAIDU_API_KEY") - BAIDU_MODEL = os.getenv("BAIDU_MODEL", "ernie-3.5-8k") - BAIDU_SEARCH_SOURCE = os.getenv("BAIDU_SEARCH_SOURCE", "baidu_search_v2") - SURFSENSE_PUBLIC_URL = os.getenv("SURFSENSE_PUBLIC_URL") NEXT_FRONTEND_URL = os.getenv("NEXT_FRONTEND_URL") or SURFSENSE_PUBLIC_URL # Backend URL to override the http to https in the OAuth redirect URI diff --git a/surfsense_backend/app/observability/metrics.py b/surfsense_backend/app/observability/metrics.py index 4fd4e8c1f..fa66a9636 100644 --- a/surfsense_backend/app/observability/metrics.py +++ b/surfsense_backend/app/observability/metrics.py @@ -36,7 +36,11 @@ _ERROR_CATEGORY_HINTS: tuple[tuple[str, tuple[str, ...]], ...] = ( def _package_version() -> str: - with contextlib.suppress(metadata.PackageNotFoundError): + # Best-effort telemetry tag only: never let a version lookup crash the + # request path. Besides PackageNotFoundError, a malformed/dynamic editable + # install can have distribution metadata with no "Version" field, which + # raises KeyError deep in importlib.metadata. Suppress broadly. + with contextlib.suppress(Exception): return metadata.version("surf-new-backend") return "unknown" diff --git a/surfsense_backend/app/routes/anonymous_chat_routes.py b/surfsense_backend/app/routes/anonymous_chat_routes.py index f6f984c20..3ff2fd38a 100644 --- a/surfsense_backend/app/routes/anonymous_chat_routes.py +++ b/surfsense_backend/app/routes/anonymous_chat_routes.py @@ -337,11 +337,6 @@ async def stream_anonymous_chat( await TokenQuotaService.anon_release(session_key, ip_key, request_id) raise HTTPException(status_code=500, detail="Failed to create LLM instance") - # Server-side tool allow-list enforcement - anon_allowed_tools = {"web_search"} - client_disabled = set(body.disabled_tools) if body.disabled_tools else set() - enabled_for_agent = anon_allowed_tools - client_disabled - except HTTPException: await TokenQuotaService.anon_release_stream_slot(client_ip) raise @@ -369,15 +364,14 @@ async def stream_anonymous_chat( # Load the optional uploaded document as read-only context. anon_doc = await _load_anon_document(session_id) - # Minimal Q/A agent: web_search only (when enabled), no - # filesystem / persistence / subagents. The uploaded document - # is injected into the system prompt as read-only context. + # Minimal Q/A agent: no tools, no filesystem / persistence / + # subagents. The uploaded document is injected into the system + # prompt as read-only context. agent = await create_anonymous_chat_agent( llm=llm, checkpointer=checkpointer, anon_session_id=session_id, anon_doc=anon_doc, - enable_web_search="web_search" in enabled_for_agent, ) langchain_messages = [] diff --git a/surfsense_backend/app/routes/discord_add_connector_route.py b/surfsense_backend/app/routes/discord_add_connector_route.py index 6b335a885..e29617f93 100644 --- a/surfsense_backend/app/routes/discord_add_connector_route.py +++ b/surfsense_backend/app/routes/discord_add_connector_route.py @@ -31,6 +31,7 @@ from app.utils.connector_naming import ( ) from app.utils.oauth_security import OAuthStateManager, TokenEncryption from app.utils.rbac import check_workspace_access +from app.utils.validators import raise_if_connector_deprecated logger = logging.getLogger(__name__) @@ -94,6 +95,8 @@ async def connect_discord( """ user = auth.user try: + raise_if_connector_deprecated(SearchSourceConnectorType.DISCORD_CONNECTOR) + if not space_id: raise HTTPException(status_code=400, detail="space_id is required") diff --git a/surfsense_backend/app/routes/luma_add_connector_route.py b/surfsense_backend/app/routes/luma_add_connector_route.py index 84bfa41ee..103cfd793 100644 --- a/surfsense_backend/app/routes/luma_add_connector_route.py +++ b/surfsense_backend/app/routes/luma_add_connector_route.py @@ -13,6 +13,7 @@ from app.db import ( get_async_session, ) from app.users import require_session_context +from app.utils.validators import raise_if_connector_deprecated logger = logging.getLogger(__name__) @@ -48,6 +49,8 @@ async def add_luma_connector( """ user = auth.user try: + raise_if_connector_deprecated(SearchSourceConnectorType.LUMA_CONNECTOR) + # Check if a Luma connector already exists for this workspace and user result = await session.execute( select(SearchSourceConnector).filter( diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 1aec3f172..4cfa78af2 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -73,6 +73,7 @@ from app.utils.periodic_scheduler import ( update_periodic_schedule, ) from app.utils.rbac import check_permission +from app.utils.validators import raise_if_connector_deprecated # Set up logging logger = logging.getLogger(__name__) @@ -185,6 +186,12 @@ async def create_search_source_connector( The config must contain the appropriate keys for the connector type. """ try: + # Refuse new connections for deprecated connector types (HTTP 410). The + # search APIs (Tavily/SearXNG/Linkup/Baidu) are created through this + # generic route rather than a dedicated OAuth route, so this is the + # single choke point that must enforce the deprecation. + raise_if_connector_deprecated(connector.connector_type) + # Check if user has permission to create connectors await check_permission( session, @@ -827,7 +834,10 @@ async def index_connector_content( # For non-calendar connectors, cap at today indexing_to = end_date if end_date else today_str - from app.services.mcp_oauth.registry import LIVE_CONNECTOR_TYPES + from app.services.mcp_oauth.registry import ( + DEPRECATED_INDEXING_CONNECTOR_TYPES, + LIVE_CONNECTOR_TYPES, + ) if connector.connector_type in LIVE_CONNECTOR_TYPES: return { @@ -842,6 +852,20 @@ async def index_connector_content( "indexing_to": indexing_to, } + if connector.connector_type in DEPRECATED_INDEXING_CONNECTOR_TYPES: + return { + "message": ( + f"Indexing for {connector.connector_type.value} has been " + "deprecated. The knowledge base now stores files, notes, and " + "uploads only." + ), + "indexing_started": False, + "connector_id": connector_id, + "workspace_id": workspace_id, + "indexing_from": indexing_from, + "indexing_to": indexing_to, + } + if connector.connector_type == SearchSourceConnectorType.NOTION_CONNECTOR: from app.tasks.celery_tasks.connector_tasks import index_notion_pages_task diff --git a/surfsense_backend/app/routes/teams_add_connector_route.py b/surfsense_backend/app/routes/teams_add_connector_route.py index 88ffd63d1..21c089be0 100644 --- a/surfsense_backend/app/routes/teams_add_connector_route.py +++ b/surfsense_backend/app/routes/teams_add_connector_route.py @@ -29,6 +29,7 @@ from app.utils.connector_naming import ( generate_unique_connector_name, ) from app.utils.oauth_security import OAuthStateManager, TokenEncryption +from app.utils.validators import raise_if_connector_deprecated logger = logging.getLogger(__name__) @@ -90,6 +91,8 @@ async def connect_teams( """ user = auth.user try: + raise_if_connector_deprecated(SearchSourceConnectorType.TEAMS_CONNECTOR) + if not space_id: raise HTTPException(status_code=400, detail="space_id is required") diff --git a/surfsense_backend/app/services/connector_service.py b/surfsense_backend/app/services/connector_service.py index ff7ded854..7c3f0543d 100644 --- a/surfsense_backend/app/services/connector_service.py +++ b/surfsense_backend/app/services/connector_service.py @@ -4,12 +4,9 @@ from datetime import datetime from threading import Lock from typing import Any -import httpx -from linkup import LinkupClient from sqlalchemy import func from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from tavily import TavilyClient from app.config import config from app.db import ( @@ -388,363 +385,6 @@ class ConnectorService: result = await self.session.execute(query) return result.scalars().first() - async def search_tavily( - self, user_query: str, workspace_id: int, top_k: int = 20 - ) -> tuple: - """ - Search using Tavily API and return both the source information and documents - - Args: - user_query: The user's query - workspace_id: The workspace ID - top_k: Maximum number of results to return - - Returns: - tuple: (sources_info, documents) - """ - # Get Tavily connector configuration - tavily_connector = await self.get_connector_by_type( - SearchSourceConnectorType.TAVILY_API, workspace_id - ) - - if not tavily_connector: - # Return empty results if no Tavily connector is configured - return { - "id": 3, - "name": "Tavily Search", - "type": "TAVILY_API", - "sources": [], - }, [] - - # Initialize Tavily client with API key from connector config - tavily_api_key = tavily_connector.config.get("TAVILY_API_KEY") - tavily_client = TavilyClient(api_key=tavily_api_key) - - # Perform search with Tavily - try: - response = tavily_client.search( - query=user_query, - max_results=top_k, - search_depth="advanced", # Use advanced search for better results - ) - - # Extract results from Tavily response - tavily_results = response.get("results", []) - - # Early return if no results - if not tavily_results: - return { - "id": 3, - "name": "Tavily Search", - "type": "TAVILY_API", - "sources": [], - }, [] - - # Process each result and create sources directly without deduplication - sources_list = [] - documents = [] - - async with self.counter_lock: - for _i, result in enumerate(tavily_results): - # Create a source entry - source = { - "id": self.source_id_counter, - "title": result.get("title", "Tavily Result"), - "description": result.get("content", ""), - "url": result.get("url", ""), - } - sources_list.append(source) - - # Create a document entry - document = { - "chunk_id": self.source_id_counter, - "content": result.get("content", ""), - "score": result.get("score", 0.0), - "document": { - "id": self.source_id_counter, - "title": result.get("title", "Tavily Result"), - "document_type": "TAVILY_API", - "metadata": { - "url": result.get("url", ""), - "published_date": result.get("published_date", ""), - "source": "TAVILY_API", - }, - }, - } - documents.append(document) - self.source_id_counter += 1 - - # Create result object - result_object = { - "id": 3, - "name": "Tavily Search", - "type": "TAVILY_API", - "sources": sources_list, - } - - return result_object, documents - - except Exception as e: - # Log the error and return empty results - print(f"Error searching with Tavily: {e!s}") - return { - "id": 3, - "name": "Tavily Search", - "type": "TAVILY_API", - "sources": [], - }, [] - - async def search_searxng( - self, - user_query: str, - workspace_id: int, - top_k: int = 20, - ) -> tuple: - """Search using the platform SearXNG instance. - - Delegates to ``WebSearchService`` which handles caching, circuit - breaking, and retries. SearXNG configuration comes from the - docker/searxng/settings.yml file. - """ - from app.services import web_search_service - - if not web_search_service.is_available(): - return { - "id": 11, - "name": "Web Search", - "type": "SEARXNG_API", - "sources": [], - }, [] - - return await web_search_service.search( - query=user_query, - top_k=top_k, - ) - - async def search_baidu( - self, - user_query: str, - workspace_id: int, - top_k: int = 20, - ) -> tuple: - """ - Search using Baidu AI Search API and return both sources and documents. - - Baidu AI Search provides intelligent search with automatic summarization. - We extract the raw search results (references) from the API response. - - Args: - user_query: User's search query - workspace_id: Workspace ID - top_k: Maximum number of results to return - - Returns: - tuple: (sources_info_dict, documents_list) - """ - # Get Baidu connector configuration - baidu_connector = await self.get_connector_by_type( - SearchSourceConnectorType.BAIDU_SEARCH_API, workspace_id - ) - - if not baidu_connector: - return { - "id": 12, - "name": "Baidu Search", - "type": "BAIDU_SEARCH_API", - "sources": [], - }, [] - - config = baidu_connector.config or {} - api_key = config.get("BAIDU_API_KEY") - - if not api_key: - print("ERROR: Baidu connector is missing BAIDU_API_KEY configuration") - print(f"Connector config: {config}") - return { - "id": 12, - "name": "Baidu Search", - "type": "BAIDU_SEARCH_API", - "sources": [], - }, [] - - # Optional configuration parameters - model = config.get("BAIDU_MODEL", "ernie-3.5-8k") - search_source = config.get("BAIDU_SEARCH_SOURCE", "baidu_search_v2") - enable_deep_search = config.get("BAIDU_ENABLE_DEEP_SEARCH", False) - - # Baidu AI Search API endpoint - baidu_endpoint = "https://qianfan.baidubce.com/v2/ai_search/chat/completions" - - # Prepare request headers - # Note: Baidu uses X-Appbuilder-Authorization instead of standard Authorization header - headers = { - "X-Appbuilder-Authorization": f"Bearer {api_key}", - "Content-Type": "application/json", - } - - # Prepare request payload - # Calculate resource_type_filter top_k values - # Baidu v2 supports max 20 per type - max_per_type = min(top_k, 20) - - payload = { - "messages": [{"role": "user", "content": user_query}], - "model": model, - "search_source": search_source, - "resource_type_filter": [ - {"type": "web", "top_k": max_per_type}, - {"type": "video", "top_k": max(1, max_per_type // 4)}, # Fewer videos - ], - "stream": False, # Non-streaming for simpler processing - "enable_deep_search": enable_deep_search, - "enable_corner_markers": True, # Enable reference markers - } - - try: - # Baidu AI Search may take longer as it performs search + summarization - # Increase timeout to 90 seconds - async with httpx.AsyncClient(timeout=90.0) as client: - response = await client.post( - baidu_endpoint, - headers=headers, - json=payload, - ) - response.raise_for_status() - except httpx.TimeoutException as exc: - print(f"ERROR: Baidu API request timeout after 90s: {exc!r}") - print(f"Endpoint: {baidu_endpoint}") - return { - "id": 12, - "name": "Baidu Search", - "type": "BAIDU_SEARCH_API", - "sources": [], - }, [] - except httpx.HTTPStatusError as exc: - print(f"ERROR: Baidu API HTTP Status Error: {exc.response.status_code}") - print(f"Response text: {exc.response.text[:500]}") - print(f"Request URL: {exc.request.url}") - return { - "id": 12, - "name": "Baidu Search", - "type": "BAIDU_SEARCH_API", - "sources": [], - }, [] - except httpx.RequestError as exc: - print(f"ERROR: Baidu API Request Error: {type(exc).__name__}: {exc!r}") - print(f"Endpoint: {baidu_endpoint}") - return { - "id": 12, - "name": "Baidu Search", - "type": "BAIDU_SEARCH_API", - "sources": [], - }, [] - except Exception as exc: - print( - f"ERROR: Unexpected error calling Baidu API: {type(exc).__name__}: {exc!r}" - ) - print(f"Endpoint: {baidu_endpoint}") - print(f"Payload: {payload}") - return { - "id": 12, - "name": "Baidu Search", - "type": "BAIDU_SEARCH_API", - "sources": [], - }, [] - - try: - data = response.json() - except ValueError as e: - print(f"ERROR: Failed to decode JSON response from Baidu AI Search: {e}") - print(f"Response status: {response.status_code}") - print(f"Response text: {response.text[:500]}") # First 500 chars - return { - "id": 12, - "name": "Baidu Search", - "type": "BAIDU_SEARCH_API", - "sources": [], - }, [] - - # Extract references (search results) from the response - baidu_references = data.get("references", []) - - if "code" in data or "message" in data: - print( - f"WARNING: Baidu API returned error - Code: {data.get('code')}, Message: {data.get('message')}" - ) - - if not baidu_references: - print("WARNING: No references found in Baidu API response") - print(f"Response keys: {list(data.keys())}") - return { - "id": 12, - "name": "Baidu Search", - "type": "BAIDU_SEARCH_API", - "sources": [], - }, [] - - sources_list: list[dict[str, Any]] = [] - documents: list[dict[str, Any]] = [] - - async with self.counter_lock: - for reference in baidu_references: - # Extract basic fields - title = reference.get("title", "Baidu Search Result") - url = reference.get("url", "") - content = reference.get("content", "") - date = reference.get("date", "") - ref_type = reference.get("type", "web") # web, image, video - - # Create a source entry - source = { - "id": self.source_id_counter, - "title": title, - "description": content[:300] - if content - else "", # Limit description length - "url": url, - } - sources_list.append(source) - - # Prepare metadata - metadata = { - "url": url, - "date": date, - "type": ref_type, - "source": "BAIDU_SEARCH_API", - "web_anchor": reference.get("web_anchor", ""), - "website": reference.get("website", ""), - } - - # Add type-specific metadata - if ref_type == "image" and reference.get("image"): - metadata["image"] = reference["image"] - elif ref_type == "video" and reference.get("video"): - metadata["video"] = reference["video"] - - # Create a document entry - document = { - "chunk_id": self.source_id_counter, - "content": content, - "score": 1.0, # Baidu doesn't provide relevance scores - "document": { - "id": self.source_id_counter, - "title": title, - "document_type": "BAIDU_SEARCH_API", - "metadata": metadata, - }, - } - documents.append(document) - self.source_id_counter += 1 - - result_object = { - "id": 12, - "name": "Baidu Search", - "type": "BAIDU_SEARCH_API", - "sources": sources_list, - } - - return result_object, documents - async def search_slack( self, user_query: str, @@ -1875,127 +1515,6 @@ class ConnectorService: return result_object, clickup_docs - async def search_linkup( - self, - user_query: str, - workspace_id: int, - mode: str = "standard", - ) -> tuple: - """ - Search using Linkup API and return both the source information and documents - - Args: - user_query: The user's query - workspace_id: The workspace ID - mode: Search depth mode, can be "standard" or "deep" - - Returns: - tuple: (sources_info, documents) - """ - # Get Linkup connector configuration - linkup_connector = await self.get_connector_by_type( - SearchSourceConnectorType.LINKUP_API, workspace_id - ) - - if not linkup_connector: - # Return empty results if no Linkup connector is configured - return { - "id": 10, - "name": "Linkup Search", - "type": "LINKUP_API", - "sources": [], - }, [] - - # Initialize Linkup client with API key from connector config - linkup_api_key = linkup_connector.config.get("LINKUP_API_KEY") - linkup_client = LinkupClient(api_key=linkup_api_key) - - # Perform search with Linkup - try: - response = linkup_client.search( - query=user_query, - depth=mode, # Use the provided mode ("standard" or "deep") - output_type="searchResults", # Default to search results - ) - - # Extract results from Linkup response - access as attribute instead of using .get() - linkup_results = response.results if hasattr(response, "results") else [] - - # Only proceed if we have results - if not linkup_results: - return { - "id": 10, - "name": "Linkup Search", - "type": "LINKUP_API", - "sources": [], - }, [] - - # Process each result and create sources directly without deduplication - sources_list = [] - documents = [] - - async with self.counter_lock: - for _i, result in enumerate(linkup_results): - # Only process results that have content - if not hasattr(result, "content") or not result.content: - continue - - # Create a source entry - source = { - "id": self.source_id_counter, - "title": ( - result.name if hasattr(result, "name") else "Linkup Result" - ), - "description": ( - result.content if hasattr(result, "content") else "" - ), - "url": result.url if hasattr(result, "url") else "", - } - sources_list.append(source) - - # Create a document entry - document = { - "chunk_id": self.source_id_counter, - "content": result.content if hasattr(result, "content") else "", - "score": 1.0, # Default score since not provided by Linkup - "document": { - "id": self.source_id_counter, - "title": ( - result.name - if hasattr(result, "name") - else "Linkup Result" - ), - "document_type": "LINKUP_API", - "metadata": { - "url": result.url if hasattr(result, "url") else "", - "type": result.type if hasattr(result, "type") else "", - "source": "LINKUP_API", - }, - }, - } - documents.append(document) - self.source_id_counter += 1 - - # Create result object - result_object = { - "id": 10, - "name": "Linkup Search", - "type": "LINKUP_API", - "sources": sources_list, - } - - return result_object, documents - - except Exception as e: - # Log the error and return empty results - print(f"Error searching with Linkup: {e!s}") - return { - "id": 10, - "name": "Linkup Search", - "type": "LINKUP_API", - "sources": [], - }, [] - async def search_discord( self, user_query: str, diff --git a/surfsense_backend/app/services/mcp_oauth/registry.py b/surfsense_backend/app/services/mcp_oauth/registry.py index 310c3f6e8..2a8819e0c 100644 --- a/surfsense_backend/app/services/mcp_oauth/registry.py +++ b/surfsense_backend/app/services/mcp_oauth/registry.py @@ -185,6 +185,46 @@ MCP_SERVICES: dict[str, MCPServiceConfig] = { ), account_metadata_keys=["user_id", "user_email"], ), + "notion": MCPServiceConfig( + name="Notion", + mcp_url="https://mcp.notion.com/mcp", + connector_type="NOTION_CONNECTOR", + # DCR (RFC 7591): Notion issues its own client credentials. It expires + # DCR registrations, but refresh reuses the original persisted + # ``mcp_oauth.client_id`` (see _refresh_connector_token). + allowed_tools=[ + "search", + "fetch", + "create-pages", + "update-page", + ], + readonly_tools=frozenset({"search", "fetch"}), + account_metadata_keys=["workspace_name"], + ), + "confluence": MCPServiceConfig( + name="Confluence", + # Same Atlassian Rovo server as Jira; tool sets are kept disjoint by + # curation so a workspace can connect both as separate connectors. + mcp_url="https://mcp.atlassian.com/v1/mcp", + connector_type="CONFLUENCE_CONNECTOR", + allowed_tools=[ + "getAccessibleAtlassianResources", + "getConfluenceSpaces", + "getConfluencePage", + "searchConfluenceUsingCql", + "createConfluencePage", + "updateConfluencePage", + ], + readonly_tools=frozenset( + { + "getAccessibleAtlassianResources", + "getConfluenceSpaces", + "getConfluencePage", + "searchConfluenceUsingCql", + } + ), + account_metadata_keys=["cloud_id", "site_name", "base_url"], + ), } _CONNECTOR_TYPE_TO_SERVICE: dict[str, MCPServiceConfig] = { @@ -205,6 +245,25 @@ LIVE_CONNECTOR_TYPES: frozenset[SearchSourceConnectorType] = frozenset( SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR, SearchSourceConnectorType.DISCORD_CONNECTOR, SearchSourceConnectorType.LUMA_CONNECTOR, + # Migrated to hosted MCP — indexing pipelines deprecated (KB is + # files/notes/uploads only). LIVE membership blocks the index route + # and auto-disables periodic indexing. + SearchSourceConnectorType.NOTION_CONNECTOR, + SearchSourceConnectorType.CONFLUENCE_CONNECTOR, + } +) + +# Indexing-only connectors retired with the KB "files, notes, and uploads only" +# shift: their ingestion pipelines are deprecated. Like LIVE membership, this +# blocks the index route and auto-disables periodic indexing — but the message +# frames it as a deprecation, not a real-time-tools swap. Obsidian is +# intentionally excluded (file-like vault content still enriches the KB). +DEPRECATED_INDEXING_CONNECTOR_TYPES: frozenset[SearchSourceConnectorType] = frozenset( + { + SearchSourceConnectorType.GITHUB_CONNECTOR, + SearchSourceConnectorType.BOOKSTACK_CONNECTOR, + SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR, + SearchSourceConnectorType.CIRCLEBACK_CONNECTOR, } ) diff --git a/surfsense_backend/app/services/web_search_service.py b/surfsense_backend/app/services/web_search_service.py deleted file mode 100644 index a5c776323..000000000 --- a/surfsense_backend/app/services/web_search_service.py +++ /dev/null @@ -1,290 +0,0 @@ -""" -Platform-level web search service backed by SearXNG. - -Redis is used only for result caching (graceful degradation if unavailable). -The circuit breaker is fully in-process — no external dependency, zero -latency overhead. -""" - -from __future__ import annotations - -import contextlib -import hashlib -import json -import logging -import threading -import time -from typing import Any -from urllib.parse import urljoin - -import httpx -import redis - -from app.config import config - -logger = logging.getLogger(__name__) - -_EMPTY_RESULT: dict[str, Any] = { - "id": 11, - "name": "Web Search", - "type": "SEARXNG_API", - "sources": [], -} - -# --------------------------------------------------------------------------- -# Redis — used only for result caching -# --------------------------------------------------------------------------- - -_redis_client: redis.Redis | None = None - - -def _get_redis() -> redis.Redis: - global _redis_client - if _redis_client is None: - _redis_client = redis.from_url(config.REDIS_APP_URL, decode_responses=True) - return _redis_client - - -# --------------------------------------------------------------------------- -# In-process Circuit Breaker (no Redis dependency) -# --------------------------------------------------------------------------- - -_CB_FAILURE_THRESHOLD = 5 -_CB_FAILURE_WINDOW_SECONDS = 60 -_CB_COOLDOWN_SECONDS = 30 - -_cb_lock = threading.Lock() -_cb_failure_count: int = 0 -_cb_last_failure_time: float = 0.0 -_cb_open_until: float = 0.0 - - -def _circuit_is_open() -> bool: - return time.monotonic() < _cb_open_until - - -def _record_failure() -> None: - global _cb_failure_count, _cb_last_failure_time, _cb_open_until - now = time.monotonic() - with _cb_lock: - if now - _cb_last_failure_time > _CB_FAILURE_WINDOW_SECONDS: - _cb_failure_count = 0 - _cb_failure_count += 1 - _cb_last_failure_time = now - if _cb_failure_count >= _CB_FAILURE_THRESHOLD: - _cb_open_until = now + _CB_COOLDOWN_SECONDS - logger.warning( - "Circuit breaker OPENED after %d failures — " - "SearXNG calls paused for %ds", - _cb_failure_count, - _CB_COOLDOWN_SECONDS, - ) - - -def _record_success() -> None: - global _cb_failure_count, _cb_open_until - with _cb_lock: - _cb_failure_count = 0 - _cb_open_until = 0.0 - - -# --------------------------------------------------------------------------- -# Result Caching (Redis, graceful degradation) -# --------------------------------------------------------------------------- - -_CACHE_TTL_SECONDS = 300 # 5 minutes -_CACHE_PREFIX = "websearch:cache:" - - -def _cache_key(query: str, engines: str | None, language: str | None) -> str: - raw = f"{query}|{engines or ''}|{language or ''}" - digest = hashlib.sha256(raw.encode()).hexdigest()[:24] - return f"{_CACHE_PREFIX}{digest}" - - -def _cache_get(key: str) -> dict | None: - try: - data = _get_redis().get(key) - if data: - return json.loads(data) - except (redis.RedisError, json.JSONDecodeError): - pass - return None - - -def _cache_set(key: str, value: dict) -> None: - with contextlib.suppress(redis.RedisError): - _get_redis().setex(key, _CACHE_TTL_SECONDS, json.dumps(value)) - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - - -def is_available() -> bool: - """Return ``True`` when the platform SearXNG host is configured.""" - return bool(config.SEARXNG_DEFAULT_HOST) - - -async def health_check() -> dict[str, Any]: - """Ping the SearXNG ``/healthz`` endpoint and return status info.""" - host = config.SEARXNG_DEFAULT_HOST - if not host: - return {"status": "unavailable", "error": "SEARXNG_DEFAULT_HOST not set"} - - healthz_url = urljoin(host if host.endswith("/") else f"{host}/", "healthz") - t0 = time.perf_counter() - try: - async with httpx.AsyncClient(timeout=5.0, verify=False) as client: - resp = await client.get(healthz_url) - resp.raise_for_status() - elapsed_ms = round((time.perf_counter() - t0) * 1000) - return { - "status": "healthy", - "response_time_ms": elapsed_ms, - "circuit_breaker": "open" if _circuit_is_open() else "closed", - } - except Exception as exc: - elapsed_ms = round((time.perf_counter() - t0) * 1000) - return { - "status": "unhealthy", - "error": str(exc), - "response_time_ms": elapsed_ms, - "circuit_breaker": "open" if _circuit_is_open() else "closed", - } - - -async def search( - query: str, - top_k: int = 20, - *, - engines: str | None = None, - language: str | None = None, - safesearch: int | None = None, -) -> tuple[dict[str, Any], list[dict[str, Any]]]: - """Execute a web search against the platform SearXNG instance. - - Returns the standard ``(result_object, documents)`` tuple expected by - ``ConnectorService.search_searxng``. - """ - host = config.SEARXNG_DEFAULT_HOST - if not host: - return dict(_EMPTY_RESULT), [] - - if _circuit_is_open(): - logger.info("Web search skipped — circuit breaker is open") - result = dict(_EMPTY_RESULT) - result["error"] = "Web search temporarily unavailable (circuit open)" - result["status"] = "degraded" - return result, [] - - ck = _cache_key(query, engines, language) - cached = _cache_get(ck) - if cached is not None: - logger.debug("Web search cache HIT for query=%r", query[:60]) - return cached["result"], cached["documents"] - - params: dict[str, Any] = { - "q": query, - "format": "json", - "limit": max(1, min(top_k, 50)), - } - if engines: - params["engines"] = engines - if language: - params["language"] = language - if safesearch is not None and 0 <= safesearch <= 2: - params["safesearch"] = safesearch - - searx_endpoint = urljoin(host if host.endswith("/") else f"{host}/", "search") - headers = {"Accept": "application/json"} - - data: dict[str, Any] | None = None - last_error: Exception | None = None - - for attempt in range(2): - try: - async with httpx.AsyncClient(timeout=15.0, verify=False) as client: - response = await client.get( - searx_endpoint, - params=params, - headers=headers, - ) - response.raise_for_status() - data = response.json() - break - except (httpx.HTTPStatusError, httpx.TimeoutException) as exc: - last_error = exc - if attempt == 0 and ( - isinstance(exc, httpx.TimeoutException) - or ( - isinstance(exc, httpx.HTTPStatusError) - and exc.response.status_code >= 500 - ) - ): - continue - break - except httpx.HTTPError as exc: - last_error = exc - break - except ValueError as exc: - last_error = exc - break - - if data is None: - _record_failure() - logger.warning("Web search failed after retries: %s", last_error) - return dict(_EMPTY_RESULT), [] - - _record_success() - - searx_results = data.get("results", []) - if not searx_results: - return dict(_EMPTY_RESULT), [] - - sources_list: list[dict[str, Any]] = [] - documents: list[dict[str, Any]] = [] - - for idx, result in enumerate(searx_results): - source_id = 200_000 + idx - description = result.get("content") or result.get("snippet") or "" - - sources_list.append( - { - "id": source_id, - "title": result.get("title", "Web Search Result"), - "description": description, - "url": result.get("url", ""), - } - ) - - documents.append( - { - "chunk_id": source_id, - "content": description or result.get("content", ""), - "score": result.get("score", 0.0), - "document": { - "id": source_id, - "title": result.get("title", "Web Search Result"), - "document_type": "SEARXNG_API", - "metadata": { - "url": result.get("url", ""), - "engines": result.get("engines", []), - "category": result.get("category"), - "source": "SEARXNG_API", - }, - }, - } - ) - - result_object: dict[str, Any] = { - "id": 11, - "name": "Web Search", - "type": "SEARXNG_API", - "sources": sources_list, - } - - _cache_set(ck, {"result": result_object, "documents": documents}) - - return result_object, documents diff --git a/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py b/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py index 55aa04d22..fc896005c 100644 --- a/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py +++ b/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py @@ -64,12 +64,20 @@ async def _check_and_trigger_schedules(): SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR: index_google_drive_files_task, } - from app.services.mcp_oauth.registry import LIVE_CONNECTOR_TYPES + from app.services.mcp_oauth.registry import ( + DEPRECATED_INDEXING_CONNECTOR_TYPES, + LIVE_CONNECTOR_TYPES, + ) - # Disable obsolete periodic indexing for live connectors in one batch. + # Disable obsolete periodic indexing in one batch: live connectors + # (now real-time agent tools) and deprecated-indexing connectors + # (KB is files/notes/uploads only) no longer index on a schedule. live_disabled = [] for connector in due_connectors: - if connector.connector_type in LIVE_CONNECTOR_TYPES: + if ( + connector.connector_type in LIVE_CONNECTOR_TYPES + or connector.connector_type in DEPRECATED_INDEXING_CONNECTOR_TYPES + ): connector.periodic_indexing_enabled = False connector.next_scheduled_at = None live_disabled.append(connector) @@ -77,7 +85,7 @@ async def _check_and_trigger_schedules(): await session.commit() for c in live_disabled: logger.info( - "Disabled obsolete periodic indexing for live connector %s (%s)", + "Disabled obsolete periodic indexing for connector %s (%s)", c.id, c.connector_type.value, ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/assistant_finalize.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/assistant_finalize.py index 5622a42b8..04cba6f5f 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/shared/assistant_finalize.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/assistant_finalize.py @@ -54,7 +54,7 @@ def _resolve_citations( ) -> list[dict[str, Any]]: """Rewrite ``[n]`` -> ``[citation:<payload>]`` in each text part before persisting. - No-op when the turn registered no citable sources; ``web_search``'s existing + No-op when the turn registered no citable sources; any pre-existing ``[citation:url]`` markers pass through untouched (the regex matches bare ``[n]``). """ registry = _as_registry(raw_registry) diff --git a/surfsense_backend/app/tasks/chat/streaming/handlers/tools/web_search/emission.py b/surfsense_backend/app/tasks/chat/streaming/handlers/tools/web_search/emission.py deleted file mode 100644 index 3efe45d0c..000000000 --- a/surfsense_backend/app/tasks/chat/streaming/handlers/tools/web_search/emission.py +++ /dev/null @@ -1,37 +0,0 @@ -"""web_search: citations parsed from provider XML.""" - -from __future__ import annotations - -import re -from collections.abc import Iterator - -from app.tasks.chat.streaming.handlers.tools.emission_context import ( - ToolCompletionEmissionContext, -) - - -def iter_completion_emission_frames( - ctx: ToolCompletionEmissionContext, -) -> Iterator[str]: - out = ctx.tool_output - xml = out.get("result", str(out)) if isinstance(out, dict) else str(out) - citations: dict[str, dict[str, str]] = {} - for m in re.finditer( - r"<title><!\[CDATA\[(.*?)\]\]>\s*", - xml, - ): - title, url = m.group(1).strip(), m.group(2).strip() - if url.startswith("http") and url not in citations: - citations[url] = {"title": title} - for m in re.finditer( - r"", - xml, - ): - chunk_url, content = m.group(1).strip(), m.group(2).strip() - if chunk_url.startswith("http") and chunk_url in citations and content: - citations[chunk_url]["snippet"] = ( - content[:200] + "…" if len(content) > 200 else content - ) - yield ctx.emit_tool_output_card( - {"status": "completed", "citations": citations}, - ) diff --git a/surfsense_backend/app/utils/validators.py b/surfsense_backend/app/utils/validators.py index 6e71abf79..aec59cf7b 100644 --- a/surfsense_backend/app/utils/validators.py +++ b/surfsense_backend/app/utils/validators.py @@ -12,6 +12,52 @@ from typing import Any import validators from fastapi import HTTPException +# Connectors retired during the MCP migration: no viable official MCP server +# exists yet, so new connections are refused. Existing rows keep working until +# the user removes them. If demand returns, reinstate the connector by dropping +# it from this set and re-enabling its subagent/route. +# +# The four search APIs (TAVILY/SEARXNG/LINKUP/BAIDU) are deprecated alongside the +# Google-only web-search consolidation: public web search now runs through the +# google_search subagent, and users who still want Tavily/Linkup can add them via +# the generic Custom MCP connector (API-key headers). +DEPRECATED_CONNECTOR_TYPES: frozenset[str] = frozenset( + { + "DISCORD_CONNECTOR", + "TEAMS_CONNECTOR", + "LUMA_CONNECTOR", + "TAVILY_API", + "SEARXNG_API", + "LINKUP_API", + "BAIDU_SEARCH_API", + } +) + + +def raise_if_connector_deprecated(connector_type: str | Any) -> None: + """Refuse new connections for a deprecated connector type (HTTP 410 Gone).""" + connector_type_str = ( + connector_type.value + if hasattr(connector_type, "value") + else str(connector_type) + ) + if connector_type_str in DEPRECATED_CONNECTOR_TYPES: + pretty = ( + connector_type_str.replace("_CONNECTOR", "") + .replace("_SEARCH_API", "") + .replace("_API", "") + .replace("_", " ") + .title() + ) + raise HTTPException( + status_code=410, + detail=( + f"The {pretty} connector has been deprecated and can no longer be " + "connected. If you were relying on it heavily, let us know and we'll " + "consider bringing it back." + ), + ) + def validate_workspace_id(workspace_id: Any) -> int: """ diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 7244b48e5..002c527c0 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -18,7 +18,6 @@ dependencies = [ "google-api-python-client>=2.156.0", "google-auth-oauthlib>=1.2.1", "kokoro>=0.9.4", - "linkup-sdk>=0.2.4", "llama-cloud-services>=0.6.25", "Markdown>=3.7", "markdownify>=0.14.1", @@ -34,7 +33,6 @@ dependencies = [ "spacy>=3.8.7", "en-core-web-sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl", "static-ffmpeg>=2.13", - "tavily-python>=0.3.2", "uvicorn[standard]>=0.34.0", "validators>=0.34.0", "youtube-transcript-api>=1.0.3", diff --git a/surfsense_backend/tests/e2e/README.md b/surfsense_backend/tests/e2e/README.md index caa0f89b0..30a9bcd3d 100644 --- a/surfsense_backend/tests/e2e/README.md +++ b/surfsense_backend/tests/e2e/README.md @@ -72,7 +72,7 @@ pnpm exec playwright install --with-deps chromium ### Each run **1. Bring up Postgres + Redis** from the repo root (the other deps-only -services (SearXNG, Zero, pgAdmin) are not needed for E2E): +services (Zero, pgAdmin) are not needed for E2E): ```bash docker compose -f docker/docker-compose.deps-only.yml up -d db redis diff --git a/surfsense_backend/tests/e2e/fakes/chat_llm.py b/surfsense_backend/tests/e2e/fakes/chat_llm.py index 234a18ec1..4e6be0995 100644 --- a/surfsense_backend/tests/e2e/fakes/chat_llm.py +++ b/surfsense_backend/tests/e2e/fakes/chat_llm.py @@ -142,6 +142,13 @@ class FakeChatLLM(BaseChatModel): and CLICKUP_CANARY_TOKEN in latest_tool_text ): return f"ClickUp live tool content found: {CLICKUP_CANARY_TOKEN}" + if latest_tool_name == "search" and NOTION_CANARY_TOKEN in latest_tool_text: + return f"Notion live tool content found: {NOTION_CANARY_TOKEN}" + if ( + latest_tool_name == "searchConfluenceUsingCql" + and CONFLUENCE_CANARY_TOKEN in latest_tool_text + ): + return f"Confluence live tool content found: {CONFLUENCE_CANARY_TOKEN}" wants_gmail = _contains_any( latest_human, @@ -556,7 +563,7 @@ class FakeChatLLM(BaseChatModel): # Marker unique to a connector subagent's prompt: the main agent must # delegate via ``task``; only the subagent has connector tools registered. in_connector_subagent = ( - "specialist for the user's connected" in _messages_to_text(messages) + "connected-apps specialist" in _messages_to_text(messages) ) # Main agent: delegate live-tool connector work to its subagent (which @@ -579,8 +586,12 @@ class FakeChatLLM(BaseChatModel): ("linear", ("linear", "issue", LINEAR_CANARY_TITLE)), ("slack", ("slack", SLACK_CANARY_TOKEN)), ("clickup", ("clickup", CLICKUP_CANARY_TITLE)), + ("notion", ("notion", NOTION_CANARY_TITLE)), + ("confluence", ("confluence", CONFLUENCE_CANARY_TITLE)), ) - for subagent_type, needles in connector_delegations: + # Every MCP-backed connector is now one ``mcp_discovery`` route; the + # needle set only decides which canary the delegation targets. + for connector_key, needles in connector_delegations: if _contains_any(latest_human, needles): return AIMessage( content="", @@ -588,10 +599,10 @@ class FakeChatLLM(BaseChatModel): { "name": "task", "args": { - "subagent_type": subagent_type, + "subagent_type": "mcp_discovery", "description": latest_human, }, - "id": f"call_e2e_task_{subagent_type}", + "id": f"call_e2e_task_{connector_key}", } ], ) @@ -718,6 +729,38 @@ class FakeChatLLM(BaseChatModel): ], ) + # Confluence check precedes Notion: the Confluence prompt also contains + # the word "page", so Notion's needle omits it to avoid cross-matching. + if latest_tool is None and _contains_any( + latest_human, + ("confluence", CONFLUENCE_CANARY_TITLE), + ): + return AIMessage( + content="", + tool_calls=[ + { + "name": "searchConfluenceUsingCql", + "args": {"cql": f'text ~ "{CONFLUENCE_CANARY_TITLE}"'}, + "id": "call_e2e_search_confluence", + } + ], + ) + + if latest_tool is None and _contains_any( + latest_human, + ("notion", NOTION_CANARY_TITLE), + ): + return AIMessage( + content="", + tool_calls=[ + { + "name": "search", + "args": {"query": NOTION_CANARY_TITLE}, + "id": "call_e2e_search_notion", + } + ], + ) + return None def _generate( diff --git a/surfsense_backend/tests/e2e/fakes/jira_module.py b/surfsense_backend/tests/e2e/fakes/jira_module.py index d67531218..c9c03e2f1 100644 --- a/surfsense_backend/tests/e2e/fakes/jira_module.py +++ b/surfsense_backend/tests/e2e/fakes/jira_module.py @@ -1,4 +1,13 @@ -"""Strict Jira MCP OAuth/tool fakes for Playwright E2E.""" +"""Strict Atlassian (Jira + Confluence) MCP OAuth/tool fakes for Playwright E2E. + +Jira and Confluence share one hosted Atlassian Rovo MCP server +(``https://mcp.atlassian.com/v1/mcp``) and one OAuth surface. Because the MCP +OAuth/runtime dispatchers are keyed by URL, a single handler here serves both +connectors; production keeps their tool sets disjoint via each service's +``allowed_tools`` curation. The OAuth ``redirect_uri`` substring is therefore +relaxed to the shared ``/api/v1/auth/mcp/`` prefix so both the +``.../mcp/jira/...`` and ``.../mcp/confluence/...`` callbacks validate. +""" from __future__ import annotations @@ -22,6 +31,13 @@ _ACCESS_TOKEN = "fake-jira-mcp-access-token" _REFRESH_TOKEN = "fake-jira-mcp-refresh-token" _OAUTH_CODE = "fake-jira-oauth-code" +# Confluence canary — keep in sync with FAKE_CONFLUENCE_PAGES / +# CANARY_TOKENS.confluenceCanary in surfsense_web/tests/helpers/canary.ts. +_CONFLUENCE_TOKEN = "SURFSENSE_E2E_CANARY_TOKEN_CONFLUENCE_001" +_CONFLUENCE_PAGE_ID = "fake-confluence-page-canary-001" +_CONFLUENCE_TITLE = "E2E Canary Confluence Page" +_CONFLUENCE_SPACE_ID = "fake-confluence-space-001" + def _load_fixture() -> dict[str, Any]: with _FIXTURE_PATH.open() as f: @@ -69,6 +85,34 @@ async def _list_tools() -> SimpleNamespace: "required": ["jql"], }, ), + SimpleNamespace( + name="searchConfluenceUsingCql", + description="Search Confluence content using a CQL expression.", + inputSchema={ + "type": "object", + "properties": { + "cql": { + "type": "string", + "description": "CQL query used to search Confluence content.", + } + }, + "required": ["cql"], + }, + ), + SimpleNamespace( + name="getConfluencePage", + description="Fetch a Confluence page's body by id.", + inputSchema={ + "type": "object", + "properties": { + "pageId": { + "type": "string", + "description": "The Confluence page id to fetch.", + } + }, + "required": ["pageId"], + }, + ), ] ) @@ -98,11 +142,30 @@ async def _call_tool( text = _issue_text(issue) return SimpleNamespace(content=[SimpleNamespace(text=text)]) - raise NotImplementedError(f"Unexpected Jira MCP tool call: {tool_name!r}") + if tool_name == "searchConfluenceUsingCql": + cql = str(arguments.get("cql", "")) + if _CONFLUENCE_TITLE.lower() not in cql.lower(): + raise ValueError(f"Unexpected Confluence CQL query: {cql!r}") + text = ( + f"{_CONFLUENCE_TITLE}\n" + f"id: {_CONFLUENCE_PAGE_ID}\n" + f"space: {_CONFLUENCE_SPACE_ID}\n" + f"excerpt: {_CONFLUENCE_TOKEN}" + ) + return SimpleNamespace(content=[SimpleNamespace(text=text)]) + + if tool_name == "getConfluencePage": + page_id = str(arguments.get("pageId", "")) + if page_id != _CONFLUENCE_PAGE_ID: + raise ValueError(f"Unexpected Confluence page id: {page_id!r}") + text = f"{_CONFLUENCE_TITLE}\n\n{_CONFLUENCE_TOKEN}" + return SimpleNamespace(content=[SimpleNamespace(text=text)]) + + raise NotImplementedError(f"Unexpected Atlassian MCP tool call: {tool_name!r}") def install(active_patches: list[Any]) -> None: - """Register Jira MCP OAuth/tool handlers with the shared dispatchers.""" + """Register the shared Atlassian (Jira + Confluence) MCP handlers.""" del active_patches mcp_oauth_runtime.register_service( mcp_url=_MCP_URL, @@ -122,8 +185,10 @@ def install(active_patches: list[Any]) -> None: oauth_code=_OAUTH_CODE, access_token=_ACCESS_TOKEN, refresh_token=_REFRESH_TOKEN, - scope="read:jira-work read:me write:jira-work", - redirect_uri_substring="/api/v1/auth/mcp/jira/connector/callback", + scope="read:jira-work read:confluence-content.all read:me write:jira-work", + # Shared Atlassian server serves both Jira and Confluence connectors, so + # accept either service's callback under the common MCP OAuth prefix. + redirect_uri_substring="/api/v1/auth/mcp/", ) mcp_runtime.register( url=_MCP_URL, diff --git a/surfsense_backend/tests/e2e/fakes/native_google.py b/surfsense_backend/tests/e2e/fakes/native_google.py index 1afcaf9c3..cdbc17828 100644 --- a/surfsense_backend/tests/e2e/fakes/native_google.py +++ b/surfsense_backend/tests/e2e/fakes/native_google.py @@ -430,15 +430,15 @@ def install(active_patches: list[Any]) -> None: ("app.connectors.google_gmail_connector.build", _fake_build), ("app.connectors.google_calendar_connector.build", _fake_build), ( - "app.agents.chat.multi_agent_chat.subagents.connectors.calendar.tools.create_event.build", + "app.agents.chat.multi_agent_chat.subagents.builtins.mcp_discovery.tools.calendar.create_event.build", _fake_build, ), ( - "app.agents.chat.multi_agent_chat.subagents.connectors.calendar.tools.update_event.build", + "app.agents.chat.multi_agent_chat.subagents.builtins.mcp_discovery.tools.calendar.update_event.build", _fake_build, ), ( - "app.agents.chat.multi_agent_chat.subagents.connectors.calendar.tools.delete_event.build", + "app.agents.chat.multi_agent_chat.subagents.builtins.mcp_discovery.tools.calendar.delete_event.build", _fake_build, ), ("googleapiclient.http.MediaIoBaseDownload", _FakeMediaIoBaseDownload), diff --git a/surfsense_backend/tests/e2e/fakes/notion_mcp_module.py b/surfsense_backend/tests/e2e/fakes/notion_mcp_module.py new file mode 100644 index 000000000..64d615c5c --- /dev/null +++ b/surfsense_backend/tests/e2e/fakes/notion_mcp_module.py @@ -0,0 +1,127 @@ +"""Strict Notion MCP OAuth/tool fakes for Playwright E2E. + +Notion migrated from indexed OAuth to the hosted Notion MCP server +(``https://mcp.notion.com/mcp``, DCR/RFC 7591). This fake mirrors +``jira_module`` for the generic MCP OAuth + streamable-HTTP tool boundaries; +the older ``notion_module`` (a ``notion_client`` SDK stand-in) stays only to +satisfy production's import-time ``import notion_client``. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +from tests.e2e.fakes import mcp_oauth_runtime, mcp_runtime + +_AUTHORIZATION_URL = "https://mcp.notion.com/authorize" +_REGISTRATION_URL = "https://mcp.notion.com/register" +_TOKEN_URL = "https://mcp.notion.com/token" +_MCP_URL = "https://mcp.notion.com/mcp" + +_CLIENT_ID = "fake-notion-mcp-client-id" +_CLIENT_SECRET = "fake-notion-mcp-client-secret" +_ACCESS_TOKEN = "fake-notion-mcp-access-token" +_REFRESH_TOKEN = "fake-notion-mcp-refresh-token" +_OAUTH_CODE = "fake-notion-oauth-code" + +# Keep in sync with FAKE_NOTION_PAGES / CANARY_TOKENS.notionCanary in +# surfsense_web/tests/helpers/canary.ts. +_CANARY_TOKEN = "SURFSENSE_E2E_CANARY_TOKEN_NOTION_001" +_CANARY_PAGE_ID = "fake-notion-page-canary-001" +_CANARY_TITLE = "E2E Canary Notion Page" +_WORKSPACE_NAME = "SurfSense E2E Notion Workspace" + + +async def _list_tools() -> SimpleNamespace: + return SimpleNamespace( + tools=[ + SimpleNamespace( + name="search", + description="Search the connected Notion workspace for pages and databases.", + inputSchema={ + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Text to search Notion pages for.", + } + }, + "required": ["query"], + }, + ), + SimpleNamespace( + name="fetch", + description="Fetch the full contents of a Notion page by id.", + inputSchema={ + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The Notion page id to fetch.", + } + }, + "required": ["id"], + }, + ), + ] + ) + + +async def _call_tool( + tool_name: str, arguments: dict[str, Any] | None = None +) -> SimpleNamespace: + arguments = arguments or {} + + if tool_name == "search": + query = str(arguments.get("query", "")) + if _CANARY_TITLE.lower() not in query.lower(): + raise ValueError(f"Unexpected Notion search query: {query!r}") + text = ( + f"{_CANARY_TITLE}\n" + f"id: {_CANARY_PAGE_ID}\n" + f"workspace: {_WORKSPACE_NAME}\n" + f"snippet: {_CANARY_TOKEN}" + ) + return SimpleNamespace(content=[SimpleNamespace(text=text)]) + + if tool_name == "fetch": + page_id = str(arguments.get("id", "")) + if page_id != _CANARY_PAGE_ID: + raise ValueError(f"Unexpected Notion fetch id: {page_id!r}") + text = f"{_CANARY_TITLE}\n\n{_CANARY_TOKEN}" + return SimpleNamespace(content=[SimpleNamespace(text=text)]) + + raise NotImplementedError(f"Unexpected Notion MCP tool call: {tool_name!r}") + + +def install(active_patches: list[Any]) -> None: + """Register Notion MCP OAuth/tool handlers with the shared dispatchers.""" + del active_patches + mcp_oauth_runtime.register_service( + mcp_url=_MCP_URL, + discovery_metadata={ + "issuer": "https://mcp.notion.com", + "authorization_endpoint": _AUTHORIZATION_URL, + "token_endpoint": _TOKEN_URL, + "registration_endpoint": _REGISTRATION_URL, + "code_challenge_methods_supported": ["S256"], + "grant_types_supported": ["authorization_code", "refresh_token"], + "response_types_supported": ["code"], + }, + client_id=_CLIENT_ID, + client_secret=_CLIENT_SECRET, + token_endpoint=_TOKEN_URL, + registration_endpoint=_REGISTRATION_URL, + oauth_code=_OAUTH_CODE, + access_token=_ACCESS_TOKEN, + refresh_token=_REFRESH_TOKEN, + scope="read write", + redirect_uri_substring="/api/v1/auth/mcp/notion/connector/callback", + ) + mcp_runtime.register( + url=_MCP_URL, + expected_bearer=_ACCESS_TOKEN, + list_tools=_list_tools, + call_tool=_call_tool, + ) diff --git a/surfsense_backend/tests/e2e/run_backend.py b/surfsense_backend/tests/e2e/run_backend.py index 87977626f..c1f9aad42 100644 --- a/surfsense_backend/tests/e2e/run_backend.py +++ b/surfsense_backend/tests/e2e/run_backend.py @@ -282,6 +282,7 @@ def _install_runtime_fakes() -> None: mcp_oauth_runtime as _fake_mcp_oauth_runtime, mcp_runtime as _fake_mcp_runtime, native_google as _fake_native_google, + notion_mcp_module as _fake_notion_mcp_module, notion_module as _fake_notion_module, onedrive_graph as _fake_onedrive_graph, slack_module as _fake_slack_module, @@ -295,6 +296,7 @@ def _install_runtime_fakes() -> None: _fake_onedrive_graph.install(_active_patches) _fake_dropbox_api.install(_active_patches) _fake_notion_module.install(_active_patches) + _fake_notion_mcp_module.install(_active_patches) _fake_linear_module.install(_active_patches) _fake_jira_module.install(_active_patches) _fake_clickup_module.install(_active_patches) diff --git a/surfsense_backend/tests/integration/agents/multi_agent_chat/test_web_search_delegation.py b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_web_search_delegation.py new file mode 100644 index 000000000..04b09af7d --- /dev/null +++ b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_web_search_delegation.py @@ -0,0 +1,86 @@ +"""Backend E2E: public web search now flows through the ``google_search`` route. + +After the Google-only consolidation the main agent has no ``web_search`` tool; +a web query must be delegated to the ``google_search`` subagent via ``task``. +This drives the *assembled* production graph (real DB, scripted LLM) end to end: +main agent emits a ``task(google_search, ...)`` call, the subagent runs a turn, +and the main agent resumes to a final answer. Proves the delegation path the +teardown relies on actually executes -- not just that the constants changed. +""" + +from __future__ import annotations + +import pytest +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langgraph.checkpoint.memory import InMemorySaver + +from app.agents.chat.multi_agent_chat import create_multi_agent_chat_deep_agent +from app.services.connector_service import ConnectorService +from tests.integration.harness import ScriptedTurn, build_scripted_harness + +pytestmark = pytest.mark.integration + + +def _last_ai_text(messages: list) -> str | None: + for m in reversed(messages): + if isinstance(m, AIMessage) and isinstance(m.content, str) and m.content: + return m.content + return None + + +@pytest.mark.asyncio +async def test_web_query_delegates_to_google_search( + db_session, db_user, db_workspace +): + """A web-search query routes through ``task(google_search)`` and resumes. + + Scripted sequence (the fake model is shared and consumed in order across the + main agent and the delegated subagent): + 1. main agent -> task(subagent_type="google_search") + 2. subagent -> plain text (never touches the scrape tool, so no network) + 3. main agent -> final answer + """ + harness = build_scripted_harness( + turns=[ + ScriptedTurn( + tool_calls=[ + { + "name": "task", + "args": { + "subagent_type": "google_search", + "description": "latest SurfSense release notes", + }, + "id": "call_ws_task", + } + ] + ), + ScriptedTurn(text="SERP: SurfSense v2 shipped Google-only web search."), + ScriptedTurn(text="SurfSense v2 shipped Google-only web search."), + ] + ) + + agent = await create_multi_agent_chat_deep_agent( + llm=harness.model, + workspace_id=db_workspace.id, + db_session=db_session, + connector_service=ConnectorService(db_session), + checkpointer=InMemorySaver(), + user_id=str(db_user.id), + thread_id=db_workspace.id, + agent_config=None, + ) + + result = await agent.ainvoke( + {"messages": [HumanMessage(content="search the web for SurfSense news")]}, + config={"configurable": {"thread_id": "ws-google-delegation-1"}}, + ) + + task_tool_messages = [ + m + for m in result["messages"] + if isinstance(m, ToolMessage) and m.name == "task" + ] + assert task_tool_messages, "web query did not delegate through the task tool" + assert _last_ai_text(result["messages"]) == ( + "SurfSense v2 shipped Google-only web search." + ) diff --git a/surfsense_backend/tests/unit/agents/chat/shared/tools/test_web_search.py b/surfsense_backend/tests/unit/agents/chat/shared/tools/test_web_search.py deleted file mode 100644 index 7137bfdfc..000000000 --- a/surfsense_backend/tests/unit/agents/chat/shared/tools/test_web_search.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Tests for the shared ``web_search`` tool's citable-result adaptation. - -The tool's network path (SearXNG + live connectors) is out of scope here; these -cover the pure mapping from raw web results to renderable, citable documents and -the end-to-end registration of ``WEB_RESULT`` ``[n]`` labels. -""" - -from __future__ import annotations - -import pytest - -from app.agents.chat.multi_agent_chat.shared.citations import ( - CitationRegistry, - CitationSourceType, -) -from app.agents.chat.multi_agent_chat.shared.document_render import render_web_results -from app.agents.chat.shared.tools.web_search import ( - _to_renderable_web_documents, - _web_source_label, -) - -pytestmark = pytest.mark.unit - - -def _raw_result(url: str, title: str, content: str) -> dict: - return { - "document": {"title": title, "metadata": {"url": url}}, - "content": content, - } - - -def test_web_source_label_strips_scheme_and_www() -> None: - assert _web_source_label("https://www.example.com/path") == "Web · example.com" - assert _web_source_label("http://news.site.org/a/b") == "Web · news.site.org" - assert _web_source_label("") == "Web" - - -def test_adapter_maps_each_result_to_one_web_passage() -> None: - docs = _to_renderable_web_documents( - [ - _raw_result("https://a.com/x", "Alpha", "alpha body"), - _raw_result("https://b.com/y", "Beta", "beta body"), - ] - ) - - assert [d.title for d in docs] == ["Alpha", "Beta"] - passages = [p for d in docs for p in d.passages] - assert all(p.source_type is CitationSourceType.WEB_RESULT for p in passages) - assert passages[0].locator == {"url": "https://a.com/x"} - assert passages[0].content == "alpha body" - - -def test_adapter_skips_results_without_url_or_content() -> None: - docs = _to_renderable_web_documents( - [ - _raw_result("", "No URL", "has content"), - _raw_result("https://c.com/z", "Empty", " "), - _raw_result("https://d.com/w", "Good", "real content"), - ] - ) - - assert [d.title for d in docs] == ["Good"] - - -def test_adapter_truncates_on_char_budget() -> None: - big = "x" * 30 - docs = _to_renderable_web_documents( - [ - _raw_result("https://a.com", "A", big), - _raw_result("https://b.com", "B", big), - _raw_result("https://c.com", "C", big), - ], - max_chars=50, - ) - - # First fits (30), second crosses 50 and stops the loop. - assert [d.title for d in docs] == ["A"] - - -def test_end_to_end_registers_web_results_for_citation() -> None: - registry = CitationRegistry() - docs = _to_renderable_web_documents( - [_raw_result("https://example.com/a", "Example", "the answer is 42")] - ) - - block = render_web_results(docs, registry) - - assert block is not None - assert "[1] the answer is 42" in block - entry = registry.resolve(1) - assert entry is not None - assert entry.source_type is CitationSourceType.WEB_RESULT - assert entry.locator == {"url": "https://example.com/a"} diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/shared/document_render/test_web_results.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/shared/document_render/test_web_results.py deleted file mode 100644 index f96473667..000000000 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/shared/document_render/test_web_results.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Tests for the ```` wrapper around web-result excerpt documents.""" - -from __future__ import annotations - -import pytest - -from app.agents.chat.multi_agent_chat.shared.citations import ( - CitationRegistry, - CitationSourceType, -) -from app.agents.chat.multi_agent_chat.shared.document_render import ( - RenderableDocument, - RenderablePassage, - render_web_results, -) - -pytestmark = pytest.mark.unit - - -def _web_doc(url: str, title: str, content: str) -> RenderableDocument: - return RenderableDocument( - title=title, - source=f"Web · {url.split('//', 1)[-1].split('/', 1)[0]}", - passages=[ - RenderablePassage( - content=content, - locator={"url": url}, - source_type=CitationSourceType.WEB_RESULT, - ) - ], - ) - - -def test_returns_none_when_nothing_to_show() -> None: - registry = CitationRegistry() - - assert render_web_results([], registry) is None - - -def test_wraps_in_web_results_container() -> None: - registry = CitationRegistry() - - block = render_web_results( - [_web_doc("https://example.com/a", "Example", "the answer is 42")], - registry, - ) - - assert block is not None - assert block.startswith("") - assert block.endswith("") - assert "cite a result with its [n]" in block - assert ( - '' in block - ) - assert "[1] the answer is 42" in block - - -def test_registers_each_result_as_web_result_with_url_locator() -> None: - registry = CitationRegistry() - - render_web_results( - [ - _web_doc("https://a.com/x", "A", "alpha"), - _web_doc("https://b.com/y", "B", "beta"), - ], - registry, - ) - - first = registry.resolve(1) - second = registry.resolve(2) - assert first is not None and second is not None - assert first.source_type is CitationSourceType.WEB_RESULT - assert first.locator == {"url": "https://a.com/x"} - assert second.locator == {"url": "https://b.com/y"} - - -def test_same_url_reuses_label_across_calls() -> None: - registry = CitationRegistry() - doc = _web_doc("https://example.com/a", "Example", "stable fact") - - render_web_results([doc], registry) - render_web_results([doc], registry) - - assert registry.next_n == 2 diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py index 2f3553a27..364569ae8 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py @@ -191,25 +191,25 @@ def test_missing_user_allowlist_keeps_coded_behaviour(): def test_user_allowlist_for_different_subagent_does_not_leak(): - """User trust for ``linear`` must not affect a ``jira`` subagent compile.""" + """User trust for one subagent must not affect a different subagent compile.""" coded = Ruleset( - origin="jira", + origin="mcp_discovery", rules=[Rule(permission="save_issue", pattern="*", action="ask")], ) - linear_allowlist = Ruleset( - origin="user_allowlist:linear", + other_allowlist = Ruleset( + origin="user_allowlist:knowledge_base", rules=[Rule(permission="save_issue", pattern="*", action="allow")], ) result = pack_subagent( - name="jira", + name="mcp_discovery", description="test", system_prompt="x", tools=[], ruleset=coded, dependencies={ "flags": AgentFeatureFlags(), - "user_allowlist_by_subagent": {"linear": linear_allowlist}, + "user_allowlist_by_subagent": {"knowledge_base": other_allowlist}, }, ) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_discovery_migration.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_discovery_migration.py new file mode 100644 index 000000000..5205e363a --- /dev/null +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_discovery_migration.py @@ -0,0 +1,153 @@ +"""Guardrails for the MCP-consolidation migration. + +Every MCP-backed connector now routes to the single ``mcp_discovery`` subagent +(file connectors stay native; Discord/Teams/Luma are deprecated). These tests +pin the pieces that make that safe: connector→route mapping, any-of gating, +legacy checkpoint aliasing, tool-name collision prefixing, the metadata-derived +approval ruleset, and the KB indexing-deprecation sets. +""" + +from __future__ import annotations + +import pytest +from langchain_core.tools import StructuredTool + +from app.agents.chat.multi_agent_chat.constants import ( + CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS, + LEGACY_SUBAGENT_ALIASES, + SUBAGENT_TO_REQUIRED_CONNECTOR_MAP, +) +from app.agents.chat.multi_agent_chat.subagents.builtins.mcp_discovery.tools.index import ( + NAME as MCP_DISCOVERY_NAME, + build_ruleset, +) +from app.agents.chat.multi_agent_chat.subagents.mcp_tools.index import ( + resolve_tool_name_collisions, +) +from app.agents.chat.multi_agent_chat.subagents.registry import ( + SUBAGENT_BUILDERS_BY_NAME, + get_subagents_to_exclude, +) +from app.services.mcp_oauth.registry import ( + DEPRECATED_INDEXING_CONNECTOR_TYPES, + LIVE_CONNECTOR_TYPES, +) + +pytestmark = pytest.mark.unit + +# Connectors that must all funnel into ``mcp_discovery`` (not their own routes). +_MCP_ROUTED = { + "SLACK_CONNECTOR", + "JIRA_CONNECTOR", + "LINEAR_CONNECTOR", + "CLICKUP_CONNECTOR", + "AIRTABLE_CONNECTOR", + "NOTION_CONNECTOR", + "CONFLUENCE_CONNECTOR", + "GOOGLE_GMAIL_CONNECTOR", + "GOOGLE_CALENDAR_CONNECTOR", + "MCP_CONNECTOR", +} + + +def _tool(name: str, metadata: dict) -> StructuredTool: + return StructuredTool.from_function( + func=lambda: name, + name=name, + description=name, + metadata=metadata, + ) + + +def test_all_mcp_connectors_route_to_discovery(): + for connector_type in _MCP_ROUTED: + assert ( + CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS[connector_type] + == MCP_DISCOVERY_NAME + ), connector_type + + +def test_file_connectors_keep_native_routes(): + assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["GOOGLE_DRIVE_CONNECTOR"] == "google_drive" + assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["DROPBOX_CONNECTOR"] == "dropbox" + assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["ONEDRIVE_CONNECTOR"] == "onedrive" + + +def test_deprecated_connectors_have_no_route(): + for connector_type in ("DISCORD_CONNECTOR", "TEAMS_CONNECTOR", "LUMA_CONNECTOR"): + assert connector_type not in CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS + + +def test_discovery_gating_is_any_of(): + """One connected app is enough to keep ``mcp_discovery``; none excludes it.""" + assert MCP_DISCOVERY_NAME not in get_subagents_to_exclude(["SLACK_CONNECTOR"]) + assert MCP_DISCOVERY_NAME in get_subagents_to_exclude([]) + # A generic user MCP server alone still unlocks it. + assert MCP_DISCOVERY_NAME not in get_subagents_to_exclude(["MCP_CONNECTOR"]) + + +def test_discovery_gating_tokens_match_routed_connectors(): + assert SUBAGENT_TO_REQUIRED_CONNECTOR_MAP[MCP_DISCOVERY_NAME] == frozenset(_MCP_ROUTED) + + +def test_legacy_aliases_resolve_to_a_live_subagent(): + """Old per-connector task names must alias onto a subagent that still exists.""" + for legacy, target in LEGACY_SUBAGENT_ALIASES.items(): + assert legacy not in SUBAGENT_BUILDERS_BY_NAME, legacy + assert target in SUBAGENT_BUILDERS_BY_NAME, target + + +def test_collision_only_prefixes_shared_names(): + """A name on two connectors is prefixed; a unique name is left untouched.""" + tools = [ + _tool("search", {"mcp_connector_id": 1, "mcp_transport": "http"}), + _tool("search", {"mcp_connector_id": 2, "mcp_transport": "http"}), + _tool("list_bases", {"mcp_connector_id": 2, "mcp_transport": "http"}), + ] + resolved = {t.name: t for t in resolve_tool_name_collisions(tools, {1: "NOTION_CONNECTOR", 2: "AIRTABLE_CONNECTOR"})} + + # The unique tool keeps its bare name (trusted_tools / history stay valid). + assert "list_bases" in resolved + # The colliding name is gone; both are prefixed with service + connector id. + assert "search" not in resolved + assert "notion_1_search" in resolved + assert "airtable_2_search" in resolved + # Original name preserved for the "Always Allow" fallback key. + for name in ("notion_1_search", "airtable_2_search"): + meta = resolved[name].metadata or {} + assert meta["mcp_original_tool_name"] == "search" + assert meta["mcp_collision_prefixed"] is True + + +def test_collision_noop_without_collisions(): + tools = [ + _tool("a", {"mcp_connector_id": 1}), + _tool("b", {"mcp_connector_id": 2}), + ] + assert [t.name for t in resolve_tool_name_collisions(tools, {})] == ["a", "b"] + + +def test_ruleset_reads_hitl_from_metadata(): + """Read-only MCP tools ``allow``; every other MCP tool ``ask``; natives skip.""" + tools = [ + _tool("readonly_search", {"mcp_transport": "http", "hitl": False}), + _tool("mutating_create", {"mcp_transport": "http", "hitl": True}), + _tool("native_helper", {}), # no mcp_transport => no rule + ] + rules = {r.permission: r.action for r in build_ruleset(tools).rules} + assert rules == {"readonly_search": "allow", "mutating_create": "ask"} + + +def test_indexing_deprecation_sets(): + """Indexing-only connectors are deprecated; migrated ones are LIVE; Obsidian stays.""" + deprecated = {t.value for t in DEPRECATED_INDEXING_CONNECTOR_TYPES} + assert deprecated == { + "GITHUB_CONNECTOR", + "BOOKSTACK_CONNECTOR", + "ELASTICSEARCH_CONNECTOR", + "CIRCLEBACK_CONNECTOR", + } + assert "OBSIDIAN_CONNECTOR" not in deprecated + + live = {t.value for t in LIVE_CONNECTOR_TYPES} + assert {"NOTION_CONNECTOR", "CONFLUENCE_CONNECTOR"} <= live diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_prompt_resources.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_prompt_resources.py index ccdfc0b98..65fbe0ec1 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_prompt_resources.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_prompt_resources.py @@ -45,7 +45,7 @@ def test_every_subagent_has_description_md(name: str): [ "core_behavior.md", "routing.md", - "tools/web_search/description.md", + "tools/scrape_webpage/description.md", ], ) def test_main_agent_prompt_fragments_resolve(filename: str): diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py index 1996afe1e..c3ad04250 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py @@ -19,32 +19,24 @@ from app.agents.chat.multi_agent_chat.subagents.registry import ( pytestmark = pytest.mark.unit -# The full specialist roster the main agent composes from: 8 builtins + 15 -# connector routes. Adding/removing a specialist is a deliberate product change -# and must be reflected here. +# The full specialist roster the main agent composes from after the MCP +# migration: builtins + the three file-connector routes (Drive/Dropbox/ +# OneDrive). Every MCP-backed connector (Slack/Jira/Linear/ClickUp/Airtable/ +# Notion/Confluence/Gmail/Calendar) now lives behind the single +# ``mcp_discovery`` route; Discord/Teams/Luma were deprecated. Adding/removing a +# specialist is a deliberate product change and must be reflected here. _EXPECTED_SUBAGENTS = frozenset( { - "airtable", - "calendar", - "clickup", - "confluence", "deliverables", - "discord", "dropbox", - "gmail", "google_drive", "google_maps", "google_search", - "jira", "knowledge_base", - "linear", - "luma", + "mcp_discovery", "memory", - "notion", "onedrive", "reddit", - "slack", - "teams", "web_crawler", "youtube", } diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_web_search_removed.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_web_search_removed.py new file mode 100644 index 000000000..2d65cce7e --- /dev/null +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_web_search_removed.py @@ -0,0 +1,45 @@ +"""Guardrail: the multi-engine ``web_search`` tool is fully retired. + +Public web search now runs exclusively through the ``google_search`` subagent, +and the four search connector types (Tavily/SearXNG/Linkup/Baidu) are soft- +deprecated. This pins those invariants so a future change can't quietly bring +the ``web_search`` tool back or un-deprecate a search connector. +""" + +from __future__ import annotations + +import pytest +from fastapi import HTTPException + +from app.agents.chat.multi_agent_chat.main_agent.tools.index import ( + MAIN_AGENT_SURFSENSE_TOOL_NAMES, +) +from app.agents.chat.multi_agent_chat.subagents.registry import ( + SUBAGENT_BUILDERS_BY_NAME, +) +from app.utils.validators import DEPRECATED_CONNECTOR_TYPES, raise_if_connector_deprecated + +pytestmark = pytest.mark.unit + +_DEPRECATED_SEARCH_TYPES = ( + "TAVILY_API", + "SEARXNG_API", + "LINKUP_API", + "BAIDU_SEARCH_API", +) + + +def test_web_search_tool_removed_from_main_agent(): + assert "web_search" not in MAIN_AGENT_SURFSENSE_TOOL_NAMES + + +def test_google_search_specialist_present(): + assert "google_search" in SUBAGENT_BUILDERS_BY_NAME + + +@pytest.mark.parametrize("connector_type", _DEPRECATED_SEARCH_TYPES) +def test_search_connectors_deprecated(connector_type): + assert connector_type in DEPRECATED_CONNECTOR_TYPES + with pytest.raises(HTTPException) as excinfo: + raise_if_connector_deprecated(connector_type) + assert excinfo.value.status_code == 410 diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_default_permissions_layering.py b/surfsense_backend/tests/unit/agents/new_chat/test_default_permissions_layering.py index b6341bfec..59e687b82 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_default_permissions_layering.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_default_permissions_layering.py @@ -9,7 +9,7 @@ earliest -> latest: 3. (future) user-defined rules from the Agent Permissions UI Without #1 every read-only built-in (``ls``, ``read_file``, ``grep``, -``glob``, ``web_search`` …) defaulted to ``ask`` because +``glob``, ``scrape_webpage`` …) defaulted to ``ask`` because ``permissions.evaluate`` returns ``ask`` when no rule matches. That caused two production-painful behaviors: @@ -58,7 +58,6 @@ class TestReadOnlyToolsAllowed: "read_file", "grep", "glob", - "web_search", "scrape_webpage", "get_connected_accounts", "write_todos", diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_otel_span.py b/surfsense_backend/tests/unit/agents/new_chat/test_otel_span.py index e2978d277..07d05d8a4 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_otel_span.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_otel_span.py @@ -65,8 +65,8 @@ class TestResolveToolName: def test_falls_back_to_tool_call_name(self) -> None: request = MagicMock() request.tool = None - request.tool_call = {"name": "web_search", "args": {}} - assert _resolve_tool_name(request) == "web_search" + request.tool_call = {"name": "create_automation", "args": {}} + assert _resolve_tool_name(request) == "create_automation" def test_unknown_when_nothing_resolves(self) -> None: request = MagicMock() @@ -278,8 +278,8 @@ class TestMiddlewareIntegration: request = MagicMock() request.tool = MagicMock() - request.tool.name = "web_search" + request.tool.name = "scrape_webpage" await mw.awrap_tool_call(request, handler) - assert errors == ["web_search"] + assert errors == ["scrape_webpage"] finally: ot.reload_for_tests() diff --git a/surfsense_backend/tests/unit/observability/test_otel.py b/surfsense_backend/tests/unit/observability/test_otel.py index 535f191fa..84c6225e2 100644 --- a/surfsense_backend/tests/unit/observability/test_otel.py +++ b/surfsense_backend/tests/unit/observability/test_otel.py @@ -166,8 +166,8 @@ class TestMetricHelpers: model="gpt-4o", provider="openai", ) - metrics.record_tool_call_duration(3.0, tool_name="web_search") - metrics.record_tool_call_error(tool_name="web_search") + metrics.record_tool_call_duration(3.0, tool_name="scrape_webpage") + metrics.record_tool_call_error(tool_name="scrape_webpage") metrics.record_kb_search_duration( 4.0, workspace_id=1, @@ -278,3 +278,30 @@ class TestEnabledIntegration: sp.set_attribute("tool.truncated", False) with otel.model_call_span(model_id="m", provider="p") as sp: sp.set_attribute("retry.count", 3) + + +class TestPackageVersionResilience: + """A version-tag lookup must never crash the request path (e.g. subagents). + + An editable/dynamic install can have distribution metadata with no + ``Version`` field, which raises ``KeyError`` deep inside importlib.metadata. + ``_package_version`` must swallow that and every other lookup failure. + """ + + def test_missing_version_key_falls_back( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + def _raise_key_error(_name: str) -> str: + raise KeyError("Version") + + monkeypatch.setattr(metrics.metadata, "version", _raise_key_error) + assert metrics._package_version() == "unknown" + + def test_package_not_found_falls_back( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + def _raise_not_found(_name: str) -> str: + raise metrics.metadata.PackageNotFoundError("surf-new-backend") + + monkeypatch.setattr(metrics.metadata, "version", _raise_not_found) + assert metrics._package_version() == "unknown" diff --git a/surfsense_backend/tests/unit/tasks/chat/streaming/test_task_span.py b/surfsense_backend/tests/unit/tasks/chat/streaming/test_task_span.py index df680018d..756d397cd 100644 --- a/surfsense_backend/tests/unit/tasks/chat/streaming/test_task_span.py +++ b/surfsense_backend/tests/unit/tasks/chat/streaming/test_task_span.py @@ -28,7 +28,7 @@ def test_clear_ignored_for_non_task_tool() -> None: open_task_span(state, run_id="run-1") sid = state.active_span_id clear_task_span_if_delegating_task_ended( - state, tool_name="web_search", run_id="run-1" + state, tool_name="scrape_webpage", run_id="run-1" ) assert state.active_span_id == sid assert state.active_task_run_id == "run-1" diff --git a/surfsense_backend/tests/unit/tasks/chat/test_content_builder.py b/surfsense_backend/tests/unit/tasks/chat/test_content_builder.py index 42e62d26b..662fb8adf 100644 --- a/surfsense_backend/tests/unit/tasks/chat/test_content_builder.py +++ b/surfsense_backend/tests/unit/tasks/chat/test_content_builder.py @@ -128,14 +128,14 @@ class TestToolHeavyTurn: b.on_tool_input_start( ui_id="call_run123", - tool_name="web_search", + tool_name="scrape_webpage", langchain_tool_call_id="lc_tool_abc", ) b.on_tool_input_delta("call_run123", '{"query":') b.on_tool_input_delta("call_run123", '"surfsense"}') b.on_tool_input_available( ui_id="call_run123", - tool_name="web_search", + tool_name="scrape_webpage", args={"query": "surfsense"}, langchain_tool_call_id="lc_tool_abc", ) @@ -150,7 +150,7 @@ class TestToolHeavyTurn: tool_part = snap[1] assert tool_part["type"] == "tool-call" assert tool_part["toolCallId"] == "call_run123" - assert tool_part["toolName"] == "web_search" + assert tool_part["toolName"] == "scrape_webpage" assert tool_part["args"] == {"query": "surfsense"} # ``argsText`` is the pretty-printed final JSON, not the raw # streaming buffer (FE ``stream-pipeline.ts:128``). diff --git a/surfsense_backend/tests/unit/utils/test_validators.py b/surfsense_backend/tests/unit/utils/test_validators.py index a359ee550..2f1e2aa96 100644 --- a/surfsense_backend/tests/unit/utils/test_validators.py +++ b/surfsense_backend/tests/unit/utils/test_validators.py @@ -4,6 +4,7 @@ import pytest from fastapi import HTTPException from app.utils.validators import ( + raise_if_connector_deprecated, validate_connector_config, validate_connectors, validate_document_ids, @@ -331,3 +332,31 @@ def test_validate_connector_config_invalid(): # Invalid URL format in SEARXNG_API with pytest.raises(ValueError): validate_connector_config("SEARXNG_API", {"SEARXNG_HOST": "not-a-url"}) + + +@pytest.mark.parametrize( + "connector_type", + [ + "DISCORD_CONNECTOR", + "TEAMS_CONNECTOR", + "LUMA_CONNECTOR", + "TAVILY_API", + "SEARXNG_API", + "LINKUP_API", + "BAIDU_SEARCH_API", + ], +) +def test_raise_if_connector_deprecated_blocks(connector_type): + """Deprecated connector types are refused with HTTP 410.""" + with pytest.raises(HTTPException) as excinfo: + raise_if_connector_deprecated(connector_type) + assert excinfo.value.status_code == 410 + + +@pytest.mark.parametrize( + "connector_type", + ["SLACK_CONNECTOR", "NOTION_CONNECTOR", "SERPER_API", "MCP_CONNECTOR"], +) +def test_raise_if_connector_deprecated_allows_active(connector_type): + """Active connector types pass through without raising.""" + raise_if_connector_deprecated(connector_type) diff --git a/surfsense_web/components/assistant-ui/citation-metadata-context.tsx b/surfsense_web/components/assistant-ui/citation-metadata-context.tsx index 0bf5dd946..9e8db901c 100644 --- a/surfsense_web/components/assistant-ui/citation-metadata-context.tsx +++ b/surfsense_web/components/assistant-ui/citation-metadata-context.tsx @@ -1,7 +1,6 @@ "use client"; -import { useAuiState } from "@assistant-ui/react"; -import { createContext, type FC, type ReactNode, useContext, useMemo } from "react"; +import { createContext, type FC, type ReactNode, useContext } from "react"; export interface CitationMeta { title: string; @@ -10,50 +9,16 @@ export interface CitationMeta { type CitationMetadataMap = ReadonlyMap; -const CitationMetadataContext = createContext(new Map()); +const EMPTY_CITATION_METADATA: CitationMetadataMap = new Map(); -interface ToolCallResult { - status?: string; - citations?: Record; -} - -interface MessageContent { - type: string; - toolName?: string; - result?: unknown; -} +const CitationMetadataContext = createContext(EMPTY_CITATION_METADATA); +// The previous web-search citation spine (WEB_RESULT hover cards) was removed +// with the multi-engine web_search tool. Agent citation logic is being reworked +// wholesale, so this provider currently yields no web citation metadata. export const CitationMetadataProvider: FC<{ children: ReactNode }> = ({ children }) => { - const content = useAuiState( - ({ message }) => (message as { content?: MessageContent[] })?.content - ); - - const metadataMap = useMemo(() => { - if (!content || !Array.isArray(content)) return new Map(); - - const merged = new Map(); - - for (const part of content) { - if (part.type !== "tool-call" || part.toolName !== "web_search" || !part.result) { - continue; - } - - const result = part.result as ToolCallResult; - const citations = result.citations; - if (!citations || typeof citations !== "object") continue; - - for (const [url, meta] of Object.entries(citations)) { - if (url.startsWith("http") && meta.title && !merged.has(url)) { - merged.set(url, { title: meta.title, snippet: meta.snippet }); - } - } - } - - return merged; - }, [content]); - return ( - + {children} ); diff --git a/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx index dadfe32ab..80930f632 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx @@ -23,6 +23,7 @@ interface ConnectorCardProps { accountCount?: number; connectorCount?: number; isIndexing?: boolean; + deprecated?: boolean; onConnect?: () => void; onManage?: () => void; } @@ -52,11 +53,15 @@ export const ConnectorCard: FC = ({ accountCount, connectorCount, isIndexing = false, + deprecated = false, onConnect, onManage, }) => { const isMCP = connectorType === EnumConnectorName.MCP_CONNECTOR; const isLive = !!connectorType && LIVE_CONNECTOR_TYPES.has(connectorType); + // Deprecated connectors can no longer be connected, but existing rows stay + // manageable (so users can disconnect them). + const isDeprecatedForConnect = deprecated && !isConnected; // Get connector status const { getConnectorStatus, isConnectorEnabled, getConnectorStatusMessage, shouldShowWarnings } = useConnectorStatus(); @@ -73,6 +78,10 @@ export const ConnectorCard: FC = ({ return null; } + if (isDeprecatedForConnect) { + return "Deprecated — no longer available to connect."; + } + return description; }; @@ -104,12 +113,19 @@ export const ConnectorCard: FC = ({
{title} - {showWarnings && status.status !== "active" && ( - + {deprecated ? ( + + Deprecated + + ) : ( + showWarnings && + status.status !== "active" && ( + + ) )}
{isIndexing ? ( @@ -151,18 +167,20 @@ export const ConnectorCard: FC = ({ !isConnected && "shadow-xs" )} onClick={isConnected ? onManage : onConnect} - disabled={isConnecting || !isEnabled} + disabled={isConnecting || !isEnabled || isDeprecatedForConnect} > - {!isEnabled - ? "Unavailable" - : isConnected - ? "Manage" - : id === "youtube-crawler" - ? "Add" - : connectorType - ? "Connect" - : "Add"} + {isDeprecatedForConnect + ? "Deprecated" + : !isEnabled + ? "Unavailable" + : isConnected + ? "Manage" + : id === "youtube-crawler" + ? "Add" + : connectorType + ? "Connect" + : "Add"} {isConnecting && } diff --git a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts index 315574aa9..18e6be136 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts @@ -1,5 +1,25 @@ import { EnumConnectorName } from "@/contracts/enums/connector"; +/** + * Connectors retired during the MCP migration (no viable official MCP server). + * The catalog card is shown disabled with a "Deprecated" badge so existing + * users understand why; the backend `/add` routes also refuse with HTTP 410. + * Reinstate by removing the type here and in the backend + * `DEPRECATED_CONNECTOR_TYPES` if demand returns. + */ +export const DEPRECATED_CONNECTOR_TYPES = new Set([ + EnumConnectorName.DISCORD_CONNECTOR, + EnumConnectorName.TEAMS_CONNECTOR, + EnumConnectorName.LUMA_CONNECTOR, + // Search APIs retired by the Google-only web-search consolidation. Public + // web search now runs through the google_search subagent; Tavily/Linkup can + // still be added via the generic Custom MCP connector (API-key headers). + EnumConnectorName.TAVILY_API, + EnumConnectorName.SEARXNG_API, + EnumConnectorName.LINKUP_API, + EnumConnectorName.BAIDU_SEARCH_API, +]); + /** * Connectors that operate in real time (no background indexing). * Used to adjust UI: hide sync controls, show "Connected" instead of doc counts. @@ -17,6 +37,9 @@ export const LIVE_CONNECTOR_TYPES = new Set([ EnumConnectorName.GOOGLE_GMAIL_CONNECTOR, EnumConnectorName.COMPOSIO_GMAIL_CONNECTOR, EnumConnectorName.LUMA_CONNECTOR, + // Migrated to hosted MCP: real-time agent tools, no background indexing. + EnumConnectorName.NOTION_CONNECTOR, + EnumConnectorName.CONFLUENCE_CONNECTOR, ]); // OAuth Connectors (Quick Connect) @@ -55,9 +78,9 @@ export const OAUTH_CONNECTORS = [ { id: "notion-connector", title: "Notion", - description: "Search your Notion pages", + description: "Search, read, and create pages", connectorType: EnumConnectorName.NOTION_CONNECTOR, - authEndpoint: "/api/v1/auth/notion/connector/add", + authEndpoint: "/api/v1/auth/mcp/notion/connector/add/", }, { id: "linear-connector", @@ -104,16 +127,16 @@ export const OAUTH_CONNECTORS = [ { id: "jira-connector", title: "Jira", - description: "Rework in progress.", + description: "Search, read, and manage issues", connectorType: EnumConnectorName.JIRA_CONNECTOR, authEndpoint: "/api/v1/auth/mcp/jira/connector/add/", }, { id: "confluence-connector", title: "Confluence", - description: "Rework in progress.", + description: "Search, read, and create pages", connectorType: EnumConnectorName.CONFLUENCE_CONNECTOR, - authEndpoint: "/api/v1/auth/confluence/connector/add/", + authEndpoint: "/api/v1/auth/mcp/confluence/connector/add/", }, { id: "clickup-connector", diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx index f7b27441b..39286b6cd 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx @@ -12,6 +12,7 @@ import { CONNECTOR_CATEGORY_LABELS, type ConnectorCategory, CRAWLERS, + DEPRECATED_CONNECTOR_TYPES, getConnectorCategory, OAUTH_CONNECTORS, OTHER_CONNECTORS, @@ -146,6 +147,7 @@ export const AllConnectorsTab: FC = ({ documentCount={documentCount} accountCount={accountCount} isIndexing={isIndexing} + deprecated={DEPRECATED_CONNECTOR_TYPES.has(connector.connectorType)} onConnect={() => onConnectOAuth(connector)} onManage={ isConnected && onViewAccountsList @@ -194,6 +196,7 @@ export const AllConnectorsTab: FC = ({ documentCount={documentCount} connectorCount={mcpConnectorCount} isIndexing={isIndexing} + deprecated={DEPRECATED_CONNECTOR_TYPES.has(connector.connectorType)} onConnect={handleConnect} onManage={actualConnector && onManage ? () => onManage(actualConnector) : undefined} /> diff --git a/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx index cc04af859..26a89b97d 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx @@ -11,7 +11,7 @@ import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import { authenticatedFetch } from "@/lib/auth-fetch"; -import { getReauthEndpoint } from "@/lib/connector-telemetry"; +import { getReauthEndpoint, needsMcpReconnect } from "@/lib/connector-telemetry"; import { buildBackendUrl } from "@/lib/env-config"; import { formatRelativeDate } from "@/lib/format-date"; import { cn } from "@/lib/utils"; @@ -192,6 +192,9 @@ export const ConnectorAccountsListView: FC = ({ const connectorReauthEndpoint = getReauthEndpoint(connector); const isAuthExpired = !!connectorReauthEndpoint && connector.config?.auth_expired === true; + // Migrated type still on legacy native config: reconnect via MCP to + // start producing agent tools again. + const needsReconnect = !isAuthExpired && needsMcpReconnect(connector); const isLive = LIVE_CONNECTOR_TYPES.has(connector.connector_type) || Boolean(connector.config?.server_config); @@ -245,6 +248,19 @@ export const ConnectorAccountsListView: FC = ({ /> Re-authenticate + ) : needsReconnect ? ( + ) : isLive && onDisconnect ? ( confirmDisconnectId === connector.id ? (
diff --git a/surfsense_web/components/assistant-ui/inline-citation.tsx b/surfsense_web/components/assistant-ui/inline-citation.tsx index 59a10739c..a66c22fa3 100644 --- a/surfsense_web/components/assistant-ui/inline-citation.tsx +++ b/surfsense_web/components/assistant-ui/inline-citation.tsx @@ -115,9 +115,9 @@ interface UrlCitationProps { } /** - * Inline citation for live web search results (URL-based chunk IDs). + * Inline citation for URL-based chunk IDs (e.g. scraped/linked web pages). * Renders a compact chip with favicon + domain and a hover popover showing the - * page title and snippet (extracted deterministically from web_search tool results). + * page title and snippet when citation metadata is available. */ export const UrlCitation: FC = ({ url }) => { const reactId = useId(); diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index a0c44c9a0..3716c59aa 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -14,7 +14,6 @@ import { ChevronDown, ChevronRight, Clipboard, - Globe, Plus, Settings2, SquareIcon, @@ -1057,12 +1056,7 @@ const ComposerAction: FC = ({ }); }, []); - const hasWebSearchTool = agentTools?.some((t) => t.name === "web_search") ?? false; - const isWebSearchEnabled = hasWebSearchTool && !disabledToolsSet.has("web_search"); - const filteredTools = useMemo( - () => agentTools?.filter((t) => t.name !== "web_search"), - [agentTools] - ); + const filteredTools = agentTools; const groupedTools = useMemo(() => { if (!filteredTools) return []; const toolsByName = new Map(filteredTools.map((t) => [t.name, t])); @@ -1143,22 +1137,6 @@ const ComposerAction: FC = ({ Upload Files - {hasWebSearchTool && ( - { - event.preventDefault(); - toggleTool("web_search"); - }} - > - - Web Search - - - )} setConnectorDialogOpen(true)}> Manage Connectors @@ -1379,26 +1357,6 @@ const ComposerAction: FC = ({ Take a screenshot - {hasWebSearchTool && ( - { - event.preventDefault(); - toggleTool("web_search"); - }} - className={cn( - "hover:bg-accent hover:text-accent-foreground", - isWebSearchEnabled && "text-primary" - )} - > - - Web Search - - - )} { diff --git a/surfsense_web/components/free-chat/free-chat-page.tsx b/surfsense_web/components/free-chat/free-chat-page.tsx index 966aaee60..0d47eeeb5 100644 --- a/surfsense_web/components/free-chat/free-chat-page.tsx +++ b/surfsense_web/components/free-chat/free-chat-page.tsx @@ -117,7 +117,6 @@ export function FreeChatPage() { const anonMode = useAnonymousMode(); const modelSlug = anonMode.isAnonymous ? anonMode.modelSlug : ""; const resetKey = anonMode.isAnonymous ? anonMode.resetKey : 0; - const webSearchEnabled = anonMode.isAnonymous ? anonMode.webSearchEnabled : true; const [messages, setMessages] = useState([]); const [isRunning, setIsRunning] = useState(false); @@ -173,7 +172,6 @@ export function FreeChatPage() { model_slug: modelSlug, messages: messageHistory, }; - if (!webSearchEnabled) reqBody.disabled_tools = ["web_search"]; if (turnstileToken) reqBody.turnstile_token = turnstileToken; const response = await fetch(buildBackendUrl("/api/v1/public/anon-chat/stream"), { @@ -323,7 +321,7 @@ export function FreeChatPage() { throw err; } }, - [modelSlug, tokenUsageStore, webSearchEnabled] + [modelSlug, tokenUsageStore] ); const onNew = useCallback( diff --git a/surfsense_web/components/free-chat/free-composer.tsx b/surfsense_web/components/free-chat/free-composer.tsx index d4523a4f9..228d81b27 100644 --- a/surfsense_web/components/free-chat/free-composer.tsx +++ b/surfsense_web/components/free-chat/free-composer.tsx @@ -1,13 +1,11 @@ "use client"; import { ComposerPrimitive, useAui, useAuiState } from "@assistant-ui/react"; -import { ArrowUpIcon, Globe, Paperclip, SquareIcon } from "lucide-react"; +import { ArrowUpIcon, Paperclip, SquareIcon } from "lucide-react"; import { type FC, useCallback, useRef, useState } from "react"; import { toast } from "sonner"; import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; import { Button } from "@/components/ui/button"; -import { Separator } from "@/components/ui/separator"; -import { Switch } from "@/components/ui/switch"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { useAnonymousMode } from "@/contexts/anonymous-mode"; import { useLoginGate } from "@/contexts/login-gate"; @@ -76,8 +74,6 @@ export const FreeComposer: FC = () => { const fileInputRef = useRef(null); const hasUploadedDoc = anonMode.isAnonymous && anonMode.uploadedDoc !== null; - const webSearchEnabled = anonMode.isAnonymous ? anonMode.webSearchEnabled : true; - const setWebSearchEnabled = anonMode.isAnonymous ? anonMode.setWebSearchEnabled : () => {}; const handleTextChange = useCallback( (e: React.ChangeEvent) => { @@ -208,26 +204,6 @@ export const FreeComposer: FC = () => { : "Upload a document (text files only)"} - - - - - - - - Toggle web search -
diff --git a/surfsense_web/content/docs/connectors/baidu-search.mdx b/surfsense_web/content/docs/connectors/baidu-search.mdx index 56d048d5b..b6e33f310 100644 --- a/surfsense_web/content/docs/connectors/baidu-search.mdx +++ b/surfsense_web/content/docs/connectors/baidu-search.mdx @@ -1,10 +1,14 @@ --- -title: Baidu Search -description: Search the Chinese web with Baidu AI Search in SurfSense +title: Baidu Search (Deprecated) +description: The Baidu Search connector is deprecated; public web search now runs through Google Search --- # Baidu Search Integration Setup Guide + +The Baidu Search connector is **deprecated** and can no longer be connected (the backend refuses new connections with HTTP 410). Public web search in SurfSense now runs through the built-in **Google Search** specialist — see [Web Search](/docs/how-to/web-search). Existing Baidu connector rows remain manageable, and you can add an alternative provider through a Custom MCP connector. The guide below is retained for reference only. + + This guide walks you through connecting Baidu AI Search to SurfSense for Chinese web search and AI-powered research. ## How it works diff --git a/surfsense_web/content/docs/how-to/web-search.mdx b/surfsense_web/content/docs/how-to/web-search.mdx index cb8007f52..3e17416b3 100644 --- a/surfsense_web/content/docs/how-to/web-search.mdx +++ b/surfsense_web/content/docs/how-to/web-search.mdx @@ -1,189 +1,53 @@ --- title: Web Search -description: How SurfSense web search works and how to configure it for production with residential proxies +description: How SurfSense web search works and how to add Tavily or Linkup via a Custom MCP connector --- # Web Search -SurfSense uses [SearXNG](https://docs.searxng.org/) as a bundled meta-search engine to provide web search across all search spaces. SearXNG aggregates results from multiple search engines (Google, DuckDuckGo, Brave, Bing, and more) without requiring any API keys. +SurfSense performs public web search through the built-in **Google Search** specialist. When the assistant needs real-time or public information (news, prices, weather, exchange rates, current events, or "who ranks for X"), it delegates the query to the Google Search subagent, which returns ranked results the assistant can cite and hand off for crawling. -You can also add live search connectors such as Baidu Search, Tavily, and Linkup to a search space. When those connectors are active, SurfSense queries them in parallel with SearXNG and merges the results before passing sources to the assistant. +There is nothing to configure — Google web search is available in every workspace out of the box. + + +SurfSense previously bundled a multi-engine web-search tool backed by SearXNG, Tavily, Linkup, and Baidu. Those have been consolidated: public web search now runs exclusively through the Google Search specialist. The **Tavily**, **Linkup**, **Baidu Search**, and **SearXNG** connectors are deprecated and can no longer be connected. + ## How It Works -When a user triggers a web search in SurfSense: +When the assistant decides a request needs the open web: -1. The backend sends a query to the bundled SearXNG instance via its JSON API -2. SearXNG fans out the query to all enabled search engines simultaneously -3. Results are aggregated, deduplicated, and ranked by engine weight -4. If the current search space has live search connectors, the backend queries them in parallel -5. The backend deduplicates the merged results and presents them to the user +1. The main agent delegates the query to the `google_search` specialist. +2. The specialist runs a Google search and returns ranked result items (title, URL, snippet). +3. The assistant summarizes the findings and can follow up by crawling a specific result page. -SearXNG runs as a Docker container alongside the backend. It is never exposed to the internet. Only the backend communicates with it over the internal Docker network. +Google web search is workspace-scoped and is not available in the free / anonymous (no-login) chat, which answers purely from the model's own knowledge. -## Live Search Connectors +## Still want Tavily or Linkup? -Live search connectors are optional API-backed search providers configured per search space. They are useful when you need a specialized index, authenticated search API, or stronger regional coverage. +If you relied on Tavily or Linkup, you can add them back yourself through the generic **Custom MCP connector** (the **MCPs** card in the Connectors dashboard). Both providers expose a hosted MCP server that authenticates with an API key sent as a request header: -| Connector | Best For | Setup | -|-----------|----------|-------| -| Baidu Search | Chinese web search and China-focused current information | [Baidu Search connector](/docs/connectors/baidu-search) | -| Tavily | General web research through Tavily's search API | Add the Tavily connector from the Connectors dashboard | -| Linkup | General web search through Linkup's search API | Add the Linkup connector from the Connectors dashboard | +| Provider | MCP server URL | Auth header | +|----------|----------------|-------------| +| Tavily | `https://mcp.tavily.com/mcp/` | `Authorization: Bearer ` | +| Linkup | `https://mcp.linkup.so/mcp` | `Authorization: Bearer ` | - -Live search connectors only run for the search space where they are configured. They do not replace SearXNG globally. +Steps: + +1. Open the Connectors dashboard and choose **MCPs** (Custom MCP connector). +2. Enter the provider's MCP server URL from the table above. +3. Add an `Authorization` header with the value `Bearer `. +4. Save. The provider's search tools become available to the assistant through the connected-apps specialist. + + +Tavily/Linkup keys live only in your Custom MCP connector configuration. SurfSense no longer ships first-party Tavily, Linkup, Baidu, or SearXNG integrations. -## Docker Setup +## Deprecated connectors -SearXNG is included in both `docker-compose.yml` and `docker-compose.dev.yml` and works out of the box with no configuration needed. +The following connectors are deprecated. Existing rows remain readable/manageable, but new connections are refused (HTTP 410), and their catalog cards show a **Deprecated** badge: -The backend connects to SearXNG automatically via the `SEARXNG_DEFAULT_HOST` environment variable (defaults to `http://searxng:8080`). - -### Disabling SearXNG - -If you don't need web search, you can skip the SearXNG container entirely: - -```bash -docker compose up --scale searxng=0 -``` - -### Using Your Own SearXNG Instance - -To point SurfSense at an external SearXNG instance instead of the bundled one, set in your `docker/.env`: - -```bash -SEARXNG_DEFAULT_HOST=http://your-searxng:8080 -``` - -## Configuration - -SearXNG is configured via `docker/searxng/settings.yml`. The key sections are: - -### Engines - -SearXNG queries multiple search engines in parallel. Each engine has a **weight** that influences how its results rank in the merged output: - -| Engine | Weight | Notes | -|--------|--------|-------| -| Google | 1.2 | Highest priority, best general results | -| DuckDuckGo | 1.1 | Strong privacy-focused alternative | -| Brave | 1.0 | Independent search index | -| Bing | 0.9 | Different index from Google | -| Wikipedia | 0.8 | Encyclopedic results | -| StackOverflow | 0.7 | Technical/programming results | -| Yahoo | 0.7 | Powered by Bing's index | -| Wikidata | 0.6 | Structured data results | -| Currency | default | Currency conversion | -| DDG Definitions | default | Instant answers from DuckDuckGo | - -All engines are free. SearXNG scrapes public search pages, no API keys required. - -### Engine Suspension - -When a search engine returns an error (CAPTCHA, rate limit, access denied), SearXNG suspends it for a configurable duration. After the suspension expires, the engine is automatically retried. - -The default suspension times are tuned for use with rotating residential proxies (shorter bans since each retry goes through a different IP): - -| Error Type | Suspension | Default (without override) | -|------------|-----------|---------------------------| -| Access Denied (403) | 1 hour | 24 hours | -| CAPTCHA | 1 hour | 24 hours | -| Too Many Requests (429) | 10 minutes | 1 hour | -| Cloudflare CAPTCHA | 2 hours | 15 days | -| Cloudflare Access Denied | 1 hour | 24 hours | -| reCAPTCHA | 2 hours | 7 days | - -### Timeouts - -| Setting | Value | Description | -|---------|-------|-------------| -| `request_timeout` | 12s | Default timeout per engine request | -| `max_request_timeout` | 20s | Maximum allowed timeout (must be ≥ `request_timeout`) | -| `extra_proxy_timeout` | 10s | Extra seconds added when using a proxy | -| `retries` | 1 | Retries on HTTP error (uses a different proxy IP per retry) | - -## Production: Residential Proxies - -In production, search engines may rate-limit or block your server's IP. To avoid this, configure a residential proxy so SearXNG's outgoing requests appear to come from rotating residential IPs. - -### Step 1: Build the Proxy URL - -Use any residential proxy that exposes a standard `http://user:pass@host:port` endpoint. For example, [DataImpulse](https://dataimpulse.com/) encodes country routing as a username suffix (`__cr.`): - -``` -http://__cr.us:@gw.dataimpulse.com:823 -``` - -The general proxy URL format is: - -``` -http://:@:/ -``` - -### Step 2: Add to SearXNG Settings - -In `docker/searxng/settings.yml`, add the proxy URL under `outgoing.proxies`: - -```yaml -outgoing: - proxies: - all://: - - http://username:base64password@proxy-host:port/ -``` - -The `all://:` key routes both HTTP and HTTPS requests through the proxy. If you have multiple proxy endpoints, list them and SearXNG will round-robin between them: - -```yaml - proxies: - all://: - - http://user:pass@proxy1:port/ - - http://user:pass@proxy2:port/ -``` - -### Step 3: Restart SearXNG - -```bash -docker compose restart searxng -``` - -### Verify - -Check that SearXNG is healthy: - -```bash -curl http://localhost:8888/healthz -``` - -## Troubleshooting - -### SearXNG Fails to Start - -**`ValueError: Invalid settings.yml`** - Check the error line above the traceback. Common causes: -- `extra_proxy_timeout` must be an integer (use `10`, not `10.0`) -- `KeyError: 'engine_name'` means an engine was removed but other engines reference its network. Remove all variants (e.g., removing `qwant` also requires removing `qwant news`, `qwant images`, `qwant videos`) - -### Engines Getting Suspended - -If an engine is suspended (visible in SearXNG logs as `suspended_time=N`), it will automatically recover after the suspension period. With residential proxies, the next request after recovery goes through a different IP and typically succeeds. - -### No Web Search Results - -1. Check SearXNG health: `curl http://localhost:8888/healthz` -2. Check SearXNG logs: `docker compose logs searxng` -3. Verify the backend can reach SearXNG: the `SEARXNG_DEFAULT_HOST` env var should point to `http://searxng:8080` (Docker) or `http://localhost:8888` (local dev) - -### Proxy Not Working - -- Verify the base64 password is correctly encoded -- Check that `extra_proxy_timeout` is set (proxies add latency) -- Ensure `max_request_timeout` is high enough to accommodate `request_timeout + extra_proxy_timeout` - -## Environment Variables Reference - -| Variable | Location | Description | Default | -|----------|----------|-------------|---------| -| `SEARXNG_DEFAULT_HOST` | `docker/.env` | URL of the SearXNG instance | `http://searxng:8080` | -| `SEARXNG_SECRET` | `docker/.env` | Secret key for SearXNG | `surfsense-searxng-secret` | -| `SEARXNG_PORT` | `docker/.env` | Port to expose SearXNG UI on the host | `8888` | +- **Tavily** — add via Custom MCP connector (see above). +- **Linkup** — add via Custom MCP connector (see above). +- **Baidu Search** — no longer bundled; use the built-in Google Search or a Custom MCP connector for a provider of your choice. +- **SearXNG** — the bundled SearXNG container and its `SEARXNG_*` environment variables have been removed from all Docker Compose files. diff --git a/surfsense_web/contexts/anonymous-mode.tsx b/surfsense_web/contexts/anonymous-mode.tsx index c24a5cb1b..eaf0996c3 100644 --- a/surfsense_web/contexts/anonymous-mode.tsx +++ b/surfsense_web/contexts/anonymous-mode.tsx @@ -9,8 +9,6 @@ export interface AnonymousModeContextValue { setModelSlug: (slug: string) => void; uploadedDoc: { filename: string; sizeBytes: number } | null; setUploadedDoc: (doc: { filename: string; sizeBytes: number } | null) => void; - webSearchEnabled: boolean; - setWebSearchEnabled: (enabled: boolean) => void; resetKey: number; resetChat: () => void; } @@ -36,7 +34,6 @@ export function AnonymousModeProvider({ const [uploadedDoc, setUploadedDoc] = useState<{ filename: string; sizeBytes: number } | null>( null ); - const [webSearchEnabled, setWebSearchEnabled] = useState(true); const [resetKey, setResetKey] = useState(0); const resetChat = () => setResetKey((k) => k + 1); @@ -59,12 +56,10 @@ export function AnonymousModeProvider({ setModelSlug, uploadedDoc, setUploadedDoc, - webSearchEnabled, - setWebSearchEnabled, resetKey, resetChat, }), - [modelSlug, uploadedDoc, webSearchEnabled, resetKey] + [modelSlug, uploadedDoc, resetKey] ); return {children}; diff --git a/surfsense_web/contracts/enums/toolIcons.tsx b/surfsense_web/contracts/enums/toolIcons.tsx index 8b2f08cd1..2638af2ce 100644 --- a/surfsense_web/contracts/enums/toolIcons.tsx +++ b/surfsense_web/contracts/enums/toolIcons.tsx @@ -11,7 +11,6 @@ import { FolderPlus, FolderTree, FolderX, - Globe, ImageIcon, ListTodo, type LucideIcon, @@ -45,7 +44,6 @@ const TOOL_ICONS: Record = { display_image: ImageIcon, // Web / search scrape_webpage: ScanLine, - web_search: Globe, // Automations create_automation: AlarmClock, // Memory @@ -149,7 +147,6 @@ const TOOL_DISPLAY_NAMES: Record = { display_image: "Show image", // Web / search scrape_webpage: "Read webpage", - web_search: "Search the web", // Automations create_automation: "Create automation", // Memory diff --git a/surfsense_web/features/chat-messages/timeline/subagent-rename.ts b/surfsense_web/features/chat-messages/timeline/subagent-rename.ts index 87accd8d7..7c6543f6b 100644 --- a/surfsense_web/features/chat-messages/timeline/subagent-rename.ts +++ b/surfsense_web/features/chat-messages/timeline/subagent-rename.ts @@ -4,6 +4,15 @@ function asNonEmptyString(v: unknown): string | undefined { return typeof v === "string" && v.trim().length > 0 ? v.trim() : undefined; } +/** + * Explicit display labels for subagents whose title-cased id reads poorly. + * Legacy per-connector names (gmail, slack, ...) intentionally fall through + * to {@link titleCaseSubagent} so historical conversations keep their labels. + */ +const SUBAGENT_DISPLAY_LABELS: Record = { + mcp_discovery: "Connected Apps", +}; + /** * Title-case a subagent identifier: * "notion" → "Notion" @@ -27,7 +36,8 @@ export function titleCaseSubagent(raw: string): string { export function resolveSubagentTitle(item: ToolCallItem): string | undefined { if (item.toolName !== "task") return undefined; const subagent = asNonEmptyString(item.args?.subagent_type); - return subagent ? titleCaseSubagent(subagent) : undefined; + if (!subagent) return undefined; + return SUBAGENT_DISPLAY_LABELS[subagent] ?? titleCaseSubagent(subagent); } /** diff --git a/surfsense_web/features/chat-messages/timeline/tool-registry/registry.ts b/surfsense_web/features/chat-messages/timeline/tool-registry/registry.ts index c4cfe7cd3..f00e94174 100644 --- a/surfsense_web/features/chat-messages/timeline/tool-registry/registry.ts +++ b/surfsense_web/features/chat-messages/timeline/tool-registry/registry.ts @@ -165,10 +165,9 @@ const DeleteConfluencePageToolUI = dynamic( * - **Structural primitives** (``task``): the row IS the parent of a * delegation span; its job is to label the group. Children render * as their own indented entries. - * - **Suppressed connectors** (``web_search``, ``link_preview``, - * ``multi_link_preview``, ``scrape_webpage``): citations they - * produce render inline in markdown; a separate card would be - * redundant noise. + * - **Suppressed connectors** (``link_preview``, ``multi_link_preview``, + * ``scrape_webpage``): citations they produce render inline in + * markdown; a separate card would be redundant noise. */ const NullTimelineBody: TimelineToolComponent = () => null; @@ -218,7 +217,6 @@ const TOOLS_BY_NAME = { create_confluence_page: CreateConfluencePageToolUI, update_confluence_page: UpdateConfluencePageToolUI, delete_confluence_page: DeleteConfluencePageToolUI, - web_search: NullTimelineBody, link_preview: NullTimelineBody, multi_link_preview: NullTimelineBody, scrape_webpage: NullTimelineBody, diff --git a/surfsense_web/lib/connector-telemetry.ts b/surfsense_web/lib/connector-telemetry.ts index eeccea1e8..ab4e63baf 100644 --- a/surfsense_web/lib/connector-telemetry.ts +++ b/surfsense_web/lib/connector-telemetry.ts @@ -112,17 +112,49 @@ const LEGACY_REAUTH_ENDPOINTS: Partial> = { [EnumConnectorName.DISCORD_CONNECTOR]: "/api/v1/auth/discord/connector/reauth", }; +/** + * Connector types migrated to hosted MCP, mapped to their MCP service key. + * A legacy native row of one of these types (no ``config.server_config``) + * re-authenticates through the MCP flow so the reconnect writes an MCP + * ``server_config`` and the row starts producing agent tools again. + * + * Gmail/Calendar/Drive/Dropbox/OneDrive are intentionally absent: they stay + * on their native OAuth (Google Workspace MCP is not GA yet; file connectors + * remain native for knowledge-base enrichment). + */ +const MIGRATED_TYPE_TO_MCP_SERVICE: Partial> = { + [EnumConnectorName.LINEAR_CONNECTOR]: "linear", + [EnumConnectorName.JIRA_CONNECTOR]: "jira", + [EnumConnectorName.NOTION_CONNECTOR]: "notion", + [EnumConnectorName.CONFLUENCE_CONNECTOR]: "confluence", +}; + +/** + * True when a row is a migrated connector type still on its legacy native + * config (no MCP ``server_config``). Such rows appear "connected" but produce + * no agent tools until reconnected via MCP — the UI surfaces a nudge. + */ +export function needsMcpReconnect(connector: SearchSourceConnector): boolean { + if (!(connector.connector_type in MIGRATED_TYPE_TO_MCP_SERVICE)) return false; + return !connector.config?.server_config; +} + /** * Resolve the reauth endpoint for a connector. * * MCP OAuth connectors (those with ``config.mcp_service``) dynamically build - * the URL from the service key. Legacy OAuth connectors fall back to the - * static ``LEGACY_REAUTH_ENDPOINTS`` map. + * the URL from the service key. Migrated native rows are routed to the MCP + * reauth flow so reconnecting converts them in place. Everything else falls + * back to the static ``LEGACY_REAUTH_ENDPOINTS`` map. */ export function getReauthEndpoint(connector: SearchSourceConnector): string | undefined { const mcpService = connector.config?.mcp_service as string | undefined; if (mcpService) { return `/api/v1/auth/mcp/${mcpService}/connector/reauth`; } + const migratedService = MIGRATED_TYPE_TO_MCP_SERVICE[connector.connector_type]; + if (migratedService) { + return `/api/v1/auth/mcp/${migratedService}/connector/reauth`; + } return LEGACY_REAUTH_ENDPOINTS[connector.connector_type]; } diff --git a/surfsense_web/lib/posthog/events.ts b/surfsense_web/lib/posthog/events.ts index 41ac7e7b2..948c436de 100644 --- a/surfsense_web/lib/posthog/events.ts +++ b/surfsense_web/lib/posthog/events.ts @@ -200,14 +200,12 @@ export function trackAnonymousChatMessageSent(options: { modelSlug: string; messageLength?: number; hasUploadedDoc?: boolean; - webSearchEnabled?: boolean; surface?: "free_chat_page" | "free_model_page"; }) { safeCapture("anonymous_chat_message_sent", { model_slug: options.modelSlug, message_length: options.messageLength, has_uploaded_doc: options.hasUploadedDoc ?? false, - web_search_enabled: options.webSearchEnabled, surface: options.surface, }); } diff --git a/surfsense_web/messages/en.json b/surfsense_web/messages/en.json index b8c50c701..df29de9de 100644 --- a/surfsense_web/messages/en.json +++ b/surfsense_web/messages/en.json @@ -336,7 +336,6 @@ "add_connector": { "title": "Connect Your Tools", "subtitle": "Integrate with your favorite services to enhance your research capabilities.", - "web_search": "Web Search", "messaging": "Messaging", "project_management": "Project Management", "documentation": "Documentation", @@ -759,27 +758,7 @@ "general_reset": "Reset Changes", "general_save": "Save Changes", "general_saving": "Saving", - "general_unsaved_changes": "You have unsaved changes. Click \"Save Changes\" to apply them.", - "nav_web_search": "Web Search", - "nav_web_search_desc": "Built-in web search settings", - "web_search_title": "Web Search", - "web_search_description": "Web search is powered by a built-in SearXNG instance. All queries are proxied through your server — no data is sent to third parties.", - "web_search_enabled_label": "Enable Web Search", - "web_search_enabled_description": "When enabled, the AI agent can search the web for real-time information like news, prices, and current events.", - "web_search_status_healthy": "Web search service is healthy", - "web_search_status_unhealthy": "Web search service is unavailable", - "web_search_status_not_configured": "Web search service is not configured", - "web_search_engines_label": "Search Engines", - "web_search_engines_placeholder": "google,brave,duckduckgo", - "web_search_engines_description": "Comma-separated list of SearXNG engines to use. Leave empty for defaults.", - "web_search_language_label": "Preferred Language", - "web_search_language_placeholder": "en", - "web_search_language_description": "IETF language tag (e.g. en, en-US). Leave empty for auto-detect.", - "web_search_safesearch_label": "SafeSearch Level", - "web_search_safesearch_description": "0 = off, 1 = moderate, 2 = strict", - "web_search_save": "Save Web Search Settings", - "web_search_saving": "Saving...", - "web_search_saved": "Web search settings saved" + "general_unsaved_changes": "You have unsaved changes. Click \"Save Changes\" to apply them." }, "homepage": { "hero_title_part1": "The AI Workspace", diff --git a/surfsense_web/messages/es.json b/surfsense_web/messages/es.json index 06b309df1..a231584f7 100644 --- a/surfsense_web/messages/es.json +++ b/surfsense_web/messages/es.json @@ -336,7 +336,6 @@ "add_connector": { "title": "Conecta tus herramientas", "subtitle": "Integra con tus servicios favoritos para mejorar tus capacidades de investigación.", - "web_search": "Búsqueda web", "messaging": "Mensajería", "project_management": "Gestión de proyectos", "documentation": "Documentación", @@ -759,27 +758,7 @@ "general_reset": "Restablecer cambios", "general_save": "Guardar cambios", "general_saving": "Guardando", - "general_unsaved_changes": "Tienes cambios sin guardar. Haz clic en \"Guardar cambios\" para aplicarlos.", - "nav_web_search": "Búsqueda web", - "nav_web_search_desc": "Configuración de búsqueda web integrada", - "web_search_title": "Búsqueda web", - "web_search_description": "La búsqueda web funciona con una instancia SearXNG integrada. Todas las consultas se procesan a través de tu servidor; no se envían datos a terceros.", - "web_search_enabled_label": "Activar búsqueda web", - "web_search_enabled_description": "Cuando está activada, el agente de IA puede buscar en la web información en tiempo real como noticias, precios y eventos actuales.", - "web_search_status_healthy": "El servicio de búsqueda web está funcionando", - "web_search_status_unhealthy": "El servicio de búsqueda web no está disponible", - "web_search_status_not_configured": "El servicio de búsqueda web no está configurado", - "web_search_engines_label": "Motores de búsqueda", - "web_search_engines_placeholder": "google,brave,duckduckgo", - "web_search_engines_description": "Lista separada por comas de motores SearXNG a usar. Déjalo vacío para usar los valores predeterminados.", - "web_search_language_label": "Idioma preferido", - "web_search_language_placeholder": "es", - "web_search_language_description": "Etiqueta de idioma IETF (por ejemplo, es, es-ES). Déjalo vacío para detección automática.", - "web_search_safesearch_label": "Nivel de SafeSearch", - "web_search_safesearch_description": "0 = desactivado, 1 = moderado, 2 = estricto", - "web_search_save": "Guardar configuración de búsqueda web", - "web_search_saving": "Guardando...", - "web_search_saved": "Configuración de búsqueda web guardada" + "general_unsaved_changes": "Tienes cambios sin guardar. Haz clic en \"Guardar cambios\" para aplicarlos." }, "homepage": { "hero_title_part1": "El espacio de trabajo con IA", diff --git a/surfsense_web/messages/hi.json b/surfsense_web/messages/hi.json index 73a025803..3586669dd 100644 --- a/surfsense_web/messages/hi.json +++ b/surfsense_web/messages/hi.json @@ -336,7 +336,6 @@ "add_connector": { "title": "अपने टूल कनेक्ट करें", "subtitle": "अपनी शोध क्षमताओं को बढ़ाने के लिए अपनी पसंदीदा सेवाओं के साथ एकीकृत करें।", - "web_search": "वेब खोज", "messaging": "मैसेजिंग", "project_management": "प्रोजेक्ट प्रबंधन", "documentation": "दस्तावेज़ीकरण", @@ -759,27 +758,7 @@ "general_reset": "परिवर्तन रीसेट करें", "general_save": "परिवर्तन सहेजें", "general_saving": "सहेजा जा रहा है", - "general_unsaved_changes": "आपके पास सहेजे नहीं गए परिवर्तन हैं। उन्हें लागू करने के लिए \"परिवर्तन सहेजें\" पर क्लिक करें।", - "nav_web_search": "वेब खोज", - "nav_web_search_desc": "बिल्ट-इन वेब खोज सेटिंग्स", - "web_search_title": "वेब खोज", - "web_search_description": "वेब खोज एक बिल्ट-इन SearXNG इंस्टेंस द्वारा संचालित है। सभी क्वेरी आपके सर्वर के माध्यम से प्रॉक्सी की जाती हैं; कोई डेटा तृतीय पक्षों को नहीं भेजा जाता।", - "web_search_enabled_label": "वेब खोज सक्षम करें", - "web_search_enabled_description": "सक्षम होने पर, AI एजेंट समाचार, कीमतों और वर्तमान घटनाओं जैसी वास्तविक समय की जानकारी के लिए वेब खोज सकता है।", - "web_search_status_healthy": "वेब खोज सेवा स्वस्थ है", - "web_search_status_unhealthy": "वेब खोज सेवा उपलब्ध नहीं है", - "web_search_status_not_configured": "वेब खोज सेवा कॉन्फ़िगर नहीं है", - "web_search_engines_label": "खोज इंजन", - "web_search_engines_placeholder": "google,brave,duckduckgo", - "web_search_engines_description": "उपयोग करने के लिए SearXNG इंजनों की कॉमा-सेपरेटेड सूची। डिफ़ॉल्ट के लिए खाली छोड़ें।", - "web_search_language_label": "पसंदीदा भाषा", - "web_search_language_placeholder": "hi", - "web_search_language_description": "IETF भाषा टैग (जैसे hi, hi-IN)। ऑटो-डिटेक्ट के लिए खाली छोड़ें।", - "web_search_safesearch_label": "SafeSearch स्तर", - "web_search_safesearch_description": "0 = बंद, 1 = मध्यम, 2 = सख्त", - "web_search_save": "वेब खोज सेटिंग्स सहेजें", - "web_search_saving": "सहेजा जा रहा है...", - "web_search_saved": "वेब खोज सेटिंग्स सहेजी गईं" + "general_unsaved_changes": "आपके पास सहेजे नहीं गए परिवर्तन हैं। उन्हें लागू करने के लिए \"परिवर्तन सहेजें\" पर क्लिक करें।" }, "homepage": { "hero_title_part1": "AI कार्यक्षेत्र", diff --git a/surfsense_web/messages/pt.json b/surfsense_web/messages/pt.json index 00b8242f7..83d2c0ad3 100644 --- a/surfsense_web/messages/pt.json +++ b/surfsense_web/messages/pt.json @@ -336,7 +336,6 @@ "add_connector": { "title": "Conecte suas ferramentas", "subtitle": "Integre com seus serviços favoritos para melhorar suas capacidades de pesquisa.", - "web_search": "Pesquisa web", "messaging": "Mensagens", "project_management": "Gerenciamento de projetos", "documentation": "Documentação", @@ -759,27 +758,7 @@ "general_reset": "Redefinir alterações", "general_save": "Salvar alterações", "general_saving": "Salvando", - "general_unsaved_changes": "Você tem alterações não salvas. Clique em \"Salvar alterações\" para aplicá-las.", - "nav_web_search": "Pesquisa na web", - "nav_web_search_desc": "Configurações integradas de pesquisa na web", - "web_search_title": "Pesquisa na web", - "web_search_description": "A pesquisa na web é alimentada por uma instância SearXNG integrada. Todas as consultas passam pelo seu servidor; nenhum dado é enviado a terceiros.", - "web_search_enabled_label": "Ativar pesquisa na web", - "web_search_enabled_description": "Quando ativado, o agente de IA pode pesquisar na web informações em tempo real, como notícias, preços e eventos atuais.", - "web_search_status_healthy": "O serviço de pesquisa na web está saudável", - "web_search_status_unhealthy": "O serviço de pesquisa na web está indisponível", - "web_search_status_not_configured": "O serviço de pesquisa na web não está configurado", - "web_search_engines_label": "Mecanismos de pesquisa", - "web_search_engines_placeholder": "google,brave,duckduckgo", - "web_search_engines_description": "Lista separada por vírgulas de mecanismos SearXNG a usar. Deixe vazio para os padrões.", - "web_search_language_label": "Idioma preferido", - "web_search_language_placeholder": "pt", - "web_search_language_description": "Tag de idioma IETF (por exemplo, pt, pt-BR). Deixe vazio para detecção automática.", - "web_search_safesearch_label": "Nível de SafeSearch", - "web_search_safesearch_description": "0 = desativado, 1 = moderado, 2 = rigoroso", - "web_search_save": "Salvar configurações de pesquisa na web", - "web_search_saving": "Salvando...", - "web_search_saved": "Configurações de pesquisa na web salvas" + "general_unsaved_changes": "Você tem alterações não salvas. Clique em \"Salvar alterações\" para aplicá-las." }, "homepage": { "hero_title_part1": "O espaço de trabalho com IA", diff --git a/surfsense_web/messages/zh.json b/surfsense_web/messages/zh.json index fd4147e66..cb319c7aa 100644 --- a/surfsense_web/messages/zh.json +++ b/surfsense_web/messages/zh.json @@ -335,7 +335,6 @@ "add_connector": { "title": "连接您的工具", "subtitle": "集成您喜欢的服务以增强研究能力。", - "web_search": "网络搜索", "messaging": "即时通讯", "project_management": "项目管理", "documentation": "文档协作", @@ -758,27 +757,7 @@ "general_reset": "重置更改", "general_save": "保存更改", "general_saving": "保存中...", - "general_unsaved_changes": "您有未保存的更改。点击\"保存更改\"以应用它们。", - "nav_web_search": "网页搜索", - "nav_web_search_desc": "内置网页搜索设置", - "web_search_title": "网页搜索", - "web_search_description": "网页搜索由内置 SearXNG 实例提供支持。所有查询都通过您的服务器代理；不会向第三方发送数据。", - "web_search_enabled_label": "启用网页搜索", - "web_search_enabled_description": "启用后，AI 代理可以搜索网页以获取新闻、价格和当前事件等实时信息。", - "web_search_status_healthy": "网页搜索服务运行正常", - "web_search_status_unhealthy": "网页搜索服务不可用", - "web_search_status_not_configured": "网页搜索服务未配置", - "web_search_engines_label": "搜索引擎", - "web_search_engines_placeholder": "google,brave,duckduckgo", - "web_search_engines_description": "要使用的 SearXNG 引擎的逗号分隔列表。留空则使用默认值。", - "web_search_language_label": "首选语言", - "web_search_language_placeholder": "zh", - "web_search_language_description": "IETF 语言标签（例如 zh、zh-CN）。留空则自动检测。", - "web_search_safesearch_label": "SafeSearch 级别", - "web_search_safesearch_description": "0 = 关闭，1 = 中等，2 = 严格", - "web_search_save": "保存网页搜索设置", - "web_search_saving": "保存中...", - "web_search_saved": "网页搜索设置已保存" + "general_unsaved_changes": "您有未保存的更改。点击\"保存更改\"以应用它们。" }, "homepage": { "hero_title_part1": "AI 工作空间", diff --git a/surfsense_web/tests/connectors/confluence/journey.spec.ts b/surfsense_web/tests/connectors/confluence/journey.spec.ts index bcd33095d..9ee900872 100644 --- a/surfsense_web/tests/connectors/confluence/journey.spec.ts +++ b/surfsense_web/tests/connectors/confluence/journey.spec.ts @@ -1,21 +1,20 @@ import { expect, confluenceWithChatTest as test } from "../../fixtures"; import { streamChatToCompletion } from "../../helpers/api/chat"; -import { listConnectors, triggerIndex } from "../../helpers/api/connectors"; -import { getEditorContent, listDocuments } from "../../helpers/api/documents"; +import { listConnectors, triggerIndexExpectDisabled } from "../../helpers/api/connectors"; +import { listDocuments } from "../../helpers/api/documents"; import { CANARY_TOKENS, FAKE_CONFLUENCE_PAGES } from "../../helpers/canary"; import { openConnectorPopup } from "../../helpers/ui/connector-popup"; -import { waitForDocumentByTitle, waitForIndexingComplete } from "../../helpers/waits/indexing"; /** - * Proves Confluence OAuth -> indexed Confluence pages -> stored source_markdown -> chat. + * Proves Confluence MCP OAuth -> live MCP tool discovery/call -> chat. * - * The external Atlassian provider is faked at the OAuth token/resource and - * Confluence page-fetch boundaries; SurfSense's own add/callback routes, - * encrypted config storage, indexing endpoint, indexing pipeline, and chat - * retrieval remain real. + * Confluence migrated to the hosted Atlassian Rovo MCP server (shared with + * Jira), so it is live-tool only: the public indexing route returns + * indexing_started=false and chat should call Confluence MCP tools + * (`searchConfluenceUsingCql`) instead of retrieving from the KB. */ test.describe("Confluence connector journey", () => { - test("user connects Confluence through OAuth, indexes a page, and chats with the canary token", async ({ + test("user connects Confluence and chats through live MCP tools with indexing disabled", async ({ page, request, apiToken, @@ -23,17 +22,27 @@ test.describe("Confluence connector journey", () => { confluenceConnector, chatThread, }) => { - test.setTimeout(180_000); // worker cold-start + summarize + embed + chunk + test.setTimeout(90_000); // worker cold-start + live tool chat expect(confluenceConnector.connector_type).toBe("CONFLUENCE_CONNECTOR"); - expect(confluenceConnector.is_indexable).toBe(true); + expect(confluenceConnector.is_indexable).toBe(false); expect(confluenceConnector.config._token_encrypted).toBe(true); - expect(confluenceConnector.config.cloud_id).toBe(FAKE_CONFLUENCE_PAGES.canary.cloudId); - expect(confluenceConnector.config.base_url).toBe(FAKE_CONFLUENCE_PAGES.canary.baseUrl); - expect(confluenceConnector.config.access_token).toBeTruthy(); - expect(confluenceConnector.config.CONFLUENCE_BASE_URL).toBeUndefined(); - expect(confluenceConnector.config.CONFLUENCE_EMAIL).toBeUndefined(); - expect(confluenceConnector.config.CONFLUENCE_API_TOKEN).toBeUndefined(); + expect(confluenceConnector.config.mcp_service).toBe("confluence"); + expect(confluenceConnector.config.server_config).toMatchObject({ + transport: "streamable-http", + url: "https://mcp.atlassian.com/v1/mcp", + }); + // Shared Atlassian server issues the token via the same cf.mcp endpoint + // as Jira; assert the endpoint + a live token rather than the (Jira-named) + // shared client id. + expect(confluenceConnector.config.mcp_oauth).toMatchObject({ + token_endpoint: "https://cf.mcp.atlassian.com/v1/token", + }); + expect( + (confluenceConnector.config.mcp_oauth as Record).access_token + ).toBeTruthy(); + expect(confluenceConnector.config.access_token).toBeUndefined(); + expect(confluenceConnector.config.refresh_token).toBeUndefined(); await page.goto(`/dashboard/${searchSpace.id}/new-chat`, { waitUntil: "domcontentloaded", @@ -44,45 +53,17 @@ test.describe("Confluence connector journey", () => { await connectorDialog.getByPlaceholder("Search").fill("Confluence"); await expect(connectorDialog.getByText("Confluence", { exact: true })).toBeVisible(); - await triggerIndex(request, apiToken, confluenceConnector.id, searchSpace.id, {}); + const beforeDocs = await listDocuments(request, apiToken, searchSpace.id); + expect(beforeDocs).toHaveLength(0); - await waitForIndexingComplete(request, apiToken, confluenceConnector.id, searchSpace.id, { - timeoutMs: 150_000, - intervalMs: 1_500, - minDocuments: 1, - }); - - await waitForDocumentByTitle( + const disabledIndex = await triggerIndexExpectDisabled( request, apiToken, - searchSpace.id, - FAKE_CONFLUENCE_PAGES.canary.title, - { - timeoutMs: 30_000, - } + confluenceConnector.id, + searchSpace.id ); - - const docs = await listDocuments(request, apiToken, searchSpace.id); - const canaryDoc = docs.find((d) => d.title === FAKE_CONFLUENCE_PAGES.canary.title); - - expect(canaryDoc, "Confluence canary document must exist after indexing").toBeDefined(); - if (!canaryDoc) throw new Error("unreachable: canaryDoc asserted defined above"); - expect(canaryDoc.document_type).toBe("CONFLUENCE_CONNECTOR"); - - const editor = await getEditorContent(request, apiToken, searchSpace.id, canaryDoc.id); - expect( - editor.source_markdown, - `canary token ${CANARY_TOKENS.confluenceCanary} should appear in editor source_markdown; ` + - `got first 200 chars: ${editor.source_markdown.slice(0, 200)}` - ).toContain(CANARY_TOKENS.confluenceCanary); - expect(editor.document_type).toBe("CONFLUENCE_CONNECTOR"); - expect(editor.chunk_count).toBeGreaterThan(0); - - const refreshedConnectors = await listConnectors(request, apiToken, searchSpace.id); - const refreshed = refreshedConnectors.find((c) => c.id === confluenceConnector.id); - expect(refreshed?.connector_type).toBe("CONFLUENCE_CONNECTOR"); - expect(refreshed?.is_indexable).toBe(true); - expect(refreshed?.last_indexed_at).not.toBeNull(); + expect(disabledIndex.message ?? "").toContain("real-time agent tools"); + expect(disabledIndex.message ?? "").toContain("background indexing is disabled"); const chat = await streamChatToCompletion(request, apiToken, { searchSpaceId: searchSpace.id, @@ -91,7 +72,19 @@ test.describe("Confluence connector journey", () => { }); expect( chat.assistantText, - `chat agent should surface Confluence canary token after indexing; got: ${chat.assistantText.slice(0, 200)}` + `chat agent should surface Confluence canary token from live MCP tools; got: ${chat.assistantText.slice(0, 200)}` ).toContain(CANARY_TOKENS.confluenceCanary); + + const eventText = JSON.stringify(chat.events); + expect(eventText).toContain("searchConfluenceUsingCql"); + + const refreshedConnectors = await listConnectors(request, apiToken, searchSpace.id); + const refreshed = refreshedConnectors.find((c) => c.id === confluenceConnector.id); + expect(refreshed?.connector_type).toBe("CONFLUENCE_CONNECTOR"); + expect(refreshed?.is_indexable).toBe(false); + expect(refreshed?.last_indexed_at).toBeNull(); + + const afterDocs = await listDocuments(request, apiToken, searchSpace.id); + expect(afterDocs).toHaveLength(0); }); }); diff --git a/surfsense_web/tests/connectors/notion/journey.spec.ts b/surfsense_web/tests/connectors/notion/journey.spec.ts index 948353011..e19d3d9f7 100644 --- a/surfsense_web/tests/connectors/notion/journey.spec.ts +++ b/surfsense_web/tests/connectors/notion/journey.spec.ts @@ -1,20 +1,19 @@ import { expect, notionWithChatTest as test } from "../../fixtures"; import { streamChatToCompletion } from "../../helpers/api/chat"; -import { listConnectors, triggerIndex } from "../../helpers/api/connectors"; -import { getEditorContent, listDocuments } from "../../helpers/api/documents"; +import { listConnectors, triggerIndexExpectDisabled } from "../../helpers/api/connectors"; +import { listDocuments } from "../../helpers/api/documents"; import { CANARY_TOKENS, FAKE_NOTION_PAGES } from "../../helpers/canary"; import { openConnectorPopup } from "../../helpers/ui/connector-popup"; -import { waitForDocumentByTitle, waitForIndexingComplete } from "../../helpers/waits/indexing"; /** - * Proves Notion OAuth -> indexed Notion API pages -> stored source_markdown -> chat. + * Proves Notion MCP OAuth -> live MCP tool discovery/call -> chat. * - * The external Notion provider is faked at the OAuth token/API boundary; - * SurfSense's own add/callback routes, encrypted config storage, connector - * indexing endpoint, indexing pipeline, and chat retrieval remain real. + * Notion migrated from indexed OAuth to the hosted Notion MCP server, so it is + * live-tool only: the public indexing route returns indexing_started=false and + * chat should call Notion MCP tools (`search`) instead of retrieving from the KB. */ test.describe("Notion connector journey", () => { - test("user connects Notion through OAuth, indexes a page, and chats with the canary token", async ({ + test("user connects Notion and chats through live MCP tools with indexing disabled", async ({ page, request, apiToken, @@ -22,16 +21,25 @@ test.describe("Notion connector journey", () => { notionConnector, chatThread, }) => { - test.setTimeout(180_000); // worker cold-start + summarize + embed + chunk + test.setTimeout(90_000); // worker cold-start + live tool chat expect(notionConnector.connector_type).toBe("NOTION_CONNECTOR"); - expect(notionConnector.is_indexable).toBe(true); + expect(notionConnector.is_indexable).toBe(false); expect(notionConnector.config._token_encrypted).toBe(true); - expect(notionConnector.config.NOTION_INTEGRATION_TOKEN).toBeUndefined(); - expect(notionConnector.config.access_token).toBeTruthy(); - expect(notionConnector.config.workspace_id).toBe(FAKE_NOTION_PAGES.canary.workspaceId); - expect(notionConnector.config.workspace_name).toBe(FAKE_NOTION_PAGES.canary.workspaceName); - expect(notionConnector.config.bot_id).toBe(FAKE_NOTION_PAGES.canary.botId); + expect(notionConnector.config.mcp_service).toBe("notion"); + expect(notionConnector.config.server_config).toMatchObject({ + transport: "streamable-http", + url: "https://mcp.notion.com/mcp", + }); + expect(notionConnector.config.mcp_oauth).toMatchObject({ + client_id: "fake-notion-mcp-client-id", + token_endpoint: "https://mcp.notion.com/token", + }); + expect( + (notionConnector.config.mcp_oauth as Record).access_token + ).toBeTruthy(); + expect(notionConnector.config.access_token).toBeUndefined(); + expect(notionConnector.config.refresh_token).toBeUndefined(); await page.goto(`/dashboard/${searchSpace.id}/new-chat`, { waitUntil: "domcontentloaded", @@ -39,46 +47,20 @@ test.describe("Notion connector journey", () => { await openConnectorPopup(page); const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); await expect(connectorDialog).toBeVisible(); + await connectorDialog.getByPlaceholder("Search").fill("Notion"); + await expect(connectorDialog.getByText("Notion", { exact: true })).toBeVisible(); - await triggerIndex(request, apiToken, notionConnector.id, searchSpace.id, {}); + const beforeDocs = await listDocuments(request, apiToken, searchSpace.id); + expect(beforeDocs).toHaveLength(0); - await waitForIndexingComplete(request, apiToken, notionConnector.id, searchSpace.id, { - timeoutMs: 150_000, - intervalMs: 1_500, - minDocuments: 1, - }); - - await waitForDocumentByTitle( + const disabledIndex = await triggerIndexExpectDisabled( request, apiToken, - searchSpace.id, - FAKE_NOTION_PAGES.canary.title, - { - timeoutMs: 30_000, - } + notionConnector.id, + searchSpace.id ); - - const docs = await listDocuments(request, apiToken, searchSpace.id); - const canaryDoc = docs.find((d) => d.title === FAKE_NOTION_PAGES.canary.title); - - expect(canaryDoc, "Notion canary document must exist after indexing").toBeDefined(); - if (!canaryDoc) throw new Error("unreachable: canaryDoc asserted defined above"); - expect(canaryDoc.document_type).toBe("NOTION_CONNECTOR"); - - const editor = await getEditorContent(request, apiToken, searchSpace.id, canaryDoc.id); - expect( - editor.source_markdown, - `canary token ${CANARY_TOKENS.notionCanary} should appear in editor source_markdown; ` + - `got first 200 chars: ${editor.source_markdown.slice(0, 200)}` - ).toContain(CANARY_TOKENS.notionCanary); - expect(editor.document_type).toBe("NOTION_CONNECTOR"); - expect(editor.chunk_count).toBeGreaterThan(0); - - const refreshedConnectors = await listConnectors(request, apiToken, searchSpace.id); - const refreshed = refreshedConnectors.find((c) => c.id === notionConnector.id); - expect(refreshed?.connector_type).toBe("NOTION_CONNECTOR"); - expect(refreshed?.is_indexable).toBe(true); - expect(refreshed?.last_indexed_at).not.toBeNull(); + expect(disabledIndex.message ?? "").toContain("real-time agent tools"); + expect(disabledIndex.message ?? "").toContain("background indexing is disabled"); const chat = await streamChatToCompletion(request, apiToken, { searchSpaceId: searchSpace.id, @@ -87,7 +69,19 @@ test.describe("Notion connector journey", () => { }); expect( chat.assistantText, - `chat agent should surface Notion canary token after indexing; got: ${chat.assistantText.slice(0, 200)}` + `chat agent should surface Notion canary token from live MCP tools; got: ${chat.assistantText.slice(0, 200)}` ).toContain(CANARY_TOKENS.notionCanary); + + const eventText = JSON.stringify(chat.events); + expect(eventText).toContain(FAKE_NOTION_PAGES.canary.id); + + const refreshedConnectors = await listConnectors(request, apiToken, searchSpace.id); + const refreshed = refreshedConnectors.find((c) => c.id === notionConnector.id); + expect(refreshed?.connector_type).toBe("NOTION_CONNECTOR"); + expect(refreshed?.is_indexable).toBe(false); + expect(refreshed?.last_indexed_at).toBeNull(); + + const afterDocs = await listDocuments(request, apiToken, searchSpace.id); + expect(afterDocs).toHaveLength(0); }); }); diff --git a/surfsense_web/tests/helpers/api/connectors.ts b/surfsense_web/tests/helpers/api/connectors.ts index 051b026e0..3d9b2ee59 100644 --- a/surfsense_web/tests/helpers/api/connectors.ts +++ b/surfsense_web/tests/helpers/api/connectors.ts @@ -454,12 +454,12 @@ export async function runNativeGoogleCalendarOAuth( } /** - * Drives the Notion OAuth flow programmatically. + * Drives the Notion MCP OAuth flow programmatically. * - * The E2E backend keeps SurfSense's OAuth add/callback routes real and - * patches only Notion's external token endpoint. Notion's authorization - * URL stays off-origin, so this helper extracts the signed state and calls - * the backend callback directly with the deterministic fake code. + * Notion migrated from indexed OAuth to the hosted Notion MCP server, so this + * uses SurfSense's generic MCP OAuth routes. The E2E backend keeps those routes + * real and patches Notion's external discovery/DCR/token/MCP tool boundaries + * (see surfsense_backend/tests/e2e/fakes/notion_mcp_module.py). */ export async function runNotionOAuth( request: APIRequestContext, @@ -471,26 +471,26 @@ export async function runNotionOAuth( connector: ConnectorRow | null; }> { const initiateResp = await request.get( - `${BACKEND_URL}/api/v1/auth/notion/connector/add?space_id=${searchSpaceId}`, + `${BACKEND_URL}/api/v1/auth/mcp/notion/connector/add?space_id=${searchSpaceId}`, { headers: authHeaders(token) } ); if (!initiateResp.ok()) { throw new Error( - `Notion initiate failed (${initiateResp.status()}): ${await initiateResp.text()}` + `Notion MCP initiate failed (${initiateResp.status()}): ${await initiateResp.text()}` ); } const { auth_url } = (await initiateResp.json()) as { auth_url: string }; if (!auth_url) { - throw new Error("Notion initiate response missing auth_url"); + throw new Error("Notion MCP initiate response missing auth_url"); } const state = new URL(auth_url).searchParams.get("state"); if (!state) { - throw new Error(`Notion auth_url missing state: ${auth_url}`); + throw new Error(`Notion MCP auth_url missing state: ${auth_url}`); } const callbackResp = await request.get( - `${BACKEND_URL}/api/v1/auth/notion/connector/callback?code=fake-notion-oauth-code&state=${encodeURIComponent(state)}`, + `${BACKEND_URL}/api/v1/auth/mcp/notion/connector/callback?code=fake-notion-oauth-code&state=${encodeURIComponent(state)}`, { headers: authHeaders(token), maxRedirects: 0, @@ -506,10 +506,14 @@ export async function runNotionOAuth( } /** - * Drives the Confluence OAuth flow programmatically. + * Drives the Confluence MCP OAuth flow programmatically. * - * The E2E backend keeps SurfSense's OAuth add/callback routes real and - * patches only Atlassian's external token/resource endpoints. + * Confluence migrated to the hosted Atlassian Rovo MCP server (shared with + * Jira), so this uses SurfSense's generic MCP OAuth routes. The E2E backend + * keeps those routes real and patches the shared Atlassian + * discovery/DCR/token/MCP tool boundaries (see + * surfsense_backend/tests/e2e/fakes/jira_module.py). The OAuth code is the + * shared Atlassian one because both connectors hit the same token endpoint. */ export async function runConfluenceOAuth( request: APIRequestContext, @@ -521,26 +525,26 @@ export async function runConfluenceOAuth( connector: ConnectorRow | null; }> { const initiateResp = await request.get( - `${BACKEND_URL}/api/v1/auth/confluence/connector/add?space_id=${searchSpaceId}`, + `${BACKEND_URL}/api/v1/auth/mcp/confluence/connector/add?space_id=${searchSpaceId}`, { headers: authHeaders(token) } ); if (!initiateResp.ok()) { throw new Error( - `Confluence initiate failed (${initiateResp.status()}): ${await initiateResp.text()}` + `Confluence MCP initiate failed (${initiateResp.status()}): ${await initiateResp.text()}` ); } const { auth_url } = (await initiateResp.json()) as { auth_url: string }; if (!auth_url) { - throw new Error("Confluence initiate response missing auth_url"); + throw new Error("Confluence MCP initiate response missing auth_url"); } const state = new URL(auth_url).searchParams.get("state"); if (!state) { - throw new Error(`Confluence auth_url missing state: ${auth_url}`); + throw new Error(`Confluence MCP auth_url missing state: ${auth_url}`); } const callbackResp = await request.get( - `${BACKEND_URL}/api/v1/auth/confluence/connector/callback?code=fake-confluence-oauth-code&state=${encodeURIComponent(state)}`, + `${BACKEND_URL}/api/v1/auth/mcp/confluence/connector/callback?code=fake-jira-oauth-code&state=${encodeURIComponent(state)}`, { headers: authHeaders(token), maxRedirects: 0,