diff --git a/README.es.md b/README.es.md index 82cfcae1d..70ccd5eb1 100644 --- a/README.es.md +++ b/README.es.md @@ -138,7 +138,7 @@ Agrega el servidor MCP de SurfSense a Claude, Cursor o tu propio framework de ag { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/README.hi.md b/README.hi.md index 1c9cfa4a7..cbf9ae8fa 100644 --- a/README.hi.md +++ b/README.hi.md @@ -138,7 +138,7 @@ SurfSense MCP सर्वर को Claude, Cursor या अपने एज { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/README.md b/README.md index a05548d66..8f2e65118 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ Add the SurfSense MCP server to Claude, Cursor, or your own agent framework: { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/README.pt-BR.md b/README.pt-BR.md index 4f2b945f4..076736bcf 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -138,7 +138,7 @@ Adicione o servidor MCP do SurfSense ao Claude, ao Cursor ou ao seu próprio fra { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/README.zh-CN.md b/README.zh-CN.md index 22410a727..7d2ae35c3 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -138,7 +138,7 @@ curl -X POST "$SURFSENSE_API_URL/workspaces/$WORKSPACE_ID/scrapers/reddit/scrape { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/surfsense_mcp/.dockerignore b/surfsense_mcp/.dockerignore new file mode 100644 index 000000000..28abd4591 --- /dev/null +++ b/surfsense_mcp/.dockerignore @@ -0,0 +1,11 @@ +.git +.gitignore +.venv +__pycache__/ +*.py[cod] +*$py.class +*.so +.pytest_cache/ +.ruff_cache/ +.env +tests/ diff --git a/surfsense_mcp/Dockerfile b/surfsense_mcp/Dockerfile new file mode 100644 index 000000000..2233411ed --- /dev/null +++ b/surfsense_mcp/Dockerfile @@ -0,0 +1,32 @@ +# syntax=docker.io/docker/dockerfile:1 +# SurfSense MCP Server — remote (streamable-http) image. +# Serves /mcp (per-request API key, no baked secret) and a public /health probe. + +# Stage 1: deps frozen from uv.lock so rebuilds never drift. +FROM python:3.12-slim AS deps + +WORKDIR /app + +COPY pyproject.toml uv.lock ./ +RUN pip install --no-cache-dir uv && \ + uv export --frozen --no-dev --no-emit-project --no-hashes \ + --format requirements-txt -o /tmp/requirements.txt && \ + uv pip install --system --no-cache-dir -r /tmp/requirements.txt && \ + rm /tmp/requirements.txt + + +# Stage 2: project source; --no-deps since deps are already installed above. +FROM deps AS production + +COPY . . +RUN uv pip install --system --no-cache-dir --no-deps -e . + +ENV PYTHONUNBUFFERED=1 \ + SURFSENSE_MCP_TRANSPORT=streamable-http \ + SURFSENSE_MCP_HOST=0.0.0.0 \ + SURFSENSE_MCP_PORT=8080 \ + SURFSENSE_BASE_URL=https://api.surfsense.com + +EXPOSE 8080 + +CMD ["python", "-m", "surfsense_mcp"] diff --git a/surfsense_mcp/README.md b/surfsense_mcp/README.md index 47dabf55c..cc5ef19c5 100644 --- a/surfsense_mcp/README.md +++ b/surfsense_mcp/README.md @@ -2,9 +2,15 @@ A [Model Context Protocol](https://modelcontextprotocol.io/) server that exposes SurfSense to MCP clients like **Claude Code**, **Cursor**, and **Claude Desktop**. -It talks to a running SurfSense backend purely over its REST API using a SurfSense -API key — it imports no backend code and can point at any instance (local or -hosted) by changing two environment variables. +It talks to a SurfSense backend purely over its REST API using a SurfSense API +key — it imports no backend code. + +Connect it two ways: + +- **Hosted** (recommended) — point your client at `https://mcp.surfsense.com/mcp` + and pass your API key in a header. Nothing to install or keep running. +- **Self-host (stdio)** — run the server yourself against any backend (cloud or + your own). Best for self-hosters and clients without remote-server support. ## Tools @@ -29,15 +35,42 @@ Workspace-scoped tools default to the active workspace; pass `workspace` (a name or id) to override for a single call. Ids never need to be typed by hand — the model carries them between calls. -## Prerequisites +## Get an API key -1. A running SurfSense backend (default `http://localhost:8000`). -2. A **SurfSense API key**: SurfSense → Settings → API → create key (`ss_pat_…`). -3. **API access enabled** on the workspace(s) you want to use (workspace settings). +1. SurfSense → **API Playground → API Keys**: create a personal key (`ss_pat_…`). + It is shown only once. +2. Toggle **API key access** on for the workspace(s) you want to use. -## Setup +## Connect (hosted) -Uses [uv](https://github.com/astral-sh/uv): +Point your client at the hosted server and send the key as a Bearer token. For +clients that read an `mcpServers` map (Cursor, Claude Desktop, and others): + +```json +{ + "mcpServers": { + "surfsense": { + "url": "https://mcp.surfsense.com/mcp", + "headers": { "Authorization": "Bearer ss_pat_your_key_here" } + } + } +} +``` + +Claude Code, from a terminal: + +```bash +claude mcp add --transport http surfsense https://mcp.surfsense.com/mcp \ + --header "Authorization: Bearer ss_pat_your_key_here" +``` + +Most MCP clients accept this `url` + `headers` form; check your client's docs for +its exact remote-server field. + +## Self-host (stdio) + +Run the server yourself when you host your own backend or use a client without +remote support. It uses [uv](https://github.com/astral-sh/uv): ```bash cd surfsense_mcp @@ -45,11 +78,8 @@ uv sync uv run python -m surfsense_mcp.selfcheck # verify tools register correctly ``` -## Connect it to a client - -### Cursor - -Add to `~/.cursor/mcp.json` (or a project `.cursor/mcp.json`): +Then add it to your client. Cursor (`~/.cursor/mcp.json` or a project +`.cursor/mcp.json`): ```json { @@ -66,7 +96,7 @@ Add to `~/.cursor/mcp.json` (or a project `.cursor/mcp.json`): } ``` -### Claude Code +Claude Code: ```bash claude mcp add surfsense \ @@ -75,15 +105,13 @@ claude mcp add surfsense \ -- uv run --directory /absolute/path/to/SurfSense/surfsense_mcp python -m surfsense_mcp ``` -### Claude Desktop - -Add the same `mcpServers` block as Cursor to +Claude Desktop: add the same `mcpServers` block as Cursor to `claude_desktop_config.json` (Settings → Developer → Edit Config). ## Configuration -See `.env.example`. Secrets are passed as environment variables by the client; -never commit tokens. +See `.env.example`. For self-host, secrets are passed as environment variables by +the client; never commit tokens. ## Backend dependency diff --git a/surfsense_mcp/pyproject.toml b/surfsense_mcp/pyproject.toml index 87db0c223..489bbf455 100644 --- a/surfsense_mcp/pyproject.toml +++ b/surfsense_mcp/pyproject.toml @@ -8,6 +8,8 @@ license = { text = "Apache-2.0" } dependencies = [ "mcp>=1.26.0", "httpx>=0.27.0", + "starlette>=0.37", + "uvicorn>=0.30", ] [project.scripts] diff --git a/surfsense_mcp/src/surfsense_mcp/__main__.py b/surfsense_mcp/src/surfsense_mcp/__main__.py index 6c878ebfa..37a2a1785 100644 --- a/surfsense_mcp/src/surfsense_mcp/__main__.py +++ b/surfsense_mcp/src/surfsense_mcp/__main__.py @@ -1,7 +1,12 @@ """Entry point: load settings from the environment and run the MCP server. -Defaults to stdio (what Cursor, Claude Code, and Claude Desktop launch). stdout -is the protocol channel, so every log line goes to stderr. +Two transports share one build: +- ``stdio`` (default): Cursor/Claude launch one process per user; the key comes + from the environment, so it is required here. +- ``streamable-http``: one process serves many users, each passing their own key + per request; the key is enforced by the transport's auth middleware instead. + +For stdio, stdout is the protocol channel, so every log line goes to stderr. """ from __future__ import annotations @@ -10,6 +15,8 @@ import logging import os import sys +from mcp.server.fastmcp import FastMCP + from .config import Settings from .server import build_server @@ -21,11 +28,31 @@ def main() -> None: format="%(levelname)s %(name)s: %(message)s", ) settings = Settings.from_env() + transport = os.environ.get("SURFSENSE_MCP_TRANSPORT", "stdio").strip() or "stdio" mcp, _client = build_server(settings) - transport = os.environ.get("SURFSENSE_MCP_TRANSPORT", "stdio").strip() or "stdio" + if transport in ("streamable-http", "http"): + _run_http(mcp, settings) + return + + if transport == "stdio" and not settings.api_key: + raise SystemExit( + "SURFSENSE_API_KEY is required for stdio transport. Create an API " + "key in SurfSense (Settings -> API) and pass it via the " + "SURFSENSE_API_KEY environment variable." + ) mcp.run(transport=transport) +def _run_http(mcp: FastMCP, settings: Settings) -> None: + """Serve the streamable-http app directly, so the per-request identity + middleware wraps the SDK's MCP endpoint.""" + import uvicorn + + from .core.transport import build_http_app + + uvicorn.run(build_http_app(mcp), host=settings.host, port=settings.port) + + if __name__ == "__main__": main() diff --git a/surfsense_mcp/src/surfsense_mcp/config.py b/surfsense_mcp/src/surfsense_mcp/config.py index 5d3646545..18d0f1514 100644 --- a/surfsense_mcp/src/surfsense_mcp/config.py +++ b/surfsense_mcp/src/surfsense_mcp/config.py @@ -12,6 +12,8 @@ from dataclasses import dataclass DEFAULT_BASE_URL = "http://localhost:8000" DEFAULT_API_PREFIX = "/api/v1" DEFAULT_TIMEOUT_SECONDS = 180.0 +DEFAULT_HTTP_HOST = "127.0.0.1" +DEFAULT_HTTP_PORT = 8080 @dataclass(frozen=True) @@ -19,10 +21,12 @@ class Settings: """Resolved configuration for a server process.""" base_url: str - api_key: str + api_key: str | None api_prefix: str timeout: float default_workspace: str | None + host: str + port: int @property def api_base(self) -> str: @@ -30,13 +34,9 @@ class Settings: @classmethod def from_env(cls) -> Settings: - api_key = os.environ.get("SURFSENSE_API_KEY", "").strip() - if not api_key: - raise SystemExit( - "SURFSENSE_API_KEY is required. Create an API key in SurfSense " - "(Settings -> API) and pass it via the SURFSENSE_API_KEY " - "environment variable." - ) + # Optional here: remote (http) callers pass their own key per request in + # a header. ``__main__`` enforces it for stdio, its only source of a key. + api_key = os.environ.get("SURFSENSE_API_KEY", "").strip() or None base_url = ( os.environ.get("SURFSENSE_BASE_URL", DEFAULT_BASE_URL).strip().rstrip("/") @@ -53,10 +53,19 @@ class Settings: default_workspace = os.environ.get("SURFSENSE_WORKSPACE", "").strip() or None + host = os.environ.get("SURFSENSE_MCP_HOST", "").strip() or DEFAULT_HTTP_HOST + raw_port = os.environ.get("SURFSENSE_MCP_PORT", "").strip() + try: + port = int(raw_port) if raw_port else DEFAULT_HTTP_PORT + except ValueError: + port = DEFAULT_HTTP_PORT + return cls( base_url=base_url, api_key=api_key, api_prefix=api_prefix, timeout=timeout, default_workspace=default_workspace, + host=host, + port=port, ) diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py b/surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py new file mode 100644 index 000000000..e837c6d61 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py @@ -0,0 +1,7 @@ +"""Per-request caller identity: header parsing, request-scoped storage, and the +ASGI middleware that binds them together for the remote transport.""" + +from .identity import current_api_key, current_identity +from .middleware import ApiKeyIdentityMiddleware + +__all__ = ["current_api_key", "current_identity", "ApiKeyIdentityMiddleware"] diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py b/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py new file mode 100644 index 000000000..3bca45967 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py @@ -0,0 +1,24 @@ +"""Extract a SurfSense API key from request headers. + +Pure header parsing, kept separate from transport and state. +""" + +from __future__ import annotations + +from starlette.datastructures import Headers + +_BEARER_PREFIX = "bearer " + + +def extract_api_key(headers: Headers) -> str | None: + """Return the caller's key from the ``Authorization: Bearer`` slot the + backend already expects, falling back to ``X-API-Key`` for clients that can + only send custom headers.""" + authorization = headers.get("authorization", "") + if authorization[: len(_BEARER_PREFIX)].lower() == _BEARER_PREFIX: + token = authorization[len(_BEARER_PREFIX) :].strip() + if token: + return token + + fallback = headers.get("x-api-key", "").strip() + return fallback or None diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py b/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py new file mode 100644 index 000000000..0c8f6b315 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py @@ -0,0 +1,34 @@ +"""Request-scoped caller identity. + +Over streamable-http one process serves many users, so the caller's key lives in +a contextvar for the life of a request: the auth middleware binds it and the +client reads it when calling the backend. Under stdio there is no request, so the +contextvar is empty and the env key is used instead. +""" + +from __future__ import annotations + +from contextvars import ContextVar, Token + +_LOCAL_IDENTITY = "__local__" + +_api_key: ContextVar[str | None] = ContextVar("surfsense_api_key", default=None) + + +def bind_api_key(api_key: str | None) -> Token: + """Bind the caller's key to the current request; returns a reset token.""" + return _api_key.set(api_key) + + +def unbind_api_key(token: Token) -> None: + _api_key.reset(token) + + +def current_api_key() -> str | None: + """The caller's key for the in-flight request, or ``None`` under stdio.""" + return _api_key.get() + + +def current_identity() -> str: + """Stable per-caller key for scoping request state; shared under stdio.""" + return _api_key.get() or _LOCAL_IDENTITY diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py new file mode 100644 index 000000000..8ba6ce939 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py @@ -0,0 +1,58 @@ +"""ASGI middleware that establishes the caller's identity for each request. + +A pure ASGI middleware, deliberately not Starlette's ``BaseHTTPMiddleware``: +the latter runs the endpoint in a separate task, so a contextvar set in it does +not reach the tool handler. A pure middleware binds the key in the request's own +task, from which the SDK's per-request handling inherits it. + +Requests without a key are rejected here so no tool ever runs unauthenticated. +Paths in ``public_paths`` (e.g. the health probe) skip the check entirely. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +from starlette.datastructures import Headers +from starlette.responses import JSONResponse +from starlette.types import ASGIApp, Receive, Scope, Send + +from .headers import extract_api_key +from .identity import bind_api_key, unbind_api_key + + +class ApiKeyIdentityMiddleware: + """Binds the per-request API key into the identity contextvar, or 401s.""" + + def __init__(self, app: ASGIApp, public_paths: Iterable[str] = ()) -> None: + self._app = app + self._public_paths = frozenset(public_paths) + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http" or scope["path"] in self._public_paths: + await self._app(scope, receive, send) + return + + api_key = extract_api_key(Headers(scope=scope)) + if api_key is None: + await _unauthenticated()(scope, receive, send) + return + + token = bind_api_key(api_key) + try: + await self._app(scope, receive, send) + finally: + unbind_api_key(token) + + +def _unauthenticated() -> JSONResponse: + return JSONResponse( + { + "error": "unauthorized", + "message": ( + "Missing SurfSense API key. Send 'Authorization: Bearer " + "ss_pat_...' (or an X-API-Key header)." + ), + }, + status_code=401, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/core/client.py b/surfsense_mcp/src/surfsense_mcp/core/client.py index 55f9d6048..9ae446ae2 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/client.py +++ b/surfsense_mcp/src/surfsense_mcp/core/client.py @@ -10,10 +10,11 @@ from typing import Any import httpx +from .auth.identity import current_api_key from .errors import ToolError _FAILURE_HINTS: dict[int, str] = { - 401: "Authentication failed — check that SURFSENSE_API_KEY is a valid, unexpired key.", + 401: "Authentication failed — the SurfSense API key is invalid or expired.", 402: "The workspace is out of credits for this operation.", 403: ( "Access denied — the token lacks permission, or API access is disabled " @@ -27,17 +28,30 @@ _FAILURE_HINTS: dict[int, str] = { class SurfSenseClient: """Issues authenticated requests against ``{base_url}{api_prefix}``.""" - def __init__(self, *, api_base: str, api_key: str, timeout: float) -> None: + def __init__( + self, *, api_base: str, timeout: float, fallback_api_key: str | None = None + ) -> None: self._api_base = api_base + # Resolved per request, so no key is baked into the shared client. The + # fallback is the env key used under stdio, where there is no header. + self._fallback_api_key = fallback_api_key self._http = httpx.AsyncClient( base_url=api_base, - headers={ - "Authorization": f"Bearer {api_key}", - "Accept": "application/json", - }, + headers={"Accept": "application/json"}, timeout=timeout, ) + def _auth_headers(self) -> dict[str, str]: + """Resolve the caller's key: the per-request header, else the env key.""" + api_key = current_api_key() or self._fallback_api_key + if not api_key: + raise ToolError( + "No SurfSense API key supplied. Send it as an 'Authorization: " + "Bearer ss_pat_...' header (remote server), or set the " + "SURFSENSE_API_KEY environment variable (stdio)." + ) + return {"Authorization": f"Bearer {api_key}"} + async def request( self, method: str, @@ -53,9 +67,16 @@ class SurfSenseClient: # as a value (e.g. int("") on folder_id) and fail. if params is not None: params = {key: value for key, value in params.items() if value is not None} + headers = self._auth_headers() try: response = await self._http.request( - method, path, params=params, json=json, data=data, files=files + method, + path, + params=params, + json=json, + data=data, + files=files, + headers=headers, ) except httpx.RequestError as exc: raise ToolError( diff --git a/surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py b/surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py new file mode 100644 index 000000000..53aaa6b8c --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py @@ -0,0 +1,5 @@ +"""Transport wiring for the MCP server (streamable-http app assembly).""" + +from .http import build_http_app + +__all__ = ["build_http_app"] diff --git a/surfsense_mcp/src/surfsense_mcp/core/transport/http.py b/surfsense_mcp/src/surfsense_mcp/core/transport/http.py new file mode 100644 index 000000000..e375e8b98 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/transport/http.py @@ -0,0 +1,36 @@ +"""Wrap the SDK's MCP endpoint with identity + CORS for the remote transport. + +CORS is outermost so keyless browser preflight is answered before the identity +middleware. ``/health`` is a public path, exempt from the key check. +""" + +from __future__ import annotations + +from mcp.server.fastmcp import FastMCP +from starlette.requests import Request +from starlette.responses import JSONResponse +from starlette.middleware.cors import CORSMiddleware +from starlette.types import ASGIApp + +from ..auth.middleware import ApiKeyIdentityMiddleware + +HEALTH_PATH = "/health" + + +async def _health(_request: Request) -> JSONResponse: + return JSONResponse({"status": "ok"}) + + +def build_http_app(mcp: FastMCP) -> ASGIApp: + """Return the MCP streamable-http app wrapped with identity + CORS.""" + mcp.custom_route(HEALTH_PATH, methods=["GET"])(_health) + app: ASGIApp = ApiKeyIdentityMiddleware( + mcp.streamable_http_app(), public_paths={HEALTH_PATH} + ) + return CORSMiddleware( + app, + allow_origins=["*"], + allow_methods=["GET", "POST", "DELETE", "OPTIONS"], + allow_headers=["*"], + expose_headers=["Mcp-Session-Id"], + ) diff --git a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py index 6682d53b5..0c5423e6b 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py +++ b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py @@ -7,13 +7,22 @@ speaks a name, we resolve it, and remember the choice for later calls. from __future__ import annotations +from collections import OrderedDict from dataclasses import dataclass from typing import Annotated from pydantic import Field +from .auth.identity import current_identity from .client import SurfSenseClient from .errors import ToolError +from .workspace_matching import as_int, match_by_name, name_list + +# ponytail: one small entry per distinct caller (API token). Bounded so a flood +# of keys can't grow memory without limit; an evicted caller just re-resolves +# its default workspace on the next call. Upgrade path: a TTL/LRU store if +# per-caller state ever grows past this one field. +_MAX_TRACKED_IDENTITIES = 2048 # Shared parameter type for every workspace-scoped tool. WorkspaceParam = Annotated[ @@ -44,15 +53,21 @@ class WorkspaceContext: ) -> None: self._client = client self._preferred_reference = preferred_reference - self._active: Workspace | None = None + # Active selection is per caller: one shared slot would leak one user's + # choice to every other user on a shared server. + self._active_by_identity: OrderedDict[str, Workspace] = OrderedDict() @property def active(self) -> Workspace | None: - return self._active + return self._active_by_identity.get(current_identity()) def remember(self, workspace: Workspace) -> Workspace: - """Make ``workspace`` the default for later scoped calls.""" - self._active = workspace + """Make ``workspace`` the default for the current caller's later calls.""" + identity = current_identity() + self._active_by_identity[identity] = workspace + self._active_by_identity.move_to_end(identity) + while len(self._active_by_identity) > _MAX_TRACKED_IDENTITIES: + self._active_by_identity.popitem(last=False) return workspace async def fetch_all(self) -> list[Workspace]: @@ -67,8 +82,9 @@ class WorkspaceContext: return self.remember(await self._match(reference)) async def _resolve_default(self) -> Workspace: - if self._active is not None: - return self._active + active = self.active + if active is not None: + return active if self._preferred_reference: return self.remember(await self._match(self._preferred_reference)) return self.remember(await self._only_workspace_or_prompt()) @@ -84,43 +100,20 @@ class WorkspaceContext: ) raise ToolError( "No workspace selected. Choose one first with surfsense_select_workspace, " - f"or pass 'workspace'. Available: {_name_list(workspaces)}." + f"or pass 'workspace'. Available: {name_list(workspaces)}." ) async def _match(self, reference: str | int) -> Workspace: workspaces = await self.fetch_all() - as_id = _as_int(reference) + as_id = as_int(reference) if as_id is not None: found = next((w for w in workspaces if w.id == as_id), None) if found is None: raise ToolError( - f"No workspace with id {as_id}. Available: {_name_list(workspaces)}." + f"No workspace with id {as_id}. Available: {name_list(workspaces)}." ) return found - return _match_by_name(str(reference), workspaces) - - -def _match_by_name(reference: str, workspaces: list[Workspace]) -> Workspace: - """Match on name: exact, then case-insensitive, then unique substring.""" - needle = reference.strip() - exact = [w for w in workspaces if w.name == needle] - if exact: - return exact[0] - lowered = needle.casefold() - insensitive = [w for w in workspaces if w.name.casefold() == lowered] - if insensitive: - return insensitive[0] - partial = [w for w in workspaces if lowered in w.name.casefold()] - if len(partial) == 1: - return partial[0] - if len(partial) > 1: - raise ToolError( - f"'{reference}' matches several workspaces: {_name_list(partial)}. " - "Use a more specific name or the id." - ) - raise ToolError( - f"No workspace named '{reference}'. Available: {_name_list(workspaces)}." - ) + return match_by_name(str(reference), workspaces) def _to_workspace(row: dict) -> Workspace: @@ -131,14 +124,3 @@ def _to_workspace(row: dict) -> Workspace: is_owner=row.get("is_owner", False), member_count=row.get("member_count", 1), ) - - -def _as_int(reference: str | int) -> int | None: - if isinstance(reference, int): - return reference - text = reference.strip() - return int(text) if text.isdigit() else None - - -def _name_list(workspaces: list[Workspace]) -> str: - return ", ".join(f"{w.name} (id {w.id})" for w in workspaces) diff --git a/surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py b/surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py new file mode 100644 index 000000000..33bfc50a5 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py @@ -0,0 +1,51 @@ +"""Resolve a user-supplied workspace reference to a single workspace. + +Pure matching over an already-fetched list: name (exact, then case-insensitive, +then unique substring) or numeric id. Kept apart from WorkspaceContext so the +resolution rules can be read and tested without the network. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from .errors import ToolError + +if TYPE_CHECKING: + from .workspace_context import Workspace + + +def match_by_name(reference: str, workspaces: list[Workspace]) -> Workspace: + """Match on name: exact, then case-insensitive, then unique substring.""" + needle = reference.strip() + exact = [w for w in workspaces if w.name == needle] + if exact: + return exact[0] + lowered = needle.casefold() + insensitive = [w for w in workspaces if w.name.casefold() == lowered] + if insensitive: + return insensitive[0] + partial = [w for w in workspaces if lowered in w.name.casefold()] + if len(partial) == 1: + return partial[0] + if len(partial) > 1: + raise ToolError( + f"'{reference}' matches several workspaces: {name_list(partial)}. " + "Use a more specific name or the id." + ) + raise ToolError( + f"No workspace named '{reference}'. Available: {name_list(workspaces)}." + ) + + +def as_int(reference: str | int) -> int | None: + """Return the reference as an id, or None when it isn't numeric.""" + if isinstance(reference, int): + return reference + text = reference.strip() + return int(text) if text.isdigit() else None + + +def name_list(workspaces: list[Workspace]) -> str: + """Render workspaces as a human-readable 'name (id N)' list.""" + return ", ".join(f"{w.name} (id {w.id})" for w in workspaces) diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py index 7fb1802ae..1a971bfe4 100644 --- a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py +++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py @@ -1,379 +1,22 @@ """Knowledge-base tools: search the KB and manage its documents. Semantic search plus the document lifecycle — list, read, add text, upload a -file, update, and delete — over a workspace's knowledge base. Search and reads -default to the active workspace; document ids identify a single document across -the whole account, so id-addressed tools need no workspace. +file, update, and delete — over a workspace's knowledge base. Read tools live in +search_tools, mutations in document_tools. """ from __future__ import annotations -import mimetypes -from pathlib import Path -from typing import Annotated - from mcp.server.fastmcp import FastMCP -from mcp.types import ToolAnnotations -from pydantic import Field from ...core.client import SurfSenseClient -from ...core.errors import ToolError -from ...core.rendering import ResponseFormatParam, clip, to_json -from ...core.workspace_context import WorkspaceContext, WorkspaceParam -from .note_ingestion import build_note_document - -_READ = ToolAnnotations( - readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False -) -_WRITE = ToolAnnotations( - readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=False -) -_DELETE = ToolAnnotations( - readOnlyHint=False, destructiveHint=True, idempotentHint=False, openWorldHint=False -) - -_DOCUMENT_ID = Annotated[ - int, - Field( - description="Document id from surfsense_search_knowledge_base or " - "surfsense_list_documents results." - ), -] - -_DOCUMENT_TYPES = Annotated[ - list[str] | None, - Field( - description="Restrict to these document types, e.g. " - "['FILE', 'CRAWLED_URL', 'YOUTUBE_VIDEO']. Omit for all types." - ), -] +from ...core.workspace_context import WorkspaceContext +from . import document_tools, search_tools def register( mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext ) -> None: - """Register the knowledge-base tools on the server.""" - - @mcp.tool( - name="surfsense_search_knowledge_base", - title="Search knowledge base", - annotations=_READ, - structured_output=False, - ) - async def search_knowledge_base( - query: Annotated[ - str, - Field( - min_length=1, - description="Natural-language search, e.g. " - "'notebooklm user complaints'.", - ), - ], - top_k: Annotated[ - int, Field(ge=1, le=20, description="Maximum documents to return.") - ] = 5, - document_types: _DOCUMENT_TYPES = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Search the workspace's knowledge base by meaning and keywords. - - Use this FIRST when a question might be answered by content already - stored in SurfSense — notes, uploaded files, saved pages, past - research. Do NOT use it to fetch new data from the web; use the - scraper tools for that. Returns the most relevant documents with the - passages that matched, ranked by relevance score. - Example: query='pricing feedback', top_k=5. - """ - resolved = await context.resolve(workspace) - hits = await client.request( - "POST", - "/documents/search-semantic", - json={ - "workspace_id": resolved.id, - "query": query, - "top_k": max(1, min(top_k, 20)), - "document_types": document_types, - }, - ) - items = (hits or {}).get("items", []) - if response_format == "json": - return to_json(items) - return _render_search(query, items) - - @mcp.tool( - name="surfsense_list_documents", - title="List documents", - annotations=_READ, - structured_output=False, - ) - async def list_documents( - document_types: _DOCUMENT_TYPES = None, - folder_id: Annotated[ - int | None, - Field(description="Only documents in this folder. Omit for all."), - ] = None, - page: Annotated[ - int, Field(ge=0, description="Zero-based page number.") - ] = 0, - page_size: Annotated[ - int, Field(ge=1, description="Documents per page.") - ] = 20, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """List documents in the workspace's knowledge base, newest first. - - Use this to browse or inventory what is stored; to find documents - about a topic, prefer surfsense_search_knowledge_base. Returns each - document's title, id, type, and update time, plus a has_more flag — - request the next page by increasing page. - Example: document_types=['FILE'], page=0, page_size=20. - """ - resolved = await context.resolve(workspace) - result = await client.request( - "GET", - "/documents", - params={ - "workspace_id": resolved.id, - "page": page, - "page_size": page_size, - "document_types": _join(document_types), - "folder_id": folder_id, - }, - ) - if response_format == "json": - return to_json(result) - return _render_document_list(result) - - @mcp.tool( - name="surfsense_get_document", - title="Read one document", - annotations=_READ, - structured_output=False, - ) - async def get_document( - document_id: _DOCUMENT_ID, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Read one document's full content and metadata by id. - - Use this after surfsense_search_knowledge_base or - surfsense_list_documents to open a specific document — search results - only include the matching passages, this returns the whole text. - """ - document = await client.request("GET", f"/documents/{document_id}") - if response_format == "json": - return clip(to_json(document)) - return _render_document(document) - - @mcp.tool( - name="surfsense_add_document", - title="Add a note", - annotations=_WRITE, - structured_output=False, - ) - async def add_document( - title: Annotated[ - str, - Field(min_length=1, description="Short descriptive title for the note."), - ], - content: Annotated[ - str, - Field( - min_length=1, - description="The note's body; plain text or markdown.", - ), - ], - source_url: Annotated[ - str | None, - Field(description="Where the text came from, if anywhere."), - ] = None, - workspace: WorkspaceParam = None, - ) -> str: - """Save a text or markdown note into the workspace's knowledge base. - - Use this to store notes, summaries, or findings so they become - searchable later — e.g. after finishing a piece of research. For files - on disk use surfsense_upload_file instead. Indexing is asynchronous, - so the note may take a moment to appear in search. - Example: title='NotebookLM subreddits', content='- r/notebooklm ...'. - """ - resolved = await context.resolve(workspace) - await client.request( - "POST", - "/documents", - json=build_note_document( - workspace_id=resolved.id, - title=title, - content=content, - source_url=source_url, - ), - ) - return ( - f"Queued '{title}' for indexing in '{resolved.name}'. " - "It will be searchable once processing completes." - ) - - @mcp.tool( - name="surfsense_upload_file", - title="Upload a file", - annotations=_WRITE, - structured_output=False, - ) - async def upload_file( - file_path: Annotated[ - str, - Field( - description="Path to a local file, e.g. " - "'C:/Users/me/report.pdf' or '~/notes/summary.md'." - ), - ], - use_vision_llm: Annotated[ - bool, - Field( - description="True reads scanned or image-heavy files with a " - "vision model (slower)." - ), - ] = False, - workspace: WorkspaceParam = None, - ) -> str: - """Upload a local file (PDF, docx, markdown, etc.) into the knowledge base. - - Use this to ingest a file from disk so its content becomes searchable; - for text you already have in hand use surfsense_add_document instead. - The file is parsed, chunked, and indexed asynchronously. Duplicate - files are detected and skipped. - Example: file_path='C:/Users/me/report.pdf'. - """ - resolved = await context.resolve(workspace) - payload = _read_upload(file_path) - result = await client.request( - "POST", - "/documents/fileupload", - data={ - "workspace_id": str(resolved.id), - "use_vision_llm": str(use_vision_llm).lower(), - "processing_mode": "basic", - }, - files=[("files", payload)], - ) - pending = (result or {}).get("pending_files", 0) - skipped = (result or {}).get("skipped_duplicates", 0) - note = " (already present, skipped)" if skipped and not pending else "" - return ( - f"Uploaded '{Path(file_path).name}' to '{resolved.name}'{note}. " - "It will be searchable once processing completes." - ) - - @mcp.tool( - name="surfsense_update_document", - title="Replace a document's content", - annotations=_WRITE, - structured_output=False, - ) - async def update_document( - document_id: _DOCUMENT_ID, - content: Annotated[ - str, - Field( - min_length=1, - description="New full text; replaces the existing content " - "entirely.", - ), - ], - ) -> str: - """Replace a document's stored content by id. - - Use this to correct or rewrite a document's text. The new content - REPLACES the old entirely — to append, read the document first with - surfsense_get_document and resend the combined text. Search chunks are - not re-indexed by this call. - """ - existing = await client.request("GET", f"/documents/{document_id}") - await client.request( - "PUT", - f"/documents/{document_id}", - json={ - "document_type": existing["document_type"], - "workspace_id": existing["workspace_id"], - "content": content, - }, - ) - return f"Updated document {document_id} ('{existing.get('title', '')}')." - - @mcp.tool( - name="surfsense_delete_document", - title="Delete a document", - annotations=_DELETE, - structured_output=False, - ) - async def delete_document(document_id: _DOCUMENT_ID) -> str: - """Permanently delete a document from the knowledge base by id. - - Use this only when the user explicitly asks to remove a document — - deletion cannot be undone. The document stops appearing in searches - immediately. - """ - await client.request("DELETE", f"/documents/{document_id}") - return f"Deleted document {document_id}." - - -def _read_upload(file_path: str) -> tuple[str, bytes, str]: - path = Path(file_path).expanduser() - if not path.is_file(): - raise ToolError(f"No file at '{file_path}'.") - mime, _ = mimetypes.guess_type(path.name) - return (path.name, path.read_bytes(), mime or "application/octet-stream") - - -def _join(values: list[str] | None) -> str | None: - return ",".join(values) if values else None - - -def _render_search(query: str, items: list[dict]) -> str: - if not items: - return f'No matches for "{query}".' - lines = [f'# {len(items)} result(s) for "{query}"', ""] - for hit in items: - lines.append( - f"## {hit.get('title', 'Untitled')} " - f"(id {hit.get('document_id')}) — score {hit.get('score', 0):.3f}" - ) - for chunk in hit.get("chunks", []): - excerpt = clip(chunk.get("content", "").strip(), 500) - lines.append(f"> {excerpt}") - lines.append("") - return "\n".join(lines).strip() - - -def _render_document_list(result: dict | None) -> str: - items = (result or {}).get("items", []) - if not items: - return "No documents found." - lines = ["# Documents", ""] - for doc in items: - lines.append( - f"- **{doc.get('title', 'Untitled')}** (id {doc.get('id')}) · " - f"{doc.get('document_type')} · updated {doc.get('updated_at')}" - ) - total = (result or {}).get("total", len(items)) - page = (result or {}).get("page", 0) - has_more = (result or {}).get("has_more", False) - lines.append("") - lines.append( - f"_Page {page} · showing {len(items)} of {total}" - + (" · more available_" if has_more else "_") - ) - return "\n".join(lines) - - -def _render_document(document: dict) -> str: - content = clip(document.get("content", "") or "(empty)") - return ( - f"# {document.get('title', 'Untitled')} (id {document.get('id')})\n" - f"- type: {document.get('document_type')}\n" - f"- workspace: {document.get('workspace_id')}\n" - f"- updated: {document.get('updated_at')}\n\n" - f"{content}" - ) + """Register every knowledge-base tool on the server.""" + search_tools.register(mcp, client, context) + document_tools.register(mcp, client, context) diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py new file mode 100644 index 000000000..16322506c --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py @@ -0,0 +1,34 @@ +"""Tool-call policy hints and shared parameter types for knowledge-base tools.""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.types import ToolAnnotations +from pydantic import Field + +READ = ToolAnnotations( + readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False +) +WRITE = ToolAnnotations( + readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=False +) +DELETE = ToolAnnotations( + readOnlyHint=False, destructiveHint=True, idempotentHint=False, openWorldHint=False +) + +DocumentId = Annotated[ + int, + Field( + description="Document id from surfsense_search_knowledge_base or " + "surfsense_list_documents results." + ), +] + +DocumentTypes = Annotated[ + list[str] | None, + Field( + description="Restrict to these document types, e.g. " + "['FILE', 'CRAWLED_URL', 'YOUTUBE_VIDEO']. Omit for all types." + ), +] diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py new file mode 100644 index 000000000..497a2526c --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py @@ -0,0 +1,185 @@ +"""Knowledge-base write tools: add a note, upload a file, update, and delete. + +Add and upload target the active workspace; update and delete address a document +by its account-unique id, so they need no workspace. +""" + +from __future__ import annotations + +import mimetypes +from pathlib import Path +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ...core.client import SurfSenseClient +from ...core.errors import ToolError +from ...core.workspace_context import WorkspaceContext, WorkspaceParam +from .annotations import DELETE, WRITE, DocumentId +from .note_ingestion import build_note_document + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the knowledge-base write and delete tools.""" + + @mcp.tool( + name="surfsense_add_document", + title="Add a note", + annotations=WRITE, + structured_output=False, + ) + async def add_document( + title: Annotated[ + str, + Field(min_length=1, description="Short descriptive title for the note."), + ], + content: Annotated[ + str, + Field( + min_length=1, + description="The note's body; plain text or markdown.", + ), + ], + source_url: Annotated[ + str | None, + Field(description="Where the text came from, if anywhere."), + ] = None, + workspace: WorkspaceParam = None, + ) -> str: + """Save a text or markdown note into the workspace's knowledge base. + + Use this to store notes, summaries, or findings so they become + searchable later — e.g. after finishing a piece of research. For files + on disk use surfsense_upload_file instead. Indexing is asynchronous, + so the note may take a moment to appear in search. + Example: title='NotebookLM subreddits', content='- r/notebooklm ...'. + """ + resolved = await context.resolve(workspace) + await client.request( + "POST", + "/documents", + json=build_note_document( + workspace_id=resolved.id, + title=title, + content=content, + source_url=source_url, + ), + ) + return ( + f"Queued '{title}' for indexing in '{resolved.name}'. " + "It will be searchable once processing completes." + ) + + @mcp.tool( + name="surfsense_upload_file", + title="Upload a file", + annotations=WRITE, + structured_output=False, + ) + async def upload_file( + file_path: Annotated[ + str, + Field( + description="Path to a local file, e.g. " + "'C:/Users/me/report.pdf' or '~/notes/summary.md'." + ), + ], + use_vision_llm: Annotated[ + bool, + Field( + description="True reads scanned or image-heavy files with a " + "vision model (slower)." + ), + ] = False, + workspace: WorkspaceParam = None, + ) -> str: + """Upload a local file (PDF, docx, markdown, etc.) into the knowledge base. + + Use this to ingest a file from disk so its content becomes searchable; + for text you already have in hand use surfsense_add_document instead. + The file is parsed, chunked, and indexed asynchronously. Duplicate + files are detected and skipped. + Example: file_path='C:/Users/me/report.pdf'. + """ + resolved = await context.resolve(workspace) + payload = _read_upload(file_path) + result = await client.request( + "POST", + "/documents/fileupload", + data={ + "workspace_id": str(resolved.id), + "use_vision_llm": str(use_vision_llm).lower(), + "processing_mode": "basic", + }, + files=[("files", payload)], + ) + pending = (result or {}).get("pending_files", 0) + skipped = (result or {}).get("skipped_duplicates", 0) + note = " (already present, skipped)" if skipped and not pending else "" + return ( + f"Uploaded '{Path(file_path).name}' to '{resolved.name}'{note}. " + "It will be searchable once processing completes." + ) + + @mcp.tool( + name="surfsense_update_document", + title="Replace a document's content", + annotations=WRITE, + structured_output=False, + ) + async def update_document( + document_id: DocumentId, + content: Annotated[ + str, + Field( + min_length=1, + description="New full text; replaces the existing content " + "entirely.", + ), + ], + ) -> str: + """Replace a document's stored content by id. + + Use this to correct or rewrite a document's text. The new content + REPLACES the old entirely — to append, read the document first with + surfsense_get_document and resend the combined text. Search chunks are + not re-indexed by this call. + """ + existing = await client.request("GET", f"/documents/{document_id}") + await client.request( + "PUT", + f"/documents/{document_id}", + json={ + "document_type": existing["document_type"], + "workspace_id": existing["workspace_id"], + "content": content, + }, + ) + return f"Updated document {document_id} ('{existing.get('title', '')}')." + + @mcp.tool( + name="surfsense_delete_document", + title="Delete a document", + annotations=DELETE, + structured_output=False, + ) + async def delete_document(document_id: DocumentId) -> str: + """Permanently delete a document from the knowledge base by id. + + Use this only when the user explicitly asks to remove a document — + deletion cannot be undone. The document stops appearing in searches + immediately. + """ + await client.request("DELETE", f"/documents/{document_id}") + return f"Deleted document {document_id}." + + +def _read_upload(file_path: str) -> tuple[str, bytes, str]: + path = Path(file_path).expanduser() + if not path.is_file(): + raise ToolError(f"No file at '{file_path}'.") + mime, _ = mimetypes.guess_type(path.name) + return (path.name, path.read_bytes(), mime or "application/octet-stream") diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py new file mode 100644 index 000000000..a9e60810d --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py @@ -0,0 +1,188 @@ +"""Knowledge-base read tools: semantic search, list, and read one document. + +Search and list default to the active workspace; a document read is addressed by +id, which is unique across the account, so it needs no workspace. +""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ...core.client import SurfSenseClient +from ...core.rendering import ResponseFormatParam, clip, to_json +from ...core.workspace_context import WorkspaceContext, WorkspaceParam +from .annotations import READ, DocumentId, DocumentTypes + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the knowledge-base read tools.""" + + @mcp.tool( + name="surfsense_search_knowledge_base", + title="Search knowledge base", + annotations=READ, + structured_output=False, + ) + async def search_knowledge_base( + query: Annotated[ + str, + Field( + min_length=1, + description="Natural-language search, e.g. " + "'notebooklm user complaints'.", + ), + ], + top_k: Annotated[ + int, Field(ge=1, le=20, description="Maximum documents to return.") + ] = 5, + document_types: DocumentTypes = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Search the workspace's knowledge base by meaning and keywords. + + Use this FIRST when a question might be answered by content already + stored in SurfSense — notes, uploaded files, saved pages, past + research. Do NOT use it to fetch new data from the web; use the + scraper tools for that. Returns the most relevant documents with the + passages that matched, ranked by relevance score. + Example: query='pricing feedback', top_k=5. + """ + resolved = await context.resolve(workspace) + hits = await client.request( + "POST", + "/documents/search-semantic", + json={ + "workspace_id": resolved.id, + "query": query, + "top_k": max(1, min(top_k, 20)), + "document_types": document_types, + }, + ) + items = (hits or {}).get("items", []) + if response_format == "json": + return to_json(items) + return _render_search(query, items) + + @mcp.tool( + name="surfsense_list_documents", + title="List documents", + annotations=READ, + structured_output=False, + ) + async def list_documents( + document_types: DocumentTypes = None, + folder_id: Annotated[ + int | None, + Field(description="Only documents in this folder. Omit for all."), + ] = None, + page: Annotated[ + int, Field(ge=0, description="Zero-based page number.") + ] = 0, + page_size: Annotated[ + int, Field(ge=1, description="Documents per page.") + ] = 20, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """List documents in the workspace's knowledge base, newest first. + + Use this to browse or inventory what is stored; to find documents + about a topic, prefer surfsense_search_knowledge_base. Returns each + document's title, id, type, and update time, plus a has_more flag — + request the next page by increasing page. + Example: document_types=['FILE'], page=0, page_size=20. + """ + resolved = await context.resolve(workspace) + result = await client.request( + "GET", + "/documents", + params={ + "workspace_id": resolved.id, + "page": page, + "page_size": page_size, + "document_types": _join(document_types), + "folder_id": folder_id, + }, + ) + if response_format == "json": + return to_json(result) + return _render_document_list(result) + + @mcp.tool( + name="surfsense_get_document", + title="Read one document", + annotations=READ, + structured_output=False, + ) + async def get_document( + document_id: DocumentId, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Read one document's full content and metadata by id. + + Use this after surfsense_search_knowledge_base or + surfsense_list_documents to open a specific document — search results + only include the matching passages, this returns the whole text. + """ + document = await client.request("GET", f"/documents/{document_id}") + if response_format == "json": + return clip(to_json(document)) + return _render_document(document) + + +def _join(values: list[str] | None) -> str | None: + return ",".join(values) if values else None + + +def _render_search(query: str, items: list[dict]) -> str: + if not items: + return f'No matches for "{query}".' + lines = [f'# {len(items)} result(s) for "{query}"', ""] + for hit in items: + lines.append( + f"## {hit.get('title', 'Untitled')} " + f"(id {hit.get('document_id')}) — score {hit.get('score', 0):.3f}" + ) + for chunk in hit.get("chunks", []): + excerpt = clip(chunk.get("content", "").strip(), 500) + lines.append(f"> {excerpt}") + lines.append("") + return "\n".join(lines).strip() + + +def _render_document_list(result: dict | None) -> str: + items = (result or {}).get("items", []) + if not items: + return "No documents found." + lines = ["# Documents", ""] + for doc in items: + lines.append( + f"- **{doc.get('title', 'Untitled')}** (id {doc.get('id')}) · " + f"{doc.get('document_type')} · updated {doc.get('updated_at')}" + ) + total = (result or {}).get("total", len(items)) + page = (result or {}).get("page", 0) + has_more = (result or {}).get("has_more", False) + lines.append("") + lines.append( + f"_Page {page} · showing {len(items)} of {total}" + + (" · more available_" if has_more else "_") + ) + return "\n".join(lines) + + +def _render_document(document: dict) -> str: + content = clip(document.get("content", "") or "(empty)") + return ( + f"# {document.get('title', 'Untitled')} (id {document.get('id')})\n" + f"- type: {document.get('document_type')}\n" + f"- workspace: {document.get('workspace_id')}\n" + f"- updated: {document.get('updated_at')}\n\n" + f"{content}" + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py index aa8265148..dfa2f3ab2 100644 --- a/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py @@ -1,570 +1,26 @@ """Scraper tools: one MCP surface per SurfSense platform capability. Web crawl, Google Search, Reddit, YouTube, and Google Maps each get a tool that -maps a natural-language request to the workspace's scraper door. Two more tools +maps a natural-language request to the workspace's scraper. Two run-history tools list and fetch past runs, so a large result truncated inline can be retrieved in -full later. +full later. Each platform lives in its own module under platforms/. """ from __future__ import annotations -from typing import Annotated, Literal - from mcp.server.fastmcp import FastMCP -from mcp.types import ToolAnnotations -from pydantic import Field from ...core.client import SurfSenseClient -from ...core.rendering import ResponseFormatParam, clip, to_json -from ...core.workspace_context import WorkspaceContext, WorkspaceParam -from .capability import run_scraper +from ...core.workspace_context import WorkspaceContext +from . import run_history +from .platforms import google_maps, google_search, reddit, web, youtube -# Scrapers reach the open web and record a billable run; they are neither -# read-only nor idempotent, but they do not mutate the knowledge base. -_SCRAPE = ToolAnnotations( - readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=True -) -_READ_RUNS = ToolAnnotations( - readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False -) - -RedditSort = Literal["relevance", "hot", "top", "new", "rising", "comments"] -RedditTime = Literal["hour", "day", "week", "month", "year", "all"] -CommentSort = Literal["TOP_COMMENTS", "NEWEST_FIRST"] -ReviewSort = Literal["newest", "mostRelevant", "highestRanking", "lowestRanking"] +_REGISTRARS = (web, google_search, reddit, youtube, google_maps, run_history) def register( mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext ) -> None: - """Register the scraper and run-history tools on the server.""" - - @mcp.tool( - name="surfsense_web_crawl", - title="Crawl web pages", - annotations=_SCRAPE, - structured_output=False, - ) - async def web_crawl( - start_urls: Annotated[ - list[str], - Field( - min_length=1, - description="Full URLs to fetch, e.g. " - "['https://example.com/blog/post'].", - ), - ], - max_crawl_depth: Annotated[ - int, - Field( - ge=0, - description="Link-hops to follow from start_urls within the " - "same site. 0 fetches only start_urls.", - ), - ] = 0, - max_crawl_pages: Annotated[ - int, Field(ge=1, description="Stop after this many pages in total.") - ] = 10, - max_length: Annotated[ - int, Field(ge=1, description="Max characters kept per page.") - ] = 50_000, - include_url_patterns: Annotated[ - list[str] | None, - Field( - description="Regexes; only discovered links matching one are " - "followed, e.g. ['/docs/.*']." - ), - ] = None, - exclude_url_patterns: Annotated[ - list[str] | None, - Field(description="Regexes; discovered links matching one are skipped."), - ] = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Fetch specific web pages and return their cleaned content as markdown. - - Use this to read a page the user names, or to spider a site from a - starting URL. Do NOT use it to find pages on a topic — use - surfsense_google_search for discovery. Returns one item per crawled - page: url, title, and the page text as markdown. - Example: start_urls=['https://blog.example.com'], max_crawl_depth=1, - include_url_patterns=['/2026/']. - """ - return await run_scraper( - client, - context, - platform="web", - verb="crawl", - payload={ - "startUrls": start_urls, - "maxCrawlDepth": max_crawl_depth, - "maxCrawlPages": max_crawl_pages, - "maxLength": max_length, - "includeUrlPatterns": include_url_patterns, - "excludeUrlPatterns": exclude_url_patterns, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_google_search", - title="Scrape Google Search", - annotations=_SCRAPE, - structured_output=False, - ) - async def google_search( - queries: Annotated[ - list[str], - Field( - min_length=1, - description="Search terms or full Google Search URLs, e.g. " - "['best rss readers 2026'].", - ), - ], - max_pages_per_query: Annotated[ - int, Field(ge=1, description="Result pages to fetch per query.") - ] = 1, - country_code: Annotated[ - str | None, - Field(description="Two-letter country to search from, e.g. 'us'."), - ] = None, - language_code: Annotated[ - str, Field(description="Results language, e.g. 'en'. Empty for default.") - ] = "", - site: Annotated[ - str | None, - Field( - description="Restrict results to one domain, e.g. 'example.com'." - ), - ] = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Scrape Google Search result pages for one or more queries. - - Use this to discover pages on the open web by topic; follow up with - surfsense_web_crawl to read a result in full. Do NOT use it for - Reddit, YouTube, or Google Maps research — the dedicated tools return - richer data. Returns each query's parsed results: title, url, and - snippet per organic result. - Example: queries=['notebooklm review'], site='news.ycombinator.com'. - """ - return await run_scraper( - client, - context, - platform="google_search", - verb="scrape", - payload={ - "queries": queries, - "max_pages_per_query": max_pages_per_query, - "country_code": country_code, - "language_code": language_code, - "site": site, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_reddit_scrape", - title="Search or scrape Reddit", - annotations=_SCRAPE, - structured_output=False, - ) - async def reddit_scrape( - urls: Annotated[ - list[str] | None, - Field( - description="Reddit URLs: a post, a subreddit like " - "'https://reddit.com/r/LocalLLaMA', a user page, or a search " - "URL. Provide urls OR search_queries." - ), - ] = None, - search_queries: Annotated[ - list[str] | None, - Field( - description="Terms to search Reddit for, e.g. " - "['NotebookLM alternatives']. Provide search_queries OR urls." - ), - ] = None, - community: Annotated[ - str | None, - Field( - description="Restrict a search to one subreddit, name without " - "'r/', e.g. 'ArtificialInteligence'." - ), - ] = None, - sort: Annotated[RedditSort, Field(description="Post ordering.")] = "new", - time_filter: Annotated[ - RedditTime | None, - Field(description="Time window; only valid with sort='top'."), - ] = None, - max_items: Annotated[ - int, Field(ge=1, description="Maximum posts to return.") - ] = 10, - skip_comments: Annotated[ - bool, - Field( - description="True fetches posts only (faster); False also " - "fetches each post's comment thread." - ), - ] = False, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Search or scrape Reddit: posts, comments, subreddits, and users. - - Use this for ANY Reddit research — finding relevant subreddits or - communities for a topic, top posts, or discussions — instead of a - generic web search. Returns posts (title, text, score, subreddit, url) - with comment threads unless skip_comments is set. Every post carries - its subreddit, so to find communities for a topic, search posts and - aggregate their subreddits. - Example: search_queries=['NotebookLM'], sort='top', time_filter='month'. - """ - return await run_scraper( - client, - context, - platform="reddit", - verb="scrape", - payload={ - "urls": urls, - "search_queries": search_queries, - "community": community, - "sort": sort, - "time_filter": time_filter, - "max_items": max_items, - "skip_comments": skip_comments, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_youtube_scrape", - title="Search or scrape YouTube", - annotations=_SCRAPE, - structured_output=False, - ) - async def youtube_scrape( - urls: Annotated[ - list[str] | None, - Field( - description="YouTube URLs: video, channel, playlist, shorts, " - "or hashtag pages. Provide urls OR search_queries." - ), - ] = None, - search_queries: Annotated[ - list[str] | None, - Field( - description="Terms to search YouTube for, e.g. " - "['NotebookLM tutorial']. Provide search_queries OR urls." - ), - ] = None, - max_results: Annotated[ - int, Field(ge=1, description="Maximum videos to return.") - ] = 10, - download_subtitles: Annotated[ - bool, - Field(description="True also fetches each video's transcript."), - ] = False, - subtitles_language: Annotated[ - str, Field(description="Transcript language code, e.g. 'en'.") - ] = "en", - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Search or scrape YouTube videos, optionally with transcripts. - - Use this for YouTube research: finding videos on a topic, or reading a - video's details or transcript. For a video's comment section use - surfsense_youtube_comments instead. Returns per-video metadata (title, - channel, views, description, url) and, if requested, the transcript. - Example: search_queries=['NotebookLM tutorial'], download_subtitles=True. - """ - return await run_scraper( - client, - context, - platform="youtube", - verb="scrape", - payload={ - "urls": urls, - "search_queries": search_queries, - "max_results": max_results, - "download_subtitles": download_subtitles, - "subtitles_language": subtitles_language, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_youtube_comments", - title="Fetch YouTube comments", - annotations=_SCRAPE, - structured_output=False, - ) - async def youtube_comments( - urls: Annotated[ - list[str], - Field( - min_length=1, - description="YouTube video URLs, e.g. " - "['https://www.youtube.com/watch?v=abc123'].", - ), - ], - max_comments: Annotated[ - int, - Field( - ge=1, - description="Maximum comments per video, counting top-level " - "comments and replies together.", - ), - ] = 20, - sort_by: Annotated[ - CommentSort, Field(description="Comment ordering.") - ] = "NEWEST_FIRST", - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Fetch the comments (and replies) on one or more YouTube videos. - - Use this when the user wants a video's discussion or audience reaction - rather than the video itself; get video URLs from - surfsense_youtube_scrape if you only have a topic. Returns comment - text, author, likes, and replies. - Example: urls=['https://www.youtube.com/watch?v=abc123'], max_comments=50. - """ - return await run_scraper( - client, - context, - platform="youtube", - verb="comments", - payload={ - "urls": urls, - "max_comments": max_comments, - "sort_by": sort_by, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_google_maps_scrape", - title="Find places on Google Maps", - annotations=_SCRAPE, - structured_output=False, - ) - async def google_maps_scrape( - search_queries: Annotated[ - list[str] | None, - Field( - description="Place searches, e.g. ['coffee shops']. Provide " - "search_queries OR urls OR place_ids." - ), - ] = None, - urls: Annotated[ - list[str] | None, - Field(description="Google Maps URLs of specific places."), - ] = None, - place_ids: Annotated[ - list[str] | None, - Field(description="Google place ids, e.g. ['ChIJj61dQgK6j4AR...']."), - ] = None, - location: Annotated[ - str | None, - Field( - description="Geographic scope for a search, e.g. " - "'Seattle, USA'." - ), - ] = None, - max_places: Annotated[ - int, Field(ge=1, description="Maximum places to return.") - ] = 10, - include_details: Annotated[ - bool, - Field( - description="True adds opening hours and extra contact info " - "(slower)." - ), - ] = False, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Find places on Google Maps by search, URL, or place id. - - Use this for local-business and location research: names, addresses, - ratings, categories, coordinates, place ids. For a place's customer - reviews use surfsense_google_maps_reviews instead. - Example: search_queries=['ramen'], location='Osaka, Japan', max_places=5. - """ - return await run_scraper( - client, - context, - platform="google_maps", - verb="scrape", - payload={ - "search_queries": search_queries, - "urls": urls, - "place_ids": place_ids, - "location": location, - "max_places": max_places, - "include_details": include_details, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_google_maps_reviews", - title="Fetch Google Maps reviews", - annotations=_SCRAPE, - structured_output=False, - ) - async def google_maps_reviews( - urls: Annotated[ - list[str] | None, - Field( - description="Google Maps URLs of places. Provide urls OR " - "place_ids." - ), - ] = None, - place_ids: Annotated[ - list[str] | None, - Field( - description="Google place ids from surfsense_google_maps_scrape." - ), - ] = None, - max_reviews: Annotated[ - int, Field(ge=1, description="Maximum reviews per place.") - ] = 20, - sort_by: Annotated[ - ReviewSort, Field(description="Review ordering.") - ] = "newest", - language: Annotated[ - str, Field(description="Reviews language code, e.g. 'en'.") - ] = "en", - start_date: Annotated[ - str | None, - Field( - description="ISO date like '2026-01-01'; keeps only reviews on " - "or after that day." - ), - ] = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Fetch customer reviews for Google Maps places by URL or place id. - - Use this to read feedback on specific places; get urls or place_ids - from surfsense_google_maps_scrape first if you only have a name. - Returns review text, rating, author, and date per review. - Example: place_ids=['ChIJj61dQgK6j4AR...'], sort_by='newest'. - """ - return await run_scraper( - client, - context, - platform="google_maps", - verb="reviews", - payload={ - "urls": urls, - "place_ids": place_ids, - "max_reviews": max_reviews, - "sort_by": sort_by, - "language": language, - "start_date": start_date, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_list_scraper_runs", - title="List past scraper runs", - annotations=_READ_RUNS, - structured_output=False, - ) - async def list_scraper_runs( - limit: Annotated[ - int, Field(ge=1, description="Maximum runs to list.") - ] = 20, - capability: Annotated[ - str | None, - Field( - description="Filter by capability slug, e.g. 'web.crawl' or " - "'reddit.scrape'." - ), - ] = None, - status: Annotated[ - str | None, - Field(description="Filter by run status: 'success' or 'error'."), - ] = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """List recent scraper runs in the workspace, newest first. - - Use this to find the run_id of an earlier scrape — for example when an - inline result was truncated — then fetch it in full with - surfsense_get_scraper_run. Returns each run's id, capability, status, - item count, and creation time. - Example: capability='reddit.scrape', status='success'. - """ - resolved = await context.resolve(workspace) - runs = await client.request( - "GET", - f"/workspaces/{resolved.id}/scrapers/runs", - params={ - "limit": limit, - "capability": capability, - "status": status, - }, - ) - if response_format == "json": - return to_json(runs) - return _render_runs(runs) - - @mcp.tool( - name="surfsense_get_scraper_run", - title="Fetch one scraper run in full", - annotations=_READ_RUNS, - structured_output=False, - ) - async def get_scraper_run( - run_id: Annotated[ - str, - Field( - description="Run id from surfsense_list_scraper_runs or a " - "prior scrape's output." - ), - ], - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Fetch a single scraper run in full, including its stored output. - - Use this to retrieve the complete, untruncated result of an earlier - scrape. Do NOT re-run a scraper just to recover a truncated result — - fetch the stored run instead. - """ - resolved = await context.resolve(workspace) - run = await client.request( - "GET", f"/workspaces/{resolved.id}/scrapers/runs/{run_id}" - ) - if response_format == "json": - return clip(to_json(run)) - return f"# Run {run.get('id', run_id)}\n\n```json\n{clip(to_json(run))}\n```" - - -def _render_runs(runs: list[dict] | None) -> str: - if not runs: - return "No scraper runs found." - lines = ["# Scraper runs", ""] - for run in runs: - lines.append( - f"- **{run.get('id')}** — {run.get('capability')} · {run.get('status')} · " - f"{run.get('item_count', 0)} item(s) · {run.get('created_at')}" - ) - return "\n".join(lines) + """Register every scraper and run-history tool on the server.""" + for module in _REGISTRARS: + module.register(mcp, client, context) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py new file mode 100644 index 000000000..cb872cc90 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py @@ -0,0 +1,13 @@ +"""Tool-call policy hints shared across scraper tools.""" + +from __future__ import annotations + +from mcp.types import ToolAnnotations + +SCRAPE = ToolAnnotations( + readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=True +) + +READ_RUNS = ToolAnnotations( + readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False +) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py new file mode 100644 index 000000000..f61704380 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py @@ -0,0 +1 @@ +"""One module per scraper platform; each exposes register(mcp, client, context).""" diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py new file mode 100644 index 000000000..e1613ca4e --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py @@ -0,0 +1,151 @@ +"""Google Maps scraper tools: places and reviews.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +ReviewSort = Literal["newest", "mostRelevant", "highestRanking", "lowestRanking"] + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the Google Maps place and review tools.""" + + @mcp.tool( + name="surfsense_google_maps_scrape", + title="Find places on Google Maps", + annotations=SCRAPE, + structured_output=False, + ) + async def google_maps_scrape( + search_queries: Annotated[ + list[str] | None, + Field( + description="Place searches, e.g. ['coffee shops']. Provide " + "search_queries OR urls OR place_ids." + ), + ] = None, + urls: Annotated[ + list[str] | None, + Field(description="Google Maps URLs of specific places."), + ] = None, + place_ids: Annotated[ + list[str] | None, + Field(description="Google place ids, e.g. ['ChIJj61dQgK6j4AR...']."), + ] = None, + location: Annotated[ + str | None, + Field( + description="Geographic scope for a search, e.g. " + "'Seattle, USA'." + ), + ] = None, + max_places: Annotated[ + int, Field(ge=1, description="Maximum places to return.") + ] = 10, + include_details: Annotated[ + bool, + Field( + description="True adds opening hours and extra contact info " + "(slower)." + ), + ] = False, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Find places on Google Maps by search, URL, or place id. + + Use this for local-business and location research: names, addresses, + ratings, categories, coordinates, place ids. For a place's customer + reviews use surfsense_google_maps_reviews instead. + Example: search_queries=['ramen'], location='Osaka, Japan', max_places=5. + """ + return await run_scraper( + client, + context, + platform="google_maps", + verb="scrape", + payload={ + "search_queries": search_queries, + "urls": urls, + "place_ids": place_ids, + "location": location, + "max_places": max_places, + "include_details": include_details, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_google_maps_reviews", + title="Fetch Google Maps reviews", + annotations=SCRAPE, + structured_output=False, + ) + async def google_maps_reviews( + urls: Annotated[ + list[str] | None, + Field( + description="Google Maps URLs of places. Provide urls OR " + "place_ids." + ), + ] = None, + place_ids: Annotated[ + list[str] | None, + Field( + description="Google place ids from surfsense_google_maps_scrape." + ), + ] = None, + max_reviews: Annotated[ + int, Field(ge=1, description="Maximum reviews per place.") + ] = 20, + sort_by: Annotated[ + ReviewSort, Field(description="Review ordering.") + ] = "newest", + language: Annotated[ + str, Field(description="Reviews language code, e.g. 'en'.") + ] = "en", + start_date: Annotated[ + str | None, + Field( + description="ISO date like '2026-01-01'; keeps only reviews on " + "or after that day." + ), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch customer reviews for Google Maps places by URL or place id. + + Use this to read feedback on specific places; get urls or place_ids + from surfsense_google_maps_scrape first if you only have a name. + Returns review text, rating, author, and date per review. + Example: place_ids=['ChIJj61dQgK6j4AR...'], sort_by='newest'. + """ + return await run_scraper( + client, + context, + platform="google_maps", + verb="reviews", + payload={ + "urls": urls, + "place_ids": place_ids, + "max_reviews": max_reviews, + "sort_by": sort_by, + "language": language, + "start_date": start_date, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py new file mode 100644 index 000000000..cc1a1f8ed --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py @@ -0,0 +1,79 @@ +"""Google Search scraper tool.""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the Google Search tool.""" + + @mcp.tool( + name="surfsense_google_search", + title="Scrape Google Search", + annotations=SCRAPE, + structured_output=False, + ) + async def google_search( + queries: Annotated[ + list[str], + Field( + min_length=1, + description="Search terms or full Google Search URLs, e.g. " + "['best rss readers 2026'].", + ), + ], + max_pages_per_query: Annotated[ + int, Field(ge=1, description="Result pages to fetch per query.") + ] = 1, + country_code: Annotated[ + str | None, + Field(description="Two-letter country to search from, e.g. 'us'."), + ] = None, + language_code: Annotated[ + str, Field(description="Results language, e.g. 'en'. Empty for default.") + ] = "", + site: Annotated[ + str | None, + Field( + description="Restrict results to one domain, e.g. 'example.com'." + ), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Scrape Google Search result pages for one or more queries. + + Use this to discover pages on the open web by topic; follow up with + surfsense_web_crawl to read a result in full. Do NOT use it for + Reddit, YouTube, or Google Maps research — the dedicated tools return + richer data. Returns each query's parsed results: title, url, and + snippet per organic result. + Example: queries=['notebooklm review'], site='news.ycombinator.com'. + """ + return await run_scraper( + client, + context, + platform="google_search", + verb="scrape", + payload={ + "queries": queries, + "max_pages_per_query": max_pages_per_query, + "country_code": country_code, + "language_code": language_code, + "site": site, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py new file mode 100644 index 000000000..035193ebc --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py @@ -0,0 +1,98 @@ +"""Reddit scraper tool.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +RedditSort = Literal["relevance", "hot", "top", "new", "rising", "comments"] +RedditTime = Literal["hour", "day", "week", "month", "year", "all"] + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the Reddit tool.""" + + @mcp.tool( + name="surfsense_reddit_scrape", + title="Search or scrape Reddit", + annotations=SCRAPE, + structured_output=False, + ) + async def reddit_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="Reddit URLs: a post, a subreddit like " + "'https://reddit.com/r/LocalLLaMA', a user page, or a search " + "URL. Provide urls OR search_queries." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Terms to search Reddit for, e.g. " + "['NotebookLM alternatives']. Provide search_queries OR urls." + ), + ] = None, + community: Annotated[ + str | None, + Field( + description="Restrict a search to one subreddit, name without " + "'r/', e.g. 'ArtificialInteligence'." + ), + ] = None, + sort: Annotated[RedditSort, Field(description="Post ordering.")] = "new", + time_filter: Annotated[ + RedditTime | None, + Field(description="Time window; only valid with sort='top'."), + ] = None, + max_items: Annotated[ + int, Field(ge=1, description="Maximum posts to return.") + ] = 10, + skip_comments: Annotated[ + bool, + Field( + description="True fetches posts only (faster); False also " + "fetches each post's comment thread." + ), + ] = False, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Search or scrape Reddit: posts, comments, subreddits, and users. + + Use this for ANY Reddit research — finding relevant subreddits or + communities for a topic, top posts, or discussions — instead of a + generic web search. Returns posts (title, text, score, subreddit, url) + with comment threads unless skip_comments is set. Every post carries + its subreddit, so to find communities for a topic, search posts and + aggregate their subreddits. + Example: search_queries=['NotebookLM'], sort='top', time_filter='month'. + """ + return await run_scraper( + client, + context, + platform="reddit", + verb="scrape", + payload={ + "urls": urls, + "search_queries": search_queries, + "community": community, + "sort": sort, + "time_filter": time_filter, + "max_items": max_items, + "skip_comments": skip_comments, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py new file mode 100644 index 000000000..9c24a4352 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py @@ -0,0 +1,89 @@ +"""Web crawl scraper tool.""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the web crawl tool.""" + + @mcp.tool( + name="surfsense_web_crawl", + title="Crawl web pages", + annotations=SCRAPE, + structured_output=False, + ) + async def web_crawl( + start_urls: Annotated[ + list[str], + Field( + min_length=1, + description="Full URLs to fetch, e.g. " + "['https://example.com/blog/post'].", + ), + ], + max_crawl_depth: Annotated[ + int, + Field( + ge=0, + description="Link-hops to follow from start_urls within the " + "same site. 0 fetches only start_urls.", + ), + ] = 0, + max_crawl_pages: Annotated[ + int, Field(ge=1, description="Stop after this many pages in total.") + ] = 10, + max_length: Annotated[ + int, Field(ge=1, description="Max characters kept per page.") + ] = 50_000, + include_url_patterns: Annotated[ + list[str] | None, + Field( + description="Regexes; only discovered links matching one are " + "followed, e.g. ['/docs/.*']." + ), + ] = None, + exclude_url_patterns: Annotated[ + list[str] | None, + Field(description="Regexes; discovered links matching one are skipped."), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch specific web pages and return their cleaned content as markdown. + + Use this to read a page the user names, or to spider a site from a + starting URL. Do NOT use it to find pages on a topic — use + surfsense_google_search for discovery. Returns one item per crawled + page: url, title, and the page text as markdown. + Example: start_urls=['https://blog.example.com'], max_crawl_depth=1, + include_url_patterns=['/2026/']. + """ + return await run_scraper( + client, + context, + platform="web", + verb="crawl", + payload={ + "startUrls": start_urls, + "maxCrawlDepth": max_crawl_depth, + "maxCrawlPages": max_crawl_pages, + "maxLength": max_length, + "includeUrlPatterns": include_url_patterns, + "excludeUrlPatterns": exclude_url_patterns, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py new file mode 100644 index 000000000..5582c82bb --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py @@ -0,0 +1,131 @@ +"""YouTube scraper tools: videos and comments.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +CommentSort = Literal["TOP_COMMENTS", "NEWEST_FIRST"] + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the YouTube video and comment tools.""" + + @mcp.tool( + name="surfsense_youtube_scrape", + title="Search or scrape YouTube", + annotations=SCRAPE, + structured_output=False, + ) + async def youtube_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="YouTube URLs: video, channel, playlist, shorts, " + "or hashtag pages. Provide urls OR search_queries." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Terms to search YouTube for, e.g. " + "['NotebookLM tutorial']. Provide search_queries OR urls." + ), + ] = None, + max_results: Annotated[ + int, Field(ge=1, description="Maximum videos to return.") + ] = 10, + download_subtitles: Annotated[ + bool, + Field(description="True also fetches each video's transcript."), + ] = False, + subtitles_language: Annotated[ + str, Field(description="Transcript language code, e.g. 'en'.") + ] = "en", + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Search or scrape YouTube videos, optionally with transcripts. + + Use this for YouTube research: finding videos on a topic, or reading a + video's details or transcript. For a video's comment section use + surfsense_youtube_comments instead. Returns per-video metadata (title, + channel, views, description, url) and, if requested, the transcript. + Example: search_queries=['NotebookLM tutorial'], download_subtitles=True. + """ + return await run_scraper( + client, + context, + platform="youtube", + verb="scrape", + payload={ + "urls": urls, + "search_queries": search_queries, + "max_results": max_results, + "download_subtitles": download_subtitles, + "subtitles_language": subtitles_language, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_youtube_comments", + title="Fetch YouTube comments", + annotations=SCRAPE, + structured_output=False, + ) + async def youtube_comments( + urls: Annotated[ + list[str], + Field( + min_length=1, + description="YouTube video URLs, e.g. " + "['https://www.youtube.com/watch?v=abc123'].", + ), + ], + max_comments: Annotated[ + int, + Field( + ge=1, + description="Maximum comments per video, counting top-level " + "comments and replies together.", + ), + ] = 20, + sort_by: Annotated[ + CommentSort, Field(description="Comment ordering.") + ] = "NEWEST_FIRST", + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch the comments (and replies) on one or more YouTube videos. + + Use this when the user wants a video's discussion or audience reaction + rather than the video itself; get video URLs from + surfsense_youtube_scrape if you only have a topic. Returns comment + text, author, likes, and replies. + Example: urls=['https://www.youtube.com/watch?v=abc123'], max_comments=50. + """ + return await run_scraper( + client, + context, + platform="youtube", + verb="comments", + payload={ + "urls": urls, + "max_comments": max_comments, + "sort_by": sort_by, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py new file mode 100644 index 000000000..9274a1a69 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py @@ -0,0 +1,112 @@ +"""Scraper run history: list past runs and fetch one in full. + +A scrape whose inline result was truncated is retrievable here by run id, so the +model never re-runs a scraper just to recover output. +""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ...core.client import SurfSenseClient +from ...core.rendering import ResponseFormatParam, clip, to_json +from ...core.workspace_context import WorkspaceContext, WorkspaceParam +from .annotations import READ_RUNS + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the run-history tools.""" + + @mcp.tool( + name="surfsense_list_scraper_runs", + title="List past scraper runs", + annotations=READ_RUNS, + structured_output=False, + ) + async def list_scraper_runs( + limit: Annotated[ + int, Field(ge=1, description="Maximum runs to list.") + ] = 20, + capability: Annotated[ + str | None, + Field( + description="Filter by capability slug, e.g. 'web.crawl' or " + "'reddit.scrape'." + ), + ] = None, + status: Annotated[ + str | None, + Field(description="Filter by run status: 'success' or 'error'."), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """List recent scraper runs in the workspace, newest first. + + Use this to find the run_id of an earlier scrape — for example when an + inline result was truncated — then fetch it in full with + surfsense_get_scraper_run. Returns each run's id, capability, status, + item count, and creation time. + Example: capability='reddit.scrape', status='success'. + """ + resolved = await context.resolve(workspace) + runs = await client.request( + "GET", + f"/workspaces/{resolved.id}/scrapers/runs", + params={ + "limit": limit, + "capability": capability, + "status": status, + }, + ) + if response_format == "json": + return to_json(runs) + return _render_runs(runs) + + @mcp.tool( + name="surfsense_get_scraper_run", + title="Fetch one scraper run in full", + annotations=READ_RUNS, + structured_output=False, + ) + async def get_scraper_run( + run_id: Annotated[ + str, + Field( + description="Run id from surfsense_list_scraper_runs or a " + "prior scrape's output." + ), + ], + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch a single scraper run in full, including its stored output. + + Use this to retrieve the complete, untruncated result of an earlier + scrape. Do NOT re-run a scraper just to recover a truncated result — + fetch the stored run instead. + """ + resolved = await context.resolve(workspace) + run = await client.request( + "GET", f"/workspaces/{resolved.id}/scrapers/runs/{run_id}" + ) + if response_format == "json": + return clip(to_json(run)) + return f"# Run {run.get('id', run_id)}\n\n```json\n{clip(to_json(run))}\n```" + + +def _render_runs(runs: list[dict] | None) -> str: + if not runs: + return "No scraper runs found." + lines = ["# Scraper runs", ""] + for run in runs: + lines.append( + f"- **{run.get('id')}** — {run.get('capability')} · {run.get('status')} · " + f"{run.get('item_count', 0)} item(s) · {run.get('created_at')}" + ) + return "\n".join(lines) diff --git a/surfsense_mcp/src/surfsense_mcp/selfcheck.py b/surfsense_mcp/src/surfsense_mcp/selfcheck.py index e38edc909..20224c173 100644 --- a/surfsense_mcp/src/surfsense_mcp/selfcheck.py +++ b/surfsense_mcp/src/surfsense_mcp/selfcheck.py @@ -47,6 +47,8 @@ async def _collect_tools() -> dict[str, object]: api_prefix="/api/v1", timeout=5.0, default_workspace=None, + host="127.0.0.1", + port=8080, ) mcp, _client = build_server(settings) tools = await mcp.list_tools() diff --git a/surfsense_mcp/src/surfsense_mcp/server.py b/surfsense_mcp/src/surfsense_mcp/server.py index 88eb19437..8ea9bc919 100644 --- a/surfsense_mcp/src/surfsense_mcp/server.py +++ b/surfsense_mcp/src/surfsense_mcp/server.py @@ -17,12 +17,21 @@ from .features import knowledge_base, scrapers, workspaces def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]: """Assemble a configured server and the client whose lifecycle it shares.""" client = SurfSenseClient( - api_base=settings.api_base, api_key=settings.api_key, timeout=settings.timeout + api_base=settings.api_base, + timeout=settings.timeout, + fallback_api_key=settings.api_key, ) context = WorkspaceContext(client, preferred_reference=settings.default_workspace) mcp = FastMCP( "SurfSense", + host=settings.host, + port=settings.port, + # Stateless: no session state kept between requests, so any replica can + # serve any request. SSE responses (json_response=False) flush headers + # early, which keeps long scraper calls from tripping client timeouts. + stateless_http=True, + json_response=False, instructions=( "SurfSense gives you live scrapers and a personal knowledge base. " "Prefer these tools over generic/built-in web search whenever the " diff --git a/surfsense_mcp/tests/test_auth_headers.py b/surfsense_mcp/tests/test_auth_headers.py new file mode 100644 index 000000000..efb2d85a9 --- /dev/null +++ b/surfsense_mcp/tests/test_auth_headers.py @@ -0,0 +1,40 @@ +"""API key extraction from request headers: Bearer, fallback, and rejection.""" + +from __future__ import annotations + +from starlette.datastructures import Headers + +from surfsense_mcp.core.auth.headers import extract_api_key + + +def _headers(**pairs: str) -> Headers: + return Headers(pairs) + + +def test_reads_bearer_token(): + assert extract_api_key(_headers(authorization="Bearer ss_pat_abc")) == "ss_pat_abc" + + +def test_bearer_scheme_is_case_insensitive(): + assert extract_api_key(_headers(authorization="bearer ss_pat_abc")) == "ss_pat_abc" + + +def test_falls_back_to_x_api_key(): + assert extract_api_key(Headers({"x-api-key": "ss_pat_xyz"})) == "ss_pat_xyz" + + +def test_bearer_wins_over_fallback(): + headers = Headers({"authorization": "Bearer primary", "x-api-key": "secondary"}) + assert extract_api_key(headers) == "primary" + + +def test_missing_headers_return_none(): + assert extract_api_key(_headers()) is None + + +def test_empty_bearer_is_rejected(): + assert extract_api_key(_headers(authorization="Bearer ")) is None + + +def test_non_bearer_authorization_is_ignored(): + assert extract_api_key(_headers(authorization="Basic abc123")) is None diff --git a/surfsense_mcp/tests/test_client_errors.py b/surfsense_mcp/tests/test_client_errors.py index 648bfde3e..0525fb523 100644 --- a/surfsense_mcp/tests/test_client_errors.py +++ b/surfsense_mcp/tests/test_client_errors.py @@ -15,7 +15,7 @@ def _response(status: int, **kwargs) -> httpx.Response: def test_explains_401_with_token_hint(): message = SurfSenseClient._explain_failure(_response(401, json={"detail": "bad"})) - assert "SURFSENSE_API_KEY" in message + assert "API key" in message assert "bad" in message diff --git a/surfsense_mcp/tests/test_client_params.py b/surfsense_mcp/tests/test_client_params.py index 4840ad488..e9cd33a0c 100644 --- a/surfsense_mcp/tests/test_client_params.py +++ b/surfsense_mcp/tests/test_client_params.py @@ -25,7 +25,9 @@ def _capture(client: SurfSenseClient) -> dict: def test_none_params_are_dropped(): - client = SurfSenseClient(api_base="http://test/api/v1", api_key="ss_pat_x", timeout=5) + client = SurfSenseClient( + api_base="http://test/api/v1", timeout=5, fallback_api_key="ss_pat_x" + ) seen = _capture(client) asyncio.run( client.request( diff --git a/surfsense_mcp/tests/test_request_auth.py b/surfsense_mcp/tests/test_request_auth.py new file mode 100644 index 000000000..c548dff7a --- /dev/null +++ b/surfsense_mcp/tests/test_request_auth.py @@ -0,0 +1,100 @@ +"""Per-request key resolution and the Authorization header the backend receives. + +Covers the security-critical behaviors: the per-request key wins over the env +fallback, the fallback covers stdio, a missing key is refused, and concurrent +callers never see each other's key. +""" + +from __future__ import annotations + +import asyncio + +import httpx +import pytest + +from surfsense_mcp.core.auth import identity +from surfsense_mcp.core.client import SurfSenseClient +from surfsense_mcp.core.errors import ToolError + + +def _client_recording_auth(seen: dict, *, fallback: str | None) -> SurfSenseClient: + async def handler(request: httpx.Request) -> httpx.Response: + seen["authorization"] = request.headers.get("authorization") + return httpx.Response(200, json={"ok": True}) + + client = SurfSenseClient( + api_base="http://test/api/v1", timeout=5, fallback_api_key=fallback + ) + client._http = httpx.AsyncClient( + base_url="http://test/api/v1", transport=httpx.MockTransport(handler) + ) + return client + + +async def _get(client: SurfSenseClient) -> None: + await client.request("GET", "/workspaces") + + +def test_request_key_is_sent_as_bearer(): + seen: dict = {} + client = _client_recording_auth(seen, fallback=None) + + async def run() -> None: + token = identity.bind_api_key("ss_pat_request") + try: + await _get(client) + finally: + identity.unbind_api_key(token) + + asyncio.run(run()) + assert seen["authorization"] == "Bearer ss_pat_request" + + +def test_request_key_overrides_env_fallback(): + seen: dict = {} + client = _client_recording_auth(seen, fallback="ss_pat_env") + + async def run() -> None: + token = identity.bind_api_key("ss_pat_request") + try: + await _get(client) + finally: + identity.unbind_api_key(token) + + asyncio.run(run()) + assert seen["authorization"] == "Bearer ss_pat_request" + + +def test_env_fallback_used_without_request_key(): + seen: dict = {} + client = _client_recording_auth(seen, fallback="ss_pat_env") + asyncio.run(_get(client)) + assert seen["authorization"] == "Bearer ss_pat_env" + + +def test_missing_key_is_refused(): + client = _client_recording_auth({}, fallback=None) + with pytest.raises(ToolError): + asyncio.run(_get(client)) + + +def test_concurrent_callers_do_not_leak_keys(): + seen_by_caller: dict[str, str | None] = {} + + async def call_as(key: str) -> None: + # Each caller runs in its own task, so the contextvar is isolated. + recorded: dict = {} + client = _client_recording_auth(recorded, fallback=None) + token = identity.bind_api_key(key) + try: + await _get(client) + finally: + identity.unbind_api_key(token) + seen_by_caller[key] = recorded["authorization"] + + async def run() -> None: + await asyncio.gather(call_as("ss_pat_A"), call_as("ss_pat_B")) + + asyncio.run(run()) + assert seen_by_caller["ss_pat_A"] == "Bearer ss_pat_A" + assert seen_by_caller["ss_pat_B"] == "Bearer ss_pat_B" diff --git a/surfsense_mcp/tests/test_workspace_context.py b/surfsense_mcp/tests/test_workspace_context.py index 9ba9c2ca3..d2024e5d5 100644 --- a/surfsense_mcp/tests/test_workspace_context.py +++ b/surfsense_mcp/tests/test_workspace_context.py @@ -6,6 +6,7 @@ import asyncio import pytest +from surfsense_mcp.core.auth import identity from surfsense_mcp.core.errors import ToolError from surfsense_mcp.core.workspace_context import WorkspaceContext @@ -96,3 +97,30 @@ def test_resolution_is_remembered_as_active(): assert ctx.active is not None and ctx.active.id == 2 # a later default call reuses the active selection without re-choosing assert asyncio.run(ctx.resolve(None)).id == 2 + + +def test_active_workspace_is_isolated_per_identity(): + ctx = _context(_rows(("A", 1), ("B", 2))) + + async def select_as(key: str, reference: str) -> None: + token = identity.bind_api_key(key) + try: + await ctx.resolve(reference) + finally: + identity.unbind_api_key(token) + + async def active_for(key: str) -> int | None: + token = identity.bind_api_key(key) + try: + return ctx.active.id if ctx.active else None + finally: + identity.unbind_api_key(token) + + asyncio.run(select_as("ss_pat_A", "A")) + asyncio.run(select_as("ss_pat_B", "B")) + + # Each caller keeps its own selection; no bleed across identities. + assert asyncio.run(active_for("ss_pat_A")) == 1 + assert asyncio.run(active_for("ss_pat_B")) == 2 + # An unknown caller has no active selection. + assert asyncio.run(active_for("ss_pat_C")) is None diff --git a/surfsense_mcp/uv.lock b/surfsense_mcp/uv.lock index bc97a9b2a..d1bfab702 100644 --- a/surfsense_mcp/uv.lock +++ b/surfsense_mcp/uv.lock @@ -718,6 +718,8 @@ source = { editable = "." } dependencies = [ { name = "httpx" }, { name = "mcp" }, + { name = "starlette" }, + { name = "uvicorn" }, ] [package.dev-dependencies] @@ -729,6 +731,8 @@ dev = [ requires-dist = [ { name = "httpx", specifier = ">=0.27.0" }, { name = "mcp", specifier = ">=1.26.0" }, + { name = "starlette", specifier = ">=0.37" }, + { name = "uvicorn", specifier = ">=0.30" }, ] [package.metadata.requires-dev] diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index 6dfc69622..5e4961b1e 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -50,16 +50,13 @@ export const metadata: Metadata = { }, }; -/* Mirrors surfsense_mcp/README.md: the real Cursor config. */ +/* The hosted Cursor config; mirrors lib/mcp/clients.ts. */ const CURSOR_CONFIG = `{ "mcpServers": { "surfsense": { - "command": "uv", - "args": ["run", "--directory", ".../surfsense_mcp", - "python", "-m", "surfsense_mcp"], - "env": { - "SURFSENSE_BASE_URL": "https://api.surfsense.com", - "SURFSENSE_API_KEY": "ss_pat_..." + "url": "https://mcp.surfsense.com/mcp", + "headers": { + "Authorization": "Bearer ss_pat_..." } } } @@ -76,7 +73,7 @@ const STEPS = [ icon: TerminalSquare, title: "Add the server to your client", description: - "Drop the config into Cursor's mcp.json, run claude mcp add for Claude Code, or paste it into Claude Desktop. Point it at the cloud or your own self-hosted instance.", + "Point your client at https://mcp.surfsense.com/mcp with your key in an Authorization header — the hosted config for Cursor, Claude Code, and others is one paste. Prefer stdio? Switch to Self-host and run it against your own backend.", }, { icon: Server, @@ -135,7 +132,7 @@ const FAQ: FaqItem[] = [ { question: "Which MCP clients does it work with?", answer: - "Any MCP client that supports stdio servers. Claude Code, Codex, OpenCode, Cursor, Claude Desktop, VS Code, Windsurf, and Gemini CLI are documented with copy-paste configs on this page, and the same command works in custom agent harnesses built on the MCP SDK.", + "Any MCP client that speaks remote (streamable HTTP) or stdio. Claude Code, Codex, OpenCode, Cursor, Claude Desktop, VS Code, Windsurf, and Gemini CLI all have copy-paste configs on this page — Hosted for the one-paste https://mcp.surfsense.com/mcp endpoint, or Self-host for stdio against your own backend.", }, { question: "How is usage billed?", @@ -278,9 +275,9 @@ export default function McpServerPage() { Step-by-step setup for every agent

- Pick your client, follow its two steps, and paste the config. Replace the placeholder - path with your surfsense_mcp checkout and the key with one from API Playground → API - Keys — or grab a pre-filled config from the playground itself. + Pick your client, choose Hosted or Self-host, and + paste the config. Replace the key with one from API Playground → API Keys — or grab a + pre-filled config from the playground itself.

diff --git a/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx b/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx index 6ca19c47b..f25faf95b 100644 --- a/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx +++ b/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx @@ -206,7 +206,7 @@ function buildMcp({ mcpTool }: ApiSample): string { const config = { mcpServers: { surfsense: { - url: "https://mcp.surfsense.com", + url: "https://mcp.surfsense.com/mcp", headers: { Authorization: "Bearer ${SURFSENSE_API_KEY}" }, }, }, diff --git a/surfsense_web/components/mcp/agent-setup-tabs.tsx b/surfsense_web/components/mcp/agent-setup-tabs.tsx index 1701b8924..3790441f2 100644 --- a/surfsense_web/components/mcp/agent-setup-tabs.tsx +++ b/surfsense_web/components/mcp/agent-setup-tabs.tsx @@ -9,6 +9,8 @@ import { DEFAULT_SERVER_DIR, MCP_CLIENTS, type McpSnippetOptions, + type McpTransport, + REMOTE_URL, } from "@/lib/mcp/clients"; function CopyButton({ text }: { text: string }) { @@ -38,48 +40,82 @@ function CopyButton({ text }: { text: string }) { ); } +const TRANSPORTS: { id: McpTransport; label: string; hint: string }[] = [ + { id: "remote", label: "Hosted", hint: "mcp.surfsense.com — nothing to install" }, + { id: "stdio", label: "Self-host", hint: "run the server against your own backend" }, +]; + /** - * Per-agent MCP setup instructions as tabs: pick a client, follow its steps, - * copy its exact config. Used on the /mcp-server marketing page and in the - * API playground; `options` fills in real values where the caller has them. + * Per-agent MCP setup instructions as tabs: pick a client, then Hosted or + * Self-host, and copy its exact config. Used on the /mcp-server marketing page + * and in the API playground; `options` fills in real values where the caller + * has them. */ export function AgentSetupTabs({ options }: { options?: Partial }) { + const [transport, setTransport] = useState("remote"); + const resolved: McpSnippetOptions = { - baseUrl: options?.baseUrl || "https://api.surfsense.com", + remoteUrl: options?.remoteUrl || REMOTE_URL, apiKey: options?.apiKey || API_KEY_PLACEHOLDER, + baseUrl: options?.baseUrl || "https://api.surfsense.com", serverDir: options?.serverDir || DEFAULT_SERVER_DIR, }; + const active = TRANSPORTS.find((t) => t.id === transport) ?? TRANSPORTS[0]; + return ( - - - {MCP_CLIENTS.map((client) => ( - - {client.label} - - ))} - - {MCP_CLIENTS.map((client) => { - const config = client.buildConfig(resolved); - return ( - -
    - {client.steps.map((step) => ( -
  1. {step}
    2. - ))} -
-
-

{client.configFile}

-
- - 
-									{config}
-
+
+
+
+ {TRANSPORTS.map((t) => ( + + ))} +
+ {active.hint} +
+ + + + {MCP_CLIENTS.map((client) => ( + + {client.label} + + ))} + + {MCP_CLIENTS.map((client) => { + const snippet = client[transport]; + const config = snippet.build(resolved); + return ( + +
    + {snippet.steps.map((step) => ( +
  1. {step}
    2. + ))} +
+
+

+ {snippet.configFile} +

+
+ + 
+										{config}
+
+
-
- - ); - })} - + + ); + })} + +
); } diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx index ffdb4fd56..62628a659 100644 --- a/surfsense_web/content/docs/how-to/mcp-server.mdx +++ b/surfsense_web/content/docs/how-to/mcp-server.mdx @@ -4,51 +4,58 @@ description: Connect the SurfSense MCP server to Claude Code, Codex, OpenCode, C --- import { Tab, Tabs } from 'fumadocs-ui/components/tabs'; -import { Step, Steps } from 'fumadocs-ui/components/steps'; # SurfSense MCP Server The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 18 native, typed tools: every scraper (Reddit, YouTube, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. -It talks to SurfSense purely over the REST API — point it at SurfSense Cloud or your own self-hosted instance by changing one environment variable. +Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted. -## Prerequisites +## Create an API key - - +You need a SurfSense API key either way. In SurfSense, open **API Playground → API Keys** in your workspace sidebar: -### Install uv +1. Toggle **API key access** on for the workspace. +2. Create a personal API key (`ss_pat_…`) and copy it — it is shown only once. -The server runs with [uv](https://github.com/astral-sh/uv). Install it once, then from the SurfSense repository run: +## Connect (hosted) + +The hosted server runs at `https://mcp.surfsense.com/mcp`. Point your client at it and send the key as a Bearer token — there is nothing to install and no backend to run. For clients that read an `mcpServers` map (Cursor, and others): + +```json +{ + "mcpServers": { + "surfsense": { + "url": "https://mcp.surfsense.com/mcp", + "headers": { "Authorization": "Bearer ss_pat_your_key_here" } + } + } +} +``` + +Claude Code, from a terminal: + +```bash +claude mcp add --transport http surfsense https://mcp.surfsense.com/mcp \ + --header "Authorization: Bearer ss_pat_your_key_here" +``` + +Most MCP clients accept this `url` + `headers` form; check your client's docs for its exact remote-server field. + +## Self-host (stdio) + +Run the server yourself when you host your own backend or use a client without remote support. It runs with [uv](https://github.com/astral-sh/uv) — install it once, then from the SurfSense repository run: ```bash cd surfsense_mcp uv sync ``` - - - -### Create an API key - -In SurfSense, open **API Playground → API Keys** in your workspace sidebar: - -1. Toggle **API key access** on for the workspace. -2. Create a personal API key (`ss_pat_…`) and copy it — it is shown only once. - - - - -### Know your base URL +Point the server at your backend with `SURFSENSE_BASE_URL`: - **SurfSense Cloud**: `https://api.surfsense.com` - **Self-hosted**: wherever your backend runs, e.g. `http://localhost:8000` - - - -## Connect your agent - Every client below launches the same command — `uv run --directory /surfsense_mcp python -m surfsense_mcp` — and passes `SURFSENSE_BASE_URL` and `SURFSENSE_API_KEY` as environment variables. Replace the placeholder paths and key with yours. @@ -221,7 +228,7 @@ Run `/mcp` inside Gemini CLI to confirm the server and its tools. -The server uses stdio transport: your client launches the process on demand and shuts it down with the session. There is no daemon to manage — only your SurfSense backend needs to be up. +In this mode your client launches the process on demand and shuts it down with the session. There is no daemon to manage — only your SurfSense backend needs to be up. (The hosted server above needs none of this.) ## Test it @@ -236,7 +243,7 @@ That calls `surfsense_list_workspaces` — the simplest end-to-end check of the ## Configuration reference -All settings are environment variables passed by the client: +For self-host (stdio), all settings are environment variables passed by the client. The hosted server needs only your API key in the `Authorization` header: | Variable | Required | Default | Purpose | |----------|----------|---------|---------| diff --git a/surfsense_web/lib/mcp/clients.ts b/surfsense_web/lib/mcp/clients.ts index 923e99049..795832ad1 100644 --- a/surfsense_web/lib/mcp/clients.ts +++ b/surfsense_web/lib/mcp/clients.ts @@ -1,42 +1,75 @@ /** - * MCP client setup catalog: one entry per popular agent, with the exact - * config file, steps, and snippet needed to connect the SurfSense MCP server. - * Shared by the marketing /mcp-server page and the API playground so the - * instructions can never drift apart. + * MCP client setup catalog: one entry per popular agent, each with a hosted + * (remote) snippet and a self-host (stdio) snippet, plus the exact config file + * and steps. Shared by the marketing /mcp-server page and the API playground so + * the instructions can never drift apart. + * + * Remote snippets point at the hosted server and pass the key as a Bearer token; + * every client's exact remote field is verified against its own docs (Windsurf + * uses `serverUrl`, Gemini CLI `httpUrl`, VS Code needs `type: "http"`, OpenCode + * `type: "remote"` + `oauth: false`, Codex needs the rmcp flag, and Claude + * Desktop has no config-file remote support so it uses the `mcp-remote` bridge). */ +export type McpTransport = "remote" | "stdio"; + export interface McpSnippetOptions { - /** SurfSense backend URL the server should call. */ - baseUrl: string; + /** Hosted MCP endpoint (Bearer-authenticated). */ + remoteUrl: string; /** API key value or placeholder to show in the snippet. */ apiKey: string; - /** Absolute path to the surfsense_mcp directory. */ + /** SurfSense backend URL a self-hosted server should call. */ + baseUrl: string; + /** Absolute path to the surfsense_mcp directory (self-host). */ serverDir: string; } +export interface McpSnippet { + /** Where the snippet goes: a file path or "Terminal". */ + configFile: string; + language: "json" | "toml" | "bash"; + steps: string[]; + build: (options: McpSnippetOptions) => string; +} + export interface McpClient { id: string; label: string; - /** Where the snippet goes: a file path or "Terminal". */ - configFile: string; - language: "json" | "toml" | "bash"; - steps: string[]; - buildConfig: (options: McpSnippetOptions) => string; + remote: McpSnippet; + stdio: McpSnippet; } +export const REMOTE_URL = "https://mcp.surfsense.com/mcp"; export const DEFAULT_SERVER_DIR = "/path/to/SurfSense/surfsense_mcp"; export const API_KEY_PLACEHOLDER = "ss_pat_your_key_here"; -function serverArgs(serverDir: string): string[] { - return ["run", "--directory", serverDir, "python", "-m", "surfsense_mcp"]; -} - function json(value: unknown): string { return JSON.stringify(value, null, 2); } -/** The `mcpServers` JSON shape shared by Cursor, Claude Desktop, Windsurf, and Gemini CLI. */ -function standardJson({ baseUrl, apiKey, serverDir }: McpSnippetOptions): string { +function bearer(apiKey: string): string { + return `Bearer ${apiKey}`; +} + +function serverArgs(serverDir: string): string[] { + return ["run", "--directory", serverDir, "python", "-m", "surfsense_mcp"]; +} + +/** The `mcpServers` remote shape shared by Cursor, Windsurf, and Gemini CLI. */ +function remoteMcpServers(urlField: "url" | "serverUrl" | "httpUrl") { + return ({ remoteUrl, apiKey }: McpSnippetOptions): string => + json({ + mcpServers: { + surfsense: { + [urlField]: remoteUrl, + headers: { Authorization: bearer(apiKey) }, + }, + }, + }); +} + +/** The `mcpServers` stdio shape shared by Cursor, Claude Desktop, Windsurf, Gemini CLI. */ +function stdioMcpServers({ baseUrl, apiKey, serverDir }: McpSnippetOptions): string { return json({ mcpServers: { surfsense: { @@ -52,125 +85,255 @@ export const MCP_CLIENTS: McpClient[] = [ { id: "claude-code", label: "Claude Code", - configFile: "Terminal", - language: "bash", - steps: [ - "Run this command in a terminal (any directory).", - "Start Claude Code and run /mcp — surfsense should be listed as connected.", - ], - buildConfig: ({ baseUrl, apiKey, serverDir }) => - [ - "claude mcp add surfsense \\", - ` -e SURFSENSE_BASE_URL=${baseUrl} \\`, - ` -e SURFSENSE_API_KEY=${apiKey} \\`, - ` -- uv run --directory ${serverDir} python -m surfsense_mcp`, - ].join("\n"), + remote: { + configFile: "Terminal", + language: "bash", + steps: [ + "Run this command in a terminal (any directory).", + "Start Claude Code and run /mcp — surfsense should be listed as connected.", + ], + build: ({ remoteUrl, apiKey }) => + [ + `claude mcp add --transport http surfsense ${remoteUrl} \\`, + ` --header "Authorization: ${bearer(apiKey)}"`, + ].join("\n"), + }, + stdio: { + configFile: "Terminal", + language: "bash", + steps: [ + "Run this command in a terminal (any directory).", + "Start Claude Code and run /mcp — surfsense should be listed as connected.", + ], + build: ({ baseUrl, apiKey, serverDir }) => + [ + "claude mcp add surfsense \\", + ` -e SURFSENSE_BASE_URL=${baseUrl} \\`, + ` -e SURFSENSE_API_KEY=${apiKey} \\`, + ` -- uv run --directory ${serverDir} python -m surfsense_mcp`, + ].join("\n"), + }, }, { id: "codex", label: "Codex", - configFile: "~/.codex/config.toml", - language: "toml", - steps: [ - "Add this to ~/.codex/config.toml (or run `codex mcp add surfsense -- uv run --directory python -m surfsense_mcp`).", - "Restart Codex; `codex mcp list` should show surfsense.", - ], - buildConfig: ({ baseUrl, apiKey, serverDir }) => - [ - "[mcp_servers.surfsense]", - 'command = "uv"', - `args = ${JSON.stringify(serverArgs(serverDir))}`, - "", - "[mcp_servers.surfsense.env]", - `SURFSENSE_BASE_URL = "${baseUrl}"`, - `SURFSENSE_API_KEY = "${apiKey}"`, - ].join("\n"), + remote: { + configFile: "~/.codex/config.toml", + language: "toml", + steps: [ + "Add this to ~/.codex/config.toml. The rmcp flag must sit above every [mcp_servers.*] table.", + "Restart Codex; `codex mcp list` should show surfsense.", + ], + build: ({ remoteUrl, apiKey }) => + [ + "experimental_use_rmcp_client = true", + "", + "[mcp_servers.surfsense]", + `url = "${remoteUrl}"`, + "", + "[mcp_servers.surfsense.http_headers]", + `Authorization = "${bearer(apiKey)}"`, + ].join("\n"), + }, + stdio: { + configFile: "~/.codex/config.toml", + language: "toml", + steps: [ + "Add this to ~/.codex/config.toml (or run `codex mcp add surfsense -- uv run --directory python -m surfsense_mcp`).", + "Restart Codex; `codex mcp list` should show surfsense.", + ], + build: ({ baseUrl, apiKey, serverDir }) => + [ + "[mcp_servers.surfsense]", + 'command = "uv"', + `args = ${JSON.stringify(serverArgs(serverDir))}`, + "", + "[mcp_servers.surfsense.env]", + `SURFSENSE_BASE_URL = "${baseUrl}"`, + `SURFSENSE_API_KEY = "${apiKey}"`, + ].join("\n"), + }, }, { id: "opencode", label: "OpenCode", - configFile: "opencode.json", - language: "json", - steps: [ - "Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).", - "Note OpenCode's format: the key is `mcp`, the command is one array, and env vars go under `environment`.", - ], - buildConfig: ({ baseUrl, apiKey, serverDir }) => - json({ - $schema: "https://opencode.ai/config.json", - mcp: { - surfsense: { - type: "local", - command: ["uv", ...serverArgs(serverDir)], - enabled: true, - environment: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + remote: { + configFile: "opencode.json", + language: "json", + steps: [ + "Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).", + "`oauth: false` tells OpenCode to use the Bearer key instead of starting an OAuth flow.", + ], + build: ({ remoteUrl, apiKey }) => + json({ + $schema: "https://opencode.ai/config.json", + mcp: { + surfsense: { + type: "remote", + url: remoteUrl, + enabled: true, + oauth: false, + headers: { Authorization: bearer(apiKey) }, + }, }, - }, - }), + }), + }, + stdio: { + configFile: "opencode.json", + language: "json", + steps: [ + "Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).", + "Note OpenCode's format: the key is `mcp`, the command is one array, and env vars go under `environment`.", + ], + build: ({ baseUrl, apiKey, serverDir }) => + json({ + $schema: "https://opencode.ai/config.json", + mcp: { + surfsense: { + type: "local", + command: ["uv", ...serverArgs(serverDir)], + enabled: true, + environment: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + }, + }, + }), + }, }, { id: "cursor", label: "Cursor", - configFile: "~/.cursor/mcp.json", - language: "json", - steps: [ - "Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.", - "Refresh the server in Cursor Settings → MCP; its 18 tools should appear.", - ], - buildConfig: standardJson, + remote: { + configFile: "~/.cursor/mcp.json", + language: "json", + steps: [ + "Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.", + "Refresh the server in Cursor Settings → MCP; its 18 tools should appear.", + ], + build: remoteMcpServers("url"), + }, + stdio: { + configFile: "~/.cursor/mcp.json", + language: "json", + steps: [ + "Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.", + "Refresh the server in Cursor Settings → MCP; its 18 tools should appear.", + ], + build: stdioMcpServers, + }, }, { id: "claude-desktop", label: "Claude Desktop", - configFile: "claude_desktop_config.json", - language: "json", - steps: [ - "Open Settings → Developer → Edit Config to reach claude_desktop_config.json and add this.", - "Restart Claude Desktop; surfsense appears under the tools icon.", - ], - buildConfig: standardJson, + remote: { + configFile: "claude_desktop_config.json", + language: "json", + steps: [ + "Claude Desktop can't take a remote URL directly, so this uses the mcp-remote bridge (needs Node 18+).", + "Open Settings → Developer → Edit Config, add this, and restart Claude Desktop.", + ], + build: ({ remoteUrl, apiKey }) => + json({ + mcpServers: { + surfsense: { + command: "npx", + args: ["-y", "mcp-remote", remoteUrl, "--header", `Authorization: ${bearer(apiKey)}`], + }, + }, + }), + }, + stdio: { + configFile: "claude_desktop_config.json", + language: "json", + steps: [ + "Open Settings → Developer → Edit Config to reach claude_desktop_config.json and add this.", + "Restart Claude Desktop; surfsense appears under the tools icon.", + ], + build: stdioMcpServers, + }, }, { id: "vscode", label: "VS Code", - configFile: ".vscode/mcp.json", - language: "json", - steps: [ - "Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).", - "Open Copilot Chat in agent mode and click the tools icon to confirm surfsense is loaded.", - ], - buildConfig: ({ baseUrl, apiKey, serverDir }) => - json({ - servers: { - surfsense: { - type: "stdio", - command: "uv", - args: serverArgs(serverDir), - env: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + remote: { + configFile: ".vscode/mcp.json", + language: "json", + steps: [ + "Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).", + "VS Code requires an explicit `type` field — `http` for the hosted server.", + ], + build: ({ remoteUrl, apiKey }) => + json({ + servers: { + surfsense: { + type: "http", + url: remoteUrl, + headers: { Authorization: bearer(apiKey) }, + }, }, - }, - }), + }), + }, + stdio: { + configFile: ".vscode/mcp.json", + language: "json", + steps: [ + "Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).", + "Open Copilot Chat in agent mode and click the tools icon to confirm surfsense is loaded.", + ], + build: ({ baseUrl, apiKey, serverDir }) => + json({ + servers: { + surfsense: { + type: "stdio", + command: "uv", + args: serverArgs(serverDir), + env: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + }, + }, + }), + }, }, { id: "windsurf", label: "Windsurf", - configFile: "~/.codeium/windsurf/mcp_config.json", - language: "json", - steps: [ - "Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).", - "Press the refresh button in the MCP panel to pick up the server.", - ], - buildConfig: standardJson, + remote: { + configFile: "~/.codeium/windsurf/mcp_config.json", + language: "json", + steps: [ + "Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).", + "Windsurf uses `serverUrl` (not `url`) for remote servers; press refresh in the MCP panel.", + ], + build: remoteMcpServers("serverUrl"), + }, + stdio: { + configFile: "~/.codeium/windsurf/mcp_config.json", + language: "json", + steps: [ + "Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).", + "Press the refresh button in the MCP panel to pick up the server.", + ], + build: stdioMcpServers, + }, }, { id: "gemini-cli", label: "Gemini CLI", - configFile: "~/.gemini/settings.json", - language: "json", - steps: [ - "Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).", - "Run /mcp inside Gemini CLI to confirm the surfsense server and its tools.", - ], - buildConfig: standardJson, + remote: { + configFile: "~/.gemini/settings.json", + language: "json", + steps: [ + "Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).", + "Gemini CLI uses `httpUrl` for streamable-HTTP servers; run /mcp to confirm surfsense.", + ], + build: remoteMcpServers("httpUrl"), + }, + stdio: { + configFile: "~/.gemini/settings.json", + language: "json", + steps: [ + "Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).", + "Run /mcp inside Gemini CLI to confirm the surfsense server and its tools.", + ], + build: stdioMcpServers, + }, }, ];