diff --git a/docker/.env.example b/docker/.env.example index 28f9356c8..5daa0f3e8 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -445,10 +445,10 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # PLATFORM_SCRAPE_BILLING_ENABLED=FALSE # REDDIT_SCRAPE_MICROS_PER_ITEM=3500 # GOOGLE_SEARCH_MICROS_PER_SERP=5500 -# GOOGLE_MAPS_MICROS_PER_PLACE=5000 -# GOOGLE_MAPS_MICROS_PER_REVIEW=2000 -# YOUTUBE_MICROS_PER_VIDEO=3500 -# YOUTUBE_MICROS_PER_COMMENT=3500 +# GOOGLE_MAPS_MICROS_PER_PLACE=3500 +# GOOGLE_MAPS_MICROS_PER_REVIEW=1500 +# YOUTUBE_MICROS_PER_VIDEO=2500 +# YOUTUBE_MICROS_PER_COMMENT=1500 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 @@ -492,6 +492,28 @@ NOLOGIN_MODE_ENABLED=FALSE # PROXY_URL=http://user:pass@host:port # PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port +# Captcha solving — last-resort bypass tier via captchatools. Only fires on the +# stealth browser tier when a sitekey is detected AND the flag is TRUE. +# Cloudflare Turnstile is already solved free in-framework. Off by default. +# NOTE: automated solving may violate a target site's ToS — opt-in, public +# data only. See surfsense_backend/.env.example for the full option docs. +# CAPTCHA_SOLVING_ENABLED=FALSE +# CAPTCHA_SOLVER_PROVIDER=capsolver +# CAPTCHA_SOLVER_API_KEY= +# CAPTCHA_MAX_ATTEMPTS_PER_URL=1 +# CAPTCHA_SOLVE_TIMEOUT_S=120 +# CAPTCHA_TYPE_DEFAULT=v2 +# CAPTCHA_V3_MIN_SCORE=0.7 +# CAPTCHA_V3_ACTION=verify + +# Stealth hardening levers on the stealth browser tier. Defaults preserve +# current behavior; see surfsense_backend/.env.example for per-flag docs. +# CRAWL_GEOIP_MATCH_ENABLED=FALSE +# CRAWL_BLOCK_WEBRTC=TRUE +# CRAWL_HIDE_CANVAS=FALSE +# CRAWL_GOOGLE_SEARCH_REFERER=TRUE +# CRAWL_DNS_OVER_HTTPS=FALSE + # ============================================================================== # DEV / DEPS-ONLY COMPOSE OVERRIDES # These are only needed for docker-compose.dev.yml or docker-compose.deps-only.yml. diff --git a/surfsense_backend/app/agents/chat/anonymous_chat/agent.py b/surfsense_backend/app/agents/chat/anonymous_chat/agent.py index 168eaaff0..13cb33adb 100644 --- a/surfsense_backend/app/agents/chat/anonymous_chat/agent.py +++ b/surfsense_backend/app/agents/chat/anonymous_chat/agent.py @@ -41,9 +41,12 @@ _MAX_DOC_CHARS = 50_000 def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str: """Build the system prompt for the minimal anonymous chat agent. - The prompt keeps the assistant focused on plain Q/A + web search, inlines - any uploaded document as read-only context, and redirects every other - SurfSense feature to account registration. + The prompt keeps the assistant focused on plain Q/A from model knowledge, + inlines any uploaded document as read-only context, and treats the chat as + a registration funnel: every other SurfSense capability (scraping, live + data, deliverables, knowledge base, automations) redirects to sign-up, and + the assistant softly suggests an account when the conversation reveals a + competitive-intelligence need the platform serves. """ today = datetime.now(UTC).strftime("%A, %B %d, %Y") @@ -70,7 +73,13 @@ def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str return ( "You are SurfSense's free AI assistant, available to everyone without " - "login.\n\n" + "login. SurfSense is the open-source competitive intelligence platform: " + "registered users get specialist agents that pull live market data from " + "Reddit, YouTube, Google Maps, Google Search, and the open web, turn it " + "into cited briefs, reports, podcasts, and presentations, keep findings " + "in a searchable knowledge base, and run scheduled monitoring " + "automations — plus a REST scraping API and MCP server for their own " + "agents.\n\n" f"Today's date is {today}.\n\n" "## How to help\n" "- Answer the user's questions directly and conversationally. You are " @@ -78,23 +87,42 @@ def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str "- Answer from your own knowledge. You do NOT have web access here, so " "for current, real-time, or fast-changing facts (news, prices, " "weather, recent events, live data) say you can't look them up in the " - "free experience and may be out of date, and invite the user to create " - "a free account for live web search.\n" + "free experience and may be out of date.\n" "- Be concise, accurate, and helpful. Use Markdown formatting when it " "improves readability." f"{doc_section}\n\n" "## What is not available here\n" - "This is the free, no-login experience. You CANNOT search the web, save " - "files or notes, generate reports, podcasts, resumes, presentations, or " - "images, search or build a knowledge base, connect to apps (Gmail, " - "Google Drive, Notion, Slack, Calendar, Discord, and similar), set up " - "automations, or remember anything across sessions.\n\n" + "This is the free, no-login experience. You CANNOT search the web or " + "scrape any platform (Reddit, YouTube, Google Maps, Google Search, " + "websites), save files or notes, upload additional files, generate " + "reports, podcasts, resumes, presentations, or images, search or build " + "a knowledge base, connect to apps (Gmail, Google Drive, Notion, " + "Slack, Calendar, Discord, and similar), set up automations or " + "monitoring, use the scraping API or MCP server, or remember anything " + "across sessions.\n\n" "If the user asks for any of these, do NOT pretend to do them and " - "never claim you saved, created, or stored anything. Instead, briefly " - "let them know the feature requires a free SurfSense account and " - "invite them to create one at https://www.surfsense.com. Then offer to " - "help with what you can do here (answering questions from your own " - "knowledge and about any uploaded document)." + "never claim you saved, created, scraped, or stored anything. Instead, " + "briefly say that this needs a free SurfSense account and invite them " + "to sign up at https://www.surfsense.com/register — name the specific " + "capability that would solve their request (e.g. 'a SurfSense account " + "can scrape those Reddit threads live and turn them into a cited " + "brief'). Then offer to help with what you can do here (answering from " + "your own knowledge and about any uploaded document).\n\n" + "## Suggesting SurfSense when it genuinely fits\n" + "You are also the front door to SurfSense. When the conversation " + "reveals a need the full platform serves — researching competitors, " + "tracking pricing or rankings, monitoring brand mentions or reviews, " + "gauging Reddit/YouTube sentiment, generating leads, needing current " + "web data, or wanting recurring reports — first answer as well as you " + "can from your own knowledge, then add ONE short sentence pointing out " + "that a free SurfSense account can do that with live data, linking " + "https://www.surfsense.com/register.\n" + "- Be helpful first, never salesy: the answer is the product; the " + "suggestion is a footnote.\n" + "- At most one suggestion per response, and stop suggesting entirely " + "if the user declines or ignores it.\n" + "- Do not suggest it for needs SurfSense does not serve (casual chat, " + "coding help, homework, creative writing)." ) diff --git a/surfsense_backend/app/proprietary/platforms/reddit/fetch.py b/surfsense_backend/app/proprietary/platforms/reddit/fetch.py index 41193a2c9..54c7ba125 100644 --- a/surfsense_backend/app/proprietary/platforms/reddit/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/reddit/fetch.py @@ -8,7 +8,7 @@ Reddit deprecated *cold* unauthenticated ``.json`` (a bare anonymous GET now ``scripts/e2e_reddit_scraper.py`` step 0) is: warm one anonymous session cookie (``loid``) with a plain GET to - ``old.reddit.com`` (``www.reddit.com/svc/shreddit/`` fallback), then + ``www.reddit.com/svc/shreddit/`` (``old.reddit.com`` fallback), then GET ``www.reddit.com//.json?raw_json=1`` through that same Chrome-impersonated, sticky-IP session. Which warm URL mints ``loid`` is exit-IP dependent, so the order is a tiebreak — see :func:`warm_session`. @@ -84,6 +84,12 @@ _BACKOFF_BASE_S = 5.0 _MIN_INTERVAL_S = 0.5 _PACE_JITTER_S = 0.25 +# curl's default is 30s, so one dead sticky IP stalled a whole run for 30-50s +# (seen live 2026-07-06). A healthy fetch lands in ~1s; cap at 10s so a dead IP +# costs one bounded wait, then the timeout falls into the generic exception +# branch of fetch_json and rotates to a fresh IP — same treatment as a 403. +_REQUEST_TIMEOUT_S = 10.0 + _HEADERS = {"Accept-Language": "en-US,en;q=0.9"} # Age-gate opt-in, sent on every ``.json`` fetch so NSFW listings aren't blanked @@ -170,7 +176,10 @@ class _RotatingSession: self._cm = self.session = None return self._cm = FetcherSession( - proxy=proxy, stealthy_headers=True, impersonate="chrome" + proxy=proxy, + stealthy_headers=True, + impersonate="chrome", + timeout=_REQUEST_TIMEOUT_S, ) self.session = await self._cm.__aenter__() @@ -234,16 +243,15 @@ async def warm_session(session: Any, *, slug: str = _WARM_SLUG) -> bool: Returns ``True`` when a ``loid`` was issued (the session can now reach ``.json``), else ``False`` (caller rotates the IP and retries). - Tries ``old.reddit`` (yt-dlp's primary), then ``svc/shreddit``. WHICH one - mints is exit-IP dependent and roughly random: live probes 2026-07-04 saw - both directions across sessions on the rotating residential/custom proxy - (one IP 403s ``old.reddit`` but mints on ``shreddit``; another does the - reverse, sometimes with an ``rdt`` bot-interstitial). So the order is a - tiebreak, not an optimization — a fresh session pays ~one wasted warm 403 - either way. That cost is amortized: ``fan_out`` reuses one warmed session - per worker across many jobs, so warm-up runs once per worker, not per fetch. - The fallback is what actually matters — it preserves correctness whichever - way a given IP leans. + Tries ``svc/shreddit`` first, then ``old.reddit`` (yt-dlp's primary) as + the fallback. Live probes 2026-07-04 saw both directions across exit IPs, + but a live run 2026-07-06 had ``old.reddit`` 403 on 12/12 fresh IPs while + ``shreddit`` minted every time — Reddit appears to have shut old.reddit to + anonymous traffic, so shreddit-first saves one guaranteed-403 round trip + per warm-up. The fallback is what actually matters — it preserves + correctness if a given IP (or Reddit) flips back the other way. That cost + is amortized: ``fan_out`` reuses one warmed session per worker across many + jobs, so warm-up runs once per worker, not per fetch. ponytail: sequential two-source warm burns 1 wasted request on ~half of new sessions. A parallel warm (gather both, take whichever mints) removes the @@ -256,14 +264,14 @@ async def warm_session(session: Any, *, slug: str = _WARM_SLUG) -> bool: """ seen: set[str] = set() with suppress(Exception): - page = await session.get(_OLD_REDDIT_URL, headers=_HEADERS) + page = await session.get(_SHREDDIT_URL.format(slug=slug), headers=_HEADERS) seen |= _response_cookie_names(page) if _LOID_COOKIE in seen: return True - # Fallback: mints loid on exit IPs where old.reddit 403s instead. + # Fallback: mints loid on exit IPs where shreddit 403s instead. with suppress(Exception): - page = await session.get(_SHREDDIT_URL.format(slug=slug), headers=_HEADERS) + page = await session.get(_OLD_REDDIT_URL, headers=_HEADERS) seen |= _response_cookie_names(page) return _LOID_COOKIE in seen @@ -278,6 +286,7 @@ async def _get_page(session: Any, url: str) -> Any: cookies=_OVER18_COOKIES, proxy=get_proxy_url(), stealthy_headers=True, + timeout=_REQUEST_TIMEOUT_S, ) diff --git a/surfsense_backend/test_scraper_api_smoke.py b/surfsense_backend/test_scraper_api_smoke.py deleted file mode 100644 index 8b6177d76..000000000 --- a/surfsense_backend/test_scraper_api_smoke.py +++ /dev/null @@ -1,106 +0,0 @@ -"""One-shot smoke test: hit every scraper API endpoint with a PAT, print PASS/FAIL.""" - -import json -import os -import sys -import time - -import httpx - -BASE = "http://localhost:8000/api/v1/workspaces/12/scrapers" -PAT = os.environ["SURFSENSE_PAT"] # export a ss_pat_... key before running -HEADERS = {"Authorization": f"Bearer {PAT}", "Content-Type": "application/json"} - -# Minimal payloads: 1-3 items each to keep credit spend tiny. -VERBS = [ - ("google_search/scrape", {"queries": ["surfsense github"], "max_pages_per_query": 1}), - ("web/crawl", {"startUrls": ["https://example.com"], "maxCrawlDepth": 0}), - ("reddit/scrape", {"urls": ["https://www.reddit.com/r/Python/"], "max_items": 3}), - ("youtube/scrape", {"search_queries": ["python tutorial"], "max_results": 1}), - ( - "youtube/comments", - {"urls": ["https://www.youtube.com/watch?v=dQw4w9WgXcQ"], "max_comments": 3}, - ), - ( - "google_maps/scrape", - {"search_queries": ["coffee shop"], "location": "New York, USA", "max_places": 1}, - ), - ( - "google_maps/reviews", - # Google Sydney office, a stable well-known place id. - {"place_ids": ["ChIJN1t_tDeuEmsRUsoyG83frY4"], "max_reviews": 3}, - ), -] - -results = [] -client = httpx.Client(headers=HEADERS, timeout=300) -sync_run_id = None - -for verb, payload in VERBS: - t0 = time.time() - try: - r = client.post(f"{BASE}/{verb}", json=payload) - dur = time.time() - t0 - run_id = r.headers.get("x-run-id") - if r.status_code == 200: - body = r.json() - # Count items in whatever list field the output has. - items = next((len(v) for v in body.values() if isinstance(v, list)), "?") - results.append((verb, "PASS", f"{r.status_code} items={items} run={run_id} {dur:.1f}s")) - if sync_run_id is None and run_id: - sync_run_id = run_id - else: - results.append((verb, "FAIL", f"{r.status_code} {r.text[:200]} {dur:.1f}s")) - except Exception as e: - results.append((verb, "FAIL", f"{type(e).__name__}: {e}")) - print(f"[{results[-1][1]}] {verb}: {results[-1][2]}", flush=True) - -# --- Run history endpoints --- -r = client.get(f"{BASE}/runs", params={"limit": 5}) -ok = r.status_code == 200 and isinstance(r.json(), list) -results.append(("GET runs (list)", "PASS" if ok else "FAIL", f"{r.status_code} rows={len(r.json()) if ok else '?'}")) -print(f"[{results[-1][1]}] GET runs: {results[-1][2]}", flush=True) - -if sync_run_id: - r = client.get(f"{BASE}/runs/{sync_run_id}") - ok = r.status_code == 200 and r.json().get("id") == sync_run_id - results.append(("GET runs/{id} (detail)", "PASS" if ok else "FAIL", f"{r.status_code} status={r.json().get('status') if ok else r.text[:100]}")) - print(f"[{results[-1][1]}] GET run detail: {results[-1][2]}", flush=True) - -# --- Async mode + SSE events --- -r = client.post(f"{BASE}/web/crawl?mode=async", json={"startUrls": ["https://example.com"]}) -if r.status_code == 202: - async_id = r.json()["run_id"] - seen, finished = [], None - with client.stream("GET", f"{BASE}/runs/{async_id}/events", timeout=120) as s: - for line in s.iter_lines(): - if line.startswith("data: "): - ev = json.loads(line[6:]) - seen.append(ev["type"]) - if ev["type"] == "run.finished": - finished = ev.get("status") - break - ok = finished == "success" - results.append(("async + SSE events", "PASS" if ok else "FAIL", f"202 events={seen} final={finished}")) -else: - results.append(("async + SSE events", "FAIL", f"{r.status_code} {r.text[:200]}")) -print(f"[{results[-1][1]}] async+SSE: {results[-1][2]}", flush=True) - -# --- Cancel endpoint --- -r = client.post(f"{BASE}/web/crawl?mode=async", json={"startUrls": ["https://example.com"], "maxCrawlDepth": 2, "maxCrawlPages": 50}) -if r.status_code == 202: - cancel_id = r.json()["run_id"] - time.sleep(1) - r2 = client.post(f"{BASE}/runs/{cancel_id}/cancel") - ok = r2.status_code == 200 and r2.json().get("status") == "cancelled" - results.append(("POST runs/{id}/cancel", "PASS" if ok else "FAIL", f"{r2.status_code} {r2.text[:150]}")) -else: - results.append(("POST runs/{id}/cancel", "FAIL", f"setup {r.status_code}")) -print(f"[{results[-1][1]}] cancel: {results[-1][2]}", flush=True) - -print("\n===== SUMMARY =====") -for name, status_, detail in results: - print(f"{status_:4} {name}: {detail}") -failed = [r for r in results if r[1] == "FAIL"] -print(f"\n{len(results) - len(failed)}/{len(results)} passed") -sys.exit(1 if failed else 0) diff --git a/surfsense_backend/tests/unit/platforms/reddit/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/reddit/test_fetch_resilience.py index 793e63440..610855f5c 100644 --- a/surfsense_backend/tests/unit/platforms/reddit/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/reddit/test_fetch_resilience.py @@ -95,8 +95,8 @@ def _no_sleep(monkeypatch) -> None: async def test_warms_then_returns_json(): - # old.reddit is tried first and mints loid -> a single warm call. - holder = _FakeHolder([_FakeSession(200, old_loid=True)]) + # shreddit is tried first and mints loid -> a single warm call. + holder = _FakeHolder([_FakeSession(200, shreddit_loid=True)]) token = _current_session.set(holder) try: result = await fetch_json("r/python/hot") @@ -107,9 +107,9 @@ async def test_warms_then_returns_json(): assert holder.session.warm_calls == 1 # warmed exactly once -async def test_warm_falls_back_to_shreddit(): - # old.reddit doesn't mint loid, shreddit does -> still warms on the same IP. - holder = _FakeHolder([_FakeSession(200, shreddit_loid=True, old_loid=False)]) +async def test_warm_falls_back_to_old_reddit(): + # shreddit doesn't mint loid, old.reddit does -> still warms on the same IP. + holder = _FakeHolder([_FakeSession(200, shreddit_loid=False, old_loid=True)]) token = _current_session.set(holder) try: result = await fetch_json("r/python/hot") diff --git a/surfsense_mcp/src/surfsense_mcp/core/rendering.py b/surfsense_mcp/src/surfsense_mcp/core/rendering.py index 9fcb6b9af..ebfdd9e71 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/rendering.py +++ b/surfsense_mcp/src/surfsense_mcp/core/rendering.py @@ -8,11 +8,55 @@ are clipped so a single call can't blow the context window. from __future__ import annotations import json -from typing import Any, Literal +from typing import Annotated, Any, Literal + +from pydantic import Field ResponseFormat = Literal["markdown", "json"] +# Shared parameter type for every tool: same name, same semantics everywhere. +ResponseFormatParam = Annotated[ + ResponseFormat, + Field( + description="'markdown' (default, human-readable) or 'json' " + "(raw data for post-processing)." + ), +] + DEFAULT_CLIP_CHARS = 20_000 +ITEM_FIELD_CLIP_CHARS = 1_500 + +# Fields that duplicate another field verbatim (e.g. Reddit's 'html' mirrors +# 'body') and only bloat inline results. The full record stays in the run. +_REDUNDANT_ITEM_FIELDS = frozenset({"html"}) + + +def compact_items(result: Any, field_limit: int = ITEM_FIELD_CLIP_CHARS) -> Any: + """Shrink a scraper result for inline return. + + Drops redundant fields and clips overlong strings per field, so a response + keeps every item as an excerpt instead of a few items in full. The + untruncated result remains retrievable via its stored run. + """ + if isinstance(result, dict) and isinstance(result.get("items"), list): + return { + **result, + "items": [_compact_item(item, field_limit) for item in result["items"]], + } + return result + + +def _compact_item(item: Any, field_limit: int) -> Any: + # ponytail: compacts top-level string fields only; nested structures pass + # through untouched. Upgrade path is a recursive walk if a platform nests + # long text. + if not isinstance(item, dict): + return item + return { + key: clip(value, field_limit) if isinstance(value, str) else value + for key, value in item.items() + if key not in _REDUNDANT_ITEM_FIELDS + } def to_json(payload: Any) -> str: diff --git a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py index 5c034708c..6682d53b5 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py +++ b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py @@ -8,10 +8,22 @@ speaks a name, we resolve it, and remember the choice for later calls. from __future__ import annotations from dataclasses import dataclass +from typing import Annotated + +from pydantic import Field from .client import SurfSenseClient from .errors import ToolError +# Shared parameter type for every workspace-scoped tool. +WorkspaceParam = Annotated[ + str | None, + Field( + description="Workspace name or id, e.g. 'Research' or '3'. Omit to use " + "the active workspace (set with surfsense_select_workspace)." + ), +] + @dataclass(frozen=True) class Workspace: diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py index 2ff3f571c..7fb1802ae 100644 --- a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py +++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py @@ -10,14 +10,16 @@ from __future__ import annotations import mimetypes from pathlib import Path +from typing import Annotated from mcp.server.fastmcp import FastMCP from mcp.types import ToolAnnotations +from pydantic import Field from ...core.client import SurfSenseClient from ...core.errors import ToolError -from ...core.rendering import ResponseFormat, clip, to_json -from ...core.workspace_context import WorkspaceContext +from ...core.rendering import ResponseFormatParam, clip, to_json +from ...core.workspace_context import WorkspaceContext, WorkspaceParam from .note_ingestion import build_note_document _READ = ToolAnnotations( @@ -30,6 +32,22 @@ _DELETE = ToolAnnotations( readOnlyHint=False, destructiveHint=True, idempotentHint=False, openWorldHint=False ) +_DOCUMENT_ID = Annotated[ + int, + Field( + description="Document id from surfsense_search_knowledge_base or " + "surfsense_list_documents results." + ), +] + +_DOCUMENT_TYPES = Annotated[ + list[str] | None, + Field( + description="Restrict to these document types, e.g. " + "['FILE', 'CRAWLED_URL', 'YOUTUBE_VIDEO']. Omit for all types." + ), +] + def register( mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext @@ -38,21 +56,34 @@ def register( @mcp.tool( name="surfsense_search_knowledge_base", + title="Search knowledge base", annotations=_READ, structured_output=False, ) async def search_knowledge_base( - query: str, - top_k: int = 5, - document_types: list[str] | None = None, - workspace: str | None = None, - response_format: ResponseFormat = "markdown", + query: Annotated[ + str, + Field( + min_length=1, + description="Natural-language search, e.g. " + "'notebooklm user complaints'.", + ), + ], + top_k: Annotated[ + int, Field(ge=1, le=20, description="Maximum documents to return.") + ] = 5, + document_types: _DOCUMENT_TYPES = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", ) -> str: - """Search the workspace's knowledge base by meaning and keyword. + """Search the workspace's knowledge base by meaning and keywords. - Use this to answer questions from stored content: it returns the most - relevant documents with the passages that matched, ranked by relevance. - top_k caps documents (1–20). Optionally restrict to document_types. + Use this FIRST when a question might be answered by content already + stored in SurfSense — notes, uploaded files, saved pages, past + research. Do NOT use it to fetch new data from the web; use the + scraper tools for that. Returns the most relevant documents with the + passages that matched, ranked by relevance score. + Example: query='pricing feedback', top_k=5. """ resolved = await context.resolve(workspace) hits = await client.request( @@ -71,21 +102,33 @@ def register( return _render_search(query, items) @mcp.tool( - name="surfsense_list_documents", annotations=_READ, structured_output=False + name="surfsense_list_documents", + title="List documents", + annotations=_READ, + structured_output=False, ) async def list_documents( - document_types: list[str] | None = None, - folder_id: int | None = None, - page: int = 0, - page_size: int = 20, - workspace: str | None = None, - response_format: ResponseFormat = "markdown", + document_types: _DOCUMENT_TYPES = None, + folder_id: Annotated[ + int | None, + Field(description="Only documents in this folder. Omit for all."), + ] = None, + page: Annotated[ + int, Field(ge=0, description="Zero-based page number.") + ] = 0, + page_size: Annotated[ + int, Field(ge=1, description="Documents per page.") + ] = 20, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", ) -> str: """List documents in the workspace's knowledge base, newest first. - Use this to browse or inventory what is stored. Optionally filter by - document_types or a folder_id. Paginated: returns page_size items and a - has_more flag; request the next page by increasing page. + Use this to browse or inventory what is stored; to find documents + about a topic, prefer surfsense_search_knowledge_base. Returns each + document's title, id, type, and update time, plus a has_more flag — + request the next page by increasing page. + Example: document_types=['FILE'], page=0, page_size=20. """ resolved = await context.resolve(workspace) result = await client.request( @@ -104,15 +147,20 @@ def register( return _render_document_list(result) @mcp.tool( - name="surfsense_get_document", annotations=_READ, structured_output=False + name="surfsense_get_document", + title="Read one document", + annotations=_READ, + structured_output=False, ) async def get_document( - document_id: int, response_format: ResponseFormat = "markdown" + document_id: _DOCUMENT_ID, + response_format: ResponseFormatParam = "markdown", ) -> str: """Read one document's full content and metadata by id. - Use this after search or list to open a specific document. The id comes - from those tools' results. + Use this after surfsense_search_knowledge_base or + surfsense_list_documents to open a specific document — search results + only include the matching passages, this returns the whole text. """ document = await client.request("GET", f"/documents/{document_id}") if response_format == "json": @@ -120,20 +168,36 @@ def register( return _render_document(document) @mcp.tool( - name="surfsense_add_document", annotations=_WRITE, structured_output=False + name="surfsense_add_document", + title="Add a note", + annotations=_WRITE, + structured_output=False, ) async def add_document( - title: str, - content: str, - source_url: str | None = None, - workspace: str | None = None, + title: Annotated[ + str, + Field(min_length=1, description="Short descriptive title for the note."), + ], + content: Annotated[ + str, + Field( + min_length=1, + description="The note's body; plain text or markdown.", + ), + ], + source_url: Annotated[ + str | None, + Field(description="Where the text came from, if anywhere."), + ] = None, + workspace: WorkspaceParam = None, ) -> str: - """Add a text or markdown note to the workspace's knowledge base. + """Save a text or markdown note into the workspace's knowledge base. - Use this to save notes, summaries, or snippets so they become - searchable. The content is indexed asynchronously, so it may take a - moment to appear in search. source_url optionally records where the text - came from. + Use this to store notes, summaries, or findings so they become + searchable later — e.g. after finishing a piece of research. For files + on disk use surfsense_upload_file instead. Indexing is asynchronous, + so the note may take a moment to appear in search. + Example: title='NotebookLM subreddits', content='- r/notebooklm ...'. """ resolved = await context.resolve(workspace) await client.request( @@ -152,18 +216,35 @@ def register( ) @mcp.tool( - name="surfsense_upload_file", annotations=_WRITE, structured_output=False + name="surfsense_upload_file", + title="Upload a file", + annotations=_WRITE, + structured_output=False, ) async def upload_file( - file_path: str, - use_vision_llm: bool = False, - workspace: str | None = None, + file_path: Annotated[ + str, + Field( + description="Path to a local file, e.g. " + "'C:/Users/me/report.pdf' or '~/notes/summary.md'." + ), + ], + use_vision_llm: Annotated[ + bool, + Field( + description="True reads scanned or image-heavy files with a " + "vision model (slower)." + ), + ] = False, + workspace: WorkspaceParam = None, ) -> str: - """Upload a local file (PDF, doc, etc.) into the knowledge base. + """Upload a local file (PDF, docx, markdown, etc.) into the knowledge base. - Use this to ingest a file from disk; it is parsed, chunked, and indexed - asynchronously. Set use_vision_llm to read scanned or image-heavy files - with a vision model (slower). + Use this to ingest a file from disk so its content becomes searchable; + for text you already have in hand use surfsense_add_document instead. + The file is parsed, chunked, and indexed asynchronously. Duplicate + files are detected and skipped. + Example: file_path='C:/Users/me/report.pdf'. """ resolved = await context.resolve(workspace) payload = _read_upload(file_path) @@ -186,14 +267,28 @@ def register( ) @mcp.tool( - name="surfsense_update_document", annotations=_WRITE, structured_output=False + name="surfsense_update_document", + title="Replace a document's content", + annotations=_WRITE, + structured_output=False, ) - async def update_document(document_id: int, content: str) -> str: + async def update_document( + document_id: _DOCUMENT_ID, + content: Annotated[ + str, + Field( + min_length=1, + description="New full text; replaces the existing content " + "entirely.", + ), + ], + ) -> str: """Replace a document's stored content by id. - Use this to correct or rewrite a document's text. Note: this updates the - stored content; re-indexing of search chunks is not triggered by this - call. + Use this to correct or rewrite a document's text. The new content + REPLACES the old entirely — to append, read the document first with + surfsense_get_document and resend the combined text. Search chunks are + not re-indexed by this call. """ existing = await client.request("GET", f"/documents/{document_id}") await client.request( @@ -208,13 +303,17 @@ def register( return f"Updated document {document_id} ('{existing.get('title', '')}')." @mcp.tool( - name="surfsense_delete_document", annotations=_DELETE, structured_output=False + name="surfsense_delete_document", + title="Delete a document", + annotations=_DELETE, + structured_output=False, ) - async def delete_document(document_id: int) -> str: - """Delete a document from the knowledge base by id. + async def delete_document(document_id: _DOCUMENT_ID) -> str: + """Permanently delete a document from the knowledge base by id. - Use this to permanently remove a document. Deletion runs in the - background; the document stops appearing in searches immediately. + Use this only when the user explicitly asks to remove a document — + deletion cannot be undone. The document stops appearing in searches + immediately. """ await client.request("DELETE", f"/documents/{document_id}") return f"Deleted document {document_id}." diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py index 2237e13cc..aa8265148 100644 --- a/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py @@ -8,14 +8,15 @@ full later. from __future__ import annotations -from typing import Literal +from typing import Annotated, Literal from mcp.server.fastmcp import FastMCP from mcp.types import ToolAnnotations +from pydantic import Field from ...core.client import SurfSenseClient -from ...core.rendering import ResponseFormat, clip, to_json -from ...core.workspace_context import WorkspaceContext +from ...core.rendering import ResponseFormatParam, clip, to_json +from ...core.workspace_context import WorkspaceContext, WorkspaceParam from .capability import run_scraper # Scrapers reach the open web and record a billable run; they are neither @@ -38,23 +39,57 @@ def register( ) -> None: """Register the scraper and run-history tools on the server.""" - @mcp.tool(name="surfsense_web_crawl", annotations=_SCRAPE, structured_output=False) + @mcp.tool( + name="surfsense_web_crawl", + title="Crawl web pages", + annotations=_SCRAPE, + structured_output=False, + ) async def web_crawl( - start_urls: list[str], - max_crawl_depth: int = 0, - max_crawl_pages: int = 10, - max_length: int = 50_000, - include_url_patterns: list[str] | None = None, - exclude_url_patterns: list[str] | None = None, - workspace: str | None = None, - response_format: ResponseFormat = "markdown", + start_urls: Annotated[ + list[str], + Field( + min_length=1, + description="Full URLs to fetch, e.g. " + "['https://example.com/blog/post'].", + ), + ], + max_crawl_depth: Annotated[ + int, + Field( + ge=0, + description="Link-hops to follow from start_urls within the " + "same site. 0 fetches only start_urls.", + ), + ] = 0, + max_crawl_pages: Annotated[ + int, Field(ge=1, description="Stop after this many pages in total.") + ] = 10, + max_length: Annotated[ + int, Field(ge=1, description="Max characters kept per page.") + ] = 50_000, + include_url_patterns: Annotated[ + list[str] | None, + Field( + description="Regexes; only discovered links matching one are " + "followed, e.g. ['/docs/.*']." + ), + ] = None, + exclude_url_patterns: Annotated[ + list[str] | None, + Field(description="Regexes; discovered links matching one are skipped."), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", ) -> str: - """Crawl web pages and return their cleaned content as markdown. + """Fetch specific web pages and return their cleaned content as markdown. - Use this to read one page or spider a site. With max_crawl_depth=0 only - start_urls are fetched; a higher depth follows same-site links up to - max_crawl_pages. include/exclude_url_patterns are regexes that narrow - which discovered links are followed. + Use this to read a page the user names, or to spider a site from a + starting URL. Do NOT use it to find pages on a topic — use + surfsense_google_search for discovery. Returns one item per crawled + page: url, title, and the page text as markdown. + Example: start_urls=['https://blog.example.com'], max_crawl_depth=1, + include_url_patterns=['/2026/']. """ return await run_scraper( client, @@ -74,22 +109,47 @@ def register( ) @mcp.tool( - name="surfsense_google_search", annotations=_SCRAPE, structured_output=False + name="surfsense_google_search", + title="Scrape Google Search", + annotations=_SCRAPE, + structured_output=False, ) async def google_search( - queries: list[str], - max_pages_per_query: int = 1, - country_code: str | None = None, - language_code: str = "", - site: str | None = None, - workspace: str | None = None, - response_format: ResponseFormat = "markdown", + queries: Annotated[ + list[str], + Field( + min_length=1, + description="Search terms or full Google Search URLs, e.g. " + "['best rss readers 2026'].", + ), + ], + max_pages_per_query: Annotated[ + int, Field(ge=1, description="Result pages to fetch per query.") + ] = 1, + country_code: Annotated[ + str | None, + Field(description="Two-letter country to search from, e.g. 'us'."), + ] = None, + language_code: Annotated[ + str, Field(description="Results language, e.g. 'en'. Empty for default.") + ] = "", + site: Annotated[ + str | None, + Field( + description="Restrict results to one domain, e.g. 'example.com'." + ), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", ) -> str: - """Scrape Google Search results for one or more queries. + """Scrape Google Search result pages for one or more queries. - Use this to find pages on the web. Each item is a query's fetched result - page. Pass full Google Search URLs to scrape them as-is, or plain terms - to search. Optionally scope to a country, language, or single domain. + Use this to discover pages on the open web by topic; follow up with + surfsense_web_crawl to read a result in full. Do NOT use it for + Reddit, YouTube, or Google Maps research — the dedicated tools return + richer data. Returns each query's parsed results: title, url, and + snippet per organic result. + Example: queries=['notebooklm review'], site='news.ycombinator.com'. """ return await run_scraper( client, @@ -108,24 +168,61 @@ def register( ) @mcp.tool( - name="surfsense_reddit_scrape", annotations=_SCRAPE, structured_output=False + name="surfsense_reddit_scrape", + title="Search or scrape Reddit", + annotations=_SCRAPE, + structured_output=False, ) async def reddit_scrape( - urls: list[str] | None = None, - search_queries: list[str] | None = None, - community: str | None = None, - sort: RedditSort = "new", - time_filter: RedditTime | None = None, - max_items: int = 10, - skip_comments: bool = False, - workspace: str | None = None, - response_format: ResponseFormat = "markdown", + urls: Annotated[ + list[str] | None, + Field( + description="Reddit URLs: a post, a subreddit like " + "'https://reddit.com/r/LocalLLaMA', a user page, or a search " + "URL. Provide urls OR search_queries." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Terms to search Reddit for, e.g. " + "['NotebookLM alternatives']. Provide search_queries OR urls." + ), + ] = None, + community: Annotated[ + str | None, + Field( + description="Restrict a search to one subreddit, name without " + "'r/', e.g. 'ArtificialInteligence'." + ), + ] = None, + sort: Annotated[RedditSort, Field(description="Post ordering.")] = "new", + time_filter: Annotated[ + RedditTime | None, + Field(description="Time window; only valid with sort='top'."), + ] = None, + max_items: Annotated[ + int, Field(ge=1, description="Maximum posts to return.") + ] = 10, + skip_comments: Annotated[ + bool, + Field( + description="True fetches posts only (faster); False also " + "fetches each post's comment thread." + ), + ] = False, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", ) -> str: - """Scrape Reddit posts and comments from URLs or a search. + """Search or scrape Reddit: posts, comments, subreddits, and users. - Provide urls (a post, /r/subreddit, /user/name, or search URL) OR - search_queries; scope a search to one subreddit with community. Use - time_filter only with sort='top'. Set skip_comments to fetch posts only. + Use this for ANY Reddit research — finding relevant subreddits or + communities for a topic, top posts, or discussions — instead of a + generic web search. Returns posts (title, text, score, subreddit, url) + with comment threads unless skip_comments is set. Every post carries + its subreddit, so to find communities for a topic, search posts and + aggregate their subreddits. + Example: search_queries=['NotebookLM'], sort='top', time_filter='month'. """ return await run_scraper( client, @@ -146,22 +243,46 @@ def register( ) @mcp.tool( - name="surfsense_youtube_scrape", annotations=_SCRAPE, structured_output=False + name="surfsense_youtube_scrape", + title="Search or scrape YouTube", + annotations=_SCRAPE, + structured_output=False, ) async def youtube_scrape( - urls: list[str] | None = None, - search_queries: list[str] | None = None, - max_results: int = 10, - download_subtitles: bool = False, - subtitles_language: str = "en", - workspace: str | None = None, - response_format: ResponseFormat = "markdown", + urls: Annotated[ + list[str] | None, + Field( + description="YouTube URLs: video, channel, playlist, shorts, " + "or hashtag pages. Provide urls OR search_queries." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Terms to search YouTube for, e.g. " + "['NotebookLM tutorial']. Provide search_queries OR urls." + ), + ] = None, + max_results: Annotated[ + int, Field(ge=1, description="Maximum videos to return.") + ] = 10, + download_subtitles: Annotated[ + bool, + Field(description="True also fetches each video's transcript."), + ] = False, + subtitles_language: Annotated[ + str, Field(description="Transcript language code, e.g. 'en'.") + ] = "en", + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", ) -> str: - """Scrape YouTube videos from URLs or a search. + """Search or scrape YouTube videos, optionally with transcripts. - Provide urls (video, channel, playlist, shorts, or hashtag pages) OR - search_queries. Set download_subtitles to also fetch each video's - transcript in subtitles_language. + Use this for YouTube research: finding videos on a topic, or reading a + video's details or transcript. For a video's comment section use + surfsense_youtube_comments instead. Returns per-video metadata (title, + channel, views, description, url) and, if requested, the transcript. + Example: search_queries=['NotebookLM tutorial'], download_subtitles=True. """ return await run_scraper( client, @@ -180,19 +301,41 @@ def register( ) @mcp.tool( - name="surfsense_youtube_comments", annotations=_SCRAPE, structured_output=False + name="surfsense_youtube_comments", + title="Fetch YouTube comments", + annotations=_SCRAPE, + structured_output=False, ) async def youtube_comments( - urls: list[str], - max_comments: int = 20, - sort_by: CommentSort = "NEWEST_FIRST", - workspace: str | None = None, - response_format: ResponseFormat = "markdown", + urls: Annotated[ + list[str], + Field( + min_length=1, + description="YouTube video URLs, e.g. " + "['https://www.youtube.com/watch?v=abc123'].", + ), + ], + max_comments: Annotated[ + int, + Field( + ge=1, + description="Maximum comments per video, counting top-level " + "comments and replies together.", + ), + ] = 20, + sort_by: Annotated[ + CommentSort, Field(description="Comment ordering.") + ] = "NEWEST_FIRST", + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", ) -> str: - """Fetch comments (and replies) for one or more YouTube videos. + """Fetch the comments (and replies) on one or more YouTube videos. - Use this when the user wants a video's discussion rather than the video - itself. max_comments counts top-level comments and replies together. + Use this when the user wants a video's discussion or audience reaction + rather than the video itself; get video URLs from + surfsense_youtube_scrape if you only have a topic. Returns comment + text, author, likes, and replies. + Example: urls=['https://www.youtube.com/watch?v=abc123'], max_comments=50. """ return await run_scraper( client, @@ -210,24 +353,52 @@ def register( @mcp.tool( name="surfsense_google_maps_scrape", + title="Find places on Google Maps", annotations=_SCRAPE, structured_output=False, ) async def google_maps_scrape( - search_queries: list[str] | None = None, - urls: list[str] | None = None, - place_ids: list[str] | None = None, - location: str | None = None, - max_places: int = 10, - include_details: bool = False, - workspace: str | None = None, - response_format: ResponseFormat = "markdown", + search_queries: Annotated[ + list[str] | None, + Field( + description="Place searches, e.g. ['coffee shops']. Provide " + "search_queries OR urls OR place_ids." + ), + ] = None, + urls: Annotated[ + list[str] | None, + Field(description="Google Maps URLs of specific places."), + ] = None, + place_ids: Annotated[ + list[str] | None, + Field(description="Google place ids, e.g. ['ChIJj61dQgK6j4AR...']."), + ] = None, + location: Annotated[ + str | None, + Field( + description="Geographic scope for a search, e.g. " + "'Seattle, USA'." + ), + ] = None, + max_places: Annotated[ + int, Field(ge=1, description="Maximum places to return.") + ] = 10, + include_details: Annotated[ + bool, + Field( + description="True adds opening hours and extra contact info " + "(slower)." + ), + ] = False, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", ) -> str: - """Scrape places from Google Maps by search, URL, or place id. + """Find places on Google Maps by search, URL, or place id. - Provide search_queries OR urls OR place_ids. Scope a search with - location (e.g. 'New York, USA'). Set include_details for opening hours - and extra contact info (slower). + Use this for local-business and location research: names, addresses, + ratings, categories, coordinates, place ids. For a place's customer + reviews use surfsense_google_maps_reviews instead. + Example: search_queries=['ramen'], location='Osaka, Japan', max_places=5. """ return await run_scraper( client, @@ -248,23 +419,49 @@ def register( @mcp.tool( name="surfsense_google_maps_reviews", + title="Fetch Google Maps reviews", annotations=_SCRAPE, structured_output=False, ) async def google_maps_reviews( - urls: list[str] | None = None, - place_ids: list[str] | None = None, - max_reviews: int = 20, - sort_by: ReviewSort = "newest", - language: str = "en", - start_date: str | None = None, - workspace: str | None = None, - response_format: ResponseFormat = "markdown", + urls: Annotated[ + list[str] | None, + Field( + description="Google Maps URLs of places. Provide urls OR " + "place_ids." + ), + ] = None, + place_ids: Annotated[ + list[str] | None, + Field( + description="Google place ids from surfsense_google_maps_scrape." + ), + ] = None, + max_reviews: Annotated[ + int, Field(ge=1, description="Maximum reviews per place.") + ] = 20, + sort_by: Annotated[ + ReviewSort, Field(description="Review ordering.") + ] = "newest", + language: Annotated[ + str, Field(description="Reviews language code, e.g. 'en'.") + ] = "en", + start_date: Annotated[ + str | None, + Field( + description="ISO date like '2026-01-01'; keeps only reviews on " + "or after that day." + ), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", ) -> str: - """Fetch reviews for Google Maps places by URL or place id. + """Fetch customer reviews for Google Maps places by URL or place id. - Provide urls OR place_ids. start_date (ISO, e.g. '2024-01-01') keeps only - reviews on or after that day. + Use this to read feedback on specific places; get urls or place_ids + from surfsense_google_maps_scrape first if you only have a name. + Returns review text, rating, author, and date per review. + Example: place_ids=['ChIJj61dQgK6j4AR...'], sort_by='newest'. """ return await run_scraper( client, @@ -285,21 +482,35 @@ def register( @mcp.tool( name="surfsense_list_scraper_runs", + title="List past scraper runs", annotations=_READ_RUNS, structured_output=False, ) async def list_scraper_runs( - limit: int = 20, - capability: str | None = None, - status: str | None = None, - workspace: str | None = None, - response_format: ResponseFormat = "markdown", + limit: Annotated[ + int, Field(ge=1, description="Maximum runs to list.") + ] = 20, + capability: Annotated[ + str | None, + Field( + description="Filter by capability slug, e.g. 'web.crawl' or " + "'reddit.scrape'." + ), + ] = None, + status: Annotated[ + str | None, + Field(description="Filter by run status: 'success' or 'error'."), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", ) -> str: - """List recent scraper runs for the workspace, newest first. + """List recent scraper runs in the workspace, newest first. - Use this to find a run_id to fetch in full with surfsense_get_scraper_run, - e.g. when an inline result was truncated. Optionally filter by capability - (like 'web.crawl') or status ('success' / 'error'). + Use this to find the run_id of an earlier scrape — for example when an + inline result was truncated — then fetch it in full with + surfsense_get_scraper_run. Returns each run's id, capability, status, + item count, and creation time. + Example: capability='reddit.scrape', status='success'. """ resolved = await context.resolve(workspace) runs = await client.request( @@ -317,18 +528,26 @@ def register( @mcp.tool( name="surfsense_get_scraper_run", + title="Fetch one scraper run in full", annotations=_READ_RUNS, structured_output=False, ) async def get_scraper_run( - run_id: str, - workspace: str | None = None, - response_format: ResponseFormat = "markdown", + run_id: Annotated[ + str, + Field( + description="Run id from surfsense_list_scraper_runs or a " + "prior scrape's output." + ), + ], + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", ) -> str: """Fetch a single scraper run in full, including its stored output. - Use this to retrieve the complete result of an earlier scrape (its - run_id comes from surfsense_list_scraper_runs or a prior scrape). + Use this to retrieve the complete, untruncated result of an earlier + scrape. Do NOT re-run a scraper just to recover a truncated result — + fetch the stored run instead. """ resolved = await context.resolve(workspace) run = await client.request( diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/capability.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/capability.py index e3aa92653..7245c9f84 100644 --- a/surfsense_mcp/src/surfsense_mcp/features/scrapers/capability.py +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/capability.py @@ -9,7 +9,7 @@ from __future__ import annotations from typing import Any from ...core.client import SurfSenseClient -from ...core.rendering import ResponseFormat, clip, to_json +from ...core.rendering import ResponseFormat, clip, compact_items, to_json from ...core.workspace_context import WorkspaceContext @@ -29,6 +29,10 @@ async def run_scraper( result = await client.request( "POST", f"/workspaces/{resolved.id}/scrapers/{platform}/{verb}", json=body ) + # Inline results are compacted (redundant fields dropped, long fields + # excerpted) so every item survives the overall clip; the complete output + # is stored server-side and retrievable with surfsense_get_scraper_run. + result = compact_items(result) if response_format == "json": return clip(to_json(result)) return _render_markdown(platform, verb, resolved.name, result) @@ -40,7 +44,11 @@ def _render_markdown( """A readable header plus the structured payload, clipped to a safe size.""" header = f'# {platform}.{verb} — {_describe_size(result)} from "{workspace_name}"' body = clip(to_json(result)) - return f"{header}\n\n```json\n{body}\n```" + footer = ( + "\n\nFields shown as excerpts; use surfsense_get_scraper_run for the " + "full output." + ) + return f"{header}\n\n```json\n{body}\n```{footer}" def _describe_size(result: Any) -> str: diff --git a/surfsense_mcp/src/surfsense_mcp/features/workspaces/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/workspaces/__init__.py index 5f5372989..6daef8a3c 100644 --- a/surfsense_mcp/src/surfsense_mcp/features/workspaces/__init__.py +++ b/surfsense_mcp/src/surfsense_mcp/features/workspaces/__init__.py @@ -7,10 +7,13 @@ rest of the conversation needs no ids. from __future__ import annotations +from typing import Annotated + from mcp.server.fastmcp import FastMCP from mcp.types import ToolAnnotations +from pydantic import Field -from ...core.rendering import ResponseFormat, to_json +from ...core.rendering import ResponseFormatParam, to_json from ...core.workspace_context import Workspace, WorkspaceContext _READ_ONLY = ToolAnnotations( @@ -23,15 +26,19 @@ def register(mcp: FastMCP, context: WorkspaceContext) -> None: @mcp.tool( name="surfsense_list_workspaces", + title="List workspaces", annotations=_READ_ONLY, structured_output=False, ) - async def list_workspaces(response_format: ResponseFormat = "markdown") -> str: + async def list_workspaces( + response_format: ResponseFormatParam = "markdown", + ) -> str: """List the SurfSense workspaces (search spaces) the account can access. - Use this to discover which workspaces exist before selecting one, or when - the user asks what search spaces they have. Returns each workspace's name, - id, description, ownership, and member count. + Use this to discover which workspaces exist before selecting one, or + when the user asks what search spaces they have. Returns each + workspace's name, id, description, ownership, and member count, and + marks the currently active one. """ workspaces = await context.fetch_all() if response_format == "json": @@ -40,16 +47,27 @@ def register(mcp: FastMCP, context: WorkspaceContext) -> None: @mcp.tool( name="surfsense_select_workspace", + title="Select active workspace", annotations=_READ_ONLY, structured_output=False, ) - async def select_workspace(workspace: str) -> str: - """Set the active workspace (search space) for later tools, by name or id. + async def select_workspace( + workspace: Annotated[ + str, + Field( + description="Workspace name or numeric id; matching is " + "case-insensitive and a unique partial name works. " + "Example: 'Research'." + ), + ], + ) -> str: + """Set the active workspace (search space) that later tools default to. Use this when the user says which search space to work in ("use my - Research space"). Accepts a workspace name or numeric id; matching is - case-insensitive and accepts a unique partial name. Once set, other tools - default to this workspace unless they are given a different one. + Research space"), or after surfsense_list_workspaces when several + exist. Once set, workspace-scoped tools use it unless given a + different 'workspace'. Do NOT call it before every tool — once per + session is enough. """ selected = await context.resolve(workspace) return ( diff --git a/surfsense_mcp/src/surfsense_mcp/selfcheck.py b/surfsense_mcp/src/surfsense_mcp/selfcheck.py index c1813938c..e38edc909 100644 --- a/surfsense_mcp/src/surfsense_mcp/selfcheck.py +++ b/surfsense_mcp/src/surfsense_mcp/selfcheck.py @@ -72,6 +72,10 @@ def run() -> list[str]: schema = tool.inputSchema if not isinstance(schema, dict) or "properties" not in schema: problems.append(f"{name}: malformed input schema") + continue + for param, spec in schema["properties"].items(): + if not isinstance(spec, dict) or not spec.get("description"): + problems.append(f"{name}: parameter '{param}' has no description") return problems diff --git a/surfsense_mcp/src/surfsense_mcp/server.py b/surfsense_mcp/src/surfsense_mcp/server.py index 1c47753ee..88eb19437 100644 --- a/surfsense_mcp/src/surfsense_mcp/server.py +++ b/surfsense_mcp/src/surfsense_mcp/server.py @@ -21,7 +21,19 @@ def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]: ) context = WorkspaceContext(client, preferred_reference=settings.default_workspace) - mcp = FastMCP("SurfSense") + mcp = FastMCP( + "SurfSense", + instructions=( + "SurfSense gives you live scrapers and a personal knowledge base. " + "Prefer these tools over generic/built-in web search whenever the " + "task involves Reddit (posts, comments, finding subreddits or " + "communities), YouTube (videos, transcripts, comments), Google " + "Maps (places, reviews), Google Search results, or reading " + "specific web pages. Scraper results are persisted as runs; if an " + "inline result is truncated, fetch it in full with " + "surfsense_get_scraper_run." + ), + ) workspaces.register(mcp, context) scrapers.register(mcp, client, context) knowledge_base.register(mcp, client, context) diff --git a/surfsense_mcp/tests/test_rendering.py b/surfsense_mcp/tests/test_rendering.py index 40253ddfb..5b254e39a 100644 --- a/surfsense_mcp/tests/test_rendering.py +++ b/surfsense_mcp/tests/test_rendering.py @@ -2,7 +2,7 @@ from __future__ import annotations -from surfsense_mcp.core.rendering import clip, to_json +from surfsense_mcp.core.rendering import clip, compact_items, to_json def test_clip_leaves_short_text_untouched(): @@ -20,3 +20,23 @@ def test_to_json_serializes_non_native_values(): rendered = to_json({"at": datetime(2026, 1, 2, 3, 4, 5)}) assert "2026-01-02" in rendered + + +def test_compact_items_drops_html_and_excerpts_long_fields(): + result = { + "items": [ + {"title": "t", "body": "b" * 5_000, "html": "

dup

", "upVotes": 3} + ] + } + compacted = compact_items(result, field_limit=100) + item = compacted["items"][0] + assert "html" not in item + assert len(item["body"]) < 200 and "truncated" in item["body"] + assert item["upVotes"] == 3 + # original untouched + assert "html" in result["items"][0] + + +def test_compact_items_passes_through_non_item_results(): + assert compact_items({"ok": True}) == {"ok": True} + assert compact_items([1, 2]) == [1, 2] diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index f916ef582..6dfc69622 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -5,6 +5,7 @@ import Link from "next/link"; import { ConnectorFaq } from "@/components/connectors-marketing/connector-faq"; import { Reveal } from "@/components/connectors-marketing/reveal"; import { MarketingSection } from "@/components/marketing/section"; +import { AgentSetupTabs } from "@/components/mcp/agent-setup-tabs"; import { BreadcrumbNav } from "@/components/seo/breadcrumb-nav"; import { FAQJsonLd, JsonLd } from "@/components/seo/json-ld"; import { Badge } from "@/components/ui/badge"; @@ -134,7 +135,7 @@ const FAQ: FaqItem[] = [ { question: "Which MCP clients does it work with?", answer: - "Any MCP client that supports stdio servers. Claude Code, Cursor, and Claude Desktop are documented with copy-paste configs, and the same command works in custom agent harnesses built on the MCP SDK.", + "Any MCP client that supports stdio servers. Claude Code, Codex, OpenCode, Cursor, Claude Desktop, VS Code, Windsurf, and Gemini CLI are documented with copy-paste configs on this page, and the same command works in custom agent harnesses built on the MCP SDK.", }, { question: "How is usage billed?", @@ -270,6 +271,25 @@ export default function McpServerPage() { + {/* Per-agent setup */} + + +

+ Step-by-step setup for every agent +

+

+ Pick your client, follow its two steps, and paste the config. Replace the placeholder + path with your surfsense_mcp checkout and the key with one from API Playground → API + Keys — or grab a pre-filled config from the playground itself. +

+ + +
+ +
+ + + {/* Tools */} diff --git a/surfsense_web/components/layout/ui/sidebar/PlaygroundSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/PlaygroundSidebar.tsx index bb90e0530..1c2f96212 100644 --- a/surfsense_web/components/layout/ui/sidebar/PlaygroundSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/PlaygroundSidebar.tsx @@ -3,6 +3,7 @@ import { History, KeyRound } from "lucide-react"; import Link from "next/link"; import { usePathname } from "next/navigation"; +import { ConnectAgentDialog } from "@/components/mcp/connect-agent-dialog"; import { PLAYGROUND_PLATFORMS, type PlatformIcon } from "@/lib/playground/catalog"; import { cn } from "@/lib/utils"; @@ -88,6 +89,10 @@ export function PlaygroundSidebar({ workspaceId }: PlaygroundSidebarProps) { ))} + +
+ +
); } diff --git a/surfsense_web/components/mcp/agent-setup-tabs.tsx b/surfsense_web/components/mcp/agent-setup-tabs.tsx new file mode 100644 index 000000000..1701b8924 --- /dev/null +++ b/surfsense_web/components/mcp/agent-setup-tabs.tsx @@ -0,0 +1,85 @@ +"use client"; + +import { Check, Copy } from "lucide-react"; +import { useState } from "react"; +import { Button } from "@/components/ui/button"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { + API_KEY_PLACEHOLDER, + DEFAULT_SERVER_DIR, + MCP_CLIENTS, + type McpSnippetOptions, +} from "@/lib/mcp/clients"; + +function CopyButton({ text }: { text: string }) { + const [copied, setCopied] = useState(false); + + const handleCopy = async () => { + try { + await navigator.clipboard.writeText(text); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + } catch { + // Clipboard unavailable (permissions/insecure context); nothing to recover. + } + }; + + return ( + + ); +} + +/** + * Per-agent MCP setup instructions as tabs: pick a client, follow its steps, + * copy its exact config. Used on the /mcp-server marketing page and in the + * API playground; `options` fills in real values where the caller has them. + */ +export function AgentSetupTabs({ options }: { options?: Partial }) { + const resolved: McpSnippetOptions = { + baseUrl: options?.baseUrl || "https://api.surfsense.com", + apiKey: options?.apiKey || API_KEY_PLACEHOLDER, + serverDir: options?.serverDir || DEFAULT_SERVER_DIR, + }; + + return ( + + + {MCP_CLIENTS.map((client) => ( + + {client.label} + + ))} + + {MCP_CLIENTS.map((client) => { + const config = client.buildConfig(resolved); + return ( + +
    + {client.steps.map((step) => ( +
  1. {step}
    2. + ))} +
+
+

{client.configFile}

+
+ + 
+									{config}
+
+
+
+ + ); + })} + + ); +} diff --git a/surfsense_web/components/mcp/connect-agent-dialog.tsx b/surfsense_web/components/mcp/connect-agent-dialog.tsx new file mode 100644 index 000000000..b48b849c6 --- /dev/null +++ b/surfsense_web/components/mcp/connect-agent-dialog.tsx @@ -0,0 +1,47 @@ +"use client"; + +import { Cable } from "lucide-react"; +import { AgentSetupTabs } from "@/components/mcp/agent-setup-tabs"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; +import { BACKEND_URL } from "@/lib/env-config"; +import { cn } from "@/lib/utils"; + +/** + * Sidebar-footer button that opens the MCP setup guide: pick an agent + * (Claude Code, Codex, OpenCode, ...), copy its config, done. + */ +export function ConnectAgentDialog({ className }: { className?: string }) { + return ( + + + + Connect your AI Agent + + + + Connect to Claude Code, Codex, OpenCode… + + The SurfSense MCP server gives any coding agent these scrapers and your knowledge base + as native tools. You need an API key (create one under API Keys) — then pick your agent + and paste its config. + + + + + + ); +} diff --git a/surfsense_web/content/docs/connectors/native/index.mdx b/surfsense_web/content/docs/connectors/native/index.mdx index a29c6b887..5c5d24444 100644 --- a/surfsense_web/content/docs/connectors/native/index.mdx +++ b/surfsense_web/content/docs/connectors/native/index.mdx @@ -35,13 +35,14 @@ Native connectors are SurfSense's own scraper APIs — built into the platform, /> -## Three ways to use them +## Four ways to use them -Every scraper is available through the same three doors: +Every scraper is available through the same four doors: 1. **In chat** — the AI agent uses these scrapers as tools automatically. Ask "what is r/selfhosted saying about SurfSense?" and the agent runs the Reddit scraper for you. 2. **API Playground** — open **API Playground** in your workspace sidebar, pick a scraper, fill in the form, and run it interactively. Great for exploring what a scraper returns before writing code. -3. **REST API** — call the scrapers from your own code. Each one is a single `POST`: +3. **MCP server** — hand every scraper to Claude Code, Codex, OpenCode, Cursor, or any MCP client as native tools. See the [MCP server guide](/docs/how-to/mcp-server). +4. **REST API** — call the scrapers from your own code. Each one is a single `POST`: ```bash POST /api/v1/workspaces/{workspace_id}/scrapers/{platform}/{verb} diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx new file mode 100644 index 000000000..ffdb4fd56 --- /dev/null +++ b/surfsense_web/content/docs/how-to/mcp-server.mdx @@ -0,0 +1,263 @@ +--- +title: MCP Server +description: Connect the SurfSense MCP server to Claude Code, Codex, OpenCode, Cursor, and other MCP clients, step by step +--- + +import { Tab, Tabs } from 'fumadocs-ui/components/tabs'; +import { Step, Steps } from 'fumadocs-ui/components/steps'; + +# SurfSense MCP Server + +The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 18 native, typed tools: every scraper (Reddit, YouTube, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. + +It talks to SurfSense purely over the REST API — point it at SurfSense Cloud or your own self-hosted instance by changing one environment variable. + +## Prerequisites + + + + +### Install uv + +The server runs with [uv](https://github.com/astral-sh/uv). Install it once, then from the SurfSense repository run: + +```bash +cd surfsense_mcp +uv sync +``` + + + + +### Create an API key + +In SurfSense, open **API Playground → API Keys** in your workspace sidebar: + +1. Toggle **API key access** on for the workspace. +2. Create a personal API key (`ss_pat_…`) and copy it — it is shown only once. + + + + +### Know your base URL + +- **SurfSense Cloud**: `https://api.surfsense.com` +- **Self-hosted**: wherever your backend runs, e.g. `http://localhost:8000` + + + + +## Connect your agent + +Every client below launches the same command — `uv run --directory /surfsense_mcp python -m surfsense_mcp` — and passes `SURFSENSE_BASE_URL` and `SURFSENSE_API_KEY` as environment variables. Replace the placeholder paths and key with yours. + + + + +Run one command in a terminal: + +```bash +claude mcp add surfsense \ + -e SURFSENSE_BASE_URL=https://api.surfsense.com \ + -e SURFSENSE_API_KEY=ss_pat_your_key_here \ + -- uv run --directory /path/to/SurfSense/surfsense_mcp python -m surfsense_mcp +``` + +Start Claude Code and run `/mcp` — `surfsense` should be listed as connected. Add `--scope project` to share the server (without the key) via a checked-in `.mcp.json`. + + + + +Add to `~/.codex/config.toml` (or a project's `.codex/config.toml`): + +```toml +[mcp_servers.surfsense] +command = "uv" +args = ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"] + +[mcp_servers.surfsense.env] +SURFSENSE_BASE_URL = "https://api.surfsense.com" +SURFSENSE_API_KEY = "ss_pat_your_key_here" +``` + +Or use the CLI: `codex mcp add surfsense -e SURFSENSE_API_KEY=... -- uv run --directory ... python -m surfsense_mcp`. Verify with `codex mcp list`. + + + + +Add to `opencode.json` in your project root (or `~/.config/opencode/opencode.json` globally): + +```json +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "surfsense": { + "type": "local", + "command": ["uv", "run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"], + "enabled": true, + "environment": { + "SURFSENSE_BASE_URL": "https://api.surfsense.com", + "SURFSENSE_API_KEY": "ss_pat_your_key_here" + } + } + } +} +``` + +OpenCode's format differs from most clients: the root key is `mcp` (not `mcpServers`), the command is a single array, and environment variables go under `environment` (not `env`). + + + + +Add to `~/.cursor/mcp.json` (global — keeps the key out of your repo) or a project's `.cursor/mcp.json`: + +```json +{ + "mcpServers": { + "surfsense": { + "command": "uv", + "args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"], + "env": { + "SURFSENSE_BASE_URL": "https://api.surfsense.com", + "SURFSENSE_API_KEY": "ss_pat_your_key_here" + } + } + } +} +``` + +Then open **Cursor Settings → MCP** and refresh the `surfsense` server; its 18 tools should appear with a green dot. + + + + +Open **Settings → Developer → Edit Config** to reach `claude_desktop_config.json`, and add the same `mcpServers` block as Cursor: + +```json +{ + "mcpServers": { + "surfsense": { + "command": "uv", + "args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"], + "env": { + "SURFSENSE_BASE_URL": "https://api.surfsense.com", + "SURFSENSE_API_KEY": "ss_pat_your_key_here" + } + } + } +} +``` + +Restart Claude Desktop; SurfSense appears under the tools icon in the chat input. + + + + +Add to `.vscode/mcp.json` in your workspace (or run the **MCP: Add Server** command). Note VS Code uses a `servers` key: + +```json +{ + "servers": { + "surfsense": { + "type": "stdio", + "command": "uv", + "args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"], + "env": { + "SURFSENSE_BASE_URL": "https://api.surfsense.com", + "SURFSENSE_API_KEY": "ss_pat_your_key_here" + } + } + } +} +``` + +Open Copilot Chat in agent mode and click the tools icon to confirm the server is loaded. + + + + +Add the standard `mcpServers` block to `~/.codeium/windsurf/mcp_config.json` (or via **Windsurf Settings → Cascade → MCP Servers**): + +```json +{ + "mcpServers": { + "surfsense": { + "command": "uv", + "args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"], + "env": { + "SURFSENSE_BASE_URL": "https://api.surfsense.com", + "SURFSENSE_API_KEY": "ss_pat_your_key_here" + } + } + } +} +``` + +Press refresh in the MCP panel to pick up the server. + + + + +Add the standard `mcpServers` block to `~/.gemini/settings.json` (or `.gemini/settings.json` in a project): + +```json +{ + "mcpServers": { + "surfsense": { + "command": "uv", + "args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"], + "env": { + "SURFSENSE_BASE_URL": "https://api.surfsense.com", + "SURFSENSE_API_KEY": "ss_pat_your_key_here" + } + } + } +} +``` + +Run `/mcp` inside Gemini CLI to confirm the server and its tools. + + + + + +The server uses stdio transport: your client launches the process on demand and shuts it down with the session. There is no daemon to manage — only your SurfSense backend needs to be up. + + +## Test it + +In a fresh agent session, try: + +> list my SurfSense workspaces + +That calls `surfsense_list_workspaces` — the simplest end-to-end check of the key, backend, and server. Then try a real task: + +> find the top subreddits discussing NotebookLM and save a summary note to my workspace + +## Configuration reference + +All settings are environment variables passed by the client: + +| Variable | Required | Default | Purpose | +|----------|----------|---------|---------| +| `SURFSENSE_API_KEY` | Yes | — | API key from **API Playground → API Keys** | +| `SURFSENSE_BASE_URL` | No | `http://localhost:8000` | Backend to talk to | +| `SURFSENSE_WORKSPACE` | No | — | Default workspace by name or id, so agents skip selection | +| `SURFSENSE_TIMEOUT` | No | `180` | Request timeout in seconds | + +## Troubleshooting + +- **401 errors** — the API key is wrong or expired; create a new one. +- **403 errors** — API access is disabled for the workspace; toggle **API key access** on under **API Playground → API Keys**. +- **"Could not reach SurfSense"** — the backend isn't running or `SURFSENSE_BASE_URL` is wrong. +- **Server won't start** — run `uv run python -m surfsense_mcp.selfcheck` inside `surfsense_mcp`; it verifies all 18 tools register without needing a backend. + +## Tools reference + +| Group | Tools | +|-------|-------| +| Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` | +| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | +| Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` | + +Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**. diff --git a/surfsense_web/content/docs/how-to/meta.json b/surfsense_web/content/docs/how-to/meta.json index 477fcafc4..91ca0ef89 100644 --- a/surfsense_web/content/docs/how-to/meta.json +++ b/surfsense_web/content/docs/how-to/meta.json @@ -1,6 +1,6 @@ { "title": "How to", - "pages": ["zero-sync", "realtime-collaboration", "web-search"], + "pages": ["mcp-server", "zero-sync", "realtime-collaboration", "web-search"], "icon": "Compass", "defaultOpen": false } diff --git a/surfsense_web/hooks/use-run-stream.ts b/surfsense_web/hooks/use-run-stream.ts index 4f3ebadf5..6e7f9b766 100644 --- a/surfsense_web/hooks/use-run-stream.ts +++ b/surfsense_web/hooks/use-run-stream.ts @@ -4,6 +4,7 @@ import { useQueryClient } from "@tanstack/react-query"; import { useCallback, useEffect, useRef, useState } from "react"; import type { ScraperRunDetail, ScraperRunEvent } from "@/contracts/types/scraper.types"; import { scrapersApiService } from "@/lib/apis/scrapers-api.service"; +import { trackWeeklyUser } from "@/lib/posthog/events"; import { cacheKeys } from "@/lib/query-client/cache-keys"; export type RunStatus = "idle" | "running" | "success" | "error" | "cancelled"; @@ -119,6 +120,7 @@ export function useRunStream(workspaceId: number) { payload ); runIdRef.current = started.run_id; + trackWeeklyUser("api_run", workspaceId); setState((s) => ({ ...s, runId: started.run_id })); void consume(started.run_id, controller.signal); } catch (e) { diff --git a/surfsense_web/lib/mcp/clients.ts b/surfsense_web/lib/mcp/clients.ts new file mode 100644 index 000000000..923e99049 --- /dev/null +++ b/surfsense_web/lib/mcp/clients.ts @@ -0,0 +1,176 @@ +/** + * MCP client setup catalog: one entry per popular agent, with the exact + * config file, steps, and snippet needed to connect the SurfSense MCP server. + * Shared by the marketing /mcp-server page and the API playground so the + * instructions can never drift apart. + */ + +export interface McpSnippetOptions { + /** SurfSense backend URL the server should call. */ + baseUrl: string; + /** API key value or placeholder to show in the snippet. */ + apiKey: string; + /** Absolute path to the surfsense_mcp directory. */ + serverDir: string; +} + +export interface McpClient { + id: string; + label: string; + /** Where the snippet goes: a file path or "Terminal". */ + configFile: string; + language: "json" | "toml" | "bash"; + steps: string[]; + buildConfig: (options: McpSnippetOptions) => string; +} + +export const DEFAULT_SERVER_DIR = "/path/to/SurfSense/surfsense_mcp"; +export const API_KEY_PLACEHOLDER = "ss_pat_your_key_here"; + +function serverArgs(serverDir: string): string[] { + return ["run", "--directory", serverDir, "python", "-m", "surfsense_mcp"]; +} + +function json(value: unknown): string { + return JSON.stringify(value, null, 2); +} + +/** The `mcpServers` JSON shape shared by Cursor, Claude Desktop, Windsurf, and Gemini CLI. */ +function standardJson({ baseUrl, apiKey, serverDir }: McpSnippetOptions): string { + return json({ + mcpServers: { + surfsense: { + command: "uv", + args: serverArgs(serverDir), + env: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + }, + }, + }); +} + +export const MCP_CLIENTS: McpClient[] = [ + { + id: "claude-code", + label: "Claude Code", + configFile: "Terminal", + language: "bash", + steps: [ + "Run this command in a terminal (any directory).", + "Start Claude Code and run /mcp — surfsense should be listed as connected.", + ], + buildConfig: ({ baseUrl, apiKey, serverDir }) => + [ + "claude mcp add surfsense \\", + ` -e SURFSENSE_BASE_URL=${baseUrl} \\`, + ` -e SURFSENSE_API_KEY=${apiKey} \\`, + ` -- uv run --directory ${serverDir} python -m surfsense_mcp`, + ].join("\n"), + }, + { + id: "codex", + label: "Codex", + configFile: "~/.codex/config.toml", + language: "toml", + steps: [ + "Add this to ~/.codex/config.toml (or run `codex mcp add surfsense -- uv run --directory python -m surfsense_mcp`).", + "Restart Codex; `codex mcp list` should show surfsense.", + ], + buildConfig: ({ baseUrl, apiKey, serverDir }) => + [ + "[mcp_servers.surfsense]", + 'command = "uv"', + `args = ${JSON.stringify(serverArgs(serverDir))}`, + "", + "[mcp_servers.surfsense.env]", + `SURFSENSE_BASE_URL = "${baseUrl}"`, + `SURFSENSE_API_KEY = "${apiKey}"`, + ].join("\n"), + }, + { + id: "opencode", + label: "OpenCode", + configFile: "opencode.json", + language: "json", + steps: [ + "Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).", + "Note OpenCode's format: the key is `mcp`, the command is one array, and env vars go under `environment`.", + ], + buildConfig: ({ baseUrl, apiKey, serverDir }) => + json({ + $schema: "https://opencode.ai/config.json", + mcp: { + surfsense: { + type: "local", + command: ["uv", ...serverArgs(serverDir)], + enabled: true, + environment: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + }, + }, + }), + }, + { + id: "cursor", + label: "Cursor", + configFile: "~/.cursor/mcp.json", + language: "json", + steps: [ + "Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.", + "Refresh the server in Cursor Settings → MCP; its 18 tools should appear.", + ], + buildConfig: standardJson, + }, + { + id: "claude-desktop", + label: "Claude Desktop", + configFile: "claude_desktop_config.json", + language: "json", + steps: [ + "Open Settings → Developer → Edit Config to reach claude_desktop_config.json and add this.", + "Restart Claude Desktop; surfsense appears under the tools icon.", + ], + buildConfig: standardJson, + }, + { + id: "vscode", + label: "VS Code", + configFile: ".vscode/mcp.json", + language: "json", + steps: [ + "Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).", + "Open Copilot Chat in agent mode and click the tools icon to confirm surfsense is loaded.", + ], + buildConfig: ({ baseUrl, apiKey, serverDir }) => + json({ + servers: { + surfsense: { + type: "stdio", + command: "uv", + args: serverArgs(serverDir), + env: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + }, + }, + }), + }, + { + id: "windsurf", + label: "Windsurf", + configFile: "~/.codeium/windsurf/mcp_config.json", + language: "json", + steps: [ + "Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).", + "Press the refresh button in the MCP panel to pick up the server.", + ], + buildConfig: standardJson, + }, + { + id: "gemini-cli", + label: "Gemini CLI", + configFile: "~/.gemini/settings.json", + language: "json", + steps: [ + "Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).", + "Run /mcp inside Gemini CLI to confirm the surfsense server and its tools.", + ], + buildConfig: standardJson, + }, +]; diff --git a/surfsense_web/lib/posthog/events.ts b/surfsense_web/lib/posthog/events.ts index 4293716d5..33d808278 100644 --- a/surfsense_web/lib/posthog/events.ts +++ b/surfsense_web/lib/posthog/events.ts @@ -97,6 +97,23 @@ export function trackWorkspaceViewed(workspaceId: number) { }); } +// ============================================ +// ACTIVE-USER (WAU) EVENT +// ============================================ + +/** + * Single signal for active-user counting. Fired whenever a user sends a + * chat message or starts an API run, so a "weekly unique users on + * weekly_users" insight in PostHog is our WAU number. + * + * ponytail: frontend-only capture — API runs made directly against the + * backend (PAT/curl, no browser) are not counted. Upgrade path is a + * server-side capture in the backend if that ever matters. + */ +export function trackWeeklyUser(source: "chat_message" | "api_run", workspaceId?: number) { + safeCapture("weekly_users", compact({ source, workspace_id: workspaceId })); +} + // ============================================ // CHAT EVENTS // ============================================ @@ -124,6 +141,7 @@ export function trackChatMessageSent( has_mentioned_documents: options?.hasMentionedDocuments ?? false, message_length: options?.messageLength, }); + trackWeeklyUser("chat_message", workspaceId); } export function trackChatResponseReceived(workspaceId: number, chatId: number) {