mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
feat: update environment variables and enhance scraping capabilities
- Adjusted Google Maps and YouTube micro pricing in the .env.example file for better cost management. - Introduced new environment variables for captcha solving and stealth browser hardening to improve scraping resilience. - Removed outdated smoke test for scraper API endpoints to streamline testing. - Enhanced anonymous chat agent's system prompt to clarify capabilities and suggest account creation for advanced features. - Updated Reddit fetch logic to prioritize new session handling and improve resilience against IP-related issues. - Added compacting functionality for scraper results to optimize data handling and presentation. - Improved workspace and document management tools with clearer descriptions and enhanced functionality. - Introduced new UI components for agent setup guidance in the web application.
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
271a21aee6
commit
1fd58752a3
24 changed files with 1326 additions and 320 deletions
|
|
@ -445,10 +445,10 @@ SURFSENSE_ENABLE_DOOM_LOOP=true
|
|||
# PLATFORM_SCRAPE_BILLING_ENABLED=FALSE
|
||||
# REDDIT_SCRAPE_MICROS_PER_ITEM=3500
|
||||
# GOOGLE_SEARCH_MICROS_PER_SERP=5500
|
||||
# GOOGLE_MAPS_MICROS_PER_PLACE=5000
|
||||
# GOOGLE_MAPS_MICROS_PER_REVIEW=2000
|
||||
# YOUTUBE_MICROS_PER_VIDEO=3500
|
||||
# YOUTUBE_MICROS_PER_COMMENT=3500
|
||||
# GOOGLE_MAPS_MICROS_PER_PLACE=3500
|
||||
# GOOGLE_MAPS_MICROS_PER_REVIEW=1500
|
||||
# YOUTUBE_MICROS_PER_VIDEO=2500
|
||||
# YOUTUBE_MICROS_PER_COMMENT=1500
|
||||
|
||||
# Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default).
|
||||
# QUOTA_MAX_RESERVE_MICROS=1000000
|
||||
|
|
@ -492,6 +492,28 @@ NOLOGIN_MODE_ENABLED=FALSE
|
|||
# PROXY_URL=http://user:pass@host:port
|
||||
# PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port
|
||||
|
||||
# Captcha solving — last-resort bypass tier via captchatools. Only fires on the
|
||||
# stealth browser tier when a sitekey is detected AND the flag is TRUE.
|
||||
# Cloudflare Turnstile is already solved free in-framework. Off by default.
|
||||
# NOTE: automated solving may violate a target site's ToS — opt-in, public
|
||||
# data only. See surfsense_backend/.env.example for the full option docs.
|
||||
# CAPTCHA_SOLVING_ENABLED=FALSE
|
||||
# CAPTCHA_SOLVER_PROVIDER=capsolver
|
||||
# CAPTCHA_SOLVER_API_KEY=
|
||||
# CAPTCHA_MAX_ATTEMPTS_PER_URL=1
|
||||
# CAPTCHA_SOLVE_TIMEOUT_S=120
|
||||
# CAPTCHA_TYPE_DEFAULT=v2
|
||||
# CAPTCHA_V3_MIN_SCORE=0.7
|
||||
# CAPTCHA_V3_ACTION=verify
|
||||
|
||||
# Stealth hardening levers on the stealth browser tier. Defaults preserve
|
||||
# current behavior; see surfsense_backend/.env.example for per-flag docs.
|
||||
# CRAWL_GEOIP_MATCH_ENABLED=FALSE
|
||||
# CRAWL_BLOCK_WEBRTC=TRUE
|
||||
# CRAWL_HIDE_CANVAS=FALSE
|
||||
# CRAWL_GOOGLE_SEARCH_REFERER=TRUE
|
||||
# CRAWL_DNS_OVER_HTTPS=FALSE
|
||||
|
||||
# ==============================================================================
|
||||
# DEV / DEPS-ONLY COMPOSE OVERRIDES
|
||||
# These are only needed for docker-compose.dev.yml or docker-compose.deps-only.yml.
|
||||
|
|
|
|||
|
|
@ -41,9 +41,12 @@ _MAX_DOC_CHARS = 50_000
|
|||
def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str:
|
||||
"""Build the system prompt for the minimal anonymous chat agent.
|
||||
|
||||
The prompt keeps the assistant focused on plain Q/A + web search, inlines
|
||||
any uploaded document as read-only context, and redirects every other
|
||||
SurfSense feature to account registration.
|
||||
The prompt keeps the assistant focused on plain Q/A from model knowledge,
|
||||
inlines any uploaded document as read-only context, and treats the chat as
|
||||
a registration funnel: every other SurfSense capability (scraping, live
|
||||
data, deliverables, knowledge base, automations) redirects to sign-up, and
|
||||
the assistant softly suggests an account when the conversation reveals a
|
||||
competitive-intelligence need the platform serves.
|
||||
"""
|
||||
today = datetime.now(UTC).strftime("%A, %B %d, %Y")
|
||||
|
||||
|
|
@ -70,7 +73,13 @@ def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str
|
|||
|
||||
return (
|
||||
"You are SurfSense's free AI assistant, available to everyone without "
|
||||
"login.\n\n"
|
||||
"login. SurfSense is the open-source competitive intelligence platform: "
|
||||
"registered users get specialist agents that pull live market data from "
|
||||
"Reddit, YouTube, Google Maps, Google Search, and the open web, turn it "
|
||||
"into cited briefs, reports, podcasts, and presentations, keep findings "
|
||||
"in a searchable knowledge base, and run scheduled monitoring "
|
||||
"automations — plus a REST scraping API and MCP server for their own "
|
||||
"agents.\n\n"
|
||||
f"Today's date is {today}.\n\n"
|
||||
"## How to help\n"
|
||||
"- Answer the user's questions directly and conversationally. You are "
|
||||
|
|
@ -78,23 +87,42 @@ def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str
|
|||
"- Answer from your own knowledge. You do NOT have web access here, so "
|
||||
"for current, real-time, or fast-changing facts (news, prices, "
|
||||
"weather, recent events, live data) say you can't look them up in the "
|
||||
"free experience and may be out of date, and invite the user to create "
|
||||
"a free account for live web search.\n"
|
||||
"free experience and may be out of date.\n"
|
||||
"- Be concise, accurate, and helpful. Use Markdown formatting when it "
|
||||
"improves readability."
|
||||
f"{doc_section}\n\n"
|
||||
"## What is not available here\n"
|
||||
"This is the free, no-login experience. You CANNOT search the web, save "
|
||||
"files or notes, generate reports, podcasts, resumes, presentations, or "
|
||||
"images, search or build a knowledge base, connect to apps (Gmail, "
|
||||
"Google Drive, Notion, Slack, Calendar, Discord, and similar), set up "
|
||||
"automations, or remember anything across sessions.\n\n"
|
||||
"This is the free, no-login experience. You CANNOT search the web or "
|
||||
"scrape any platform (Reddit, YouTube, Google Maps, Google Search, "
|
||||
"websites), save files or notes, upload additional files, generate "
|
||||
"reports, podcasts, resumes, presentations, or images, search or build "
|
||||
"a knowledge base, connect to apps (Gmail, Google Drive, Notion, "
|
||||
"Slack, Calendar, Discord, and similar), set up automations or "
|
||||
"monitoring, use the scraping API or MCP server, or remember anything "
|
||||
"across sessions.\n\n"
|
||||
"If the user asks for any of these, do NOT pretend to do them and "
|
||||
"never claim you saved, created, or stored anything. Instead, briefly "
|
||||
"let them know the feature requires a free SurfSense account and "
|
||||
"invite them to create one at https://www.surfsense.com. Then offer to "
|
||||
"help with what you can do here (answering questions from your own "
|
||||
"knowledge and about any uploaded document)."
|
||||
"never claim you saved, created, scraped, or stored anything. Instead, "
|
||||
"briefly say that this needs a free SurfSense account and invite them "
|
||||
"to sign up at https://www.surfsense.com/register — name the specific "
|
||||
"capability that would solve their request (e.g. 'a SurfSense account "
|
||||
"can scrape those Reddit threads live and turn them into a cited "
|
||||
"brief'). Then offer to help with what you can do here (answering from "
|
||||
"your own knowledge and about any uploaded document).\n\n"
|
||||
"## Suggesting SurfSense when it genuinely fits\n"
|
||||
"You are also the front door to SurfSense. When the conversation "
|
||||
"reveals a need the full platform serves — researching competitors, "
|
||||
"tracking pricing or rankings, monitoring brand mentions or reviews, "
|
||||
"gauging Reddit/YouTube sentiment, generating leads, needing current "
|
||||
"web data, or wanting recurring reports — first answer as well as you "
|
||||
"can from your own knowledge, then add ONE short sentence pointing out "
|
||||
"that a free SurfSense account can do that with live data, linking "
|
||||
"https://www.surfsense.com/register.\n"
|
||||
"- Be helpful first, never salesy: the answer is the product; the "
|
||||
"suggestion is a footnote.\n"
|
||||
"- At most one suggestion per response, and stop suggesting entirely "
|
||||
"if the user declines or ignores it.\n"
|
||||
"- Do not suggest it for needs SurfSense does not serve (casual chat, "
|
||||
"coding help, homework, creative writing)."
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ Reddit deprecated *cold* unauthenticated ``.json`` (a bare anonymous GET now
|
|||
``scripts/e2e_reddit_scraper.py`` step 0) is:
|
||||
|
||||
warm one anonymous session cookie (``loid``) with a plain GET to
|
||||
``old.reddit.com`` (``www.reddit.com/svc/shreddit/<slug>`` fallback), then
|
||||
``www.reddit.com/svc/shreddit/<slug>`` (``old.reddit.com`` fallback), then
|
||||
GET ``www.reddit.com/<path>/.json?raw_json=1`` through that same
|
||||
Chrome-impersonated, sticky-IP session. Which warm URL mints ``loid`` is
|
||||
exit-IP dependent, so the order is a tiebreak — see :func:`warm_session`.
|
||||
|
|
@ -84,6 +84,12 @@ _BACKOFF_BASE_S = 5.0
|
|||
_MIN_INTERVAL_S = 0.5
|
||||
_PACE_JITTER_S = 0.25
|
||||
|
||||
# curl's default is 30s, so one dead sticky IP stalled a whole run for 30-50s
|
||||
# (seen live 2026-07-06). A healthy fetch lands in ~1s; cap at 10s so a dead IP
|
||||
# costs one bounded wait, then the timeout falls into the generic exception
|
||||
# branch of fetch_json and rotates to a fresh IP — same treatment as a 403.
|
||||
_REQUEST_TIMEOUT_S = 10.0
|
||||
|
||||
_HEADERS = {"Accept-Language": "en-US,en;q=0.9"}
|
||||
|
||||
# Age-gate opt-in, sent on every ``.json`` fetch so NSFW listings aren't blanked
|
||||
|
|
@ -170,7 +176,10 @@ class _RotatingSession:
|
|||
self._cm = self.session = None
|
||||
return
|
||||
self._cm = FetcherSession(
|
||||
proxy=proxy, stealthy_headers=True, impersonate="chrome"
|
||||
proxy=proxy,
|
||||
stealthy_headers=True,
|
||||
impersonate="chrome",
|
||||
timeout=_REQUEST_TIMEOUT_S,
|
||||
)
|
||||
self.session = await self._cm.__aenter__()
|
||||
|
||||
|
|
@ -234,16 +243,15 @@ async def warm_session(session: Any, *, slug: str = _WARM_SLUG) -> bool:
|
|||
Returns ``True`` when a ``loid`` was issued (the session can now reach
|
||||
``.json``), else ``False`` (caller rotates the IP and retries).
|
||||
|
||||
Tries ``old.reddit`` (yt-dlp's primary), then ``svc/shreddit``. WHICH one
|
||||
mints is exit-IP dependent and roughly random: live probes 2026-07-04 saw
|
||||
both directions across sessions on the rotating residential/custom proxy
|
||||
(one IP 403s ``old.reddit`` but mints on ``shreddit``; another does the
|
||||
reverse, sometimes with an ``rdt`` bot-interstitial). So the order is a
|
||||
tiebreak, not an optimization — a fresh session pays ~one wasted warm 403
|
||||
either way. That cost is amortized: ``fan_out`` reuses one warmed session
|
||||
per worker across many jobs, so warm-up runs once per worker, not per fetch.
|
||||
The fallback is what actually matters — it preserves correctness whichever
|
||||
way a given IP leans.
|
||||
Tries ``svc/shreddit`` first, then ``old.reddit`` (yt-dlp's primary) as
|
||||
the fallback. Live probes 2026-07-04 saw both directions across exit IPs,
|
||||
but a live run 2026-07-06 had ``old.reddit`` 403 on 12/12 fresh IPs while
|
||||
``shreddit`` minted every time — Reddit appears to have shut old.reddit to
|
||||
anonymous traffic, so shreddit-first saves one guaranteed-403 round trip
|
||||
per warm-up. The fallback is what actually matters — it preserves
|
||||
correctness if a given IP (or Reddit) flips back the other way. That cost
|
||||
is amortized: ``fan_out`` reuses one warmed session per worker across many
|
||||
jobs, so warm-up runs once per worker, not per fetch.
|
||||
|
||||
ponytail: sequential two-source warm burns 1 wasted request on ~half of new
|
||||
sessions. A parallel warm (gather both, take whichever mints) removes the
|
||||
|
|
@ -256,14 +264,14 @@ async def warm_session(session: Any, *, slug: str = _WARM_SLUG) -> bool:
|
|||
"""
|
||||
seen: set[str] = set()
|
||||
with suppress(Exception):
|
||||
page = await session.get(_OLD_REDDIT_URL, headers=_HEADERS)
|
||||
page = await session.get(_SHREDDIT_URL.format(slug=slug), headers=_HEADERS)
|
||||
seen |= _response_cookie_names(page)
|
||||
if _LOID_COOKIE in seen:
|
||||
return True
|
||||
|
||||
# Fallback: mints loid on exit IPs where old.reddit 403s instead.
|
||||
# Fallback: mints loid on exit IPs where shreddit 403s instead.
|
||||
with suppress(Exception):
|
||||
page = await session.get(_SHREDDIT_URL.format(slug=slug), headers=_HEADERS)
|
||||
page = await session.get(_OLD_REDDIT_URL, headers=_HEADERS)
|
||||
seen |= _response_cookie_names(page)
|
||||
return _LOID_COOKIE in seen
|
||||
|
||||
|
|
@ -278,6 +286,7 @@ async def _get_page(session: Any, url: str) -> Any:
|
|||
cookies=_OVER18_COOKIES,
|
||||
proxy=get_proxy_url(),
|
||||
stealthy_headers=True,
|
||||
timeout=_REQUEST_TIMEOUT_S,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,106 +0,0 @@
|
|||
"""One-shot smoke test: hit every scraper API endpoint with a PAT, print PASS/FAIL."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
BASE = "http://localhost:8000/api/v1/workspaces/12/scrapers"
|
||||
PAT = os.environ["SURFSENSE_PAT"] # export a ss_pat_... key before running
|
||||
HEADERS = {"Authorization": f"Bearer {PAT}", "Content-Type": "application/json"}
|
||||
|
||||
# Minimal payloads: 1-3 items each to keep credit spend tiny.
|
||||
VERBS = [
|
||||
("google_search/scrape", {"queries": ["surfsense github"], "max_pages_per_query": 1}),
|
||||
("web/crawl", {"startUrls": ["https://example.com"], "maxCrawlDepth": 0}),
|
||||
("reddit/scrape", {"urls": ["https://www.reddit.com/r/Python/"], "max_items": 3}),
|
||||
("youtube/scrape", {"search_queries": ["python tutorial"], "max_results": 1}),
|
||||
(
|
||||
"youtube/comments",
|
||||
{"urls": ["https://www.youtube.com/watch?v=dQw4w9WgXcQ"], "max_comments": 3},
|
||||
),
|
||||
(
|
||||
"google_maps/scrape",
|
||||
{"search_queries": ["coffee shop"], "location": "New York, USA", "max_places": 1},
|
||||
),
|
||||
(
|
||||
"google_maps/reviews",
|
||||
# Google Sydney office, a stable well-known place id.
|
||||
{"place_ids": ["ChIJN1t_tDeuEmsRUsoyG83frY4"], "max_reviews": 3},
|
||||
),
|
||||
]
|
||||
|
||||
results = []
|
||||
client = httpx.Client(headers=HEADERS, timeout=300)
|
||||
sync_run_id = None
|
||||
|
||||
for verb, payload in VERBS:
|
||||
t0 = time.time()
|
||||
try:
|
||||
r = client.post(f"{BASE}/{verb}", json=payload)
|
||||
dur = time.time() - t0
|
||||
run_id = r.headers.get("x-run-id")
|
||||
if r.status_code == 200:
|
||||
body = r.json()
|
||||
# Count items in whatever list field the output has.
|
||||
items = next((len(v) for v in body.values() if isinstance(v, list)), "?")
|
||||
results.append((verb, "PASS", f"{r.status_code} items={items} run={run_id} {dur:.1f}s"))
|
||||
if sync_run_id is None and run_id:
|
||||
sync_run_id = run_id
|
||||
else:
|
||||
results.append((verb, "FAIL", f"{r.status_code} {r.text[:200]} {dur:.1f}s"))
|
||||
except Exception as e:
|
||||
results.append((verb, "FAIL", f"{type(e).__name__}: {e}"))
|
||||
print(f"[{results[-1][1]}] {verb}: {results[-1][2]}", flush=True)
|
||||
|
||||
# --- Run history endpoints ---
|
||||
r = client.get(f"{BASE}/runs", params={"limit": 5})
|
||||
ok = r.status_code == 200 and isinstance(r.json(), list)
|
||||
results.append(("GET runs (list)", "PASS" if ok else "FAIL", f"{r.status_code} rows={len(r.json()) if ok else '?'}"))
|
||||
print(f"[{results[-1][1]}] GET runs: {results[-1][2]}", flush=True)
|
||||
|
||||
if sync_run_id:
|
||||
r = client.get(f"{BASE}/runs/{sync_run_id}")
|
||||
ok = r.status_code == 200 and r.json().get("id") == sync_run_id
|
||||
results.append(("GET runs/{id} (detail)", "PASS" if ok else "FAIL", f"{r.status_code} status={r.json().get('status') if ok else r.text[:100]}"))
|
||||
print(f"[{results[-1][1]}] GET run detail: {results[-1][2]}", flush=True)
|
||||
|
||||
# --- Async mode + SSE events ---
|
||||
r = client.post(f"{BASE}/web/crawl?mode=async", json={"startUrls": ["https://example.com"]})
|
||||
if r.status_code == 202:
|
||||
async_id = r.json()["run_id"]
|
||||
seen, finished = [], None
|
||||
with client.stream("GET", f"{BASE}/runs/{async_id}/events", timeout=120) as s:
|
||||
for line in s.iter_lines():
|
||||
if line.startswith("data: "):
|
||||
ev = json.loads(line[6:])
|
||||
seen.append(ev["type"])
|
||||
if ev["type"] == "run.finished":
|
||||
finished = ev.get("status")
|
||||
break
|
||||
ok = finished == "success"
|
||||
results.append(("async + SSE events", "PASS" if ok else "FAIL", f"202 events={seen} final={finished}"))
|
||||
else:
|
||||
results.append(("async + SSE events", "FAIL", f"{r.status_code} {r.text[:200]}"))
|
||||
print(f"[{results[-1][1]}] async+SSE: {results[-1][2]}", flush=True)
|
||||
|
||||
# --- Cancel endpoint ---
|
||||
r = client.post(f"{BASE}/web/crawl?mode=async", json={"startUrls": ["https://example.com"], "maxCrawlDepth": 2, "maxCrawlPages": 50})
|
||||
if r.status_code == 202:
|
||||
cancel_id = r.json()["run_id"]
|
||||
time.sleep(1)
|
||||
r2 = client.post(f"{BASE}/runs/{cancel_id}/cancel")
|
||||
ok = r2.status_code == 200 and r2.json().get("status") == "cancelled"
|
||||
results.append(("POST runs/{id}/cancel", "PASS" if ok else "FAIL", f"{r2.status_code} {r2.text[:150]}"))
|
||||
else:
|
||||
results.append(("POST runs/{id}/cancel", "FAIL", f"setup {r.status_code}"))
|
||||
print(f"[{results[-1][1]}] cancel: {results[-1][2]}", flush=True)
|
||||
|
||||
print("\n===== SUMMARY =====")
|
||||
for name, status_, detail in results:
|
||||
print(f"{status_:4} {name}: {detail}")
|
||||
failed = [r for r in results if r[1] == "FAIL"]
|
||||
print(f"\n{len(results) - len(failed)}/{len(results)} passed")
|
||||
sys.exit(1 if failed else 0)
|
||||
|
|
@ -95,8 +95,8 @@ def _no_sleep(monkeypatch) -> None:
|
|||
|
||||
|
||||
async def test_warms_then_returns_json():
|
||||
# old.reddit is tried first and mints loid -> a single warm call.
|
||||
holder = _FakeHolder([_FakeSession(200, old_loid=True)])
|
||||
# shreddit is tried first and mints loid -> a single warm call.
|
||||
holder = _FakeHolder([_FakeSession(200, shreddit_loid=True)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
result = await fetch_json("r/python/hot")
|
||||
|
|
@ -107,9 +107,9 @@ async def test_warms_then_returns_json():
|
|||
assert holder.session.warm_calls == 1 # warmed exactly once
|
||||
|
||||
|
||||
async def test_warm_falls_back_to_shreddit():
|
||||
# old.reddit doesn't mint loid, shreddit does -> still warms on the same IP.
|
||||
holder = _FakeHolder([_FakeSession(200, shreddit_loid=True, old_loid=False)])
|
||||
async def test_warm_falls_back_to_old_reddit():
|
||||
# shreddit doesn't mint loid, old.reddit does -> still warms on the same IP.
|
||||
holder = _FakeHolder([_FakeSession(200, shreddit_loid=False, old_loid=True)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
result = await fetch_json("r/python/hot")
|
||||
|
|
|
|||
|
|
@ -8,11 +8,55 @@ are clipped so a single call can't blow the context window.
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Literal
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
ResponseFormat = Literal["markdown", "json"]
|
||||
|
||||
# Shared parameter type for every tool: same name, same semantics everywhere.
|
||||
ResponseFormatParam = Annotated[
|
||||
ResponseFormat,
|
||||
Field(
|
||||
description="'markdown' (default, human-readable) or 'json' "
|
||||
"(raw data for post-processing)."
|
||||
),
|
||||
]
|
||||
|
||||
DEFAULT_CLIP_CHARS = 20_000
|
||||
ITEM_FIELD_CLIP_CHARS = 1_500
|
||||
|
||||
# Fields that duplicate another field verbatim (e.g. Reddit's 'html' mirrors
|
||||
# 'body') and only bloat inline results. The full record stays in the run.
|
||||
_REDUNDANT_ITEM_FIELDS = frozenset({"html"})
|
||||
|
||||
|
||||
def compact_items(result: Any, field_limit: int = ITEM_FIELD_CLIP_CHARS) -> Any:
|
||||
"""Shrink a scraper result for inline return.
|
||||
|
||||
Drops redundant fields and clips overlong strings per field, so a response
|
||||
keeps every item as an excerpt instead of a few items in full. The
|
||||
untruncated result remains retrievable via its stored run.
|
||||
"""
|
||||
if isinstance(result, dict) and isinstance(result.get("items"), list):
|
||||
return {
|
||||
**result,
|
||||
"items": [_compact_item(item, field_limit) for item in result["items"]],
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def _compact_item(item: Any, field_limit: int) -> Any:
|
||||
# ponytail: compacts top-level string fields only; nested structures pass
|
||||
# through untouched. Upgrade path is a recursive walk if a platform nests
|
||||
# long text.
|
||||
if not isinstance(item, dict):
|
||||
return item
|
||||
return {
|
||||
key: clip(value, field_limit) if isinstance(value, str) else value
|
||||
for key, value in item.items()
|
||||
if key not in _REDUNDANT_ITEM_FIELDS
|
||||
}
|
||||
|
||||
|
||||
def to_json(payload: Any) -> str:
|
||||
|
|
|
|||
|
|
@ -8,10 +8,22 @@ speaks a name, we resolve it, and remember the choice for later calls.
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Annotated
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from .client import SurfSenseClient
|
||||
from .errors import ToolError
|
||||
|
||||
# Shared parameter type for every workspace-scoped tool.
|
||||
WorkspaceParam = Annotated[
|
||||
str | None,
|
||||
Field(
|
||||
description="Workspace name or id, e.g. 'Research' or '3'. Omit to use "
|
||||
"the active workspace (set with surfsense_select_workspace)."
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Workspace:
|
||||
|
|
|
|||
|
|
@ -10,14 +10,16 @@ from __future__ import annotations
|
|||
|
||||
import mimetypes
|
||||
from pathlib import Path
|
||||
from typing import Annotated
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from mcp.types import ToolAnnotations
|
||||
from pydantic import Field
|
||||
|
||||
from ...core.client import SurfSenseClient
|
||||
from ...core.errors import ToolError
|
||||
from ...core.rendering import ResponseFormat, clip, to_json
|
||||
from ...core.workspace_context import WorkspaceContext
|
||||
from ...core.rendering import ResponseFormatParam, clip, to_json
|
||||
from ...core.workspace_context import WorkspaceContext, WorkspaceParam
|
||||
from .note_ingestion import build_note_document
|
||||
|
||||
_READ = ToolAnnotations(
|
||||
|
|
@ -30,6 +32,22 @@ _DELETE = ToolAnnotations(
|
|||
readOnlyHint=False, destructiveHint=True, idempotentHint=False, openWorldHint=False
|
||||
)
|
||||
|
||||
_DOCUMENT_ID = Annotated[
|
||||
int,
|
||||
Field(
|
||||
description="Document id from surfsense_search_knowledge_base or "
|
||||
"surfsense_list_documents results."
|
||||
),
|
||||
]
|
||||
|
||||
_DOCUMENT_TYPES = Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Restrict to these document types, e.g. "
|
||||
"['FILE', 'CRAWLED_URL', 'YOUTUBE_VIDEO']. Omit for all types."
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def register(
|
||||
mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
|
||||
|
|
@ -38,21 +56,34 @@ def register(
|
|||
|
||||
@mcp.tool(
|
||||
name="surfsense_search_knowledge_base",
|
||||
title="Search knowledge base",
|
||||
annotations=_READ,
|
||||
structured_output=False,
|
||||
)
|
||||
async def search_knowledge_base(
|
||||
query: str,
|
||||
top_k: int = 5,
|
||||
document_types: list[str] | None = None,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
query: Annotated[
|
||||
str,
|
||||
Field(
|
||||
min_length=1,
|
||||
description="Natural-language search, e.g. "
|
||||
"'notebooklm user complaints'.",
|
||||
),
|
||||
],
|
||||
top_k: Annotated[
|
||||
int, Field(ge=1, le=20, description="Maximum documents to return.")
|
||||
] = 5,
|
||||
document_types: _DOCUMENT_TYPES = None,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Search the workspace's knowledge base by meaning and keyword.
|
||||
"""Search the workspace's knowledge base by meaning and keywords.
|
||||
|
||||
Use this to answer questions from stored content: it returns the most
|
||||
relevant documents with the passages that matched, ranked by relevance.
|
||||
top_k caps documents (1–20). Optionally restrict to document_types.
|
||||
Use this FIRST when a question might be answered by content already
|
||||
stored in SurfSense — notes, uploaded files, saved pages, past
|
||||
research. Do NOT use it to fetch new data from the web; use the
|
||||
scraper tools for that. Returns the most relevant documents with the
|
||||
passages that matched, ranked by relevance score.
|
||||
Example: query='pricing feedback', top_k=5.
|
||||
"""
|
||||
resolved = await context.resolve(workspace)
|
||||
hits = await client.request(
|
||||
|
|
@ -71,21 +102,33 @@ def register(
|
|||
return _render_search(query, items)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_list_documents", annotations=_READ, structured_output=False
|
||||
name="surfsense_list_documents",
|
||||
title="List documents",
|
||||
annotations=_READ,
|
||||
structured_output=False,
|
||||
)
|
||||
async def list_documents(
|
||||
document_types: list[str] | None = None,
|
||||
folder_id: int | None = None,
|
||||
page: int = 0,
|
||||
page_size: int = 20,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
document_types: _DOCUMENT_TYPES = None,
|
||||
folder_id: Annotated[
|
||||
int | None,
|
||||
Field(description="Only documents in this folder. Omit for all."),
|
||||
] = None,
|
||||
page: Annotated[
|
||||
int, Field(ge=0, description="Zero-based page number.")
|
||||
] = 0,
|
||||
page_size: Annotated[
|
||||
int, Field(ge=1, description="Documents per page.")
|
||||
] = 20,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""List documents in the workspace's knowledge base, newest first.
|
||||
|
||||
Use this to browse or inventory what is stored. Optionally filter by
|
||||
document_types or a folder_id. Paginated: returns page_size items and a
|
||||
has_more flag; request the next page by increasing page.
|
||||
Use this to browse or inventory what is stored; to find documents
|
||||
about a topic, prefer surfsense_search_knowledge_base. Returns each
|
||||
document's title, id, type, and update time, plus a has_more flag —
|
||||
request the next page by increasing page.
|
||||
Example: document_types=['FILE'], page=0, page_size=20.
|
||||
"""
|
||||
resolved = await context.resolve(workspace)
|
||||
result = await client.request(
|
||||
|
|
@ -104,15 +147,20 @@ def register(
|
|||
return _render_document_list(result)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_get_document", annotations=_READ, structured_output=False
|
||||
name="surfsense_get_document",
|
||||
title="Read one document",
|
||||
annotations=_READ,
|
||||
structured_output=False,
|
||||
)
|
||||
async def get_document(
|
||||
document_id: int, response_format: ResponseFormat = "markdown"
|
||||
document_id: _DOCUMENT_ID,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Read one document's full content and metadata by id.
|
||||
|
||||
Use this after search or list to open a specific document. The id comes
|
||||
from those tools' results.
|
||||
Use this after surfsense_search_knowledge_base or
|
||||
surfsense_list_documents to open a specific document — search results
|
||||
only include the matching passages, this returns the whole text.
|
||||
"""
|
||||
document = await client.request("GET", f"/documents/{document_id}")
|
||||
if response_format == "json":
|
||||
|
|
@ -120,20 +168,36 @@ def register(
|
|||
return _render_document(document)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_add_document", annotations=_WRITE, structured_output=False
|
||||
name="surfsense_add_document",
|
||||
title="Add a note",
|
||||
annotations=_WRITE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def add_document(
|
||||
title: str,
|
||||
content: str,
|
||||
source_url: str | None = None,
|
||||
workspace: str | None = None,
|
||||
title: Annotated[
|
||||
str,
|
||||
Field(min_length=1, description="Short descriptive title for the note."),
|
||||
],
|
||||
content: Annotated[
|
||||
str,
|
||||
Field(
|
||||
min_length=1,
|
||||
description="The note's body; plain text or markdown.",
|
||||
),
|
||||
],
|
||||
source_url: Annotated[
|
||||
str | None,
|
||||
Field(description="Where the text came from, if anywhere."),
|
||||
] = None,
|
||||
workspace: WorkspaceParam = None,
|
||||
) -> str:
|
||||
"""Add a text or markdown note to the workspace's knowledge base.
|
||||
"""Save a text or markdown note into the workspace's knowledge base.
|
||||
|
||||
Use this to save notes, summaries, or snippets so they become
|
||||
searchable. The content is indexed asynchronously, so it may take a
|
||||
moment to appear in search. source_url optionally records where the text
|
||||
came from.
|
||||
Use this to store notes, summaries, or findings so they become
|
||||
searchable later — e.g. after finishing a piece of research. For files
|
||||
on disk use surfsense_upload_file instead. Indexing is asynchronous,
|
||||
so the note may take a moment to appear in search.
|
||||
Example: title='NotebookLM subreddits', content='- r/notebooklm ...'.
|
||||
"""
|
||||
resolved = await context.resolve(workspace)
|
||||
await client.request(
|
||||
|
|
@ -152,18 +216,35 @@ def register(
|
|||
)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_upload_file", annotations=_WRITE, structured_output=False
|
||||
name="surfsense_upload_file",
|
||||
title="Upload a file",
|
||||
annotations=_WRITE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def upload_file(
|
||||
file_path: str,
|
||||
use_vision_llm: bool = False,
|
||||
workspace: str | None = None,
|
||||
file_path: Annotated[
|
||||
str,
|
||||
Field(
|
||||
description="Path to a local file, e.g. "
|
||||
"'C:/Users/me/report.pdf' or '~/notes/summary.md'."
|
||||
),
|
||||
],
|
||||
use_vision_llm: Annotated[
|
||||
bool,
|
||||
Field(
|
||||
description="True reads scanned or image-heavy files with a "
|
||||
"vision model (slower)."
|
||||
),
|
||||
] = False,
|
||||
workspace: WorkspaceParam = None,
|
||||
) -> str:
|
||||
"""Upload a local file (PDF, doc, etc.) into the knowledge base.
|
||||
"""Upload a local file (PDF, docx, markdown, etc.) into the knowledge base.
|
||||
|
||||
Use this to ingest a file from disk; it is parsed, chunked, and indexed
|
||||
asynchronously. Set use_vision_llm to read scanned or image-heavy files
|
||||
with a vision model (slower).
|
||||
Use this to ingest a file from disk so its content becomes searchable;
|
||||
for text you already have in hand use surfsense_add_document instead.
|
||||
The file is parsed, chunked, and indexed asynchronously. Duplicate
|
||||
files are detected and skipped.
|
||||
Example: file_path='C:/Users/me/report.pdf'.
|
||||
"""
|
||||
resolved = await context.resolve(workspace)
|
||||
payload = _read_upload(file_path)
|
||||
|
|
@ -186,14 +267,28 @@ def register(
|
|||
)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_update_document", annotations=_WRITE, structured_output=False
|
||||
name="surfsense_update_document",
|
||||
title="Replace a document's content",
|
||||
annotations=_WRITE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def update_document(document_id: int, content: str) -> str:
|
||||
async def update_document(
|
||||
document_id: _DOCUMENT_ID,
|
||||
content: Annotated[
|
||||
str,
|
||||
Field(
|
||||
min_length=1,
|
||||
description="New full text; replaces the existing content "
|
||||
"entirely.",
|
||||
),
|
||||
],
|
||||
) -> str:
|
||||
"""Replace a document's stored content by id.
|
||||
|
||||
Use this to correct or rewrite a document's text. Note: this updates the
|
||||
stored content; re-indexing of search chunks is not triggered by this
|
||||
call.
|
||||
Use this to correct or rewrite a document's text. The new content
|
||||
REPLACES the old entirely — to append, read the document first with
|
||||
surfsense_get_document and resend the combined text. Search chunks are
|
||||
not re-indexed by this call.
|
||||
"""
|
||||
existing = await client.request("GET", f"/documents/{document_id}")
|
||||
await client.request(
|
||||
|
|
@ -208,13 +303,17 @@ def register(
|
|||
return f"Updated document {document_id} ('{existing.get('title', '')}')."
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_delete_document", annotations=_DELETE, structured_output=False
|
||||
name="surfsense_delete_document",
|
||||
title="Delete a document",
|
||||
annotations=_DELETE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def delete_document(document_id: int) -> str:
|
||||
"""Delete a document from the knowledge base by id.
|
||||
async def delete_document(document_id: _DOCUMENT_ID) -> str:
|
||||
"""Permanently delete a document from the knowledge base by id.
|
||||
|
||||
Use this to permanently remove a document. Deletion runs in the
|
||||
background; the document stops appearing in searches immediately.
|
||||
Use this only when the user explicitly asks to remove a document —
|
||||
deletion cannot be undone. The document stops appearing in searches
|
||||
immediately.
|
||||
"""
|
||||
await client.request("DELETE", f"/documents/{document_id}")
|
||||
return f"Deleted document {document_id}."
|
||||
|
|
|
|||
|
|
@ -8,14 +8,15 @@ full later.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from mcp.types import ToolAnnotations
|
||||
from pydantic import Field
|
||||
|
||||
from ...core.client import SurfSenseClient
|
||||
from ...core.rendering import ResponseFormat, clip, to_json
|
||||
from ...core.workspace_context import WorkspaceContext
|
||||
from ...core.rendering import ResponseFormatParam, clip, to_json
|
||||
from ...core.workspace_context import WorkspaceContext, WorkspaceParam
|
||||
from .capability import run_scraper
|
||||
|
||||
# Scrapers reach the open web and record a billable run; they are neither
|
||||
|
|
@ -38,23 +39,57 @@ def register(
|
|||
) -> None:
|
||||
"""Register the scraper and run-history tools on the server."""
|
||||
|
||||
@mcp.tool(name="surfsense_web_crawl", annotations=_SCRAPE, structured_output=False)
|
||||
@mcp.tool(
|
||||
name="surfsense_web_crawl",
|
||||
title="Crawl web pages",
|
||||
annotations=_SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def web_crawl(
|
||||
start_urls: list[str],
|
||||
max_crawl_depth: int = 0,
|
||||
max_crawl_pages: int = 10,
|
||||
max_length: int = 50_000,
|
||||
include_url_patterns: list[str] | None = None,
|
||||
exclude_url_patterns: list[str] | None = None,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
start_urls: Annotated[
|
||||
list[str],
|
||||
Field(
|
||||
min_length=1,
|
||||
description="Full URLs to fetch, e.g. "
|
||||
"['https://example.com/blog/post'].",
|
||||
),
|
||||
],
|
||||
max_crawl_depth: Annotated[
|
||||
int,
|
||||
Field(
|
||||
ge=0,
|
||||
description="Link-hops to follow from start_urls within the "
|
||||
"same site. 0 fetches only start_urls.",
|
||||
),
|
||||
] = 0,
|
||||
max_crawl_pages: Annotated[
|
||||
int, Field(ge=1, description="Stop after this many pages in total.")
|
||||
] = 10,
|
||||
max_length: Annotated[
|
||||
int, Field(ge=1, description="Max characters kept per page.")
|
||||
] = 50_000,
|
||||
include_url_patterns: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Regexes; only discovered links matching one are "
|
||||
"followed, e.g. ['/docs/.*']."
|
||||
),
|
||||
] = None,
|
||||
exclude_url_patterns: Annotated[
|
||||
list[str] | None,
|
||||
Field(description="Regexes; discovered links matching one are skipped."),
|
||||
] = None,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Crawl web pages and return their cleaned content as markdown.
|
||||
"""Fetch specific web pages and return their cleaned content as markdown.
|
||||
|
||||
Use this to read one page or spider a site. With max_crawl_depth=0 only
|
||||
start_urls are fetched; a higher depth follows same-site links up to
|
||||
max_crawl_pages. include/exclude_url_patterns are regexes that narrow
|
||||
which discovered links are followed.
|
||||
Use this to read a page the user names, or to spider a site from a
|
||||
starting URL. Do NOT use it to find pages on a topic — use
|
||||
surfsense_google_search for discovery. Returns one item per crawled
|
||||
page: url, title, and the page text as markdown.
|
||||
Example: start_urls=['https://blog.example.com'], max_crawl_depth=1,
|
||||
include_url_patterns=['/2026/'].
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
|
|
@ -74,22 +109,47 @@ def register(
|
|||
)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_google_search", annotations=_SCRAPE, structured_output=False
|
||||
name="surfsense_google_search",
|
||||
title="Scrape Google Search",
|
||||
annotations=_SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def google_search(
|
||||
queries: list[str],
|
||||
max_pages_per_query: int = 1,
|
||||
country_code: str | None = None,
|
||||
language_code: str = "",
|
||||
site: str | None = None,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
queries: Annotated[
|
||||
list[str],
|
||||
Field(
|
||||
min_length=1,
|
||||
description="Search terms or full Google Search URLs, e.g. "
|
||||
"['best rss readers 2026'].",
|
||||
),
|
||||
],
|
||||
max_pages_per_query: Annotated[
|
||||
int, Field(ge=1, description="Result pages to fetch per query.")
|
||||
] = 1,
|
||||
country_code: Annotated[
|
||||
str | None,
|
||||
Field(description="Two-letter country to search from, e.g. 'us'."),
|
||||
] = None,
|
||||
language_code: Annotated[
|
||||
str, Field(description="Results language, e.g. 'en'. Empty for default.")
|
||||
] = "",
|
||||
site: Annotated[
|
||||
str | None,
|
||||
Field(
|
||||
description="Restrict results to one domain, e.g. 'example.com'."
|
||||
),
|
||||
] = None,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Scrape Google Search results for one or more queries.
|
||||
"""Scrape Google Search result pages for one or more queries.
|
||||
|
||||
Use this to find pages on the web. Each item is a query's fetched result
|
||||
page. Pass full Google Search URLs to scrape them as-is, or plain terms
|
||||
to search. Optionally scope to a country, language, or single domain.
|
||||
Use this to discover pages on the open web by topic; follow up with
|
||||
surfsense_web_crawl to read a result in full. Do NOT use it for
|
||||
Reddit, YouTube, or Google Maps research — the dedicated tools return
|
||||
richer data. Returns each query's parsed results: title, url, and
|
||||
snippet per organic result.
|
||||
Example: queries=['notebooklm review'], site='news.ycombinator.com'.
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
|
|
@ -108,24 +168,61 @@ def register(
|
|||
)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_reddit_scrape", annotations=_SCRAPE, structured_output=False
|
||||
name="surfsense_reddit_scrape",
|
||||
title="Search or scrape Reddit",
|
||||
annotations=_SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def reddit_scrape(
|
||||
urls: list[str] | None = None,
|
||||
search_queries: list[str] | None = None,
|
||||
community: str | None = None,
|
||||
sort: RedditSort = "new",
|
||||
time_filter: RedditTime | None = None,
|
||||
max_items: int = 10,
|
||||
skip_comments: bool = False,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
urls: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Reddit URLs: a post, a subreddit like "
|
||||
"'https://reddit.com/r/LocalLLaMA', a user page, or a search "
|
||||
"URL. Provide urls OR search_queries."
|
||||
),
|
||||
] = None,
|
||||
search_queries: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Terms to search Reddit for, e.g. "
|
||||
"['NotebookLM alternatives']. Provide search_queries OR urls."
|
||||
),
|
||||
] = None,
|
||||
community: Annotated[
|
||||
str | None,
|
||||
Field(
|
||||
description="Restrict a search to one subreddit, name without "
|
||||
"'r/', e.g. 'ArtificialInteligence'."
|
||||
),
|
||||
] = None,
|
||||
sort: Annotated[RedditSort, Field(description="Post ordering.")] = "new",
|
||||
time_filter: Annotated[
|
||||
RedditTime | None,
|
||||
Field(description="Time window; only valid with sort='top'."),
|
||||
] = None,
|
||||
max_items: Annotated[
|
||||
int, Field(ge=1, description="Maximum posts to return.")
|
||||
] = 10,
|
||||
skip_comments: Annotated[
|
||||
bool,
|
||||
Field(
|
||||
description="True fetches posts only (faster); False also "
|
||||
"fetches each post's comment thread."
|
||||
),
|
||||
] = False,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Scrape Reddit posts and comments from URLs or a search.
|
||||
"""Search or scrape Reddit: posts, comments, subreddits, and users.
|
||||
|
||||
Provide urls (a post, /r/subreddit, /user/name, or search URL) OR
|
||||
search_queries; scope a search to one subreddit with community. Use
|
||||
time_filter only with sort='top'. Set skip_comments to fetch posts only.
|
||||
Use this for ANY Reddit research — finding relevant subreddits or
|
||||
communities for a topic, top posts, or discussions — instead of a
|
||||
generic web search. Returns posts (title, text, score, subreddit, url)
|
||||
with comment threads unless skip_comments is set. Every post carries
|
||||
its subreddit, so to find communities for a topic, search posts and
|
||||
aggregate their subreddits.
|
||||
Example: search_queries=['NotebookLM'], sort='top', time_filter='month'.
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
|
|
@ -146,22 +243,46 @@ def register(
|
|||
)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_youtube_scrape", annotations=_SCRAPE, structured_output=False
|
||||
name="surfsense_youtube_scrape",
|
||||
title="Search or scrape YouTube",
|
||||
annotations=_SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def youtube_scrape(
|
||||
urls: list[str] | None = None,
|
||||
search_queries: list[str] | None = None,
|
||||
max_results: int = 10,
|
||||
download_subtitles: bool = False,
|
||||
subtitles_language: str = "en",
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
urls: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="YouTube URLs: video, channel, playlist, shorts, "
|
||||
"or hashtag pages. Provide urls OR search_queries."
|
||||
),
|
||||
] = None,
|
||||
search_queries: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Terms to search YouTube for, e.g. "
|
||||
"['NotebookLM tutorial']. Provide search_queries OR urls."
|
||||
),
|
||||
] = None,
|
||||
max_results: Annotated[
|
||||
int, Field(ge=1, description="Maximum videos to return.")
|
||||
] = 10,
|
||||
download_subtitles: Annotated[
|
||||
bool,
|
||||
Field(description="True also fetches each video's transcript."),
|
||||
] = False,
|
||||
subtitles_language: Annotated[
|
||||
str, Field(description="Transcript language code, e.g. 'en'.")
|
||||
] = "en",
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Scrape YouTube videos from URLs or a search.
|
||||
"""Search or scrape YouTube videos, optionally with transcripts.
|
||||
|
||||
Provide urls (video, channel, playlist, shorts, or hashtag pages) OR
|
||||
search_queries. Set download_subtitles to also fetch each video's
|
||||
transcript in subtitles_language.
|
||||
Use this for YouTube research: finding videos on a topic, or reading a
|
||||
video's details or transcript. For a video's comment section use
|
||||
surfsense_youtube_comments instead. Returns per-video metadata (title,
|
||||
channel, views, description, url) and, if requested, the transcript.
|
||||
Example: search_queries=['NotebookLM tutorial'], download_subtitles=True.
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
|
|
@ -180,19 +301,41 @@ def register(
|
|||
)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_youtube_comments", annotations=_SCRAPE, structured_output=False
|
||||
name="surfsense_youtube_comments",
|
||||
title="Fetch YouTube comments",
|
||||
annotations=_SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def youtube_comments(
|
||||
urls: list[str],
|
||||
max_comments: int = 20,
|
||||
sort_by: CommentSort = "NEWEST_FIRST",
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
urls: Annotated[
|
||||
list[str],
|
||||
Field(
|
||||
min_length=1,
|
||||
description="YouTube video URLs, e.g. "
|
||||
"['https://www.youtube.com/watch?v=abc123'].",
|
||||
),
|
||||
],
|
||||
max_comments: Annotated[
|
||||
int,
|
||||
Field(
|
||||
ge=1,
|
||||
description="Maximum comments per video, counting top-level "
|
||||
"comments and replies together.",
|
||||
),
|
||||
] = 20,
|
||||
sort_by: Annotated[
|
||||
CommentSort, Field(description="Comment ordering.")
|
||||
] = "NEWEST_FIRST",
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Fetch comments (and replies) for one or more YouTube videos.
|
||||
"""Fetch the comments (and replies) on one or more YouTube videos.
|
||||
|
||||
Use this when the user wants a video's discussion rather than the video
|
||||
itself. max_comments counts top-level comments and replies together.
|
||||
Use this when the user wants a video's discussion or audience reaction
|
||||
rather than the video itself; get video URLs from
|
||||
surfsense_youtube_scrape if you only have a topic. Returns comment
|
||||
text, author, likes, and replies.
|
||||
Example: urls=['https://www.youtube.com/watch?v=abc123'], max_comments=50.
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
|
|
@ -210,24 +353,52 @@ def register(
|
|||
|
||||
@mcp.tool(
|
||||
name="surfsense_google_maps_scrape",
|
||||
title="Find places on Google Maps",
|
||||
annotations=_SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def google_maps_scrape(
|
||||
search_queries: list[str] | None = None,
|
||||
urls: list[str] | None = None,
|
||||
place_ids: list[str] | None = None,
|
||||
location: str | None = None,
|
||||
max_places: int = 10,
|
||||
include_details: bool = False,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
search_queries: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Place searches, e.g. ['coffee shops']. Provide "
|
||||
"search_queries OR urls OR place_ids."
|
||||
),
|
||||
] = None,
|
||||
urls: Annotated[
|
||||
list[str] | None,
|
||||
Field(description="Google Maps URLs of specific places."),
|
||||
] = None,
|
||||
place_ids: Annotated[
|
||||
list[str] | None,
|
||||
Field(description="Google place ids, e.g. ['ChIJj61dQgK6j4AR...']."),
|
||||
] = None,
|
||||
location: Annotated[
|
||||
str | None,
|
||||
Field(
|
||||
description="Geographic scope for a search, e.g. "
|
||||
"'Seattle, USA'."
|
||||
),
|
||||
] = None,
|
||||
max_places: Annotated[
|
||||
int, Field(ge=1, description="Maximum places to return.")
|
||||
] = 10,
|
||||
include_details: Annotated[
|
||||
bool,
|
||||
Field(
|
||||
description="True adds opening hours and extra contact info "
|
||||
"(slower)."
|
||||
),
|
||||
] = False,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Scrape places from Google Maps by search, URL, or place id.
|
||||
"""Find places on Google Maps by search, URL, or place id.
|
||||
|
||||
Provide search_queries OR urls OR place_ids. Scope a search with
|
||||
location (e.g. 'New York, USA'). Set include_details for opening hours
|
||||
and extra contact info (slower).
|
||||
Use this for local-business and location research: names, addresses,
|
||||
ratings, categories, coordinates, place ids. For a place's customer
|
||||
reviews use surfsense_google_maps_reviews instead.
|
||||
Example: search_queries=['ramen'], location='Osaka, Japan', max_places=5.
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
|
|
@ -248,23 +419,49 @@ def register(
|
|||
|
||||
@mcp.tool(
|
||||
name="surfsense_google_maps_reviews",
|
||||
title="Fetch Google Maps reviews",
|
||||
annotations=_SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def google_maps_reviews(
|
||||
urls: list[str] | None = None,
|
||||
place_ids: list[str] | None = None,
|
||||
max_reviews: int = 20,
|
||||
sort_by: ReviewSort = "newest",
|
||||
language: str = "en",
|
||||
start_date: str | None = None,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
urls: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Google Maps URLs of places. Provide urls OR "
|
||||
"place_ids."
|
||||
),
|
||||
] = None,
|
||||
place_ids: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Google place ids from surfsense_google_maps_scrape."
|
||||
),
|
||||
] = None,
|
||||
max_reviews: Annotated[
|
||||
int, Field(ge=1, description="Maximum reviews per place.")
|
||||
] = 20,
|
||||
sort_by: Annotated[
|
||||
ReviewSort, Field(description="Review ordering.")
|
||||
] = "newest",
|
||||
language: Annotated[
|
||||
str, Field(description="Reviews language code, e.g. 'en'.")
|
||||
] = "en",
|
||||
start_date: Annotated[
|
||||
str | None,
|
||||
Field(
|
||||
description="ISO date like '2026-01-01'; keeps only reviews on "
|
||||
"or after that day."
|
||||
),
|
||||
] = None,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Fetch reviews for Google Maps places by URL or place id.
|
||||
"""Fetch customer reviews for Google Maps places by URL or place id.
|
||||
|
||||
Provide urls OR place_ids. start_date (ISO, e.g. '2024-01-01') keeps only
|
||||
reviews on or after that day.
|
||||
Use this to read feedback on specific places; get urls or place_ids
|
||||
from surfsense_google_maps_scrape first if you only have a name.
|
||||
Returns review text, rating, author, and date per review.
|
||||
Example: place_ids=['ChIJj61dQgK6j4AR...'], sort_by='newest'.
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
|
|
@ -285,21 +482,35 @@ def register(
|
|||
|
||||
@mcp.tool(
|
||||
name="surfsense_list_scraper_runs",
|
||||
title="List past scraper runs",
|
||||
annotations=_READ_RUNS,
|
||||
structured_output=False,
|
||||
)
|
||||
async def list_scraper_runs(
|
||||
limit: int = 20,
|
||||
capability: str | None = None,
|
||||
status: str | None = None,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
limit: Annotated[
|
||||
int, Field(ge=1, description="Maximum runs to list.")
|
||||
] = 20,
|
||||
capability: Annotated[
|
||||
str | None,
|
||||
Field(
|
||||
description="Filter by capability slug, e.g. 'web.crawl' or "
|
||||
"'reddit.scrape'."
|
||||
),
|
||||
] = None,
|
||||
status: Annotated[
|
||||
str | None,
|
||||
Field(description="Filter by run status: 'success' or 'error'."),
|
||||
] = None,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""List recent scraper runs for the workspace, newest first.
|
||||
"""List recent scraper runs in the workspace, newest first.
|
||||
|
||||
Use this to find a run_id to fetch in full with surfsense_get_scraper_run,
|
||||
e.g. when an inline result was truncated. Optionally filter by capability
|
||||
(like 'web.crawl') or status ('success' / 'error').
|
||||
Use this to find the run_id of an earlier scrape — for example when an
|
||||
inline result was truncated — then fetch it in full with
|
||||
surfsense_get_scraper_run. Returns each run's id, capability, status,
|
||||
item count, and creation time.
|
||||
Example: capability='reddit.scrape', status='success'.
|
||||
"""
|
||||
resolved = await context.resolve(workspace)
|
||||
runs = await client.request(
|
||||
|
|
@ -317,18 +528,26 @@ def register(
|
|||
|
||||
@mcp.tool(
|
||||
name="surfsense_get_scraper_run",
|
||||
title="Fetch one scraper run in full",
|
||||
annotations=_READ_RUNS,
|
||||
structured_output=False,
|
||||
)
|
||||
async def get_scraper_run(
|
||||
run_id: str,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
run_id: Annotated[
|
||||
str,
|
||||
Field(
|
||||
description="Run id from surfsense_list_scraper_runs or a "
|
||||
"prior scrape's output."
|
||||
),
|
||||
],
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Fetch a single scraper run in full, including its stored output.
|
||||
|
||||
Use this to retrieve the complete result of an earlier scrape (its
|
||||
run_id comes from surfsense_list_scraper_runs or a prior scrape).
|
||||
Use this to retrieve the complete, untruncated result of an earlier
|
||||
scrape. Do NOT re-run a scraper just to recover a truncated result —
|
||||
fetch the stored run instead.
|
||||
"""
|
||||
resolved = await context.resolve(workspace)
|
||||
run = await client.request(
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from __future__ import annotations
|
|||
from typing import Any
|
||||
|
||||
from ...core.client import SurfSenseClient
|
||||
from ...core.rendering import ResponseFormat, clip, to_json
|
||||
from ...core.rendering import ResponseFormat, clip, compact_items, to_json
|
||||
from ...core.workspace_context import WorkspaceContext
|
||||
|
||||
|
||||
|
|
@ -29,6 +29,10 @@ async def run_scraper(
|
|||
result = await client.request(
|
||||
"POST", f"/workspaces/{resolved.id}/scrapers/{platform}/{verb}", json=body
|
||||
)
|
||||
# Inline results are compacted (redundant fields dropped, long fields
|
||||
# excerpted) so every item survives the overall clip; the complete output
|
||||
# is stored server-side and retrievable with surfsense_get_scraper_run.
|
||||
result = compact_items(result)
|
||||
if response_format == "json":
|
||||
return clip(to_json(result))
|
||||
return _render_markdown(platform, verb, resolved.name, result)
|
||||
|
|
@ -40,7 +44,11 @@ def _render_markdown(
|
|||
"""A readable header plus the structured payload, clipped to a safe size."""
|
||||
header = f'# {platform}.{verb} — {_describe_size(result)} from "{workspace_name}"'
|
||||
body = clip(to_json(result))
|
||||
return f"{header}\n\n```json\n{body}\n```"
|
||||
footer = (
|
||||
"\n\nFields shown as excerpts; use surfsense_get_scraper_run for the "
|
||||
"full output."
|
||||
)
|
||||
return f"{header}\n\n```json\n{body}\n```{footer}"
|
||||
|
||||
|
||||
def _describe_size(result: Any) -> str:
|
||||
|
|
|
|||
|
|
@ -7,10 +7,13 @@ rest of the conversation needs no ids.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from mcp.types import ToolAnnotations
|
||||
from pydantic import Field
|
||||
|
||||
from ...core.rendering import ResponseFormat, to_json
|
||||
from ...core.rendering import ResponseFormatParam, to_json
|
||||
from ...core.workspace_context import Workspace, WorkspaceContext
|
||||
|
||||
_READ_ONLY = ToolAnnotations(
|
||||
|
|
@ -23,15 +26,19 @@ def register(mcp: FastMCP, context: WorkspaceContext) -> None:
|
|||
|
||||
@mcp.tool(
|
||||
name="surfsense_list_workspaces",
|
||||
title="List workspaces",
|
||||
annotations=_READ_ONLY,
|
||||
structured_output=False,
|
||||
)
|
||||
async def list_workspaces(response_format: ResponseFormat = "markdown") -> str:
|
||||
async def list_workspaces(
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""List the SurfSense workspaces (search spaces) the account can access.
|
||||
|
||||
Use this to discover which workspaces exist before selecting one, or when
|
||||
the user asks what search spaces they have. Returns each workspace's name,
|
||||
id, description, ownership, and member count.
|
||||
Use this to discover which workspaces exist before selecting one, or
|
||||
when the user asks what search spaces they have. Returns each
|
||||
workspace's name, id, description, ownership, and member count, and
|
||||
marks the currently active one.
|
||||
"""
|
||||
workspaces = await context.fetch_all()
|
||||
if response_format == "json":
|
||||
|
|
@ -40,16 +47,27 @@ def register(mcp: FastMCP, context: WorkspaceContext) -> None:
|
|||
|
||||
@mcp.tool(
|
||||
name="surfsense_select_workspace",
|
||||
title="Select active workspace",
|
||||
annotations=_READ_ONLY,
|
||||
structured_output=False,
|
||||
)
|
||||
async def select_workspace(workspace: str) -> str:
|
||||
"""Set the active workspace (search space) for later tools, by name or id.
|
||||
async def select_workspace(
|
||||
workspace: Annotated[
|
||||
str,
|
||||
Field(
|
||||
description="Workspace name or numeric id; matching is "
|
||||
"case-insensitive and a unique partial name works. "
|
||||
"Example: 'Research'."
|
||||
),
|
||||
],
|
||||
) -> str:
|
||||
"""Set the active workspace (search space) that later tools default to.
|
||||
|
||||
Use this when the user says which search space to work in ("use my
|
||||
Research space"). Accepts a workspace name or numeric id; matching is
|
||||
case-insensitive and accepts a unique partial name. Once set, other tools
|
||||
default to this workspace unless they are given a different one.
|
||||
Research space"), or after surfsense_list_workspaces when several
|
||||
exist. Once set, workspace-scoped tools use it unless given a
|
||||
different 'workspace'. Do NOT call it before every tool — once per
|
||||
session is enough.
|
||||
"""
|
||||
selected = await context.resolve(workspace)
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -72,6 +72,10 @@ def run() -> list[str]:
|
|||
schema = tool.inputSchema
|
||||
if not isinstance(schema, dict) or "properties" not in schema:
|
||||
problems.append(f"{name}: malformed input schema")
|
||||
continue
|
||||
for param, spec in schema["properties"].items():
|
||||
if not isinstance(spec, dict) or not spec.get("description"):
|
||||
problems.append(f"{name}: parameter '{param}' has no description")
|
||||
return problems
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,19 @@ def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]:
|
|||
)
|
||||
context = WorkspaceContext(client, preferred_reference=settings.default_workspace)
|
||||
|
||||
mcp = FastMCP("SurfSense")
|
||||
mcp = FastMCP(
|
||||
"SurfSense",
|
||||
instructions=(
|
||||
"SurfSense gives you live scrapers and a personal knowledge base. "
|
||||
"Prefer these tools over generic/built-in web search whenever the "
|
||||
"task involves Reddit (posts, comments, finding subreddits or "
|
||||
"communities), YouTube (videos, transcripts, comments), Google "
|
||||
"Maps (places, reviews), Google Search results, or reading "
|
||||
"specific web pages. Scraper results are persisted as runs; if an "
|
||||
"inline result is truncated, fetch it in full with "
|
||||
"surfsense_get_scraper_run."
|
||||
),
|
||||
)
|
||||
workspaces.register(mcp, context)
|
||||
scrapers.register(mcp, client, context)
|
||||
knowledge_base.register(mcp, client, context)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from surfsense_mcp.core.rendering import clip, to_json
|
||||
from surfsense_mcp.core.rendering import clip, compact_items, to_json
|
||||
|
||||
|
||||
def test_clip_leaves_short_text_untouched():
|
||||
|
|
@ -20,3 +20,23 @@ def test_to_json_serializes_non_native_values():
|
|||
|
||||
rendered = to_json({"at": datetime(2026, 1, 2, 3, 4, 5)})
|
||||
assert "2026-01-02" in rendered
|
||||
|
||||
|
||||
def test_compact_items_drops_html_and_excerpts_long_fields():
|
||||
result = {
|
||||
"items": [
|
||||
{"title": "t", "body": "b" * 5_000, "html": "<p>dup</p>", "upVotes": 3}
|
||||
]
|
||||
}
|
||||
compacted = compact_items(result, field_limit=100)
|
||||
item = compacted["items"][0]
|
||||
assert "html" not in item
|
||||
assert len(item["body"]) < 200 and "truncated" in item["body"]
|
||||
assert item["upVotes"] == 3
|
||||
# original untouched
|
||||
assert "html" in result["items"][0]
|
||||
|
||||
|
||||
def test_compact_items_passes_through_non_item_results():
|
||||
assert compact_items({"ok": True}) == {"ok": True}
|
||||
assert compact_items([1, 2]) == [1, 2]
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import Link from "next/link";
|
|||
import { ConnectorFaq } from "@/components/connectors-marketing/connector-faq";
|
||||
import { Reveal } from "@/components/connectors-marketing/reveal";
|
||||
import { MarketingSection } from "@/components/marketing/section";
|
||||
import { AgentSetupTabs } from "@/components/mcp/agent-setup-tabs";
|
||||
import { BreadcrumbNav } from "@/components/seo/breadcrumb-nav";
|
||||
import { FAQJsonLd, JsonLd } from "@/components/seo/json-ld";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
|
|
@ -134,7 +135,7 @@ const FAQ: FaqItem[] = [
|
|||
{
|
||||
question: "Which MCP clients does it work with?",
|
||||
answer:
|
||||
"Any MCP client that supports stdio servers. Claude Code, Cursor, and Claude Desktop are documented with copy-paste configs, and the same command works in custom agent harnesses built on the MCP SDK.",
|
||||
"Any MCP client that supports stdio servers. Claude Code, Codex, OpenCode, Cursor, Claude Desktop, VS Code, Windsurf, and Gemini CLI are documented with copy-paste configs on this page, and the same command works in custom agent harnesses built on the MCP SDK.",
|
||||
},
|
||||
{
|
||||
question: "How is usage billed?",
|
||||
|
|
@ -270,6 +271,25 @@ export default function McpServerPage() {
|
|||
</div>
|
||||
</MarketingSection>
|
||||
|
||||
{/* Per-agent setup */}
|
||||
<MarketingSection>
|
||||
<Reveal>
|
||||
<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">
|
||||
Step-by-step setup for every agent
|
||||
</h2>
|
||||
<p className="mt-3 max-w-2xl text-muted-foreground leading-relaxed">
|
||||
Pick your client, follow its two steps, and paste the config. Replace the placeholder
|
||||
path with your surfsense_mcp checkout and the key with one from API Playground → API
|
||||
Keys — or grab a pre-filled config from the playground itself.
|
||||
</p>
|
||||
</Reveal>
|
||||
<Reveal>
|
||||
<div className="mt-8 rounded-xl border bg-card p-5 shadow-sm sm:p-6">
|
||||
<AgentSetupTabs />
|
||||
</div>
|
||||
</Reveal>
|
||||
</MarketingSection>
|
||||
|
||||
{/* Tools */}
|
||||
<MarketingSection>
|
||||
<Reveal>
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
import { History, KeyRound } from "lucide-react";
|
||||
import Link from "next/link";
|
||||
import { usePathname } from "next/navigation";
|
||||
import { ConnectAgentDialog } from "@/components/mcp/connect-agent-dialog";
|
||||
import { PLAYGROUND_PLATFORMS, type PlatformIcon } from "@/lib/playground/catalog";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
|
|
@ -88,6 +89,10 @@ export function PlaygroundSidebar({ workspaceId }: PlaygroundSidebarProps) {
|
|||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="shrink-0 py-1.5 before:mx-3 before:mb-1.5 before:block before:h-px before:bg-border">
|
||||
<ConnectAgentDialog />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
85
surfsense_web/components/mcp/agent-setup-tabs.tsx
Normal file
85
surfsense_web/components/mcp/agent-setup-tabs.tsx
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
"use client";
|
||||
|
||||
import { Check, Copy } from "lucide-react";
|
||||
import { useState } from "react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import {
|
||||
API_KEY_PLACEHOLDER,
|
||||
DEFAULT_SERVER_DIR,
|
||||
MCP_CLIENTS,
|
||||
type McpSnippetOptions,
|
||||
} from "@/lib/mcp/clients";
|
||||
|
||||
function CopyButton({ text }: { text: string }) {
|
||||
const [copied, setCopied] = useState(false);
|
||||
|
||||
const handleCopy = async () => {
|
||||
try {
|
||||
await navigator.clipboard.writeText(text);
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 2000);
|
||||
} catch {
|
||||
// Clipboard unavailable (permissions/insecure context); nothing to recover.
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="absolute top-2 right-2 h-7 gap-1.5 px-2 text-xs"
|
||||
onClick={handleCopy}
|
||||
aria-label="Copy configuration"
|
||||
>
|
||||
{copied ? <Check className="size-3.5 text-brand" /> : <Copy className="size-3.5" />}
|
||||
{copied ? "Copied" : "Copy"}
|
||||
</Button>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-agent MCP setup instructions as tabs: pick a client, follow its steps,
|
||||
* copy its exact config. Used on the /mcp-server marketing page and in the
|
||||
* API playground; `options` fills in real values where the caller has them.
|
||||
*/
|
||||
export function AgentSetupTabs({ options }: { options?: Partial<McpSnippetOptions> }) {
|
||||
const resolved: McpSnippetOptions = {
|
||||
baseUrl: options?.baseUrl || "https://api.surfsense.com",
|
||||
apiKey: options?.apiKey || API_KEY_PLACEHOLDER,
|
||||
serverDir: options?.serverDir || DEFAULT_SERVER_DIR,
|
||||
};
|
||||
|
||||
return (
|
||||
<Tabs defaultValue={MCP_CLIENTS[0].id} className="w-full">
|
||||
<TabsList className="flex h-auto flex-wrap justify-start gap-1">
|
||||
{MCP_CLIENTS.map((client) => (
|
||||
<TabsTrigger key={client.id} value={client.id}>
|
||||
{client.label}
|
||||
</TabsTrigger>
|
||||
))}
|
||||
</TabsList>
|
||||
{MCP_CLIENTS.map((client) => {
|
||||
const config = client.buildConfig(resolved);
|
||||
return (
|
||||
<TabsContent key={client.id} value={client.id} className="space-y-3">
|
||||
<ol className="list-decimal space-y-1 pl-5 text-sm leading-relaxed text-muted-foreground">
|
||||
{client.steps.map((step) => (
|
||||
<li key={step}>{step}</li>
|
||||
))}
|
||||
</ol>
|
||||
<div>
|
||||
<p className="mb-1.5 font-mono text-xs text-muted-foreground">{client.configFile}</p>
|
||||
<div className="relative">
|
||||
<CopyButton text={config} />
|
||||
<pre className="overflow-x-auto rounded-lg border bg-muted/50 p-4 font-mono text-xs leading-relaxed">
|
||||
<code>{config}</code>
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</TabsContent>
|
||||
);
|
||||
})}
|
||||
</Tabs>
|
||||
);
|
||||
}
|
||||
47
surfsense_web/components/mcp/connect-agent-dialog.tsx
Normal file
47
surfsense_web/components/mcp/connect-agent-dialog.tsx
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
"use client";
|
||||
|
||||
import { Cable } from "lucide-react";
|
||||
import { AgentSetupTabs } from "@/components/mcp/agent-setup-tabs";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
DialogTrigger,
|
||||
} from "@/components/ui/dialog";
|
||||
import { BACKEND_URL } from "@/lib/env-config";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
/**
|
||||
* Sidebar-footer button that opens the MCP setup guide: pick an agent
|
||||
* (Claude Code, Codex, OpenCode, ...), copy its config, done.
|
||||
*/
|
||||
export function ConnectAgentDialog({ className }: { className?: string }) {
|
||||
return (
|
||||
<Dialog>
|
||||
<DialogTrigger
|
||||
className={cn(
|
||||
"group/link relative flex h-9 items-center gap-2 rounded-md mx-2 px-2 text-sm text-left",
|
||||
"transition-colors hover:bg-accent hover:text-accent-foreground",
|
||||
"focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring",
|
||||
className
|
||||
)}
|
||||
>
|
||||
<Cable className="h-3.5 w-3.5 shrink-0" />
|
||||
<span className="min-w-0 flex-1 truncate">Connect your AI Agent</span>
|
||||
</DialogTrigger>
|
||||
<DialogContent className="max-h-[85vh] overflow-y-auto sm:max-w-2xl">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Connect to Claude Code, Codex, OpenCode…</DialogTitle>
|
||||
<DialogDescription>
|
||||
The SurfSense MCP server gives any coding agent these scrapers and your knowledge base
|
||||
as native tools. You need an API key (create one under API Keys) — then pick your agent
|
||||
and paste its config.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<AgentSetupTabs options={{ baseUrl: BACKEND_URL || undefined }} />
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
|
|
@ -35,13 +35,14 @@ Native connectors are SurfSense's own scraper APIs — built into the platform,
|
|||
/>
|
||||
</Cards>
|
||||
|
||||
## Three ways to use them
|
||||
## Four ways to use them
|
||||
|
||||
Every scraper is available through the same three doors:
|
||||
Every scraper is available through the same four doors:
|
||||
|
||||
1. **In chat** — the AI agent uses these scrapers as tools automatically. Ask "what is r/selfhosted saying about SurfSense?" and the agent runs the Reddit scraper for you.
|
||||
2. **API Playground** — open **API Playground** in your workspace sidebar, pick a scraper, fill in the form, and run it interactively. Great for exploring what a scraper returns before writing code.
|
||||
3. **REST API** — call the scrapers from your own code. Each one is a single `POST`:
|
||||
3. **MCP server** — hand every scraper to Claude Code, Codex, OpenCode, Cursor, or any MCP client as native tools. See the [MCP server guide](/docs/how-to/mcp-server).
|
||||
4. **REST API** — call the scrapers from your own code. Each one is a single `POST`:
|
||||
|
||||
```bash
|
||||
POST /api/v1/workspaces/{workspace_id}/scrapers/{platform}/{verb}
|
||||
|
|
|
|||
263
surfsense_web/content/docs/how-to/mcp-server.mdx
Normal file
263
surfsense_web/content/docs/how-to/mcp-server.mdx
Normal file
|
|
@ -0,0 +1,263 @@
|
|||
---
|
||||
title: MCP Server
|
||||
description: Connect the SurfSense MCP server to Claude Code, Codex, OpenCode, Cursor, and other MCP clients, step by step
|
||||
---
|
||||
|
||||
import { Tab, Tabs } from 'fumadocs-ui/components/tabs';
|
||||
import { Step, Steps } from 'fumadocs-ui/components/steps';
|
||||
|
||||
# SurfSense MCP Server
|
||||
|
||||
The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 18 native, typed tools: every scraper (Reddit, YouTube, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
|
||||
|
||||
It talks to SurfSense purely over the REST API — point it at SurfSense Cloud or your own self-hosted instance by changing one environment variable.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
<Steps>
|
||||
<Step>
|
||||
|
||||
### Install uv
|
||||
|
||||
The server runs with [uv](https://github.com/astral-sh/uv). Install it once, then from the SurfSense repository run:
|
||||
|
||||
```bash
|
||||
cd surfsense_mcp
|
||||
uv sync
|
||||
```
|
||||
|
||||
</Step>
|
||||
<Step>
|
||||
|
||||
### Create an API key
|
||||
|
||||
In SurfSense, open **API Playground → API Keys** in your workspace sidebar:
|
||||
|
||||
1. Toggle **API key access** on for the workspace.
|
||||
2. Create a personal API key (`ss_pat_…`) and copy it — it is shown only once.
|
||||
|
||||
</Step>
|
||||
<Step>
|
||||
|
||||
### Know your base URL
|
||||
|
||||
- **SurfSense Cloud**: `https://api.surfsense.com`
|
||||
- **Self-hosted**: wherever your backend runs, e.g. `http://localhost:8000`
|
||||
|
||||
</Step>
|
||||
</Steps>
|
||||
|
||||
## Connect your agent
|
||||
|
||||
Every client below launches the same command — `uv run --directory <path-to>/surfsense_mcp python -m surfsense_mcp` — and passes `SURFSENSE_BASE_URL` and `SURFSENSE_API_KEY` as environment variables. Replace the placeholder paths and key with yours.
|
||||
|
||||
<Tabs items={['Claude Code', 'Codex', 'OpenCode', 'Cursor', 'Claude Desktop', 'VS Code', 'Windsurf', 'Gemini CLI']}>
|
||||
<Tab value="Claude Code">
|
||||
|
||||
Run one command in a terminal:
|
||||
|
||||
```bash
|
||||
claude mcp add surfsense \
|
||||
-e SURFSENSE_BASE_URL=https://api.surfsense.com \
|
||||
-e SURFSENSE_API_KEY=ss_pat_your_key_here \
|
||||
-- uv run --directory /path/to/SurfSense/surfsense_mcp python -m surfsense_mcp
|
||||
```
|
||||
|
||||
Start Claude Code and run `/mcp` — `surfsense` should be listed as connected. Add `--scope project` to share the server (without the key) via a checked-in `.mcp.json`.
|
||||
|
||||
</Tab>
|
||||
<Tab value="Codex">
|
||||
|
||||
Add to `~/.codex/config.toml` (or a project's `.codex/config.toml`):
|
||||
|
||||
```toml
|
||||
[mcp_servers.surfsense]
|
||||
command = "uv"
|
||||
args = ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"]
|
||||
|
||||
[mcp_servers.surfsense.env]
|
||||
SURFSENSE_BASE_URL = "https://api.surfsense.com"
|
||||
SURFSENSE_API_KEY = "ss_pat_your_key_here"
|
||||
```
|
||||
|
||||
Or use the CLI: `codex mcp add surfsense -e SURFSENSE_API_KEY=... -- uv run --directory ... python -m surfsense_mcp`. Verify with `codex mcp list`.
|
||||
|
||||
</Tab>
|
||||
<Tab value="OpenCode">
|
||||
|
||||
Add to `opencode.json` in your project root (or `~/.config/opencode/opencode.json` globally):
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "https://opencode.ai/config.json",
|
||||
"mcp": {
|
||||
"surfsense": {
|
||||
"type": "local",
|
||||
"command": ["uv", "run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"],
|
||||
"enabled": true,
|
||||
"environment": {
|
||||
"SURFSENSE_BASE_URL": "https://api.surfsense.com",
|
||||
"SURFSENSE_API_KEY": "ss_pat_your_key_here"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
OpenCode's format differs from most clients: the root key is `mcp` (not `mcpServers`), the command is a single array, and environment variables go under `environment` (not `env`).
|
||||
|
||||
</Tab>
|
||||
<Tab value="Cursor">
|
||||
|
||||
Add to `~/.cursor/mcp.json` (global — keeps the key out of your repo) or a project's `.cursor/mcp.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"surfsense": {
|
||||
"command": "uv",
|
||||
"args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"],
|
||||
"env": {
|
||||
"SURFSENSE_BASE_URL": "https://api.surfsense.com",
|
||||
"SURFSENSE_API_KEY": "ss_pat_your_key_here"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Then open **Cursor Settings → MCP** and refresh the `surfsense` server; its 18 tools should appear with a green dot.
|
||||
|
||||
</Tab>
|
||||
<Tab value="Claude Desktop">
|
||||
|
||||
Open **Settings → Developer → Edit Config** to reach `claude_desktop_config.json`, and add the same `mcpServers` block as Cursor:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"surfsense": {
|
||||
"command": "uv",
|
||||
"args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"],
|
||||
"env": {
|
||||
"SURFSENSE_BASE_URL": "https://api.surfsense.com",
|
||||
"SURFSENSE_API_KEY": "ss_pat_your_key_here"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Restart Claude Desktop; SurfSense appears under the tools icon in the chat input.
|
||||
|
||||
</Tab>
|
||||
<Tab value="VS Code">
|
||||
|
||||
Add to `.vscode/mcp.json` in your workspace (or run the **MCP: Add Server** command). Note VS Code uses a `servers` key:
|
||||
|
||||
```json
|
||||
{
|
||||
"servers": {
|
||||
"surfsense": {
|
||||
"type": "stdio",
|
||||
"command": "uv",
|
||||
"args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"],
|
||||
"env": {
|
||||
"SURFSENSE_BASE_URL": "https://api.surfsense.com",
|
||||
"SURFSENSE_API_KEY": "ss_pat_your_key_here"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Open Copilot Chat in agent mode and click the tools icon to confirm the server is loaded.
|
||||
|
||||
</Tab>
|
||||
<Tab value="Windsurf">
|
||||
|
||||
Add the standard `mcpServers` block to `~/.codeium/windsurf/mcp_config.json` (or via **Windsurf Settings → Cascade → MCP Servers**):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"surfsense": {
|
||||
"command": "uv",
|
||||
"args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"],
|
||||
"env": {
|
||||
"SURFSENSE_BASE_URL": "https://api.surfsense.com",
|
||||
"SURFSENSE_API_KEY": "ss_pat_your_key_here"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Press refresh in the MCP panel to pick up the server.
|
||||
|
||||
</Tab>
|
||||
<Tab value="Gemini CLI">
|
||||
|
||||
Add the standard `mcpServers` block to `~/.gemini/settings.json` (or `.gemini/settings.json` in a project):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"surfsense": {
|
||||
"command": "uv",
|
||||
"args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"],
|
||||
"env": {
|
||||
"SURFSENSE_BASE_URL": "https://api.surfsense.com",
|
||||
"SURFSENSE_API_KEY": "ss_pat_your_key_here"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Run `/mcp` inside Gemini CLI to confirm the server and its tools.
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
<Callout type="info" title="stdio transport — nothing to keep running">
|
||||
The server uses stdio transport: your client launches the process on demand and shuts it down with the session. There is no daemon to manage — only your SurfSense backend needs to be up.
|
||||
</Callout>
|
||||
|
||||
## Test it
|
||||
|
||||
In a fresh agent session, try:
|
||||
|
||||
> list my SurfSense workspaces
|
||||
|
||||
That calls `surfsense_list_workspaces` — the simplest end-to-end check of the key, backend, and server. Then try a real task:
|
||||
|
||||
> find the top subreddits discussing NotebookLM and save a summary note to my workspace
|
||||
|
||||
## Configuration reference
|
||||
|
||||
All settings are environment variables passed by the client:
|
||||
|
||||
| Variable | Required | Default | Purpose |
|
||||
|----------|----------|---------|---------|
|
||||
| `SURFSENSE_API_KEY` | Yes | — | API key from **API Playground → API Keys** |
|
||||
| `SURFSENSE_BASE_URL` | No | `http://localhost:8000` | Backend to talk to |
|
||||
| `SURFSENSE_WORKSPACE` | No | — | Default workspace by name or id, so agents skip selection |
|
||||
| `SURFSENSE_TIMEOUT` | No | `180` | Request timeout in seconds |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **401 errors** — the API key is wrong or expired; create a new one.
|
||||
- **403 errors** — API access is disabled for the workspace; toggle **API key access** on under **API Playground → API Keys**.
|
||||
- **"Could not reach SurfSense"** — the backend isn't running or `SURFSENSE_BASE_URL` is wrong.
|
||||
- **Server won't start** — run `uv run python -m surfsense_mcp.selfcheck` inside `surfsense_mcp`; it verifies all 18 tools register without needing a backend.
|
||||
|
||||
## Tools reference
|
||||
|
||||
| Group | Tools |
|
||||
|-------|-------|
|
||||
| Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` |
|
||||
| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` |
|
||||
| Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` |
|
||||
|
||||
Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**.
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"title": "How to",
|
||||
"pages": ["zero-sync", "realtime-collaboration", "web-search"],
|
||||
"pages": ["mcp-server", "zero-sync", "realtime-collaboration", "web-search"],
|
||||
"icon": "Compass",
|
||||
"defaultOpen": false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { useQueryClient } from "@tanstack/react-query";
|
|||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import type { ScraperRunDetail, ScraperRunEvent } from "@/contracts/types/scraper.types";
|
||||
import { scrapersApiService } from "@/lib/apis/scrapers-api.service";
|
||||
import { trackWeeklyUser } from "@/lib/posthog/events";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
|
||||
export type RunStatus = "idle" | "running" | "success" | "error" | "cancelled";
|
||||
|
|
@ -119,6 +120,7 @@ export function useRunStream(workspaceId: number) {
|
|||
payload
|
||||
);
|
||||
runIdRef.current = started.run_id;
|
||||
trackWeeklyUser("api_run", workspaceId);
|
||||
setState((s) => ({ ...s, runId: started.run_id }));
|
||||
void consume(started.run_id, controller.signal);
|
||||
} catch (e) {
|
||||
|
|
|
|||
176
surfsense_web/lib/mcp/clients.ts
Normal file
176
surfsense_web/lib/mcp/clients.ts
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
/**
|
||||
* MCP client setup catalog: one entry per popular agent, with the exact
|
||||
* config file, steps, and snippet needed to connect the SurfSense MCP server.
|
||||
* Shared by the marketing /mcp-server page and the API playground so the
|
||||
* instructions can never drift apart.
|
||||
*/
|
||||
|
||||
export interface McpSnippetOptions {
|
||||
/** SurfSense backend URL the server should call. */
|
||||
baseUrl: string;
|
||||
/** API key value or placeholder to show in the snippet. */
|
||||
apiKey: string;
|
||||
/** Absolute path to the surfsense_mcp directory. */
|
||||
serverDir: string;
|
||||
}
|
||||
|
||||
export interface McpClient {
|
||||
id: string;
|
||||
label: string;
|
||||
/** Where the snippet goes: a file path or "Terminal". */
|
||||
configFile: string;
|
||||
language: "json" | "toml" | "bash";
|
||||
steps: string[];
|
||||
buildConfig: (options: McpSnippetOptions) => string;
|
||||
}
|
||||
|
||||
export const DEFAULT_SERVER_DIR = "/path/to/SurfSense/surfsense_mcp";
|
||||
export const API_KEY_PLACEHOLDER = "ss_pat_your_key_here";
|
||||
|
||||
function serverArgs(serverDir: string): string[] {
|
||||
return ["run", "--directory", serverDir, "python", "-m", "surfsense_mcp"];
|
||||
}
|
||||
|
||||
function json(value: unknown): string {
|
||||
return JSON.stringify(value, null, 2);
|
||||
}
|
||||
|
||||
/** The `mcpServers` JSON shape shared by Cursor, Claude Desktop, Windsurf, and Gemini CLI. */
|
||||
function standardJson({ baseUrl, apiKey, serverDir }: McpSnippetOptions): string {
|
||||
return json({
|
||||
mcpServers: {
|
||||
surfsense: {
|
||||
command: "uv",
|
||||
args: serverArgs(serverDir),
|
||||
env: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey },
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export const MCP_CLIENTS: McpClient[] = [
|
||||
{
|
||||
id: "claude-code",
|
||||
label: "Claude Code",
|
||||
configFile: "Terminal",
|
||||
language: "bash",
|
||||
steps: [
|
||||
"Run this command in a terminal (any directory).",
|
||||
"Start Claude Code and run /mcp — surfsense should be listed as connected.",
|
||||
],
|
||||
buildConfig: ({ baseUrl, apiKey, serverDir }) =>
|
||||
[
|
||||
"claude mcp add surfsense \\",
|
||||
` -e SURFSENSE_BASE_URL=${baseUrl} \\`,
|
||||
` -e SURFSENSE_API_KEY=${apiKey} \\`,
|
||||
` -- uv run --directory ${serverDir} python -m surfsense_mcp`,
|
||||
].join("\n"),
|
||||
},
|
||||
{
|
||||
id: "codex",
|
||||
label: "Codex",
|
||||
configFile: "~/.codex/config.toml",
|
||||
language: "toml",
|
||||
steps: [
|
||||
"Add this to ~/.codex/config.toml (or run `codex mcp add surfsense -- uv run --directory <dir> python -m surfsense_mcp`).",
|
||||
"Restart Codex; `codex mcp list` should show surfsense.",
|
||||
],
|
||||
buildConfig: ({ baseUrl, apiKey, serverDir }) =>
|
||||
[
|
||||
"[mcp_servers.surfsense]",
|
||||
'command = "uv"',
|
||||
`args = ${JSON.stringify(serverArgs(serverDir))}`,
|
||||
"",
|
||||
"[mcp_servers.surfsense.env]",
|
||||
`SURFSENSE_BASE_URL = "${baseUrl}"`,
|
||||
`SURFSENSE_API_KEY = "${apiKey}"`,
|
||||
].join("\n"),
|
||||
},
|
||||
{
|
||||
id: "opencode",
|
||||
label: "OpenCode",
|
||||
configFile: "opencode.json",
|
||||
language: "json",
|
||||
steps: [
|
||||
"Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).",
|
||||
"Note OpenCode's format: the key is `mcp`, the command is one array, and env vars go under `environment`.",
|
||||
],
|
||||
buildConfig: ({ baseUrl, apiKey, serverDir }) =>
|
||||
json({
|
||||
$schema: "https://opencode.ai/config.json",
|
||||
mcp: {
|
||||
surfsense: {
|
||||
type: "local",
|
||||
command: ["uv", ...serverArgs(serverDir)],
|
||||
enabled: true,
|
||||
environment: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey },
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "cursor",
|
||||
label: "Cursor",
|
||||
configFile: "~/.cursor/mcp.json",
|
||||
language: "json",
|
||||
steps: [
|
||||
"Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.",
|
||||
"Refresh the server in Cursor Settings → MCP; its 18 tools should appear.",
|
||||
],
|
||||
buildConfig: standardJson,
|
||||
},
|
||||
{
|
||||
id: "claude-desktop",
|
||||
label: "Claude Desktop",
|
||||
configFile: "claude_desktop_config.json",
|
||||
language: "json",
|
||||
steps: [
|
||||
"Open Settings → Developer → Edit Config to reach claude_desktop_config.json and add this.",
|
||||
"Restart Claude Desktop; surfsense appears under the tools icon.",
|
||||
],
|
||||
buildConfig: standardJson,
|
||||
},
|
||||
{
|
||||
id: "vscode",
|
||||
label: "VS Code",
|
||||
configFile: ".vscode/mcp.json",
|
||||
language: "json",
|
||||
steps: [
|
||||
"Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).",
|
||||
"Open Copilot Chat in agent mode and click the tools icon to confirm surfsense is loaded.",
|
||||
],
|
||||
buildConfig: ({ baseUrl, apiKey, serverDir }) =>
|
||||
json({
|
||||
servers: {
|
||||
surfsense: {
|
||||
type: "stdio",
|
||||
command: "uv",
|
||||
args: serverArgs(serverDir),
|
||||
env: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey },
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "windsurf",
|
||||
label: "Windsurf",
|
||||
configFile: "~/.codeium/windsurf/mcp_config.json",
|
||||
language: "json",
|
||||
steps: [
|
||||
"Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).",
|
||||
"Press the refresh button in the MCP panel to pick up the server.",
|
||||
],
|
||||
buildConfig: standardJson,
|
||||
},
|
||||
{
|
||||
id: "gemini-cli",
|
||||
label: "Gemini CLI",
|
||||
configFile: "~/.gemini/settings.json",
|
||||
language: "json",
|
||||
steps: [
|
||||
"Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).",
|
||||
"Run /mcp inside Gemini CLI to confirm the surfsense server and its tools.",
|
||||
],
|
||||
buildConfig: standardJson,
|
||||
},
|
||||
];
|
||||
|
|
@ -97,6 +97,23 @@ export function trackWorkspaceViewed(workspaceId: number) {
|
|||
});
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// ACTIVE-USER (WAU) EVENT
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Single signal for active-user counting. Fired whenever a user sends a
|
||||
* chat message or starts an API run, so a "weekly unique users on
|
||||
* weekly_users" insight in PostHog is our WAU number.
|
||||
*
|
||||
* ponytail: frontend-only capture — API runs made directly against the
|
||||
* backend (PAT/curl, no browser) are not counted. Upgrade path is a
|
||||
* server-side capture in the backend if that ever matters.
|
||||
*/
|
||||
export function trackWeeklyUser(source: "chat_message" | "api_run", workspaceId?: number) {
|
||||
safeCapture("weekly_users", compact({ source, workspace_id: workspaceId }));
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// CHAT EVENTS
|
||||
// ============================================
|
||||
|
|
@ -124,6 +141,7 @@ export function trackChatMessageSent(
|
|||
has_mentioned_documents: options?.hasMentionedDocuments ?? false,
|
||||
message_length: options?.messageLength,
|
||||
});
|
||||
trackWeeklyUser("chat_message", workspaceId);
|
||||
}
|
||||
|
||||
export function trackChatResponseReceived(workspaceId: number, chatId: number) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue