mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
feat: update environment variables and enhance scraping capabilities
- Adjusted Google Maps and YouTube micro pricing in the .env.example file for better cost management. - Introduced new environment variables for captcha solving and stealth browser hardening to improve scraping resilience. - Removed outdated smoke test for scraper API endpoints to streamline testing. - Enhanced anonymous chat agent's system prompt to clarify capabilities and suggest account creation for advanced features. - Updated Reddit fetch logic to prioritize new session handling and improve resilience against IP-related issues. - Added compacting functionality for scraper results to optimize data handling and presentation. - Improved workspace and document management tools with clearer descriptions and enhanced functionality. - Introduced new UI components for agent setup guidance in the web application.
This commit is contained in:
parent
271a21aee6
commit
1fd58752a3
24 changed files with 1326 additions and 320 deletions
|
|
@ -5,6 +5,7 @@ import Link from "next/link";
|
|||
import { ConnectorFaq } from "@/components/connectors-marketing/connector-faq";
|
||||
import { Reveal } from "@/components/connectors-marketing/reveal";
|
||||
import { MarketingSection } from "@/components/marketing/section";
|
||||
import { AgentSetupTabs } from "@/components/mcp/agent-setup-tabs";
|
||||
import { BreadcrumbNav } from "@/components/seo/breadcrumb-nav";
|
||||
import { FAQJsonLd, JsonLd } from "@/components/seo/json-ld";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
|
|
@ -134,7 +135,7 @@ const FAQ: FaqItem[] = [
|
|||
{
|
||||
question: "Which MCP clients does it work with?",
|
||||
answer:
|
||||
"Any MCP client that supports stdio servers. Claude Code, Cursor, and Claude Desktop are documented with copy-paste configs, and the same command works in custom agent harnesses built on the MCP SDK.",
|
||||
"Any MCP client that supports stdio servers. Claude Code, Codex, OpenCode, Cursor, Claude Desktop, VS Code, Windsurf, and Gemini CLI are documented with copy-paste configs on this page, and the same command works in custom agent harnesses built on the MCP SDK.",
|
||||
},
|
||||
{
|
||||
question: "How is usage billed?",
|
||||
|
|
@ -270,6 +271,25 @@ export default function McpServerPage() {
|
|||
</div>
|
||||
</MarketingSection>
|
||||
|
||||
{/* Per-agent setup */}
|
||||
<MarketingSection>
|
||||
<Reveal>
|
||||
<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">
|
||||
Step-by-step setup for every agent
|
||||
</h2>
|
||||
<p className="mt-3 max-w-2xl text-muted-foreground leading-relaxed">
|
||||
Pick your client, follow its two steps, and paste the config. Replace the placeholder
|
||||
path with your surfsense_mcp checkout and the key with one from API Playground → API
|
||||
Keys — or grab a pre-filled config from the playground itself.
|
||||
</p>
|
||||
</Reveal>
|
||||
<Reveal>
|
||||
<div className="mt-8 rounded-xl border bg-card p-5 shadow-sm sm:p-6">
|
||||
<AgentSetupTabs />
|
||||
</div>
|
||||
</Reveal>
|
||||
</MarketingSection>
|
||||
|
||||
{/* Tools */}
|
||||
<MarketingSection>
|
||||
<Reveal>
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
import { History, KeyRound } from "lucide-react";
|
||||
import Link from "next/link";
|
||||
import { usePathname } from "next/navigation";
|
||||
import { ConnectAgentDialog } from "@/components/mcp/connect-agent-dialog";
|
||||
import { PLAYGROUND_PLATFORMS, type PlatformIcon } from "@/lib/playground/catalog";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
|
|
@ -88,6 +89,10 @@ export function PlaygroundSidebar({ workspaceId }: PlaygroundSidebarProps) {
|
|||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="shrink-0 py-1.5 before:mx-3 before:mb-1.5 before:block before:h-px before:bg-border">
|
||||
<ConnectAgentDialog />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
85
surfsense_web/components/mcp/agent-setup-tabs.tsx
Normal file
85
surfsense_web/components/mcp/agent-setup-tabs.tsx
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
"use client";
|
||||
|
||||
import { Check, Copy } from "lucide-react";
|
||||
import { useState } from "react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import {
|
||||
API_KEY_PLACEHOLDER,
|
||||
DEFAULT_SERVER_DIR,
|
||||
MCP_CLIENTS,
|
||||
type McpSnippetOptions,
|
||||
} from "@/lib/mcp/clients";
|
||||
|
||||
function CopyButton({ text }: { text: string }) {
|
||||
const [copied, setCopied] = useState(false);
|
||||
|
||||
const handleCopy = async () => {
|
||||
try {
|
||||
await navigator.clipboard.writeText(text);
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 2000);
|
||||
} catch {
|
||||
// Clipboard unavailable (permissions/insecure context); nothing to recover.
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="absolute top-2 right-2 h-7 gap-1.5 px-2 text-xs"
|
||||
onClick={handleCopy}
|
||||
aria-label="Copy configuration"
|
||||
>
|
||||
{copied ? <Check className="size-3.5 text-brand" /> : <Copy className="size-3.5" />}
|
||||
{copied ? "Copied" : "Copy"}
|
||||
</Button>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-agent MCP setup instructions as tabs: pick a client, follow its steps,
|
||||
* copy its exact config. Used on the /mcp-server marketing page and in the
|
||||
* API playground; `options` fills in real values where the caller has them.
|
||||
*/
|
||||
export function AgentSetupTabs({ options }: { options?: Partial<McpSnippetOptions> }) {
|
||||
const resolved: McpSnippetOptions = {
|
||||
baseUrl: options?.baseUrl || "https://api.surfsense.com",
|
||||
apiKey: options?.apiKey || API_KEY_PLACEHOLDER,
|
||||
serverDir: options?.serverDir || DEFAULT_SERVER_DIR,
|
||||
};
|
||||
|
||||
return (
|
||||
<Tabs defaultValue={MCP_CLIENTS[0].id} className="w-full">
|
||||
<TabsList className="flex h-auto flex-wrap justify-start gap-1">
|
||||
{MCP_CLIENTS.map((client) => (
|
||||
<TabsTrigger key={client.id} value={client.id}>
|
||||
{client.label}
|
||||
</TabsTrigger>
|
||||
))}
|
||||
</TabsList>
|
||||
{MCP_CLIENTS.map((client) => {
|
||||
const config = client.buildConfig(resolved);
|
||||
return (
|
||||
<TabsContent key={client.id} value={client.id} className="space-y-3">
|
||||
<ol className="list-decimal space-y-1 pl-5 text-sm leading-relaxed text-muted-foreground">
|
||||
{client.steps.map((step) => (
|
||||
<li key={step}>{step}</li>
|
||||
))}
|
||||
</ol>
|
||||
<div>
|
||||
<p className="mb-1.5 font-mono text-xs text-muted-foreground">{client.configFile}</p>
|
||||
<div className="relative">
|
||||
<CopyButton text={config} />
|
||||
<pre className="overflow-x-auto rounded-lg border bg-muted/50 p-4 font-mono text-xs leading-relaxed">
|
||||
<code>{config}</code>
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</TabsContent>
|
||||
);
|
||||
})}
|
||||
</Tabs>
|
||||
);
|
||||
}
|
||||
47
surfsense_web/components/mcp/connect-agent-dialog.tsx
Normal file
47
surfsense_web/components/mcp/connect-agent-dialog.tsx
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
"use client";
|
||||
|
||||
import { Cable } from "lucide-react";
|
||||
import { AgentSetupTabs } from "@/components/mcp/agent-setup-tabs";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
DialogTrigger,
|
||||
} from "@/components/ui/dialog";
|
||||
import { BACKEND_URL } from "@/lib/env-config";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
/**
|
||||
* Sidebar-footer button that opens the MCP setup guide: pick an agent
|
||||
* (Claude Code, Codex, OpenCode, ...), copy its config, done.
|
||||
*/
|
||||
export function ConnectAgentDialog({ className }: { className?: string }) {
|
||||
return (
|
||||
<Dialog>
|
||||
<DialogTrigger
|
||||
className={cn(
|
||||
"group/link relative flex h-9 items-center gap-2 rounded-md mx-2 px-2 text-sm text-left",
|
||||
"transition-colors hover:bg-accent hover:text-accent-foreground",
|
||||
"focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring",
|
||||
className
|
||||
)}
|
||||
>
|
||||
<Cable className="h-3.5 w-3.5 shrink-0" />
|
||||
<span className="min-w-0 flex-1 truncate">Connect your AI Agent</span>
|
||||
</DialogTrigger>
|
||||
<DialogContent className="max-h-[85vh] overflow-y-auto sm:max-w-2xl">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Connect to Claude Code, Codex, OpenCode…</DialogTitle>
|
||||
<DialogDescription>
|
||||
The SurfSense MCP server gives any coding agent these scrapers and your knowledge base
|
||||
as native tools. You need an API key (create one under API Keys) — then pick your agent
|
||||
and paste its config.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<AgentSetupTabs options={{ baseUrl: BACKEND_URL || undefined }} />
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
|
|
@ -35,13 +35,14 @@ Native connectors are SurfSense's own scraper APIs — built into the platform,
|
|||
/>
|
||||
</Cards>
|
||||
|
||||
## Three ways to use them
|
||||
## Four ways to use them
|
||||
|
||||
Every scraper is available through the same three doors:
|
||||
Every scraper is available through the same four doors:
|
||||
|
||||
1. **In chat** — the AI agent uses these scrapers as tools automatically. Ask "what is r/selfhosted saying about SurfSense?" and the agent runs the Reddit scraper for you.
|
||||
2. **API Playground** — open **API Playground** in your workspace sidebar, pick a scraper, fill in the form, and run it interactively. Great for exploring what a scraper returns before writing code.
|
||||
3. **REST API** — call the scrapers from your own code. Each one is a single `POST`:
|
||||
3. **MCP server** — hand every scraper to Claude Code, Codex, OpenCode, Cursor, or any MCP client as native tools. See the [MCP server guide](/docs/how-to/mcp-server).
|
||||
4. **REST API** — call the scrapers from your own code. Each one is a single `POST`:
|
||||
|
||||
```bash
|
||||
POST /api/v1/workspaces/{workspace_id}/scrapers/{platform}/{verb}
|
||||
|
|
|
|||
263
surfsense_web/content/docs/how-to/mcp-server.mdx
Normal file
263
surfsense_web/content/docs/how-to/mcp-server.mdx
Normal file
|
|
@ -0,0 +1,263 @@
|
|||
---
|
||||
title: MCP Server
|
||||
description: Connect the SurfSense MCP server to Claude Code, Codex, OpenCode, Cursor, and other MCP clients, step by step
|
||||
---
|
||||
|
||||
import { Tab, Tabs } from 'fumadocs-ui/components/tabs';
|
||||
import { Step, Steps } from 'fumadocs-ui/components/steps';
|
||||
|
||||
# SurfSense MCP Server
|
||||
|
||||
The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 18 native, typed tools: every scraper (Reddit, YouTube, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
|
||||
|
||||
It talks to SurfSense purely over the REST API — point it at SurfSense Cloud or your own self-hosted instance by changing one environment variable.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
<Steps>
|
||||
<Step>
|
||||
|
||||
### Install uv
|
||||
|
||||
The server runs with [uv](https://github.com/astral-sh/uv). Install it once, then from the SurfSense repository run:
|
||||
|
||||
```bash
|
||||
cd surfsense_mcp
|
||||
uv sync
|
||||
```
|
||||
|
||||
</Step>
|
||||
<Step>
|
||||
|
||||
### Create an API key
|
||||
|
||||
In SurfSense, open **API Playground → API Keys** in your workspace sidebar:
|
||||
|
||||
1. Toggle **API key access** on for the workspace.
|
||||
2. Create a personal API key (`ss_pat_…`) and copy it — it is shown only once.
|
||||
|
||||
</Step>
|
||||
<Step>
|
||||
|
||||
### Know your base URL
|
||||
|
||||
- **SurfSense Cloud**: `https://api.surfsense.com`
|
||||
- **Self-hosted**: wherever your backend runs, e.g. `http://localhost:8000`
|
||||
|
||||
</Step>
|
||||
</Steps>
|
||||
|
||||
## Connect your agent
|
||||
|
||||
Every client below launches the same command — `uv run --directory <path-to>/surfsense_mcp python -m surfsense_mcp` — and passes `SURFSENSE_BASE_URL` and `SURFSENSE_API_KEY` as environment variables. Replace the placeholder paths and key with yours.
|
||||
|
||||
<Tabs items={['Claude Code', 'Codex', 'OpenCode', 'Cursor', 'Claude Desktop', 'VS Code', 'Windsurf', 'Gemini CLI']}>
|
||||
<Tab value="Claude Code">
|
||||
|
||||
Run one command in a terminal:
|
||||
|
||||
```bash
|
||||
claude mcp add surfsense \
|
||||
-e SURFSENSE_BASE_URL=https://api.surfsense.com \
|
||||
-e SURFSENSE_API_KEY=ss_pat_your_key_here \
|
||||
-- uv run --directory /path/to/SurfSense/surfsense_mcp python -m surfsense_mcp
|
||||
```
|
||||
|
||||
Start Claude Code and run `/mcp` — `surfsense` should be listed as connected. Add `--scope project` to share the server (without the key) via a checked-in `.mcp.json`.
|
||||
|
||||
</Tab>
|
||||
<Tab value="Codex">
|
||||
|
||||
Add to `~/.codex/config.toml` (or a project's `.codex/config.toml`):
|
||||
|
||||
```toml
|
||||
[mcp_servers.surfsense]
|
||||
command = "uv"
|
||||
args = ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"]
|
||||
|
||||
[mcp_servers.surfsense.env]
|
||||
SURFSENSE_BASE_URL = "https://api.surfsense.com"
|
||||
SURFSENSE_API_KEY = "ss_pat_your_key_here"
|
||||
```
|
||||
|
||||
Or use the CLI: `codex mcp add surfsense -e SURFSENSE_API_KEY=... -- uv run --directory ... python -m surfsense_mcp`. Verify with `codex mcp list`.
|
||||
|
||||
</Tab>
|
||||
<Tab value="OpenCode">
|
||||
|
||||
Add to `opencode.json` in your project root (or `~/.config/opencode/opencode.json` globally):
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "https://opencode.ai/config.json",
|
||||
"mcp": {
|
||||
"surfsense": {
|
||||
"type": "local",
|
||||
"command": ["uv", "run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"],
|
||||
"enabled": true,
|
||||
"environment": {
|
||||
"SURFSENSE_BASE_URL": "https://api.surfsense.com",
|
||||
"SURFSENSE_API_KEY": "ss_pat_your_key_here"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
OpenCode's format differs from most clients: the root key is `mcp` (not `mcpServers`), the command is a single array, and environment variables go under `environment` (not `env`).
|
||||
|
||||
</Tab>
|
||||
<Tab value="Cursor">
|
||||
|
||||
Add to `~/.cursor/mcp.json` (global — keeps the key out of your repo) or a project's `.cursor/mcp.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"surfsense": {
|
||||
"command": "uv",
|
||||
"args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"],
|
||||
"env": {
|
||||
"SURFSENSE_BASE_URL": "https://api.surfsense.com",
|
||||
"SURFSENSE_API_KEY": "ss_pat_your_key_here"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Then open **Cursor Settings → MCP** and refresh the `surfsense` server; its 18 tools should appear with a green dot.
|
||||
|
||||
</Tab>
|
||||
<Tab value="Claude Desktop">
|
||||
|
||||
Open **Settings → Developer → Edit Config** to reach `claude_desktop_config.json`, and add the same `mcpServers` block as Cursor:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"surfsense": {
|
||||
"command": "uv",
|
||||
"args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"],
|
||||
"env": {
|
||||
"SURFSENSE_BASE_URL": "https://api.surfsense.com",
|
||||
"SURFSENSE_API_KEY": "ss_pat_your_key_here"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Restart Claude Desktop; SurfSense appears under the tools icon in the chat input.
|
||||
|
||||
</Tab>
|
||||
<Tab value="VS Code">
|
||||
|
||||
Add to `.vscode/mcp.json` in your workspace (or run the **MCP: Add Server** command). Note VS Code uses a `servers` key:
|
||||
|
||||
```json
|
||||
{
|
||||
"servers": {
|
||||
"surfsense": {
|
||||
"type": "stdio",
|
||||
"command": "uv",
|
||||
"args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"],
|
||||
"env": {
|
||||
"SURFSENSE_BASE_URL": "https://api.surfsense.com",
|
||||
"SURFSENSE_API_KEY": "ss_pat_your_key_here"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Open Copilot Chat in agent mode and click the tools icon to confirm the server is loaded.
|
||||
|
||||
</Tab>
|
||||
<Tab value="Windsurf">
|
||||
|
||||
Add the standard `mcpServers` block to `~/.codeium/windsurf/mcp_config.json` (or via **Windsurf Settings → Cascade → MCP Servers**):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"surfsense": {
|
||||
"command": "uv",
|
||||
"args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"],
|
||||
"env": {
|
||||
"SURFSENSE_BASE_URL": "https://api.surfsense.com",
|
||||
"SURFSENSE_API_KEY": "ss_pat_your_key_here"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Press refresh in the MCP panel to pick up the server.
|
||||
|
||||
</Tab>
|
||||
<Tab value="Gemini CLI">
|
||||
|
||||
Add the standard `mcpServers` block to `~/.gemini/settings.json` (or `.gemini/settings.json` in a project):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"surfsense": {
|
||||
"command": "uv",
|
||||
"args": ["run", "--directory", "/path/to/SurfSense/surfsense_mcp", "python", "-m", "surfsense_mcp"],
|
||||
"env": {
|
||||
"SURFSENSE_BASE_URL": "https://api.surfsense.com",
|
||||
"SURFSENSE_API_KEY": "ss_pat_your_key_here"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Run `/mcp` inside Gemini CLI to confirm the server and its tools.
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
<Callout type="info" title="stdio transport — nothing to keep running">
|
||||
The server uses stdio transport: your client launches the process on demand and shuts it down with the session. There is no daemon to manage — only your SurfSense backend needs to be up.
|
||||
</Callout>
|
||||
|
||||
## Test it
|
||||
|
||||
In a fresh agent session, try:
|
||||
|
||||
> list my SurfSense workspaces
|
||||
|
||||
That calls `surfsense_list_workspaces` — the simplest end-to-end check of the key, backend, and server. Then try a real task:
|
||||
|
||||
> find the top subreddits discussing NotebookLM and save a summary note to my workspace
|
||||
|
||||
## Configuration reference
|
||||
|
||||
All settings are environment variables passed by the client:
|
||||
|
||||
| Variable | Required | Default | Purpose |
|
||||
|----------|----------|---------|---------|
|
||||
| `SURFSENSE_API_KEY` | Yes | — | API key from **API Playground → API Keys** |
|
||||
| `SURFSENSE_BASE_URL` | No | `http://localhost:8000` | Backend to talk to |
|
||||
| `SURFSENSE_WORKSPACE` | No | — | Default workspace by name or id, so agents skip selection |
|
||||
| `SURFSENSE_TIMEOUT` | No | `180` | Request timeout in seconds |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **401 errors** — the API key is wrong or expired; create a new one.
|
||||
- **403 errors** — API access is disabled for the workspace; toggle **API key access** on under **API Playground → API Keys**.
|
||||
- **"Could not reach SurfSense"** — the backend isn't running or `SURFSENSE_BASE_URL` is wrong.
|
||||
- **Server won't start** — run `uv run python -m surfsense_mcp.selfcheck` inside `surfsense_mcp`; it verifies all 18 tools register without needing a backend.
|
||||
|
||||
## Tools reference
|
||||
|
||||
| Group | Tools |
|
||||
|-------|-------|
|
||||
| Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` |
|
||||
| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` |
|
||||
| Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` |
|
||||
|
||||
Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**.
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"title": "How to",
|
||||
"pages": ["zero-sync", "realtime-collaboration", "web-search"],
|
||||
"pages": ["mcp-server", "zero-sync", "realtime-collaboration", "web-search"],
|
||||
"icon": "Compass",
|
||||
"defaultOpen": false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { useQueryClient } from "@tanstack/react-query";
|
|||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import type { ScraperRunDetail, ScraperRunEvent } from "@/contracts/types/scraper.types";
|
||||
import { scrapersApiService } from "@/lib/apis/scrapers-api.service";
|
||||
import { trackWeeklyUser } from "@/lib/posthog/events";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
|
||||
export type RunStatus = "idle" | "running" | "success" | "error" | "cancelled";
|
||||
|
|
@ -119,6 +120,7 @@ export function useRunStream(workspaceId: number) {
|
|||
payload
|
||||
);
|
||||
runIdRef.current = started.run_id;
|
||||
trackWeeklyUser("api_run", workspaceId);
|
||||
setState((s) => ({ ...s, runId: started.run_id }));
|
||||
void consume(started.run_id, controller.signal);
|
||||
} catch (e) {
|
||||
|
|
|
|||
176
surfsense_web/lib/mcp/clients.ts
Normal file
176
surfsense_web/lib/mcp/clients.ts
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
/**
|
||||
* MCP client setup catalog: one entry per popular agent, with the exact
|
||||
* config file, steps, and snippet needed to connect the SurfSense MCP server.
|
||||
* Shared by the marketing /mcp-server page and the API playground so the
|
||||
* instructions can never drift apart.
|
||||
*/
|
||||
|
||||
export interface McpSnippetOptions {
|
||||
/** SurfSense backend URL the server should call. */
|
||||
baseUrl: string;
|
||||
/** API key value or placeholder to show in the snippet. */
|
||||
apiKey: string;
|
||||
/** Absolute path to the surfsense_mcp directory. */
|
||||
serverDir: string;
|
||||
}
|
||||
|
||||
export interface McpClient {
|
||||
id: string;
|
||||
label: string;
|
||||
/** Where the snippet goes: a file path or "Terminal". */
|
||||
configFile: string;
|
||||
language: "json" | "toml" | "bash";
|
||||
steps: string[];
|
||||
buildConfig: (options: McpSnippetOptions) => string;
|
||||
}
|
||||
|
||||
export const DEFAULT_SERVER_DIR = "/path/to/SurfSense/surfsense_mcp";
|
||||
export const API_KEY_PLACEHOLDER = "ss_pat_your_key_here";
|
||||
|
||||
function serverArgs(serverDir: string): string[] {
|
||||
return ["run", "--directory", serverDir, "python", "-m", "surfsense_mcp"];
|
||||
}
|
||||
|
||||
function json(value: unknown): string {
|
||||
return JSON.stringify(value, null, 2);
|
||||
}
|
||||
|
||||
/** The `mcpServers` JSON shape shared by Cursor, Claude Desktop, Windsurf, and Gemini CLI. */
|
||||
function standardJson({ baseUrl, apiKey, serverDir }: McpSnippetOptions): string {
|
||||
return json({
|
||||
mcpServers: {
|
||||
surfsense: {
|
||||
command: "uv",
|
||||
args: serverArgs(serverDir),
|
||||
env: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey },
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export const MCP_CLIENTS: McpClient[] = [
|
||||
{
|
||||
id: "claude-code",
|
||||
label: "Claude Code",
|
||||
configFile: "Terminal",
|
||||
language: "bash",
|
||||
steps: [
|
||||
"Run this command in a terminal (any directory).",
|
||||
"Start Claude Code and run /mcp — surfsense should be listed as connected.",
|
||||
],
|
||||
buildConfig: ({ baseUrl, apiKey, serverDir }) =>
|
||||
[
|
||||
"claude mcp add surfsense \\",
|
||||
` -e SURFSENSE_BASE_URL=${baseUrl} \\`,
|
||||
` -e SURFSENSE_API_KEY=${apiKey} \\`,
|
||||
` -- uv run --directory ${serverDir} python -m surfsense_mcp`,
|
||||
].join("\n"),
|
||||
},
|
||||
{
|
||||
id: "codex",
|
||||
label: "Codex",
|
||||
configFile: "~/.codex/config.toml",
|
||||
language: "toml",
|
||||
steps: [
|
||||
"Add this to ~/.codex/config.toml (or run `codex mcp add surfsense -- uv run --directory <dir> python -m surfsense_mcp`).",
|
||||
"Restart Codex; `codex mcp list` should show surfsense.",
|
||||
],
|
||||
buildConfig: ({ baseUrl, apiKey, serverDir }) =>
|
||||
[
|
||||
"[mcp_servers.surfsense]",
|
||||
'command = "uv"',
|
||||
`args = ${JSON.stringify(serverArgs(serverDir))}`,
|
||||
"",
|
||||
"[mcp_servers.surfsense.env]",
|
||||
`SURFSENSE_BASE_URL = "${baseUrl}"`,
|
||||
`SURFSENSE_API_KEY = "${apiKey}"`,
|
||||
].join("\n"),
|
||||
},
|
||||
{
|
||||
id: "opencode",
|
||||
label: "OpenCode",
|
||||
configFile: "opencode.json",
|
||||
language: "json",
|
||||
steps: [
|
||||
"Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).",
|
||||
"Note OpenCode's format: the key is `mcp`, the command is one array, and env vars go under `environment`.",
|
||||
],
|
||||
buildConfig: ({ baseUrl, apiKey, serverDir }) =>
|
||||
json({
|
||||
$schema: "https://opencode.ai/config.json",
|
||||
mcp: {
|
||||
surfsense: {
|
||||
type: "local",
|
||||
command: ["uv", ...serverArgs(serverDir)],
|
||||
enabled: true,
|
||||
environment: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey },
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "cursor",
|
||||
label: "Cursor",
|
||||
configFile: "~/.cursor/mcp.json",
|
||||
language: "json",
|
||||
steps: [
|
||||
"Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.",
|
||||
"Refresh the server in Cursor Settings → MCP; its 18 tools should appear.",
|
||||
],
|
||||
buildConfig: standardJson,
|
||||
},
|
||||
{
|
||||
id: "claude-desktop",
|
||||
label: "Claude Desktop",
|
||||
configFile: "claude_desktop_config.json",
|
||||
language: "json",
|
||||
steps: [
|
||||
"Open Settings → Developer → Edit Config to reach claude_desktop_config.json and add this.",
|
||||
"Restart Claude Desktop; surfsense appears under the tools icon.",
|
||||
],
|
||||
buildConfig: standardJson,
|
||||
},
|
||||
{
|
||||
id: "vscode",
|
||||
label: "VS Code",
|
||||
configFile: ".vscode/mcp.json",
|
||||
language: "json",
|
||||
steps: [
|
||||
"Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).",
|
||||
"Open Copilot Chat in agent mode and click the tools icon to confirm surfsense is loaded.",
|
||||
],
|
||||
buildConfig: ({ baseUrl, apiKey, serverDir }) =>
|
||||
json({
|
||||
servers: {
|
||||
surfsense: {
|
||||
type: "stdio",
|
||||
command: "uv",
|
||||
args: serverArgs(serverDir),
|
||||
env: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey },
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "windsurf",
|
||||
label: "Windsurf",
|
||||
configFile: "~/.codeium/windsurf/mcp_config.json",
|
||||
language: "json",
|
||||
steps: [
|
||||
"Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).",
|
||||
"Press the refresh button in the MCP panel to pick up the server.",
|
||||
],
|
||||
buildConfig: standardJson,
|
||||
},
|
||||
{
|
||||
id: "gemini-cli",
|
||||
label: "Gemini CLI",
|
||||
configFile: "~/.gemini/settings.json",
|
||||
language: "json",
|
||||
steps: [
|
||||
"Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).",
|
||||
"Run /mcp inside Gemini CLI to confirm the surfsense server and its tools.",
|
||||
],
|
||||
buildConfig: standardJson,
|
||||
},
|
||||
];
|
||||
|
|
@ -97,6 +97,23 @@ export function trackWorkspaceViewed(workspaceId: number) {
|
|||
});
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// ACTIVE-USER (WAU) EVENT
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Single signal for active-user counting. Fired whenever a user sends a
|
||||
* chat message or starts an API run, so a "weekly unique users on
|
||||
* weekly_users" insight in PostHog is our WAU number.
|
||||
*
|
||||
* ponytail: frontend-only capture — API runs made directly against the
|
||||
* backend (PAT/curl, no browser) are not counted. Upgrade path is a
|
||||
* server-side capture in the backend if that ever matters.
|
||||
*/
|
||||
export function trackWeeklyUser(source: "chat_message" | "api_run", workspaceId?: number) {
|
||||
safeCapture("weekly_users", compact({ source, workspace_id: workspaceId }));
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// CHAT EVENTS
|
||||
// ============================================
|
||||
|
|
@ -124,6 +141,7 @@ export function trackChatMessageSent(
|
|||
has_mentioned_documents: options?.hasMentionedDocuments ?? false,
|
||||
message_length: options?.messageLength,
|
||||
});
|
||||
trackWeeklyUser("chat_message", workspaceId);
|
||||
}
|
||||
|
||||
export function trackChatResponseReceived(workspaceId: number, chatId: number) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue