apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-22 23:31:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor: update README files and system prompts to include Amazon as a data source for AI agents, enhancing the platform's capabilities for live web research

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-17 16:44:11 -07:00
parent 8a5d37a4db
commit bd7647b314
19 changed files with 55 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

11
surfsense_web/app/(home)/mcp-server/page.tsx
View file

@ -16,7 +16,7 @@ import type { FaqItem } from "@/lib/connectors-marketing/types";
const canonicalUrl = "https://www.surfsense.com/mcp-server";
const metaDescription =
"The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the web, plus full knowledge base access. One API key.";
"The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and the web, plus full knowledge base access. One API key.";
export const metadata: Metadata = {
title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools",
@ -102,6 +102,7 @@ const TOOL_GROUPS = [
"surfsense_google_maps_scrape",
"surfsense_google_maps_reviews",
"surfsense_google_search",
"surfsense_amazon_scrape",
"surfsense_web_crawl",
"surfsense_list_scraper_runs",
"surfsense_get_scraper_run",
@ -133,7 +134,7 @@ const FAQ: FaqItem[] = [
{
question: "What is the SurfSense MCP server?",
answer:
"It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.",
"It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.",
},
{
question: "Which MCP clients does it work with?",
@ -221,9 +222,9 @@ export default function McpServerPage() {
</h1>
<p className="mt-5 max-w-xl text-base leading-relaxed text-muted-foreground sm:text-lg">
The SurfSense MCP server hands Claude, Cursor, or any MCP client the whole platform:
scrape Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the open
web, and search, read, and write your knowledge base. One API key, typed tools, pay
as you go.
scrape Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and
the open web, and search, read, and write your knowledge base. One API key, typed
tools, pay as you go.
</p>
<div className="mt-8 flex flex-wrap items-center gap-3">
<Button asChild size="lg">

6
surfsense_web/app/layout.tsx
View file

@ -52,7 +52,7 @@ export const metadata: Metadata = {
},
title: "SurfSense - NotebookLM for Open Web Research",
description:
"SurfSense is an open-source open web research platform. Your AI agents research the live web with structured data from Reddit, YouTube, Google Maps, Google Search, and any page, through one API or MCP server.",
"SurfSense is an open-source open web research platform. Your AI agents research the live web with structured data from Reddit, YouTube, Amazon, Google Maps, Google Search, and any page, through one API or MCP server.",
keywords: [
"open web research platform",
"web research for AI agents",
@ -69,7 +69,7 @@ export const metadata: Metadata = {
openGraph: {
title: "SurfSense - NotebookLM for Open Web Research",
description:
"SurfSense is an open-source open web research platform. Your AI agents research the live web with structured data from Reddit, YouTube, Google Maps, Google Search, and any page, through one API or MCP server.",
"SurfSense is an open-source open web research platform. Your AI agents research the live web with structured data from Reddit, YouTube, Amazon, Google Maps, Google Search, and any page, through one API or MCP server.",
url: "https://www.surfsense.com",
siteName: "SurfSense",
type: "website",
@ -87,7 +87,7 @@ export const metadata: Metadata = {
card: "summary_large_image",
title: "SurfSense - NotebookLM for Open Web Research",
description:
"SurfSense is an open-source open web research platform. Your AI agents research the live web with structured data from Reddit, YouTube, Google Maps, Google Search, and any page, through one API or MCP server.",
"SurfSense is an open-source open web research platform. Your AI agents research the live web with structured data from Reddit, YouTube, Amazon, Google Maps, Google Search, and any page, through one API or MCP server.",
creator: "@SurfSenseAI",
site: "@SurfSenseAI",
images: [

4
surfsense_web/components/homepage/hero-section.tsx
View file

@ -70,7 +70,6 @@ type HeroCategory = {
};
const HERO_TUTORIAL = "/homepage/hero_tutorial";
const HERO_REALTIME = "/homepage/hero_realtime";
/*
* Every scripted demo below mirrors a task the SurfSense agent has actually run
@ -659,7 +658,8 @@ export function HeroSection() {
>
SurfSense is an open-source open web research platform, like NotebookLM but with live
data connectors. Your AI agents research the live web with structured data from
Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and any page on the
Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and any page on
the
open web.
</p>

2
surfsense_web/components/homepage/persona-paths.tsx
View file

@ -35,7 +35,7 @@ const PATHS: {
eyebrow: "For developers & agents",
title: "The whole platform is programmable",
description:
"Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.",
"Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, TikTok, Amazon, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.",
links: [
{ label: "Read the docs", href: "/docs" },
{ label: "SurfSense MCP server", href: "/mcp-server" },

2
surfsense_web/components/pricing/pricing-section.tsx
View file

@ -35,7 +35,7 @@ const demoPlans = [
billingText: "Your first $5 of credit is free. No subscription, ever",
features: [
"$5 of free credit to start, one balance for everything",
"Platform connectors: Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web",
"Platform connectors: Reddit, YouTube, TikTok, Amazon, Google Maps, Google Search, and the open web",
"Call every connector as a REST API with your key or through the MCP server",
"Pay per item returned and per page crawled. Failed calls are never billed",
"Premium models like GPT-5.5, Claude Sonnet 5, Gemini 3.1 Pro billed at provider cost",

4
surfsense_web/components/seo/json-ld.tsx
View file

@ -76,11 +76,11 @@ export function SoftwareApplicationJsonLd() {
"Free self-hosted from the open-source repo; cloud starts with $5 of free credit, then pay as you go",
},
description:
"SurfSense is an open-source open web research platform. AI agents research the live web with platform-native connectors for Reddit, YouTube, TikTok, Google Maps, Google Search, and any page on the open web, through one API or MCP server.",
"SurfSense is an open-source open web research platform. AI agents research the live web with platform-native connectors for Reddit, YouTube, TikTok, Amazon, Google Maps, Google Search, and any page on the open web, through one API or MCP server.",
url: "https://www.surfsense.com",
downloadUrl: "https://github.com/MODSetter/SurfSense/releases",
featureList: [
"Platform-native connectors: Reddit, YouTube, TikTok, Google Maps, Google Search, Web Crawl",
"Platform-native connectors: Reddit, YouTube, TikTok, Amazon, Google Maps, Google Search, Web Crawl",
"MCP server that exposes every connector as a native agent tool",
"Agent harness with retries, structured output, and credit metering",
"Live web research with cited briefs and alerts",

2
surfsense_web/content/docs/connectors/index.mdx
View file

@ -12,7 +12,7 @@ Connectors bring data into SurfSense — either as searchable knowledge or as li
<Card
icon={<Zap />}
title="Native Connectors"
description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST"
description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST"
href="/docs/connectors/native"
/>
<Card

7
surfsense_web/content/docs/connectors/native/index.mdx
View file

@ -1,6 +1,6 @@
---
title: Native Connectors
description: SurfSense's built-in scraper APIs for Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the web
description: SurfSense's built-in scraper APIs for Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and the web
---
import { Card, Cards } from 'fumadocs-ui/components/card';
@ -38,6 +38,11 @@ Native connectors are SurfSense's own scraper APIs — built into the platform,
description="Structured SERPs: organic results, people-also-ask, AI overviews"
href="/docs/connectors/native/google-search"
/>
<Card
title="Amazon"
description="Public product data: prices, ratings, offers, sellers, and best-seller ranks"
href="/docs/connectors/native/amazon"
/>
<Card
title="Web Crawl"
description="Scrape any page or spider a whole site into clean markdown"

6
surfsense_web/content/docs/how-to/mcp-server.mdx
View file

@ -7,7 +7,7 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs';
# SurfSense MCP Server
The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 24 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 25 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted.
@ -133,7 +133,7 @@ Add to `~/.cursor/mcp.json` (global — keeps the key out of your repo) or a pro
}
```
Then open **Cursor Settings → MCP** and refresh the `surfsense` server; its 24 tools should appear with a green dot.
Then open **Cursor Settings → MCP** and refresh the `surfsense` server; its 25 tools should appear with a green dot.
</Tab>
<Tab value="Claude Desktop">
@ -257,7 +257,7 @@ For self-host (stdio), all settings are environment variables passed by the clie
- **401 errors** — the API key is wrong or expired; create a new one.
- **403 errors** — API access is disabled for the workspace; toggle **API key access** on under **API Playground → API Keys**.
- **"Could not reach SurfSense"** — the backend isn't running or `SURFSENSE_BASE_URL` is wrong.
- **Server won't start** — run `uv run python -m mcp_server.selfcheck` inside `surfsense_mcp`; it verifies all 24 tools register without needing a backend.
- **Server won't start** — run `uv run python -m mcp_server.selfcheck` inside `surfsense_mcp`; it verifies all 25 tools register without needing a backend.
## Tools reference

1
surfsense_web/lib/chat/example-prompts.ts
View file

@ -44,6 +44,7 @@ export const CHAT_EXAMPLE_CATEGORIES: ChatExampleCategory[] = [
label: "Monitor Competitors",
prompts: [
"Extract every plan, price, and limit from [competitor]'s pricing page",
"Track the Amazon price, rating, and offers for [product] and its top rivals",
"Crawl [competitor]'s changelog and brief me on what they shipped this month",
"Measure the reaction to [competitor]'s launch across search, Reddit, and YouTube",
"Find the top-rated [category] businesses in [city], crawl their sites, and build a lead list with contacts",