mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-10 22:32:16 +02:00
feat: update environment variables and enhance scraping capabilities
- Adjusted Google Maps and YouTube micro pricing in the .env.example file for better cost management. - Introduced new environment variables for captcha solving and stealth browser hardening to improve scraping resilience. - Removed outdated smoke test for scraper API endpoints to streamline testing. - Enhanced anonymous chat agent's system prompt to clarify capabilities and suggest account creation for advanced features. - Updated Reddit fetch logic to prioritize new session handling and improve resilience against IP-related issues. - Added compacting functionality for scraper results to optimize data handling and presentation. - Improved workspace and document management tools with clearer descriptions and enhanced functionality. - Introduced new UI components for agent setup guidance in the web application.
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
271a21aee6
commit
1fd58752a3
24 changed files with 1326 additions and 320 deletions
|
|
@ -8,11 +8,55 @@ are clipped so a single call can't blow the context window.
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Literal
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
ResponseFormat = Literal["markdown", "json"]
|
||||
|
||||
# Shared parameter type for every tool: same name, same semantics everywhere.
|
||||
ResponseFormatParam = Annotated[
|
||||
ResponseFormat,
|
||||
Field(
|
||||
description="'markdown' (default, human-readable) or 'json' "
|
||||
"(raw data for post-processing)."
|
||||
),
|
||||
]
|
||||
|
||||
DEFAULT_CLIP_CHARS = 20_000
|
||||
ITEM_FIELD_CLIP_CHARS = 1_500
|
||||
|
||||
# Fields that duplicate another field verbatim (e.g. Reddit's 'html' mirrors
|
||||
# 'body') and only bloat inline results. The full record stays in the run.
|
||||
_REDUNDANT_ITEM_FIELDS = frozenset({"html"})
|
||||
|
||||
|
||||
def compact_items(result: Any, field_limit: int = ITEM_FIELD_CLIP_CHARS) -> Any:
|
||||
"""Shrink a scraper result for inline return.
|
||||
|
||||
Drops redundant fields and clips overlong strings per field, so a response
|
||||
keeps every item as an excerpt instead of a few items in full. The
|
||||
untruncated result remains retrievable via its stored run.
|
||||
"""
|
||||
if isinstance(result, dict) and isinstance(result.get("items"), list):
|
||||
return {
|
||||
**result,
|
||||
"items": [_compact_item(item, field_limit) for item in result["items"]],
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def _compact_item(item: Any, field_limit: int) -> Any:
|
||||
# ponytail: compacts top-level string fields only; nested structures pass
|
||||
# through untouched. Upgrade path is a recursive walk if a platform nests
|
||||
# long text.
|
||||
if not isinstance(item, dict):
|
||||
return item
|
||||
return {
|
||||
key: clip(value, field_limit) if isinstance(value, str) else value
|
||||
for key, value in item.items()
|
||||
if key not in _REDUNDANT_ITEM_FIELDS
|
||||
}
|
||||
|
||||
|
||||
def to_json(payload: Any) -> str:
|
||||
|
|
|
|||
|
|
@ -8,10 +8,22 @@ speaks a name, we resolve it, and remember the choice for later calls.
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Annotated
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from .client import SurfSenseClient
|
||||
from .errors import ToolError
|
||||
|
||||
# Shared parameter type for every workspace-scoped tool.
|
||||
WorkspaceParam = Annotated[
|
||||
str | None,
|
||||
Field(
|
||||
description="Workspace name or id, e.g. 'Research' or '3'. Omit to use "
|
||||
"the active workspace (set with surfsense_select_workspace)."
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Workspace:
|
||||
|
|
|
|||
|
|
@ -10,14 +10,16 @@ from __future__ import annotations
|
|||
|
||||
import mimetypes
|
||||
from pathlib import Path
|
||||
from typing import Annotated
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from mcp.types import ToolAnnotations
|
||||
from pydantic import Field
|
||||
|
||||
from ...core.client import SurfSenseClient
|
||||
from ...core.errors import ToolError
|
||||
from ...core.rendering import ResponseFormat, clip, to_json
|
||||
from ...core.workspace_context import WorkspaceContext
|
||||
from ...core.rendering import ResponseFormatParam, clip, to_json
|
||||
from ...core.workspace_context import WorkspaceContext, WorkspaceParam
|
||||
from .note_ingestion import build_note_document
|
||||
|
||||
_READ = ToolAnnotations(
|
||||
|
|
@ -30,6 +32,22 @@ _DELETE = ToolAnnotations(
|
|||
readOnlyHint=False, destructiveHint=True, idempotentHint=False, openWorldHint=False
|
||||
)
|
||||
|
||||
_DOCUMENT_ID = Annotated[
|
||||
int,
|
||||
Field(
|
||||
description="Document id from surfsense_search_knowledge_base or "
|
||||
"surfsense_list_documents results."
|
||||
),
|
||||
]
|
||||
|
||||
_DOCUMENT_TYPES = Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Restrict to these document types, e.g. "
|
||||
"['FILE', 'CRAWLED_URL', 'YOUTUBE_VIDEO']. Omit for all types."
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def register(
|
||||
mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
|
||||
|
|
@ -38,21 +56,34 @@ def register(
|
|||
|
||||
@mcp.tool(
|
||||
name="surfsense_search_knowledge_base",
|
||||
title="Search knowledge base",
|
||||
annotations=_READ,
|
||||
structured_output=False,
|
||||
)
|
||||
async def search_knowledge_base(
|
||||
query: str,
|
||||
top_k: int = 5,
|
||||
document_types: list[str] | None = None,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
query: Annotated[
|
||||
str,
|
||||
Field(
|
||||
min_length=1,
|
||||
description="Natural-language search, e.g. "
|
||||
"'notebooklm user complaints'.",
|
||||
),
|
||||
],
|
||||
top_k: Annotated[
|
||||
int, Field(ge=1, le=20, description="Maximum documents to return.")
|
||||
] = 5,
|
||||
document_types: _DOCUMENT_TYPES = None,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Search the workspace's knowledge base by meaning and keyword.
|
||||
"""Search the workspace's knowledge base by meaning and keywords.
|
||||
|
||||
Use this to answer questions from stored content: it returns the most
|
||||
relevant documents with the passages that matched, ranked by relevance.
|
||||
top_k caps documents (1–20). Optionally restrict to document_types.
|
||||
Use this FIRST when a question might be answered by content already
|
||||
stored in SurfSense — notes, uploaded files, saved pages, past
|
||||
research. Do NOT use it to fetch new data from the web; use the
|
||||
scraper tools for that. Returns the most relevant documents with the
|
||||
passages that matched, ranked by relevance score.
|
||||
Example: query='pricing feedback', top_k=5.
|
||||
"""
|
||||
resolved = await context.resolve(workspace)
|
||||
hits = await client.request(
|
||||
|
|
@ -71,21 +102,33 @@ def register(
|
|||
return _render_search(query, items)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_list_documents", annotations=_READ, structured_output=False
|
||||
name="surfsense_list_documents",
|
||||
title="List documents",
|
||||
annotations=_READ,
|
||||
structured_output=False,
|
||||
)
|
||||
async def list_documents(
|
||||
document_types: list[str] | None = None,
|
||||
folder_id: int | None = None,
|
||||
page: int = 0,
|
||||
page_size: int = 20,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
document_types: _DOCUMENT_TYPES = None,
|
||||
folder_id: Annotated[
|
||||
int | None,
|
||||
Field(description="Only documents in this folder. Omit for all."),
|
||||
] = None,
|
||||
page: Annotated[
|
||||
int, Field(ge=0, description="Zero-based page number.")
|
||||
] = 0,
|
||||
page_size: Annotated[
|
||||
int, Field(ge=1, description="Documents per page.")
|
||||
] = 20,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""List documents in the workspace's knowledge base, newest first.
|
||||
|
||||
Use this to browse or inventory what is stored. Optionally filter by
|
||||
document_types or a folder_id. Paginated: returns page_size items and a
|
||||
has_more flag; request the next page by increasing page.
|
||||
Use this to browse or inventory what is stored; to find documents
|
||||
about a topic, prefer surfsense_search_knowledge_base. Returns each
|
||||
document's title, id, type, and update time, plus a has_more flag —
|
||||
request the next page by increasing page.
|
||||
Example: document_types=['FILE'], page=0, page_size=20.
|
||||
"""
|
||||
resolved = await context.resolve(workspace)
|
||||
result = await client.request(
|
||||
|
|
@ -104,15 +147,20 @@ def register(
|
|||
return _render_document_list(result)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_get_document", annotations=_READ, structured_output=False
|
||||
name="surfsense_get_document",
|
||||
title="Read one document",
|
||||
annotations=_READ,
|
||||
structured_output=False,
|
||||
)
|
||||
async def get_document(
|
||||
document_id: int, response_format: ResponseFormat = "markdown"
|
||||
document_id: _DOCUMENT_ID,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Read one document's full content and metadata by id.
|
||||
|
||||
Use this after search or list to open a specific document. The id comes
|
||||
from those tools' results.
|
||||
Use this after surfsense_search_knowledge_base or
|
||||
surfsense_list_documents to open a specific document — search results
|
||||
only include the matching passages, this returns the whole text.
|
||||
"""
|
||||
document = await client.request("GET", f"/documents/{document_id}")
|
||||
if response_format == "json":
|
||||
|
|
@ -120,20 +168,36 @@ def register(
|
|||
return _render_document(document)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_add_document", annotations=_WRITE, structured_output=False
|
||||
name="surfsense_add_document",
|
||||
title="Add a note",
|
||||
annotations=_WRITE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def add_document(
|
||||
title: str,
|
||||
content: str,
|
||||
source_url: str | None = None,
|
||||
workspace: str | None = None,
|
||||
title: Annotated[
|
||||
str,
|
||||
Field(min_length=1, description="Short descriptive title for the note."),
|
||||
],
|
||||
content: Annotated[
|
||||
str,
|
||||
Field(
|
||||
min_length=1,
|
||||
description="The note's body; plain text or markdown.",
|
||||
),
|
||||
],
|
||||
source_url: Annotated[
|
||||
str | None,
|
||||
Field(description="Where the text came from, if anywhere."),
|
||||
] = None,
|
||||
workspace: WorkspaceParam = None,
|
||||
) -> str:
|
||||
"""Add a text or markdown note to the workspace's knowledge base.
|
||||
"""Save a text or markdown note into the workspace's knowledge base.
|
||||
|
||||
Use this to save notes, summaries, or snippets so they become
|
||||
searchable. The content is indexed asynchronously, so it may take a
|
||||
moment to appear in search. source_url optionally records where the text
|
||||
came from.
|
||||
Use this to store notes, summaries, or findings so they become
|
||||
searchable later — e.g. after finishing a piece of research. For files
|
||||
on disk use surfsense_upload_file instead. Indexing is asynchronous,
|
||||
so the note may take a moment to appear in search.
|
||||
Example: title='NotebookLM subreddits', content='- r/notebooklm ...'.
|
||||
"""
|
||||
resolved = await context.resolve(workspace)
|
||||
await client.request(
|
||||
|
|
@ -152,18 +216,35 @@ def register(
|
|||
)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_upload_file", annotations=_WRITE, structured_output=False
|
||||
name="surfsense_upload_file",
|
||||
title="Upload a file",
|
||||
annotations=_WRITE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def upload_file(
|
||||
file_path: str,
|
||||
use_vision_llm: bool = False,
|
||||
workspace: str | None = None,
|
||||
file_path: Annotated[
|
||||
str,
|
||||
Field(
|
||||
description="Path to a local file, e.g. "
|
||||
"'C:/Users/me/report.pdf' or '~/notes/summary.md'."
|
||||
),
|
||||
],
|
||||
use_vision_llm: Annotated[
|
||||
bool,
|
||||
Field(
|
||||
description="True reads scanned or image-heavy files with a "
|
||||
"vision model (slower)."
|
||||
),
|
||||
] = False,
|
||||
workspace: WorkspaceParam = None,
|
||||
) -> str:
|
||||
"""Upload a local file (PDF, doc, etc.) into the knowledge base.
|
||||
"""Upload a local file (PDF, docx, markdown, etc.) into the knowledge base.
|
||||
|
||||
Use this to ingest a file from disk; it is parsed, chunked, and indexed
|
||||
asynchronously. Set use_vision_llm to read scanned or image-heavy files
|
||||
with a vision model (slower).
|
||||
Use this to ingest a file from disk so its content becomes searchable;
|
||||
for text you already have in hand use surfsense_add_document instead.
|
||||
The file is parsed, chunked, and indexed asynchronously. Duplicate
|
||||
files are detected and skipped.
|
||||
Example: file_path='C:/Users/me/report.pdf'.
|
||||
"""
|
||||
resolved = await context.resolve(workspace)
|
||||
payload = _read_upload(file_path)
|
||||
|
|
@ -186,14 +267,28 @@ def register(
|
|||
)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_update_document", annotations=_WRITE, structured_output=False
|
||||
name="surfsense_update_document",
|
||||
title="Replace a document's content",
|
||||
annotations=_WRITE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def update_document(document_id: int, content: str) -> str:
|
||||
async def update_document(
|
||||
document_id: _DOCUMENT_ID,
|
||||
content: Annotated[
|
||||
str,
|
||||
Field(
|
||||
min_length=1,
|
||||
description="New full text; replaces the existing content "
|
||||
"entirely.",
|
||||
),
|
||||
],
|
||||
) -> str:
|
||||
"""Replace a document's stored content by id.
|
||||
|
||||
Use this to correct or rewrite a document's text. Note: this updates the
|
||||
stored content; re-indexing of search chunks is not triggered by this
|
||||
call.
|
||||
Use this to correct or rewrite a document's text. The new content
|
||||
REPLACES the old entirely — to append, read the document first with
|
||||
surfsense_get_document and resend the combined text. Search chunks are
|
||||
not re-indexed by this call.
|
||||
"""
|
||||
existing = await client.request("GET", f"/documents/{document_id}")
|
||||
await client.request(
|
||||
|
|
@ -208,13 +303,17 @@ def register(
|
|||
return f"Updated document {document_id} ('{existing.get('title', '')}')."
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_delete_document", annotations=_DELETE, structured_output=False
|
||||
name="surfsense_delete_document",
|
||||
title="Delete a document",
|
||||
annotations=_DELETE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def delete_document(document_id: int) -> str:
|
||||
"""Delete a document from the knowledge base by id.
|
||||
async def delete_document(document_id: _DOCUMENT_ID) -> str:
|
||||
"""Permanently delete a document from the knowledge base by id.
|
||||
|
||||
Use this to permanently remove a document. Deletion runs in the
|
||||
background; the document stops appearing in searches immediately.
|
||||
Use this only when the user explicitly asks to remove a document —
|
||||
deletion cannot be undone. The document stops appearing in searches
|
||||
immediately.
|
||||
"""
|
||||
await client.request("DELETE", f"/documents/{document_id}")
|
||||
return f"Deleted document {document_id}."
|
||||
|
|
|
|||
|
|
@ -8,14 +8,15 @@ full later.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from mcp.types import ToolAnnotations
|
||||
from pydantic import Field
|
||||
|
||||
from ...core.client import SurfSenseClient
|
||||
from ...core.rendering import ResponseFormat, clip, to_json
|
||||
from ...core.workspace_context import WorkspaceContext
|
||||
from ...core.rendering import ResponseFormatParam, clip, to_json
|
||||
from ...core.workspace_context import WorkspaceContext, WorkspaceParam
|
||||
from .capability import run_scraper
|
||||
|
||||
# Scrapers reach the open web and record a billable run; they are neither
|
||||
|
|
@ -38,23 +39,57 @@ def register(
|
|||
) -> None:
|
||||
"""Register the scraper and run-history tools on the server."""
|
||||
|
||||
@mcp.tool(name="surfsense_web_crawl", annotations=_SCRAPE, structured_output=False)
|
||||
@mcp.tool(
|
||||
name="surfsense_web_crawl",
|
||||
title="Crawl web pages",
|
||||
annotations=_SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def web_crawl(
|
||||
start_urls: list[str],
|
||||
max_crawl_depth: int = 0,
|
||||
max_crawl_pages: int = 10,
|
||||
max_length: int = 50_000,
|
||||
include_url_patterns: list[str] | None = None,
|
||||
exclude_url_patterns: list[str] | None = None,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
start_urls: Annotated[
|
||||
list[str],
|
||||
Field(
|
||||
min_length=1,
|
||||
description="Full URLs to fetch, e.g. "
|
||||
"['https://example.com/blog/post'].",
|
||||
),
|
||||
],
|
||||
max_crawl_depth: Annotated[
|
||||
int,
|
||||
Field(
|
||||
ge=0,
|
||||
description="Link-hops to follow from start_urls within the "
|
||||
"same site. 0 fetches only start_urls.",
|
||||
),
|
||||
] = 0,
|
||||
max_crawl_pages: Annotated[
|
||||
int, Field(ge=1, description="Stop after this many pages in total.")
|
||||
] = 10,
|
||||
max_length: Annotated[
|
||||
int, Field(ge=1, description="Max characters kept per page.")
|
||||
] = 50_000,
|
||||
include_url_patterns: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Regexes; only discovered links matching one are "
|
||||
"followed, e.g. ['/docs/.*']."
|
||||
),
|
||||
] = None,
|
||||
exclude_url_patterns: Annotated[
|
||||
list[str] | None,
|
||||
Field(description="Regexes; discovered links matching one are skipped."),
|
||||
] = None,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Crawl web pages and return their cleaned content as markdown.
|
||||
"""Fetch specific web pages and return their cleaned content as markdown.
|
||||
|
||||
Use this to read one page or spider a site. With max_crawl_depth=0 only
|
||||
start_urls are fetched; a higher depth follows same-site links up to
|
||||
max_crawl_pages. include/exclude_url_patterns are regexes that narrow
|
||||
which discovered links are followed.
|
||||
Use this to read a page the user names, or to spider a site from a
|
||||
starting URL. Do NOT use it to find pages on a topic — use
|
||||
surfsense_google_search for discovery. Returns one item per crawled
|
||||
page: url, title, and the page text as markdown.
|
||||
Example: start_urls=['https://blog.example.com'], max_crawl_depth=1,
|
||||
include_url_patterns=['/2026/'].
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
|
|
@ -74,22 +109,47 @@ def register(
|
|||
)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_google_search", annotations=_SCRAPE, structured_output=False
|
||||
name="surfsense_google_search",
|
||||
title="Scrape Google Search",
|
||||
annotations=_SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def google_search(
|
||||
queries: list[str],
|
||||
max_pages_per_query: int = 1,
|
||||
country_code: str | None = None,
|
||||
language_code: str = "",
|
||||
site: str | None = None,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
queries: Annotated[
|
||||
list[str],
|
||||
Field(
|
||||
min_length=1,
|
||||
description="Search terms or full Google Search URLs, e.g. "
|
||||
"['best rss readers 2026'].",
|
||||
),
|
||||
],
|
||||
max_pages_per_query: Annotated[
|
||||
int, Field(ge=1, description="Result pages to fetch per query.")
|
||||
] = 1,
|
||||
country_code: Annotated[
|
||||
str | None,
|
||||
Field(description="Two-letter country to search from, e.g. 'us'."),
|
||||
] = None,
|
||||
language_code: Annotated[
|
||||
str, Field(description="Results language, e.g. 'en'. Empty for default.")
|
||||
] = "",
|
||||
site: Annotated[
|
||||
str | None,
|
||||
Field(
|
||||
description="Restrict results to one domain, e.g. 'example.com'."
|
||||
),
|
||||
] = None,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Scrape Google Search results for one or more queries.
|
||||
"""Scrape Google Search result pages for one or more queries.
|
||||
|
||||
Use this to find pages on the web. Each item is a query's fetched result
|
||||
page. Pass full Google Search URLs to scrape them as-is, or plain terms
|
||||
to search. Optionally scope to a country, language, or single domain.
|
||||
Use this to discover pages on the open web by topic; follow up with
|
||||
surfsense_web_crawl to read a result in full. Do NOT use it for
|
||||
Reddit, YouTube, or Google Maps research — the dedicated tools return
|
||||
richer data. Returns each query's parsed results: title, url, and
|
||||
snippet per organic result.
|
||||
Example: queries=['notebooklm review'], site='news.ycombinator.com'.
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
|
|
@ -108,24 +168,61 @@ def register(
|
|||
)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_reddit_scrape", annotations=_SCRAPE, structured_output=False
|
||||
name="surfsense_reddit_scrape",
|
||||
title="Search or scrape Reddit",
|
||||
annotations=_SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def reddit_scrape(
|
||||
urls: list[str] | None = None,
|
||||
search_queries: list[str] | None = None,
|
||||
community: str | None = None,
|
||||
sort: RedditSort = "new",
|
||||
time_filter: RedditTime | None = None,
|
||||
max_items: int = 10,
|
||||
skip_comments: bool = False,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
urls: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Reddit URLs: a post, a subreddit like "
|
||||
"'https://reddit.com/r/LocalLLaMA', a user page, or a search "
|
||||
"URL. Provide urls OR search_queries."
|
||||
),
|
||||
] = None,
|
||||
search_queries: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Terms to search Reddit for, e.g. "
|
||||
"['NotebookLM alternatives']. Provide search_queries OR urls."
|
||||
),
|
||||
] = None,
|
||||
community: Annotated[
|
||||
str | None,
|
||||
Field(
|
||||
description="Restrict a search to one subreddit, name without "
|
||||
"'r/', e.g. 'ArtificialInteligence'."
|
||||
),
|
||||
] = None,
|
||||
sort: Annotated[RedditSort, Field(description="Post ordering.")] = "new",
|
||||
time_filter: Annotated[
|
||||
RedditTime | None,
|
||||
Field(description="Time window; only valid with sort='top'."),
|
||||
] = None,
|
||||
max_items: Annotated[
|
||||
int, Field(ge=1, description="Maximum posts to return.")
|
||||
] = 10,
|
||||
skip_comments: Annotated[
|
||||
bool,
|
||||
Field(
|
||||
description="True fetches posts only (faster); False also "
|
||||
"fetches each post's comment thread."
|
||||
),
|
||||
] = False,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Scrape Reddit posts and comments from URLs or a search.
|
||||
"""Search or scrape Reddit: posts, comments, subreddits, and users.
|
||||
|
||||
Provide urls (a post, /r/subreddit, /user/name, or search URL) OR
|
||||
search_queries; scope a search to one subreddit with community. Use
|
||||
time_filter only with sort='top'. Set skip_comments to fetch posts only.
|
||||
Use this for ANY Reddit research — finding relevant subreddits or
|
||||
communities for a topic, top posts, or discussions — instead of a
|
||||
generic web search. Returns posts (title, text, score, subreddit, url)
|
||||
with comment threads unless skip_comments is set. Every post carries
|
||||
its subreddit, so to find communities for a topic, search posts and
|
||||
aggregate their subreddits.
|
||||
Example: search_queries=['NotebookLM'], sort='top', time_filter='month'.
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
|
|
@ -146,22 +243,46 @@ def register(
|
|||
)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_youtube_scrape", annotations=_SCRAPE, structured_output=False
|
||||
name="surfsense_youtube_scrape",
|
||||
title="Search or scrape YouTube",
|
||||
annotations=_SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def youtube_scrape(
|
||||
urls: list[str] | None = None,
|
||||
search_queries: list[str] | None = None,
|
||||
max_results: int = 10,
|
||||
download_subtitles: bool = False,
|
||||
subtitles_language: str = "en",
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
urls: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="YouTube URLs: video, channel, playlist, shorts, "
|
||||
"or hashtag pages. Provide urls OR search_queries."
|
||||
),
|
||||
] = None,
|
||||
search_queries: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Terms to search YouTube for, e.g. "
|
||||
"['NotebookLM tutorial']. Provide search_queries OR urls."
|
||||
),
|
||||
] = None,
|
||||
max_results: Annotated[
|
||||
int, Field(ge=1, description="Maximum videos to return.")
|
||||
] = 10,
|
||||
download_subtitles: Annotated[
|
||||
bool,
|
||||
Field(description="True also fetches each video's transcript."),
|
||||
] = False,
|
||||
subtitles_language: Annotated[
|
||||
str, Field(description="Transcript language code, e.g. 'en'.")
|
||||
] = "en",
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Scrape YouTube videos from URLs or a search.
|
||||
"""Search or scrape YouTube videos, optionally with transcripts.
|
||||
|
||||
Provide urls (video, channel, playlist, shorts, or hashtag pages) OR
|
||||
search_queries. Set download_subtitles to also fetch each video's
|
||||
transcript in subtitles_language.
|
||||
Use this for YouTube research: finding videos on a topic, or reading a
|
||||
video's details or transcript. For a video's comment section use
|
||||
surfsense_youtube_comments instead. Returns per-video metadata (title,
|
||||
channel, views, description, url) and, if requested, the transcript.
|
||||
Example: search_queries=['NotebookLM tutorial'], download_subtitles=True.
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
|
|
@ -180,19 +301,41 @@ def register(
|
|||
)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_youtube_comments", annotations=_SCRAPE, structured_output=False
|
||||
name="surfsense_youtube_comments",
|
||||
title="Fetch YouTube comments",
|
||||
annotations=_SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def youtube_comments(
|
||||
urls: list[str],
|
||||
max_comments: int = 20,
|
||||
sort_by: CommentSort = "NEWEST_FIRST",
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
urls: Annotated[
|
||||
list[str],
|
||||
Field(
|
||||
min_length=1,
|
||||
description="YouTube video URLs, e.g. "
|
||||
"['https://www.youtube.com/watch?v=abc123'].",
|
||||
),
|
||||
],
|
||||
max_comments: Annotated[
|
||||
int,
|
||||
Field(
|
||||
ge=1,
|
||||
description="Maximum comments per video, counting top-level "
|
||||
"comments and replies together.",
|
||||
),
|
||||
] = 20,
|
||||
sort_by: Annotated[
|
||||
CommentSort, Field(description="Comment ordering.")
|
||||
] = "NEWEST_FIRST",
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Fetch comments (and replies) for one or more YouTube videos.
|
||||
"""Fetch the comments (and replies) on one or more YouTube videos.
|
||||
|
||||
Use this when the user wants a video's discussion rather than the video
|
||||
itself. max_comments counts top-level comments and replies together.
|
||||
Use this when the user wants a video's discussion or audience reaction
|
||||
rather than the video itself; get video URLs from
|
||||
surfsense_youtube_scrape if you only have a topic. Returns comment
|
||||
text, author, likes, and replies.
|
||||
Example: urls=['https://www.youtube.com/watch?v=abc123'], max_comments=50.
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
|
|
@ -210,24 +353,52 @@ def register(
|
|||
|
||||
@mcp.tool(
|
||||
name="surfsense_google_maps_scrape",
|
||||
title="Find places on Google Maps",
|
||||
annotations=_SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def google_maps_scrape(
|
||||
search_queries: list[str] | None = None,
|
||||
urls: list[str] | None = None,
|
||||
place_ids: list[str] | None = None,
|
||||
location: str | None = None,
|
||||
max_places: int = 10,
|
||||
include_details: bool = False,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
search_queries: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Place searches, e.g. ['coffee shops']. Provide "
|
||||
"search_queries OR urls OR place_ids."
|
||||
),
|
||||
] = None,
|
||||
urls: Annotated[
|
||||
list[str] | None,
|
||||
Field(description="Google Maps URLs of specific places."),
|
||||
] = None,
|
||||
place_ids: Annotated[
|
||||
list[str] | None,
|
||||
Field(description="Google place ids, e.g. ['ChIJj61dQgK6j4AR...']."),
|
||||
] = None,
|
||||
location: Annotated[
|
||||
str | None,
|
||||
Field(
|
||||
description="Geographic scope for a search, e.g. "
|
||||
"'Seattle, USA'."
|
||||
),
|
||||
] = None,
|
||||
max_places: Annotated[
|
||||
int, Field(ge=1, description="Maximum places to return.")
|
||||
] = 10,
|
||||
include_details: Annotated[
|
||||
bool,
|
||||
Field(
|
||||
description="True adds opening hours and extra contact info "
|
||||
"(slower)."
|
||||
),
|
||||
] = False,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Scrape places from Google Maps by search, URL, or place id.
|
||||
"""Find places on Google Maps by search, URL, or place id.
|
||||
|
||||
Provide search_queries OR urls OR place_ids. Scope a search with
|
||||
location (e.g. 'New York, USA'). Set include_details for opening hours
|
||||
and extra contact info (slower).
|
||||
Use this for local-business and location research: names, addresses,
|
||||
ratings, categories, coordinates, place ids. For a place's customer
|
||||
reviews use surfsense_google_maps_reviews instead.
|
||||
Example: search_queries=['ramen'], location='Osaka, Japan', max_places=5.
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
|
|
@ -248,23 +419,49 @@ def register(
|
|||
|
||||
@mcp.tool(
|
||||
name="surfsense_google_maps_reviews",
|
||||
title="Fetch Google Maps reviews",
|
||||
annotations=_SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def google_maps_reviews(
|
||||
urls: list[str] | None = None,
|
||||
place_ids: list[str] | None = None,
|
||||
max_reviews: int = 20,
|
||||
sort_by: ReviewSort = "newest",
|
||||
language: str = "en",
|
||||
start_date: str | None = None,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
urls: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Google Maps URLs of places. Provide urls OR "
|
||||
"place_ids."
|
||||
),
|
||||
] = None,
|
||||
place_ids: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Google place ids from surfsense_google_maps_scrape."
|
||||
),
|
||||
] = None,
|
||||
max_reviews: Annotated[
|
||||
int, Field(ge=1, description="Maximum reviews per place.")
|
||||
] = 20,
|
||||
sort_by: Annotated[
|
||||
ReviewSort, Field(description="Review ordering.")
|
||||
] = "newest",
|
||||
language: Annotated[
|
||||
str, Field(description="Reviews language code, e.g. 'en'.")
|
||||
] = "en",
|
||||
start_date: Annotated[
|
||||
str | None,
|
||||
Field(
|
||||
description="ISO date like '2026-01-01'; keeps only reviews on "
|
||||
"or after that day."
|
||||
),
|
||||
] = None,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Fetch reviews for Google Maps places by URL or place id.
|
||||
"""Fetch customer reviews for Google Maps places by URL or place id.
|
||||
|
||||
Provide urls OR place_ids. start_date (ISO, e.g. '2024-01-01') keeps only
|
||||
reviews on or after that day.
|
||||
Use this to read feedback on specific places; get urls or place_ids
|
||||
from surfsense_google_maps_scrape first if you only have a name.
|
||||
Returns review text, rating, author, and date per review.
|
||||
Example: place_ids=['ChIJj61dQgK6j4AR...'], sort_by='newest'.
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
|
|
@ -285,21 +482,35 @@ def register(
|
|||
|
||||
@mcp.tool(
|
||||
name="surfsense_list_scraper_runs",
|
||||
title="List past scraper runs",
|
||||
annotations=_READ_RUNS,
|
||||
structured_output=False,
|
||||
)
|
||||
async def list_scraper_runs(
|
||||
limit: int = 20,
|
||||
capability: str | None = None,
|
||||
status: str | None = None,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
limit: Annotated[
|
||||
int, Field(ge=1, description="Maximum runs to list.")
|
||||
] = 20,
|
||||
capability: Annotated[
|
||||
str | None,
|
||||
Field(
|
||||
description="Filter by capability slug, e.g. 'web.crawl' or "
|
||||
"'reddit.scrape'."
|
||||
),
|
||||
] = None,
|
||||
status: Annotated[
|
||||
str | None,
|
||||
Field(description="Filter by run status: 'success' or 'error'."),
|
||||
] = None,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""List recent scraper runs for the workspace, newest first.
|
||||
"""List recent scraper runs in the workspace, newest first.
|
||||
|
||||
Use this to find a run_id to fetch in full with surfsense_get_scraper_run,
|
||||
e.g. when an inline result was truncated. Optionally filter by capability
|
||||
(like 'web.crawl') or status ('success' / 'error').
|
||||
Use this to find the run_id of an earlier scrape — for example when an
|
||||
inline result was truncated — then fetch it in full with
|
||||
surfsense_get_scraper_run. Returns each run's id, capability, status,
|
||||
item count, and creation time.
|
||||
Example: capability='reddit.scrape', status='success'.
|
||||
"""
|
||||
resolved = await context.resolve(workspace)
|
||||
runs = await client.request(
|
||||
|
|
@ -317,18 +528,26 @@ def register(
|
|||
|
||||
@mcp.tool(
|
||||
name="surfsense_get_scraper_run",
|
||||
title="Fetch one scraper run in full",
|
||||
annotations=_READ_RUNS,
|
||||
structured_output=False,
|
||||
)
|
||||
async def get_scraper_run(
|
||||
run_id: str,
|
||||
workspace: str | None = None,
|
||||
response_format: ResponseFormat = "markdown",
|
||||
run_id: Annotated[
|
||||
str,
|
||||
Field(
|
||||
description="Run id from surfsense_list_scraper_runs or a "
|
||||
"prior scrape's output."
|
||||
),
|
||||
],
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Fetch a single scraper run in full, including its stored output.
|
||||
|
||||
Use this to retrieve the complete result of an earlier scrape (its
|
||||
run_id comes from surfsense_list_scraper_runs or a prior scrape).
|
||||
Use this to retrieve the complete, untruncated result of an earlier
|
||||
scrape. Do NOT re-run a scraper just to recover a truncated result —
|
||||
fetch the stored run instead.
|
||||
"""
|
||||
resolved = await context.resolve(workspace)
|
||||
run = await client.request(
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from __future__ import annotations
|
|||
from typing import Any
|
||||
|
||||
from ...core.client import SurfSenseClient
|
||||
from ...core.rendering import ResponseFormat, clip, to_json
|
||||
from ...core.rendering import ResponseFormat, clip, compact_items, to_json
|
||||
from ...core.workspace_context import WorkspaceContext
|
||||
|
||||
|
||||
|
|
@ -29,6 +29,10 @@ async def run_scraper(
|
|||
result = await client.request(
|
||||
"POST", f"/workspaces/{resolved.id}/scrapers/{platform}/{verb}", json=body
|
||||
)
|
||||
# Inline results are compacted (redundant fields dropped, long fields
|
||||
# excerpted) so every item survives the overall clip; the complete output
|
||||
# is stored server-side and retrievable with surfsense_get_scraper_run.
|
||||
result = compact_items(result)
|
||||
if response_format == "json":
|
||||
return clip(to_json(result))
|
||||
return _render_markdown(platform, verb, resolved.name, result)
|
||||
|
|
@ -40,7 +44,11 @@ def _render_markdown(
|
|||
"""A readable header plus the structured payload, clipped to a safe size."""
|
||||
header = f'# {platform}.{verb} — {_describe_size(result)} from "{workspace_name}"'
|
||||
body = clip(to_json(result))
|
||||
return f"{header}\n\n```json\n{body}\n```"
|
||||
footer = (
|
||||
"\n\nFields shown as excerpts; use surfsense_get_scraper_run for the "
|
||||
"full output."
|
||||
)
|
||||
return f"{header}\n\n```json\n{body}\n```{footer}"
|
||||
|
||||
|
||||
def _describe_size(result: Any) -> str:
|
||||
|
|
|
|||
|
|
@ -7,10 +7,13 @@ rest of the conversation needs no ids.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from mcp.types import ToolAnnotations
|
||||
from pydantic import Field
|
||||
|
||||
from ...core.rendering import ResponseFormat, to_json
|
||||
from ...core.rendering import ResponseFormatParam, to_json
|
||||
from ...core.workspace_context import Workspace, WorkspaceContext
|
||||
|
||||
_READ_ONLY = ToolAnnotations(
|
||||
|
|
@ -23,15 +26,19 @@ def register(mcp: FastMCP, context: WorkspaceContext) -> None:
|
|||
|
||||
@mcp.tool(
|
||||
name="surfsense_list_workspaces",
|
||||
title="List workspaces",
|
||||
annotations=_READ_ONLY,
|
||||
structured_output=False,
|
||||
)
|
||||
async def list_workspaces(response_format: ResponseFormat = "markdown") -> str:
|
||||
async def list_workspaces(
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""List the SurfSense workspaces (search spaces) the account can access.
|
||||
|
||||
Use this to discover which workspaces exist before selecting one, or when
|
||||
the user asks what search spaces they have. Returns each workspace's name,
|
||||
id, description, ownership, and member count.
|
||||
Use this to discover which workspaces exist before selecting one, or
|
||||
when the user asks what search spaces they have. Returns each
|
||||
workspace's name, id, description, ownership, and member count, and
|
||||
marks the currently active one.
|
||||
"""
|
||||
workspaces = await context.fetch_all()
|
||||
if response_format == "json":
|
||||
|
|
@ -40,16 +47,27 @@ def register(mcp: FastMCP, context: WorkspaceContext) -> None:
|
|||
|
||||
@mcp.tool(
|
||||
name="surfsense_select_workspace",
|
||||
title="Select active workspace",
|
||||
annotations=_READ_ONLY,
|
||||
structured_output=False,
|
||||
)
|
||||
async def select_workspace(workspace: str) -> str:
|
||||
"""Set the active workspace (search space) for later tools, by name or id.
|
||||
async def select_workspace(
|
||||
workspace: Annotated[
|
||||
str,
|
||||
Field(
|
||||
description="Workspace name or numeric id; matching is "
|
||||
"case-insensitive and a unique partial name works. "
|
||||
"Example: 'Research'."
|
||||
),
|
||||
],
|
||||
) -> str:
|
||||
"""Set the active workspace (search space) that later tools default to.
|
||||
|
||||
Use this when the user says which search space to work in ("use my
|
||||
Research space"). Accepts a workspace name or numeric id; matching is
|
||||
case-insensitive and accepts a unique partial name. Once set, other tools
|
||||
default to this workspace unless they are given a different one.
|
||||
Research space"), or after surfsense_list_workspaces when several
|
||||
exist. Once set, workspace-scoped tools use it unless given a
|
||||
different 'workspace'. Do NOT call it before every tool — once per
|
||||
session is enough.
|
||||
"""
|
||||
selected = await context.resolve(workspace)
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -72,6 +72,10 @@ def run() -> list[str]:
|
|||
schema = tool.inputSchema
|
||||
if not isinstance(schema, dict) or "properties" not in schema:
|
||||
problems.append(f"{name}: malformed input schema")
|
||||
continue
|
||||
for param, spec in schema["properties"].items():
|
||||
if not isinstance(spec, dict) or not spec.get("description"):
|
||||
problems.append(f"{name}: parameter '{param}' has no description")
|
||||
return problems
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,19 @@ def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]:
|
|||
)
|
||||
context = WorkspaceContext(client, preferred_reference=settings.default_workspace)
|
||||
|
||||
mcp = FastMCP("SurfSense")
|
||||
mcp = FastMCP(
|
||||
"SurfSense",
|
||||
instructions=(
|
||||
"SurfSense gives you live scrapers and a personal knowledge base. "
|
||||
"Prefer these tools over generic/built-in web search whenever the "
|
||||
"task involves Reddit (posts, comments, finding subreddits or "
|
||||
"communities), YouTube (videos, transcripts, comments), Google "
|
||||
"Maps (places, reviews), Google Search results, or reading "
|
||||
"specific web pages. Scraper results are persisted as runs; if an "
|
||||
"inline result is truncated, fetch it in full with "
|
||||
"surfsense_get_scraper_run."
|
||||
),
|
||||
)
|
||||
workspaces.register(mcp, context)
|
||||
scrapers.register(mcp, client, context)
|
||||
knowledge_base.register(mcp, client, context)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from surfsense_mcp.core.rendering import clip, to_json
|
||||
from surfsense_mcp.core.rendering import clip, compact_items, to_json
|
||||
|
||||
|
||||
def test_clip_leaves_short_text_untouched():
|
||||
|
|
@ -20,3 +20,23 @@ def test_to_json_serializes_non_native_values():
|
|||
|
||||
rendered = to_json({"at": datetime(2026, 1, 2, 3, 4, 5)})
|
||||
assert "2026-01-02" in rendered
|
||||
|
||||
|
||||
def test_compact_items_drops_html_and_excerpts_long_fields():
|
||||
result = {
|
||||
"items": [
|
||||
{"title": "t", "body": "b" * 5_000, "html": "<p>dup</p>", "upVotes": 3}
|
||||
]
|
||||
}
|
||||
compacted = compact_items(result, field_limit=100)
|
||||
item = compacted["items"][0]
|
||||
assert "html" not in item
|
||||
assert len(item["body"]) < 200 and "truncated" in item["body"]
|
||||
assert item["upVotes"] == 3
|
||||
# original untouched
|
||||
assert "html" in result["items"][0]
|
||||
|
||||
|
||||
def test_compact_items_passes_through_non_item_results():
|
||||
assert compact_items({"ok": True}) == {"ok": True}
|
||||
assert compact_items([1, 2]) == [1, 2]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue