apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-08 22:22:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat: docs and ui tweaks

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-06 19:26:35 -07:00
parent 64f2b4a6eb
commit 271a21aee6
103 changed files with 2161 additions and 2625 deletions
Download patch file Download diff file Expand all files Collapse all files

33
surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/tool.py
View file

@ -649,14 +649,31 @@ async def _load_http_mcp_tools(
total_discovered = len(tool_definitions)
if allowed_set:
tool_definitions = [td for td in tool_definitions if td["name"] in allowed_set]
logger.info(
"HTTP MCP server '%s' (connector %d): %d/%d tools after allowlist filter",
url,
connector_id,
len(tool_definitions),
total_discovered,
)
filtered = [td for td in tool_definitions if td["name"] in allowed_set]
if not filtered and total_discovered:
# The server renamed its tools out from under our allowlist
# (e.g. Notion's "notion-" prefix rename) — a fully-dead
# allowlist would silently disable the connector. Load
# everything instead: renamed tools won't match
# ``readonly_tools`` either, so every tool stays HITL-gated.
logger.warning(
"HTTP MCP server '%s' (connector %d): allowlist matched 0/%d "
"advertised tools — server likely renamed its tools. "
"Loading all tools (HITL-gated). Update the registry allowlist: %s",
url,
connector_id,
total_discovered,
sorted(td["name"] for td in tool_definitions),
)
else:
tool_definitions = filtered
logger.info(
"HTTP MCP server '%s' (connector %d): %d/%d tools after allowlist filter",
url,
connector_id,
len(tool_definitions),
total_discovered,
)
else:
logger.info(
"Discovered %d tools from HTTP MCP server '%s' (connector %d) — no allowlist, loading all",

9
surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/agent.py
View file

@ -16,6 +16,9 @@ from langchain_core.tools import BaseTool
from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode
from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset
from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec
from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import (
append_today_utc,
)
from .middleware_stack import build_kb_middleware
from .prompts import load_description, load_readonly_system_prompt, load_system_prompt
@ -57,7 +60,7 @@ def build_subagent(
{
"name": NAME,
"description": load_description(),
"system_prompt": load_system_prompt(filesystem_mode),
"system_prompt": append_today_utc(load_system_prompt(filesystem_mode)),
"model": llm,
"tools": [_build_search_knowledge_base_tool(dependencies)],
"middleware": build_kb_middleware(
@ -86,7 +89,9 @@ def build_readonly_subagent(
{
"name": READONLY_NAME,
"description": "Read-only knowledge_base specialist (invoked via ask_knowledge_base).",
"system_prompt": load_readonly_system_prompt(filesystem_mode),
"system_prompt": append_today_utc(
load_readonly_system_prompt(filesystem_mode)
),
"model": llm,
"tools": [_build_search_knowledge_base_tool(dependencies)],
"middleware": build_kb_middleware(

15
surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/subagent_builder.py
View file

@ -5,6 +5,7 @@ from __future__ import annotations
import logging
import re
import time as _perf_time
from datetime import UTC, datetime
from typing import Any, cast
from deepagents import SubAgent
@ -60,6 +61,19 @@ def _resolve_includes(prompt: str, *, subagent_name: str) -> str:
return _INCLUDE_DIRECTIVE_RE.sub(_replace, prompt)
def append_today_utc(prompt: str) -> str:
"""Append the current UTC date so subagents share the main agent's clock.
The main agent injects ``Today (UTC): ...`` into its identity prompt, but
delegated subagents (calendar/gmail/drive, date-filtered searches, etc.)
never saw it and would guess the date. Appended at build time; the compiled
graph is cache-keyed on the main prompt hash (which carries the same date),
so a day rollover rebuilds both together and they stay consistent.
"""
today = datetime.now(UTC).date().isoformat()
return f"{prompt}\n\nToday (UTC): {today}\n"
def _user_allowlist_for(
dependencies: dict[str, Any], subagent_name: str
) -> Ruleset | None:
@ -115,6 +129,7 @@ def pack_subagent(
_t0 = _perf_time.perf_counter()
system_prompt = _resolve_includes(system_prompt, subagent_name=name)
system_prompt = append_today_utc(system_prompt)
_t_resolve = _perf_time.perf_counter() - _t0
flags = dependencies["flags"]

41
surfsense_backend/app/capabilities/core/access/rest.py
View file

@ -28,7 +28,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.auth.context import AuthContext
from app.capabilities.core.access.rate_limit import enforce_capability_rate_limit
from app.capabilities.core.billing import charge_capability, gate_capability
from app.capabilities.core.billing import (
charge_capability,
gate_capability,
pricing_meters,
)
from app.capabilities.core.events import run_event_bus
from app.capabilities.core.progress import progress_scope
from app.capabilities.core.runs import (
@ -55,13 +59,22 @@ _SSE_HEADERS = {
}
class PricingMeter(BaseModel):
"""One live per-item rate a verb charges on, e.g. 3500 micro-USD per place."""
unit: str
micros_per_unit: int
class CapabilitySummary(BaseModel):
"""A verb's identity + input/output JSON schemas, for the playground UI."""
"""A verb's identity + input/output JSON schemas + pricing, for the playground UI."""
name: str
description: str
input_schema: dict
output_schema: dict
# Empty list = free (billing disabled or an unmetered verb).
pricing: list[PricingMeter] = []
class RunSummary(BaseModel):
@ -125,12 +138,17 @@ def _register_capabilities_list(
) -> None:
"""Register the ``GET`` that lists verbs + their input schemas for the UI."""
summaries = [
CapabilitySummary(
name=capability.name,
description=capability.description,
input_schema=capability.input_schema.model_json_schema(),
output_schema=capability.output_schema.model_json_schema(),
# Schemas are static; pricing is attached per request because rates are
# read live from config (env retune + restart, no rebuild).
base_summaries = [
(
CapabilitySummary(
name=capability.name,
description=capability.description,
input_schema=capability.input_schema.model_json_schema(),
output_schema=capability.output_schema.model_json_schema(),
),
capability.billing_unit,
)
for capability in capabilities
]
@ -141,7 +159,12 @@ def _register_capabilities_list(
auth: AuthContext = Depends(get_auth_context),
) -> list[CapabilitySummary]:
await check_workspace_access(session, auth, workspace_id)
return summaries
return [
summary.model_copy(
update={"pricing": [PricingMeter(**m) for m in pricing_meters(unit)]}
)
for summary, unit in base_summaries
]
router.add_api_route(
"/workspaces/{workspace_id}/scrapers/capabilities",

47
surfsense_backend/app/capabilities/core/billing.py
View file

@ -43,6 +43,53 @@ def _platform_rate(unit: BillingUnit) -> int:
return int(getattr(config, _PLATFORM_RATE_KEYS[unit]))
# Display noun for each platform meter, e.g. "$3.50 / 1k places".
_UNIT_NOUNS: dict[BillingUnit, str] = {
BillingUnit.REDDIT_ITEM: "item",
BillingUnit.GOOGLE_SEARCH_SERP: "SERP",
BillingUnit.GOOGLE_MAPS_PLACE: "place",
BillingUnit.GOOGLE_MAPS_REVIEW: "review",
BillingUnit.YOUTUBE_VIDEO: "video",
BillingUnit.YOUTUBE_COMMENT: "comment",
}
def pricing_meters(unit: BillingUnit | None) -> list[dict]:
"""The live per-item rates a verb charges, for UI display. Empty = free.
Mirrors the gate/charge logic exactly: meters whose billing flag is off are
omitted, so a self-hosted install with billing disabled reads as free.
"""
if unit is None:
return []
if unit is BillingUnit.WEB_CRAWL:
meters = []
if WebCrawlCreditService.billing_enabled():
meters.append(
{"unit": "page", "micros_per_unit": config.WEB_CRAWL_MICROS_PER_SUCCESS}
)
if WebCrawlCreditService.captcha_billing_enabled() and captcha_enabled():
meters.append(
{
"unit": "captcha solve",
"micros_per_unit": config.WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE,
}
)
return meters
if not config.PLATFORM_SCRAPE_BILLING_ENABLED:
return []
meters = [{"unit": _UNIT_NOUNS[unit], "micros_per_unit": _platform_rate(unit)}]
if unit is BillingUnit.GOOGLE_MAPS_PLACE:
# Dual-metered: attached reviews bill on their own meter.
meters.append(
{
"unit": _UNIT_NOUNS[BillingUnit.GOOGLE_MAPS_REVIEW],
"micros_per_unit": _platform_rate(BillingUnit.GOOGLE_MAPS_REVIEW),
}
)
return meters
async def gate_capability(
payload: BillableInput, unit: BillingUnit | None, ctx: CapabilityContext
) -> None:

26
surfsense_backend/app/services/mcp_oauth/registry.py
View file

@ -153,13 +153,26 @@ MCP_SERVICES: dict[str, MCPServiceConfig] = {
"mpim:history",
"im:history",
],
# Both prefixed and unprefixed variants: sources disagree on which
# names mcp.slack.com currently advertises. Unmatched entries are
# ignored at discovery time.
allowed_tools=[
"slack_search_channels",
"slack_read_channel",
"slack_read_thread",
"search_channels",
"read_channel",
"read_thread",
],
readonly_tools=frozenset(
{"slack_search_channels", "slack_read_channel", "slack_read_thread"}
{
"slack_search_channels",
"slack_read_channel",
"slack_read_thread",
"search_channels",
"read_channel",
"read_thread",
}
),
# TODO: oauth.v2.user.access only returns team.id, not team.name.
# To populate team_name, either add "team:read" scope and call
@ -192,13 +205,22 @@ MCP_SERVICES: dict[str, MCPServiceConfig] = {
# DCR (RFC 7591): Notion issues its own client credentials. It expires
# DCR registrations, but refresh reuses the original persisted
# ``mcp_oauth.client_id`` (see _refresh_connector_token).
# Notion renamed its MCP tools with a "notion-" prefix (late 2025).
# Unprefixed names kept for servers still advertising the old names —
# allowlist entries the server doesn't advertise are simply ignored.
allowed_tools=[
"notion-search",
"notion-fetch",
"notion-create-pages",
"notion-update-page",
"search",
"fetch",
"create-pages",
"update-page",
],
readonly_tools=frozenset({"search", "fetch"}),
readonly_tools=frozenset(
{"notion-search", "notion-fetch", "search", "fetch"}
),
account_metadata_keys=["workspace_name"],
),
"confluence": MCPServiceConfig(

6
surfsense_backend/app/utils/validators.py
View file

@ -30,6 +30,12 @@ DEPRECATED_CONNECTOR_TYPES: frozenset[str] = frozenset(
"SEARXNG_API",
"LINKUP_API",
"BAIDU_SEARCH_API",
# Legacy content crawlers/search superseded by the file Import menu and
# hosted MCP tooling. Created via the generic connector /add route, which
# is the single choke point enforcing this deprecation.
"YOUTUBE_CONNECTOR",
"WEBCRAWLER_CONNECTOR",
"ELASTICSEARCH_CONNECTOR",
}
)

106
surfsense_backend/test_scraper_api_smoke.py Normal file
View file

@ -0,0 +1,106 @@
"""One-shot smoke test: hit every scraper API endpoint with a PAT, print PASS/FAIL."""
import json
import os
import sys
import time
import httpx
BASE = "http://localhost:8000/api/v1/workspaces/12/scrapers"
PAT = os.environ["SURFSENSE_PAT"] # export a ss_pat_... key before running
HEADERS = {"Authorization": f"Bearer {PAT}", "Content-Type": "application/json"}
# Minimal payloads: 1-3 items each to keep credit spend tiny.
VERBS = [
("google_search/scrape", {"queries": ["surfsense github"], "max_pages_per_query": 1}),
("web/crawl", {"startUrls": ["https://example.com"], "maxCrawlDepth": 0}),
("reddit/scrape", {"urls": ["https://www.reddit.com/r/Python/"], "max_items": 3}),
("youtube/scrape", {"search_queries": ["python tutorial"], "max_results": 1}),
(
"youtube/comments",
{"urls": ["https://www.youtube.com/watch?v=dQw4w9WgXcQ"], "max_comments": 3},
),
(
"google_maps/scrape",
{"search_queries": ["coffee shop"], "location": "New York, USA", "max_places": 1},
),
(
"google_maps/reviews",
# Google Sydney office, a stable well-known place id.
{"place_ids": ["ChIJN1t_tDeuEmsRUsoyG83frY4"], "max_reviews": 3},
),
]
results = []
client = httpx.Client(headers=HEADERS, timeout=300)
sync_run_id = None
for verb, payload in VERBS:
t0 = time.time()
try:
r = client.post(f"{BASE}/{verb}", json=payload)
dur = time.time() - t0
run_id = r.headers.get("x-run-id")
if r.status_code == 200:
body = r.json()
# Count items in whatever list field the output has.
items = next((len(v) for v in body.values() if isinstance(v, list)), "?")
results.append((verb, "PASS", f"{r.status_code} items={items} run={run_id} {dur:.1f}s"))
if sync_run_id is None and run_id:
sync_run_id = run_id
else:
results.append((verb, "FAIL", f"{r.status_code} {r.text[:200]} {dur:.1f}s"))
except Exception as e:
results.append((verb, "FAIL", f"{type(e).__name__}: {e}"))
print(f"[{results[-1][1]}] {verb}: {results[-1][2]}", flush=True)
# --- Run history endpoints ---
r = client.get(f"{BASE}/runs", params={"limit": 5})
ok = r.status_code == 200 and isinstance(r.json(), list)
results.append(("GET runs (list)", "PASS" if ok else "FAIL", f"{r.status_code} rows={len(r.json()) if ok else '?'}"))
print(f"[{results[-1][1]}] GET runs: {results[-1][2]}", flush=True)
if sync_run_id:
r = client.get(f"{BASE}/runs/{sync_run_id}")
ok = r.status_code == 200 and r.json().get("id") == sync_run_id
results.append(("GET runs/{id} (detail)", "PASS" if ok else "FAIL", f"{r.status_code} status={r.json().get('status') if ok else r.text[:100]}"))
print(f"[{results[-1][1]}] GET run detail: {results[-1][2]}", flush=True)
# --- Async mode + SSE events ---
r = client.post(f"{BASE}/web/crawl?mode=async", json={"startUrls": ["https://example.com"]})
if r.status_code == 202:
async_id = r.json()["run_id"]
seen, finished = [], None
with client.stream("GET", f"{BASE}/runs/{async_id}/events", timeout=120) as s:
for line in s.iter_lines():
if line.startswith("data: "):
ev = json.loads(line[6:])
seen.append(ev["type"])
if ev["type"] == "run.finished":
finished = ev.get("status")
break
ok = finished == "success"
results.append(("async + SSE events", "PASS" if ok else "FAIL", f"202 events={seen} final={finished}"))
else:
results.append(("async + SSE events", "FAIL", f"{r.status_code} {r.text[:200]}"))
print(f"[{results[-1][1]}] async+SSE: {results[-1][2]}", flush=True)
# --- Cancel endpoint ---
r = client.post(f"{BASE}/web/crawl?mode=async", json={"startUrls": ["https://example.com"], "maxCrawlDepth": 2, "maxCrawlPages": 50})
if r.status_code == 202:
cancel_id = r.json()["run_id"]
time.sleep(1)
r2 = client.post(f"{BASE}/runs/{cancel_id}/cancel")
ok = r2.status_code == 200 and r2.json().get("status") == "cancelled"
results.append(("POST runs/{id}/cancel", "PASS" if ok else "FAIL", f"{r2.status_code} {r2.text[:150]}"))
else:
results.append(("POST runs/{id}/cancel", "FAIL", f"setup {r.status_code}"))
print(f"[{results[-1][1]}] cancel: {results[-1][2]}", flush=True)
print("\n===== SUMMARY =====")
for name, status_, detail in results:
print(f"{status_:4} {name}: {detail}")
failed = [r for r in results if r[1] == "FAIL"]
print(f"\n{len(results) - len(failed)}/{len(results)} passed")
sys.exit(1 if failed else 0)

30
surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py
View file

@ -25,6 +25,7 @@ from app.agents.chat.multi_agent_chat.shared.permissions.middleware.core import
PermissionMiddleware,
)
from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import (
append_today_utc,
pack_subagent,
)
@ -104,6 +105,35 @@ async def test_subagent_recovers_when_primary_llm_fails():
assert final.content == "recovered via fallback"
def test_packed_subagent_prompt_carries_todays_utc_date():
"""Subagents must inherit the main agent's clock, not guess the date."""
from datetime import UTC, datetime
today = datetime.now(UTC).date().isoformat()
result = pack_subagent(
name="date_test",
description="test",
system_prompt="be helpful",
tools=[],
ruleset=Ruleset(origin="date_test", rules=[]),
dependencies={"flags": AgentFeatureFlags()},
)
prompt = result.spec["system_prompt"]
assert prompt.startswith("be helpful")
assert f"Today (UTC): {today}" in prompt
def test_append_today_utc_is_idempotent_shape():
"""Helper appends exactly one dated line and preserves the original body."""
from datetime import UTC, datetime
today = datetime.now(UTC).date().isoformat()
out = append_today_utc("body")
assert out == f"body\n\nToday (UTC): {today}\n"
def _extract_permission_mw(spec) -> PermissionMiddleware:
"""Find the lone PermissionMiddleware in a subagent's middleware list."""
matches = [m for m in spec["middleware"] if isinstance(m, PermissionMiddleware)]

52
surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_allowlist_fallback.py Normal file
View file

@ -0,0 +1,52 @@
"""Allowlist filtering in ``_load_http_mcp_tools``.
Covers the fallback added after Notion renamed its MCP tools ("notion-"
prefix) and a stale allowlist silently disabled the connector: when the
allowlist matches zero advertised tools, load everything (HITL-gated)
instead of nothing.
"""
from datetime import UTC, datetime
from app.agents.chat.multi_agent_chat.shared.tools.mcp.cache import (
CachedMCPTools,
)
from app.agents.chat.multi_agent_chat.shared.tools.mcp.tool import (
_load_http_mcp_tools,
)
_CACHED = CachedMCPTools(
discovered_at=datetime.now(UTC),
tools=[
{"name": "notion-search", "description": "search", "input_schema": {}},
{"name": "notion-update-page", "description": "write", "input_schema": {}},
],
)
_SERVER_CONFIG = {"url": "https://example.com/mcp", "headers": {}}
async def test_allowlist_match_filters_and_flags_readonly():
tools = await _load_http_mcp_tools(
1,
"Notion",
_SERVER_CONFIG,
allowed_tools=["notion-search"],
readonly_tools=frozenset({"notion-search"}),
cached_tools=_CACHED,
)
assert [t.name for t in tools] == ["notion-search"]
assert tools[0].metadata["hitl"] is False
async def test_stale_allowlist_falls_back_to_all_tools_hitl_gated():
tools = await _load_http_mcp_tools(
1,
"Notion",
_SERVER_CONFIG,
allowed_tools=["search", "update-page"], # server renamed everything
readonly_tools=frozenset({"search"}),
cached_tools=_CACHED,
)
assert sorted(t.name for t in tools) == ["notion-search", "notion-update-page"]
# Renamed tools match no readonly entry -> every tool requires approval.
assert all(t.metadata["hitl"] is True for t in tools)

35
surfsense_backend/tests/unit/capabilities/access/test_rest_router.py
View file

@ -123,6 +123,41 @@ async def test_capabilities_endpoint_lists_verbs_with_input_schema(monkeypatch):
assert "properties" in entry["output_schema"]
@pytest.mark.asyncio
async def test_capabilities_endpoint_exposes_live_pricing(monkeypatch):
"""Billed verbs report their per-item rate; free verbs report an empty list."""
from app.capabilities.core.types import BillingUnit
from app.config import config
monkeypatch.setattr(config, "PLATFORM_SCRAPE_BILLING_ENABLED", True)
monkeypatch.setattr(config, "YOUTUBE_MICROS_PER_VIDEO", 2500)
billed = Capability(
name="test.billed",
description="Billed verb for tests.",
input_schema=_EchoInput,
output_schema=_EchoOutput,
executor=_echo_executor,
billing_unit=BillingUnit.YOUTUBE_VIDEO,
)
app = _build_app([_ECHO, billed], monkeypatch)
async with _client(app) as client:
resp = await client.get("/api/v1/workspaces/7/scrapers/capabilities")
assert resp.status_code == 200
by_name = {entry["name"]: entry for entry in resp.json()}
assert by_name["test.echo"]["pricing"] == []
assert by_name["test.billed"]["pricing"] == [
{"unit": "video", "micros_per_unit": 2500}
]
# Rates are read live: a config retune shows up without a router rebuild.
monkeypatch.setattr(config, "PLATFORM_SCRAPE_BILLING_ENABLED", False)
async with _client(app) as client:
resp = await client.get("/api/v1/workspaces/7/scrapers/capabilities")
assert resp.json()[1]["pricing"] == []
@pytest.mark.asyncio
async def test_over_budget_is_blocked_before_the_executor(monkeypatch):
from app.capabilities.core.access import rest

3
surfsense_backend/tests/unit/utils/test_validators.py
View file

@ -344,6 +344,9 @@ def test_validate_connector_config_invalid():
"SEARXNG_API",
"LINKUP_API",
"BAIDU_SEARCH_API",
"YOUTUBE_CONNECTOR",
"WEBCRAWLER_CONNECTOR",
"ELASTICSEARCH_CONNECTOR",
],
)
def test_raise_if_connector_deprecated_blocks(connector_type):