diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py index fff608742..1e4c4276d 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py @@ -26,7 +26,7 @@ CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS: dict[str, str] = { SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = { "deliverables": frozenset(), "knowledge_base": frozenset(), - "intelligence_agent": frozenset(), + "scraping": frozenset(), "airtable": frozenset({"AIRTABLE_CONNECTOR"}), "calendar": frozenset({"GOOGLE_CALENDAR_CONNECTOR"}), "clickup": frozenset({"CLICKUP_CONNECTOR"}), diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/description.md deleted file mode 100644 index cefad396c..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/description.md +++ /dev/null @@ -1,2 +0,0 @@ -Specialist for web intelligence: fetch live pages and search the web, then compare against earlier findings. -Use whenever a task needs current data pulled from specific URLs or discovered on the web — competitor/product monitoring, pricing, listings, "what changed since last time" — and answered from that evidence. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/system_prompt.md deleted file mode 100644 index 60bcff29c..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/system_prompt.md +++ /dev/null @@ -1,56 +0,0 @@ -You are the SurfSense intelligence sub-agent. -You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis. - - -Pull evidence from the live web with the web.* capability verbs and answer the delegated question, including "what changed" comparisons against evidence already in this conversation. - - - -- `web_discover` — search the web for a query; returns ranked hits (url, title, snippet, provider). -- `web_scrape` — fetch specific URLs; returns clean page content and metadata per URL. -- `start_watch` — (only when running inside a chat) keep re-asking a question on a schedule in this chat, so the user gets refreshed answers without asking again. Takes the self-contained question, a cron cadence, and an IANA timezone. - - - -- If the request names exact URLs, call `web_scrape` on them directly. -- If it does not, call `web_discover` first, pick the most relevant hits, then `web_scrape` those URLs to read them. -- Batch URLs into a single `web_scrape` call when reading several pages (up to its limit) instead of many one-URL calls. -- For "what changed" / monitoring requests, compare the freshly scraped values against the prior values already present in this conversation's earlier tool results, and report the concrete deltas (added, removed, changed old -> new). Do not claim a change you cannot point to. -- When the user asks to keep watching / track something on a recurring basis (e.g. "check this daily", "tell me weekly what changed"), first answer now, then call `start_watch` with the self-contained question and the cadence (cron + timezone). Ask for the cadence or timezone only if the user did not imply one. - - - -- Use only tools in ``. -- A `web_scrape` row with `status` other than `success` yielded no content — do not invent its content; report it as unavailable. -- Never fabricate facts, URLs, prices, or quotes. - - - -- Report uncertainty explicitly when evidence is incomplete or conflicting. -- Never present unverified claims as facts. - - - -- On tool failure, return `status=error` with a concise recovery `next_step`. -- On no useful evidence, return `status=blocked` with a narrower query or the URLs you still need. - - - -Return **only** one JSON object (no markdown/prose): -{ - "status": "success" | "partial" | "blocked" | "error", - "action_summary": string, - "evidence": { - "findings": string[], - "sources": string[], - "confidence": "high" | "medium" | "low" - }, - "next_step": string | null, - "missing_fields": string[] | null, - "assumptions": string[] | null -} - -Route-specific rules: -- `evidence.findings`: max 10 entries, each a single sentence stating one distinct fact or delta. Do not paste raw scraped pages. -- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once. - diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/__init__.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/__init__.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/agent.py similarity index 88% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/agent.py index 21933005b..9c6734b57 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/agent.py @@ -1,4 +1,4 @@ -"""``intelligence_agent`` route: ``SurfSenseSubagentSpec`` builder for deepagents.""" +"""``scraping`` route: ``SurfSenseSubagentSpec`` builder for deepagents.""" from __future__ import annotations @@ -28,7 +28,7 @@ def build_subagent( tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] description = ( read_md_file(__package__, "description").strip() - or "Gathers and compares web intelligence for this workspace." + or "Scrapes live public web pages and search results for this workspace." ) system_prompt = read_md_file(__package__, "system_prompt").strip() return pack_subagent( diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/description.md new file mode 100644 index 000000000..f6b2f8ada --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/description.md @@ -0,0 +1,2 @@ +Scraping specialist: fetches live public web pages, searches the web to find pages, and compares fresh data against earlier findings in this chat. +Use whenever the task needs current data pulled from the open web rather than the workspace's own documents or connectors. Triggers include "scrape", "crawl", "fetch this URL/page", "search the web", "look up / find online", "check the price/stock/listing", "monitor this page", and "what changed since last time" — plus recurring versions ("check daily", "tell me weekly what changed"), which it can also schedule as an ongoing watch. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/system_prompt.md new file mode 100644 index 000000000..7f0d6841e --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/system_prompt.md @@ -0,0 +1,62 @@ +You are the SurfSense scraping sub-agent. +You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis. + + +Answer the delegated question from live evidence gathered with your data verbs, including "what changed" comparisons against evidence already in this conversation. + + + +- `web_discover` +- `web_scrape` +- `start_watch` +- `stop_watch` +- `refresh_watch` + + + +- Named URLs: `web_scrape` them directly. Otherwise `web_discover` first, then `web_scrape` the most relevant hits. +- Read several pages in one batched `web_scrape` call rather than many single-URL calls. +- "What changed" / monitoring: scrape the current values, compare against the prior values in this conversation's earlier tool results, and report concrete deltas (added, removed, old -> new). +- Recurring intent ("check daily", "tell me weekly what changed"): answer now, then `start_watch` with a self-contained question, a cron cadence, and an IANA timezone. Use `stop_watch` / `refresh_watch` to end or immediately re-run an existing watch. + + + +- Use only tools in ``. +- A `web_scrape` row whose `status` is not `success` returned no content — report it unavailable, never invent it. +- Report only deltas you can point to in the evidence. Never fabricate facts, URLs, prices, or quotes. + + + +- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on. + + + +- Report uncertainty explicitly when evidence is incomplete or conflicting. +- Never present unverified claims as facts. + + + +- Underspecified request — including a recurring request whose cadence or timezone is neither given nor implied — return `status=blocked` with the missing fields. +- Tool failure: return `status=error` with a concise recovery `next_step`. +- No useful evidence: return `status=blocked` with a narrower query or the URLs you still need. + + + +Return **only** one JSON object (no markdown/prose): +{ + "status": "success" | "partial" | "blocked" | "error", + "action_summary": string, + "evidence": { + "findings": string[], + "sources": string[], + "confidence": "high" | "medium" | "low" + }, + "next_step": string | null, + "missing_fields": string[] | null, + "assumptions": string[] | null +} + +Route-specific rules: +- `evidence.findings`: max 10 entries, each a single sentence stating one distinct fact or delta. Do not paste raw scraped pages. +- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once. + diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/tools/__init__.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/tools/index.py similarity index 92% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/tools/index.py index 98f704755..5c89dbe30 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/tools/index.py @@ -1,4 +1,4 @@ -"""``intelligence_agent`` tools: web.* verbs generated from the capability registry.""" +"""``scraping`` sub-agent tools: scraper capability verbs + chat-watch controls.""" from __future__ import annotations @@ -15,7 +15,7 @@ from .refresh_watch import create_refresh_watch_tool from .start_watch import create_start_watch_tool from .stop_watch import create_stop_watch_tool -NAME = "intelligence_agent" +NAME = "scraping" RULESET = Ruleset(origin=NAME, rules=[]) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/tools/refresh_watch.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/tools/refresh_watch.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/tools/refresh_watch.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/tools/refresh_watch.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/tools/start_watch.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/tools/start_watch.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/tools/start_watch.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/tools/start_watch.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/tools/stop_watch.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/tools/stop_watch.py similarity index 100% rename from surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/tools/stop_watch.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/tools/stop_watch.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index 652ca079e..ae3a88bbe 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -15,9 +15,6 @@ from app.agents.chat.multi_agent_chat.constants import ( from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.agent import ( build_subagent as build_deliverables_subagent, ) -from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.agent import ( - build_subagent as build_intelligence_agent_subagent, -) from app.agents.chat.multi_agent_chat.subagents.builtins.knowledge_base.agent import ( build_subagent as build_knowledge_base_subagent, ) @@ -27,6 +24,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.memory.agent import ( from app.agents.chat.multi_agent_chat.subagents.builtins.research.agent import ( build_subagent as build_research_subagent, ) +from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.agent import ( + build_subagent as build_scraping_subagent, +) from app.agents.chat.multi_agent_chat.subagents.connectors.airtable.agent import ( build_subagent as build_airtable_subagent, ) @@ -102,7 +102,6 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { "dropbox": build_dropbox_subagent, "gmail": build_gmail_subagent, "google_drive": build_google_drive_subagent, - "intelligence_agent": build_intelligence_agent_subagent, "jira": build_jira_subagent, "knowledge_base": build_knowledge_base_subagent, "linear": build_linear_subagent, @@ -111,6 +110,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { "notion": build_notion_subagent, "onedrive": build_onedrive_subagent, "research": build_research_subagent, + "scraping": build_scraping_subagent, "slack": build_slack_subagent, "teams": build_teams_subagent, } diff --git a/surfsense_backend/tests/integration/automations/conftest.py b/surfsense_backend/tests/integration/automations/conftest.py index 74c9e1808..1d3e91b07 100644 --- a/surfsense_backend/tests/integration/automations/conftest.py +++ b/surfsense_backend/tests/integration/automations/conftest.py @@ -63,7 +63,7 @@ def tools_use_test_session(monkeypatch, db_session: AsyncSession) -> None: async def _session_cm(): yield db_session # owned by the outer fixture; do not close - from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools import ( + from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools import ( refresh_watch, start_watch, stop_watch, diff --git a/surfsense_backend/tests/integration/automations/tools/test_watch_tools.py b/surfsense_backend/tests/integration/automations/tools/test_watch_tools.py index 03d3ca93a..acec89069 100644 --- a/surfsense_backend/tests/integration/automations/tools/test_watch_tools.py +++ b/surfsense_backend/tests/integration/automations/tools/test_watch_tools.py @@ -13,13 +13,13 @@ import pytest import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools.refresh_watch import ( +from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools.refresh_watch import ( create_refresh_watch_tool, ) -from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools.start_watch import ( +from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools.start_watch import ( create_start_watch_tool, ) -from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools.stop_watch import ( +from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools.stop_watch import ( create_stop_watch_tool, ) from app.auth.context import AuthContext diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/test_start_watch_tool.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/test_start_watch_tool.py index c532cb5e6..3cc49b7f2 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/test_start_watch_tool.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/test_start_watch_tool.py @@ -1,4 +1,4 @@ -"""Unit tests for the ``start_watch`` tool on the intelligence_agent. +"""Unit tests for the ``start_watch`` tool on the scraping sub-agent. ``start_watch`` binds a recurring watch to the *current* chat: it distils the question + cadence the agent extracted and creates a ``schedule`` + @@ -26,7 +26,7 @@ class _FakeSessionCM: def _patch_deps(monkeypatch: pytest.MonkeyPatch, *, created: Any) -> AsyncMock: - from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools import ( + from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools import ( start_watch as mod, ) @@ -41,7 +41,7 @@ def _patch_deps(monkeypatch: pytest.MonkeyPatch, *, created: Any) -> AsyncMock: async def test_start_watch_binds_watch_to_current_chat( monkeypatch: pytest.MonkeyPatch, ) -> None: - from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools import ( + from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools import ( start_watch as mod, ) @@ -75,7 +75,7 @@ async def test_start_watch_binds_watch_to_current_chat( async def test_start_watch_errors_without_thread_or_auth( monkeypatch: pytest.MonkeyPatch, ) -> None: - from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools import ( + from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools import ( start_watch as mod, ) @@ -101,7 +101,7 @@ async def test_start_watch_errors_without_thread_or_auth( def test_load_tools_includes_start_watch_only_when_bindable() -> None: - from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools.index import ( + from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools.index import ( load_tools, ) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/test_watch_control_tools.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/test_watch_control_tools.py index b07f864c4..c659e289b 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/test_watch_control_tools.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/test_watch_control_tools.py @@ -33,7 +33,7 @@ def _watch(automation_id: int) -> Any: async def test_stop_watch_stops_every_watch_on_the_chat( monkeypatch: pytest.MonkeyPatch, ) -> None: - from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools import ( + from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools import ( stop_watch as mod, ) @@ -59,7 +59,7 @@ async def test_stop_watch_stops_every_watch_on_the_chat( async def test_stop_watch_reports_when_nothing_to_stop( monkeypatch: pytest.MonkeyPatch, ) -> None: - from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools import ( + from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools import ( stop_watch as mod, ) @@ -80,7 +80,7 @@ async def test_stop_watch_reports_when_nothing_to_stop( async def test_refresh_watch_runs_each_watch_now( monkeypatch: pytest.MonkeyPatch, ) -> None: - from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools import ( + from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools import ( refresh_watch as mod, ) @@ -104,7 +104,7 @@ async def test_refresh_watch_runs_each_watch_now( def test_load_tools_includes_control_tools_when_bindable() -> None: - from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools.index import ( + from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools.index import ( load_tools, ) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py index 68ae05bbd..0cba7976a 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py @@ -33,7 +33,6 @@ _EXPECTED_SUBAGENTS = frozenset( "dropbox", "gmail", "google_drive", - "intelligence_agent", "jira", "knowledge_base", "linear", @@ -42,6 +41,7 @@ _EXPECTED_SUBAGENTS = frozenset( "notion", "onedrive", "research", + "scraping", "slack", "teams", }