refactor(agents): rename intelligence_agent subagent to scraping

surfsense_backend/app/agents/chat/multi_agent_chat/constants.py

@@ -26,7 +26,7 @@ CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS: dict[str, str] = {
 SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = {
     "deliverables": frozenset(),
     "knowledge_base": frozenset(),
-    "intelligence_agent": frozenset(),
+    "scraping": frozenset(),
     "airtable": frozenset({"AIRTABLE_CONNECTOR"}),
     "calendar": frozenset({"GOOGLE_CALENDAR_CONNECTOR"}),
     "clickup": frozenset({"CLICKUP_CONNECTOR"}),

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/description.md

@@ -1,2 +0,0 @@
-Specialist for web intelligence: fetch live pages and search the web, then compare against earlier findings.
-Use whenever a task needs current data pulled from specific URLs or discovered on the web — competitor/product monitoring, pricing, listings, "what changed since last time" — and answered from that evidence.

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/intelligence_agent/system_prompt.md

@@ -1,56 +0,0 @@
-You are the SurfSense intelligence sub-agent.
-You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis.
-
-<goal>
-Pull evidence from the live web with the web.* capability verbs and answer the delegated question, including "what changed" comparisons against evidence already in this conversation.
-</goal>
-
-<available_tools>
-- `web_discover` — search the web for a query; returns ranked hits (url, title, snippet, provider).
-- `web_scrape` — fetch specific URLs; returns clean page content and metadata per URL.
-- `start_watch` — (only when running inside a chat) keep re-asking a question on a schedule in this chat, so the user gets refreshed answers without asking again. Takes the self-contained question, a cron cadence, and an IANA timezone.
-</available_tools>
-
-<playbook>
-- If the request names exact URLs, call `web_scrape` on them directly.
-- If it does not, call `web_discover` first, pick the most relevant hits, then `web_scrape` those URLs to read them.
-- Batch URLs into a single `web_scrape` call when reading several pages (up to its limit) instead of many one-URL calls.
-- For "what changed" / monitoring requests, compare the freshly scraped values against the prior values already present in this conversation's earlier tool results, and report the concrete deltas (added, removed, changed old -> new). Do not claim a change you cannot point to.
-- When the user asks to keep watching / track something on a recurring basis (e.g. "check this daily", "tell me weekly what changed"), first answer now, then call `start_watch` with the self-contained question and the cadence (cron + timezone). Ask for the cadence or timezone only if the user did not imply one.
-</playbook>
-
-<tool_policy>
-- Use only tools in `<available_tools>`.
-- A `web_scrape` row with `status` other than `success` yielded no content — do not invent its content; report it as unavailable.
-- Never fabricate facts, URLs, prices, or quotes.
-</tool_policy>
-
-<safety>
-- Report uncertainty explicitly when evidence is incomplete or conflicting.
-- Never present unverified claims as facts.
-</safety>
-
-<failure_policy>
-- On tool failure, return `status=error` with a concise recovery `next_step`.
-- On no useful evidence, return `status=blocked` with a narrower query or the URLs you still need.
-</failure_policy>
-
-<output_contract>
-Return **only** one JSON object (no markdown/prose):
-{
-  "status": "success" | "partial" | "blocked" | "error",
-  "action_summary": string,
-  "evidence": {
-    "findings": string[],
-    "sources": string[],
-    "confidence": "high" | "medium" | "low"
-  },
-  "next_step": string | null,
-  "missing_fields": string[] | null,
-  "assumptions": string[] | null
-}
-<include snippet="output_contract_base"/>
-Route-specific rules:
-- `evidence.findings`: max 10 entries, each a single sentence stating one distinct fact or delta. Do not paste raw scraped pages.
-- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once.
-</output_contract>

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/agent.py

@@ -1,4 +1,4 @@
-"""``intelligence_agent`` route: ``SurfSenseSubagentSpec`` builder for deepagents."""
+"""``scraping`` route: ``SurfSenseSubagentSpec`` builder for deepagents."""
 
 from __future__ import annotations
 
@@ -28,7 +28,7 @@ def build_subagent(
     tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])]
     description = (
         read_md_file(__package__, "description").strip()
-        or "Gathers and compares web intelligence for this workspace."
+        or "Scrapes live public web pages and search results for this workspace."
     )
     system_prompt = read_md_file(__package__, "system_prompt").strip()
     return pack_subagent(

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/description.md

@@ -0,0 +1,2 @@
+Scraping specialist: fetches live public web pages, searches the web to find pages, and compares fresh data against earlier findings in this chat.
+Use whenever the task needs current data pulled from the open web rather than the workspace's own documents or connectors. Triggers include "scrape", "crawl", "fetch this URL/page", "search the web", "look up / find online", "check the price/stock/listing", "monitor this page", and "what changed since last time" — plus recurring versions ("check daily", "tell me weekly what changed"), which it can also schedule as an ongoing watch.

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/system_prompt.md

@@ -0,0 +1,62 @@
+You are the SurfSense scraping sub-agent.
+You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis.
+
+<goal>
+Answer the delegated question from live evidence gathered with your data verbs, including "what changed" comparisons against evidence already in this conversation.
+</goal>
+
+<available_tools>
+- `web_discover`
+- `web_scrape`
+- `start_watch`
+- `stop_watch`
+- `refresh_watch`
+</available_tools>
+
+<playbook>
+- Named URLs: `web_scrape` them directly. Otherwise `web_discover` first, then `web_scrape` the most relevant hits.
+- Read several pages in one batched `web_scrape` call rather than many single-URL calls.
+- "What changed" / monitoring: scrape the current values, compare against the prior values in this conversation's earlier tool results, and report concrete deltas (added, removed, old -> new).
+- Recurring intent ("check daily", "tell me weekly what changed"): answer now, then `start_watch` with a self-contained question, a cron cadence, and an IANA timezone. Use `stop_watch` / `refresh_watch` to end or immediately re-run an existing watch.
+</playbook>
+
+<tool_policy>
+- Use only tools in `<available_tools>`.
+- A `web_scrape` row whose `status` is not `success` returned no content — report it unavailable, never invent it.
+- Report only deltas you can point to in the evidence. Never fabricate facts, URLs, prices, or quotes.
+</tool_policy>
+
+<out_of_scope>
+- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on.
+</out_of_scope>
+
+<safety>
+- Report uncertainty explicitly when evidence is incomplete or conflicting.
+- Never present unverified claims as facts.
+</safety>
+
+<failure_policy>
+- Underspecified request — including a recurring request whose cadence or timezone is neither given nor implied — return `status=blocked` with the missing fields.
+- Tool failure: return `status=error` with a concise recovery `next_step`.
+- No useful evidence: return `status=blocked` with a narrower query or the URLs you still need.
+</failure_policy>
+
+<output_contract>
+Return **only** one JSON object (no markdown/prose):
+{
+  "status": "success" | "partial" | "blocked" | "error",
+  "action_summary": string,
+  "evidence": {
+    "findings": string[],
+    "sources": string[],
+    "confidence": "high" | "medium" | "low"
+  },
+  "next_step": string | null,
+  "missing_fields": string[] | null,
+  "assumptions": string[] | null
+}
+<include snippet="output_contract_base"/>
+Route-specific rules:
+- `evidence.findings`: max 10 entries, each a single sentence stating one distinct fact or delta. Do not paste raw scraped pages.
+- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once.
+</output_contract>

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/tools/index.py

@@ -1,4 +1,4 @@
-"""``intelligence_agent`` tools: web.* verbs generated from the capability registry."""
+"""``scraping`` sub-agent tools: scraper capability verbs + chat-watch controls."""
 
 from __future__ import annotations
 
@@ -15,7 +15,7 @@ from .refresh_watch import create_refresh_watch_tool
 from .start_watch import create_start_watch_tool
 from .stop_watch import create_stop_watch_tool
 
-NAME = "intelligence_agent"
+NAME = "scraping"
 
 RULESET = Ruleset(origin=NAME, rules=[])
 

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py

@@ -15,9 +15,6 @@ from app.agents.chat.multi_agent_chat.constants import (
 from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.agent import (
     build_subagent as build_deliverables_subagent,
 )
-from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.agent import (
-    build_subagent as build_intelligence_agent_subagent,
-)
 from app.agents.chat.multi_agent_chat.subagents.builtins.knowledge_base.agent import (
     build_subagent as build_knowledge_base_subagent,
 )
@@ -27,6 +24,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.memory.agent import (
 from app.agents.chat.multi_agent_chat.subagents.builtins.research.agent import (
     build_subagent as build_research_subagent,
 )
+from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.agent import (
+    build_subagent as build_scraping_subagent,
+)
 from app.agents.chat.multi_agent_chat.subagents.connectors.airtable.agent import (
     build_subagent as build_airtable_subagent,
 )
@@ -102,7 +102,6 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = {
     "dropbox": build_dropbox_subagent,
     "gmail": build_gmail_subagent,
     "google_drive": build_google_drive_subagent,
-    "intelligence_agent": build_intelligence_agent_subagent,
     "jira": build_jira_subagent,
     "knowledge_base": build_knowledge_base_subagent,
     "linear": build_linear_subagent,
@@ -111,6 +110,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = {
     "notion": build_notion_subagent,
     "onedrive": build_onedrive_subagent,
     "research": build_research_subagent,
+    "scraping": build_scraping_subagent,
     "slack": build_slack_subagent,
     "teams": build_teams_subagent,
 }

surfsense_backend/tests/integration/automations/conftest.py

@@ -63,7 +63,7 @@ def tools_use_test_session(monkeypatch, db_session: AsyncSession) -> None:
     async def _session_cm():
         yield db_session  # owned by the outer fixture; do not close
 
-    from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools import (
+    from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools import (
         refresh_watch,
         start_watch,
         stop_watch,

surfsense_backend/tests/integration/automations/tools/test_watch_tools.py

@@ -13,13 +13,13 @@ import pytest
 import pytest_asyncio
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools.refresh_watch import (
+from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools.refresh_watch import (
     create_refresh_watch_tool,
 )
-from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools.start_watch import (
+from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools.start_watch import (
     create_start_watch_tool,
 )
-from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools.stop_watch import (
+from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools.stop_watch import (
     create_stop_watch_tool,
 )
 from app.auth.context import AuthContext

surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/test_start_watch_tool.py

@@ -1,4 +1,4 @@
-"""Unit tests for the ``start_watch`` tool on the intelligence_agent.
+"""Unit tests for the ``start_watch`` tool on the scraping sub-agent.
 
 ``start_watch`` binds a recurring watch to the *current* chat: it distils the
 question + cadence the agent extracted and creates a ``schedule`` +
@@ -26,7 +26,7 @@ class _FakeSessionCM:
 
 
 def _patch_deps(monkeypatch: pytest.MonkeyPatch, *, created: Any) -> AsyncMock:
-    from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools import (
+    from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools import (
         start_watch as mod,
     )
 
@@ -41,7 +41,7 @@ def _patch_deps(monkeypatch: pytest.MonkeyPatch, *, created: Any) -> AsyncMock:
 async def test_start_watch_binds_watch_to_current_chat(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools import (
+    from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools import (
         start_watch as mod,
     )
 
@@ -75,7 +75,7 @@ async def test_start_watch_binds_watch_to_current_chat(
 async def test_start_watch_errors_without_thread_or_auth(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools import (
+    from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools import (
         start_watch as mod,
     )
 
@@ -101,7 +101,7 @@ async def test_start_watch_errors_without_thread_or_auth(
 
 
 def test_load_tools_includes_start_watch_only_when_bindable() -> None:
-    from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools.index import (
+    from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools.index import (
         load_tools,
     )
 

surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/test_watch_control_tools.py

@@ -33,7 +33,7 @@ def _watch(automation_id: int) -> Any:
 async def test_stop_watch_stops_every_watch_on_the_chat(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools import (
+    from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools import (
         stop_watch as mod,
     )
 
@@ -59,7 +59,7 @@ async def test_stop_watch_stops_every_watch_on_the_chat(
 async def test_stop_watch_reports_when_nothing_to_stop(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools import (
+    from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools import (
         stop_watch as mod,
     )
 
@@ -80,7 +80,7 @@ async def test_stop_watch_reports_when_nothing_to_stop(
 async def test_refresh_watch_runs_each_watch_now(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools import (
+    from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools import (
         refresh_watch as mod,
     )
 
@@ -104,7 +104,7 @@ async def test_refresh_watch_runs_each_watch_now(
 
 
 def test_load_tools_includes_control_tools_when_bindable() -> None:
-    from app.agents.chat.multi_agent_chat.subagents.builtins.intelligence_agent.tools.index import (
+    from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.tools.index import (
         load_tools,
     )
 

surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py

@@ -33,7 +33,6 @@ _EXPECTED_SUBAGENTS = frozenset(
         "dropbox",
         "gmail",
         "google_drive",
-        "intelligence_agent",
         "jira",
         "knowledge_base",
         "linear",
@@ -42,6 +41,7 @@ _EXPECTED_SUBAGENTS = frozenset(
         "notion",
         "onedrive",
         "research",
+        "scraping",
         "slack",
         "teams",
     }