diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/agent.py
deleted file mode 100644
index e3c0ab9ae..000000000
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/agent.py
+++ /dev/null
@@ -1,52 +0,0 @@
-"""``research`` route: ``SurfSenseSubagentSpec`` builder for deepagents."""
-
-from __future__ import annotations
-
-from typing import Any
-
-from langchain_core.language_models import BaseChatModel
-from langchain_core.tools import BaseTool
-
-from app.agents.chat.multi_agent_chat.shared.middleware.citation_state import (
- build_citation_state_mw,
-)
-from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import (
- read_md_file,
-)
-from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec
-from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import (
- pack_subagent,
-)
-
-from .tools.index import NAME, RULESET, load_tools
-
-
-def build_subagent(
- *,
- dependencies: dict[str, Any],
- model: BaseChatModel | None = None,
- middleware_stack: dict[str, Any] | None = None,
- mcp_tools: list[BaseTool] | None = None,
-) -> SurfSenseSubagentSpec:
- tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])]
- description = (
- read_md_file(__package__, "description").strip()
- or "Handles research tasks for this workspace."
- )
- system_prompt = read_md_file(__package__, "system_prompt").strip()
- # web_search registers WEB_RESULT citations via Command(update=...); the
- # citation-state middleware declares the channel so those [n] merge back up.
- middleware_with_citations = {
- **(middleware_stack or {}),
- "citation_state": build_citation_state_mw(),
- }
- return pack_subagent(
- name=NAME,
- description=description,
- system_prompt=system_prompt,
- tools=tools,
- ruleset=RULESET,
- dependencies=dependencies,
- model=model,
- middleware_stack=middleware_with_citations,
- )
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/description.md
deleted file mode 100644
index 0a99b4140..000000000
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/description.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Specialist for external research.
-Use whenever a task requires finding sources on the web and extracting evidence to answer documentation questions.
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/system_prompt.md
deleted file mode 100644
index 3d90a4352..000000000
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/system_prompt.md
+++ /dev/null
@@ -1,62 +0,0 @@
-You are the SurfSense research operations sub-agent.
-You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis.
-
-
-Gather and synthesize evidence using SurfSense research tools with clear citations and uncertainty reporting.
-
-
-
-- `web_search`
-- `scrape_webpage`
-
-
-
-- Use only tools in ``.
-- Prefer primary and recent sources when recency matters.
-- If the delegated request is underspecified, return `status=blocked` with the missing research constraints.
-- Never fabricate facts, citations, URLs, or quote text.
-
-
-
-`web_search` returns a `` block whose results are each prefixed with a bracketed label — `[1]`, `[2]`, `[3]`. That `[n]` is the citation label. When a finding came from a specific result, append its `[n]` to that finding, copying the label **exactly** as shown. The caller relays these labels verbatim and the server resolves each one, so a wrong number silently breaks the citation.
-
-- Use the exact `[n]` shown next to the result you actually used; never renumber, guess, or invent a label.
-- Before emitting an `[n]`, confirm that bracketed label appears in the `web_search` output this turn. If you can't see it, omit it.
-- Write the bare label `[n]` only — no `[citation:…]` wrapper, no markdown links.
-- Several results behind one finding → each in its own brackets with nothing between: `[1][2]`.
-- `scrape_webpage` returns raw page text with no `[n]` labels; a fact drawn only from a scrape carries no citation (report the URL in `evidence.sources` instead).
-
-
-
-- Do not execute connector mutations (email/calendar/docs/chat writes) or deliverable generation.
-
-
-
-- Report uncertainty explicitly when evidence is incomplete or conflicting.
-- Never present unverified claims as facts.
-
-
-
-- On tool failure, return `status=error` with a concise recovery `next_step`.
-- On no useful evidence, return `status=blocked` with recommended narrower filters.
-
-
-
-Return **only** one JSON object (no markdown/prose):
-{
- "status": "success" | "partial" | "blocked" | "error",
- "action_summary": string,
- "evidence": {
- "findings": string[],
- "sources": string[],
- "confidence": "high" | "medium" | "low"
- },
- "next_step": string | null,
- "missing_fields": string[] | null,
- "assumptions": string[] | null
-}
-
-Route-specific rules:
-- `evidence.findings`: max 10 entries, each a single sentence stating one distinct fact. Append the supporting `[n]` to each finding drawn from a `web_search` result. Do not paste raw paragraphs, scraped pages, or quote blocks.
-- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once. (Citations travel as `[n]`; `sources` is for transparency and for scrape-only facts that carry no `[n]`.)
-
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/__init__.py
deleted file mode 100644
index 0c99bf222..000000000
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/__init__.py
+++ /dev/null
@@ -1,10 +0,0 @@
-"""Research-stage tools: web search (shared) and scrape."""
-
-from app.agents.chat.shared.tools.web_search import create_web_search_tool
-
-from .scrape_webpage import create_scrape_webpage_tool
-
-__all__ = [
- "create_scrape_webpage_tool",
- "create_web_search_tool",
-]
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/index.py
deleted file mode 100644
index b3ea5a6b0..000000000
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/index.py
+++ /dev/null
@@ -1,29 +0,0 @@
-"""``research`` native tools and (empty) permission ruleset."""
-
-from __future__ import annotations
-
-from typing import Any
-
-from langchain_core.tools import BaseTool
-
-from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
-from app.agents.chat.shared.tools.web_search import create_web_search_tool
-
-from .scrape_webpage import create_scrape_webpage_tool
-
-NAME = "research"
-
-RULESET = Ruleset(origin=NAME, rules=[])
-
-
-def load_tools(
- *, dependencies: dict[str, Any] | None = None, **kwargs: Any
-) -> list[BaseTool]:
- d = {**(dependencies or {}), **kwargs}
- return [
- create_web_search_tool(
- workspace_id=d.get("workspace_id"),
- available_connectors=d.get("available_connectors"),
- ),
- create_scrape_webpage_tool(),
- ]
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py
deleted file mode 100644
index af7ce44e7..000000000
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py
+++ /dev/null
@@ -1,347 +0,0 @@
-"""Scrape pages via WebCrawlerConnector; YouTube URLs use the transcript API instead of HTML crawl."""
-
-import hashlib
-import logging
-import time
-from typing import Any
-from urllib.parse import urlparse
-
-from fake_useragent import UserAgent
-from langchain_core.tools import tool
-from requests import Session
-from scrapling.fetchers import AsyncFetcher
-from youtube_transcript_api import YouTubeTranscriptApi
-
-from app.proprietary.web_crawler import (
- CrawlOutcomeStatus,
- WebCrawlerConnector,
-)
-from app.tasks.document_processors.youtube_processor import get_youtube_video_id
-from app.utils.proxy import get_proxy_url, get_requests_proxies
-
-logger = logging.getLogger(__name__)
-
-
-def _bill_successful_scrape() -> None:
- """Fold one successful crawl into the current chat turn's bill (Phase 3c).
-
- The cost rides the turn accumulator and settles at the premium
- ``finalize_credit`` step — no separate wallet hit. Free / BYOK / anonymous
- turns (which never reserve/finalize) still record the line in the
- breakdown but are never debited. No-op when crawl billing is disabled or
- there is no active turn (e.g. non-chat callers).
- """
- from app.services.token_tracking_service import get_current_accumulator
- from app.services.web_crawl_credit_service import WebCrawlCreditService
-
- if not WebCrawlCreditService.billing_enabled():
- return
- acc = get_current_accumulator()
- if acc is None:
- return
- acc.add(
- model="web_crawl",
- prompt_tokens=0,
- completion_tokens=0,
- total_tokens=0,
- cost_micros=WebCrawlCreditService.successes_to_micros(1),
- call_kind="web_crawl",
- )
-
-
-def _bill_captcha_attempts(outcome) -> None:
- """Fold captcha solve attempts (Phase 3d) into the current chat turn's bill.
-
- Per *attempt*, not per success: a solve that didn't rescue the crawl still
- cost real solver money, so this runs before the success/failure branch.
- Mirrors :func:`_bill_successful_scrape` (rides the turn accumulator, settles
- at ``finalize_credit``; record-only on free/anonymous turns). No-op when
- captcha billing is off, there were no attempts, or no turn is active.
- """
- from app.services.token_tracking_service import get_current_accumulator
- from app.services.web_crawl_credit_service import WebCrawlCreditService
-
- if not WebCrawlCreditService.captcha_billing_enabled():
- return
- attempts = getattr(outcome, "captcha_attempts", 0) or 0
- if attempts <= 0:
- return
- acc = get_current_accumulator()
- if acc is None:
- return
- acc.add(
- model="web_crawl_captcha",
- prompt_tokens=0,
- completion_tokens=0,
- total_tokens=0,
- cost_micros=WebCrawlCreditService.captcha_solves_to_micros(attempts),
- call_kind="web_crawl_captcha",
- )
-
-
-def extract_domain(url: str) -> str:
- """Extract the domain from a URL."""
- try:
- parsed = urlparse(url)
- domain = parsed.netloc
- if domain.startswith("www."):
- domain = domain[4:]
- return domain
- except Exception:
- return ""
-
-
-def generate_scrape_id(url: str) -> str:
- """Generate a unique ID for a scraped webpage."""
- hash_val = hashlib.md5(url.encode()).hexdigest()[:12]
- return f"scrape-{hash_val}"
-
-
-def truncate_content(content: str, max_length: int = 50000) -> tuple[str, bool]:
- """
- Truncate content to a maximum length.
-
- Returns:
- Tuple of (truncated_content, was_truncated)
- """
- if len(content) <= max_length:
- return content, False
-
- # Prefer truncating at a sentence/paragraph boundary.
- truncated = content[:max_length]
- last_period = truncated.rfind(".")
- last_newline = truncated.rfind("\n\n")
-
- boundary = max(last_period, last_newline)
- if boundary > max_length * 0.8: # only if the boundary isn't too far back
- truncated = content[: boundary + 1]
-
- return truncated + "\n\n[Content truncated...]", True
-
-
-async def _scrape_youtube_video(
- url: str, video_id: str, max_length: int
-) -> dict[str, Any]:
- """
- Fetch YouTube video metadata and transcript via the YouTubeTranscriptApi.
-
- Returns a result dict in the same shape as the regular scrape_webpage output.
- """
- scrape_id = generate_scrape_id(url)
- domain = "youtube.com"
-
- # --- Video metadata via oEmbed ---
- residential_proxies = get_requests_proxies()
-
- params = {
- "format": "json",
- "url": f"https://www.youtube.com/watch?v={video_id}",
- }
- oembed_url = "https://www.youtube.com/oembed"
-
- try:
- oembed_fetch_start = time.perf_counter()
- oembed_page = await AsyncFetcher.get(
- oembed_url,
- params=params,
- proxy=get_proxy_url(),
- stealthy_headers=True,
- )
- logger.info(
- "[scrape_webpage][perf] source=oembed video=%s status=%s fetch_ms=%.1f",
- video_id,
- getattr(oembed_page, "status", None),
- (time.perf_counter() - oembed_fetch_start) * 1000,
- )
- video_data = oembed_page.json()
- except Exception:
- video_data = {}
-
- title = video_data.get("title", "YouTube Video")
- author = video_data.get("author_name", "Unknown")
-
- # --- Transcript via YouTubeTranscriptApi ---
- try:
- transcript_fetch_start = time.perf_counter()
- ua = UserAgent()
- http_client = Session()
- http_client.headers.update({"User-Agent": ua.random})
- if residential_proxies:
- http_client.proxies.update(residential_proxies)
- ytt_api = YouTubeTranscriptApi(http_client=http_client)
-
- # Pick the first transcript (video's primary language) rather than
- # defaulting to English.
- transcript_list = ytt_api.list(video_id)
- transcript = next(iter(transcript_list))
- captions = transcript.fetch()
-
- logger.info(
- "[scrape_webpage][perf] source=transcript video=%s fetch_ms=%.1f",
- video_id,
- (time.perf_counter() - transcript_fetch_start) * 1000,
- )
- logger.info(
- f"[scrape_webpage] Fetched transcript for {video_id} "
- f"in {transcript.language} ({transcript.language_code})"
- )
-
- transcript_segments = []
- for line in captions:
- start_time = line.start
- duration = line.duration
- text = line.text
- timestamp = f"[{start_time:.2f}s-{start_time + duration:.2f}s]"
- transcript_segments.append(f"{timestamp} {text}")
- transcript_text = "\n".join(transcript_segments)
- except Exception as e:
- logger.warning(f"[scrape_webpage] No transcript for video {video_id}: {e}")
- transcript_text = f"No captions available for this video. Error: {e!s}"
-
- content = f"# {title}\n\n**Author:** {author}\n**Video ID:** {video_id}\n\n## Transcript\n\n{transcript_text}"
-
- content, was_truncated = truncate_content(content, max_length)
- word_count = len(content.split())
-
- description = f"YouTube video by {author}"
-
- return {
- "id": scrape_id,
- "assetId": url,
- "kind": "article",
- "href": url,
- "title": title,
- "description": description,
- "content": content,
- "domain": domain,
- "word_count": word_count,
- "was_truncated": was_truncated,
- "crawler_type": "youtube_transcript",
- "author": author,
- }
-
-
-def create_scrape_webpage_tool():
- """
- Factory function to create the scrape_webpage tool.
-
- Returns:
- A configured tool function for scraping webpages.
- """
-
- @tool
- async def scrape_webpage(
- url: str,
- max_length: int = 50000,
- ) -> dict[str, Any]:
- """
- Scrape and extract the main content from a webpage.
-
- Use this tool when the user wants you to read, summarize, or answer
- questions about a specific webpage's content. This tool actually
- fetches and reads the full page content. For YouTube video URLs it
- fetches the transcript directly instead of crawling the page.
-
- Common triggers:
- - "Read this article and summarize it"
- - "What does this page say about X?"
- - "Summarize this blog post for me"
- - "Tell me the key points from this article"
- - "What's in this webpage?"
-
- Args:
- url: The URL of the webpage to scrape (must be HTTP/HTTPS)
- max_length: Maximum content length to return (default: 50000 chars)
-
- Returns:
- A dictionary containing:
- - id: Unique identifier for this scrape
- - assetId: The URL (for deduplication)
- - kind: "article" (type of content)
- - href: The URL to open when clicked
- - title: Page title
- - description: Brief description or excerpt
- - content: The extracted main content (markdown format)
- - domain: The domain name
- - word_count: Approximate word count
- - was_truncated: Whether content was truncated
- - error: Error message (if scraping failed)
- """
- scrape_id = generate_scrape_id(url)
- domain = extract_domain(url)
-
- if not url.startswith(("http://", "https://")):
- url = f"https://{url}"
-
- try:
- # YouTube URLs use the transcript API instead of crawling.
- video_id = get_youtube_video_id(url)
- if video_id:
- return await _scrape_youtube_video(url, video_id, max_length)
-
- connector = WebCrawlerConnector()
- outcome = await connector.crawl_url(url)
-
- # 03d: bill any captcha attempts (even if the crawl ultimately failed).
- _bill_captcha_attempts(outcome)
-
- if outcome.status is not CrawlOutcomeStatus.SUCCESS or not outcome.result:
- return {
- "id": scrape_id,
- "assetId": url,
- "kind": "article",
- "href": url,
- "title": domain or "Webpage",
- "domain": domain,
- "error": outcome.error or "No content returned from crawler",
- }
-
- result = outcome.result
- _bill_successful_scrape()
- content = result.get("content", "")
- metadata = result.get("metadata", {})
-
- title = metadata.get("title", "")
- if not title:
- title = domain or url.split("/")[-1] or "Webpage"
-
- description = metadata.get("description", "")
- if not description and content:
- first_para = content.split("\n\n")[0] if content else ""
- description = (
- first_para[:300] + "..." if len(first_para) > 300 else first_para
- )
-
- content, was_truncated = truncate_content(content, max_length)
- word_count = len(content.split())
-
- return {
- "id": scrape_id,
- "assetId": url,
- "kind": "article",
- "href": url,
- "title": title,
- "description": description,
- "content": content,
- "domain": domain,
- "word_count": word_count,
- "was_truncated": was_truncated,
- "crawler_type": result.get("crawler_type", "unknown"),
- "author": metadata.get("author"),
- "date": metadata.get("date"),
- }
-
- except Exception as e:
- error_message = str(e)
- logger.error(f"[scrape_webpage] Error scraping {url}: {error_message}")
- return {
- "id": scrape_id,
- "assetId": url,
- "kind": "article",
- "href": url,
- "title": domain or "Webpage",
- "domain": domain,
- "error": f"Failed to scrape: {error_message[:100]}",
- }
-
- return scrape_webpage
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py
index 02c196086..8fbd866dc 100644
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py
@@ -21,9 +21,6 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.knowledge_base.agent im
from app.agents.chat.multi_agent_chat.subagents.builtins.memory.agent import (
build_subagent as build_memory_subagent,
)
-from app.agents.chat.multi_agent_chat.subagents.builtins.research.agent import (
- build_subagent as build_research_subagent,
-)
from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import (
build_subagent as build_web_crawler_subagent,
)
@@ -112,7 +109,6 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = {
"memory": build_memory_subagent,
"notion": build_notion_subagent,
"onedrive": build_onedrive_subagent,
- "research": build_research_subagent,
"slack": build_slack_subagent,
"teams": build_teams_subagent,
"web_crawler": build_web_crawler_subagent,
@@ -127,7 +123,7 @@ def _route_resource_package(builder: SubagentBuilder) -> str:
def main_prompt_registry_subagent_lines(exclude: list[str]) -> list[tuple[str, str]]:
"""(name, description) for registry specialists included for **task** (same rules as ``build_subagents``)."""
- banned = frozenset(("memory", "research")) | frozenset(exclude)
+ banned = frozenset(("memory",)) | frozenset(exclude)
rows: list[tuple[str, str]] = []
for name in sorted(SUBAGENT_BUILDERS_BY_NAME):
if name in banned:
@@ -197,10 +193,10 @@ def build_subagents(
disabled_tools: list[str] | None = None,
ask_kb_tool: BaseTool | None = None,
) -> list[SubAgent]:
- """Build registry subagents; skip memory/research; skip names in exclude."""
+ """Build registry subagents; skip memory; skip names in exclude."""
mcp = mcp_tools_by_agent or {}
specs: list[SubAgent] = []
- excluded = ["memory", "research"]
+ excluded = ["memory"]
if exclude:
excluded.extend(exclude)
disabled_names = frozenset(disabled_tools or ())
diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py
index dc0f35994..873fe26c5 100644
--- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py
+++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py
@@ -19,7 +19,7 @@ from app.agents.chat.multi_agent_chat.subagents.registry import (
pytestmark = pytest.mark.unit
-# The full specialist roster the main agent composes from: 6 builtins + 15
+# The full specialist roster the main agent composes from: 5 builtins + 15
# connector routes. Adding/removing a specialist is a deliberate product change
# and must be reflected here.
_EXPECTED_SUBAGENTS = frozenset(
@@ -40,7 +40,6 @@ _EXPECTED_SUBAGENTS = frozenset(
"memory",
"notion",
"onedrive",
- "research",
"slack",
"teams",
"web_crawler",
@@ -50,7 +49,7 @@ _EXPECTED_SUBAGENTS = frozenset(
# Specialists that are always available regardless of connected sources, so they
# carry no required-connector entry.
-_CONNECTORLESS = frozenset({"memory", "research"})
+_CONNECTORLESS = frozenset({"memory"})
def test_registry_contains_exactly_expected_subagents():