From e3ed3b2be391bc3ffd89b9c5c422c8156b9e2c04 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 3 Jul 2026 11:43:26 +0200 Subject: [PATCH] refactor(subagents): remove dormant research subagent --- .../subagents/builtins/research/__init__.py | 0 .../subagents/builtins/research/agent.py | 52 --- .../builtins/research/description.md | 2 - .../builtins/research/system_prompt.md | 62 ---- .../builtins/research/tools/__init__.py | 10 - .../builtins/research/tools/index.py | 29 -- .../builtins/research/tools/scrape_webpage.py | 347 ------------------ .../multi_agent_chat/subagents/registry.py | 10 +- .../test_subagent_composition.py | 5 +- 9 files changed, 5 insertions(+), 512 deletions(-) delete mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/__init__.py delete mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/agent.py delete mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/description.md delete mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/system_prompt.md delete mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/__init__.py delete mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/index.py delete mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/agent.py deleted file mode 100644 index e3c0ab9ae..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/agent.py +++ /dev/null @@ -1,52 +0,0 @@ -"""``research`` route: ``SurfSenseSubagentSpec`` builder for deepagents.""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.shared.middleware.citation_state import ( - build_citation_state_mw, -) -from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( - read_md_file, -) -from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( - pack_subagent, -) - -from .tools.index import NAME, RULESET, load_tools - - -def build_subagent( - *, - dependencies: dict[str, Any], - model: BaseChatModel | None = None, - middleware_stack: dict[str, Any] | None = None, - mcp_tools: list[BaseTool] | None = None, -) -> SurfSenseSubagentSpec: - tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] - description = ( - read_md_file(__package__, "description").strip() - or "Handles research tasks for this workspace." - ) - system_prompt = read_md_file(__package__, "system_prompt").strip() - # web_search registers WEB_RESULT citations via Command(update=...); the - # citation-state middleware declares the channel so those [n] merge back up. - middleware_with_citations = { - **(middleware_stack or {}), - "citation_state": build_citation_state_mw(), - } - return pack_subagent( - name=NAME, - description=description, - system_prompt=system_prompt, - tools=tools, - ruleset=RULESET, - dependencies=dependencies, - model=model, - middleware_stack=middleware_with_citations, - ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/description.md deleted file mode 100644 index 0a99b4140..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/description.md +++ /dev/null @@ -1,2 +0,0 @@ -Specialist for external research. -Use whenever a task requires finding sources on the web and extracting evidence to answer documentation questions. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/system_prompt.md deleted file mode 100644 index 3d90a4352..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/system_prompt.md +++ /dev/null @@ -1,62 +0,0 @@ -You are the SurfSense research operations sub-agent. -You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis. - - -Gather and synthesize evidence using SurfSense research tools with clear citations and uncertainty reporting. - - - -- `web_search` -- `scrape_webpage` - - - -- Use only tools in ``. -- Prefer primary and recent sources when recency matters. -- If the delegated request is underspecified, return `status=blocked` with the missing research constraints. -- Never fabricate facts, citations, URLs, or quote text. - - - -`web_search` returns a `` block whose results are each prefixed with a bracketed label — `[1]`, `[2]`, `[3]`. That `[n]` is the citation label. When a finding came from a specific result, append its `[n]` to that finding, copying the label **exactly** as shown. The caller relays these labels verbatim and the server resolves each one, so a wrong number silently breaks the citation. - -- Use the exact `[n]` shown next to the result you actually used; never renumber, guess, or invent a label. -- Before emitting an `[n]`, confirm that bracketed label appears in the `web_search` output this turn. If you can't see it, omit it. -- Write the bare label `[n]` only — no `[citation:…]` wrapper, no markdown links. -- Several results behind one finding → each in its own brackets with nothing between: `[1][2]`. -- `scrape_webpage` returns raw page text with no `[n]` labels; a fact drawn only from a scrape carries no citation (report the URL in `evidence.sources` instead). - - - -- Do not execute connector mutations (email/calendar/docs/chat writes) or deliverable generation. - - - -- Report uncertainty explicitly when evidence is incomplete or conflicting. -- Never present unverified claims as facts. - - - -- On tool failure, return `status=error` with a concise recovery `next_step`. -- On no useful evidence, return `status=blocked` with recommended narrower filters. - - - -Return **only** one JSON object (no markdown/prose): -{ - "status": "success" | "partial" | "blocked" | "error", - "action_summary": string, - "evidence": { - "findings": string[], - "sources": string[], - "confidence": "high" | "medium" | "low" - }, - "next_step": string | null, - "missing_fields": string[] | null, - "assumptions": string[] | null -} - -Route-specific rules: -- `evidence.findings`: max 10 entries, each a single sentence stating one distinct fact. Append the supporting `[n]` to each finding drawn from a `web_search` result. Do not paste raw paragraphs, scraped pages, or quote blocks. -- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once. (Citations travel as `[n]`; `sources` is for transparency and for scrape-only facts that carry no `[n]`.) - diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/__init__.py deleted file mode 100644 index 0c99bf222..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Research-stage tools: web search (shared) and scrape.""" - -from app.agents.chat.shared.tools.web_search import create_web_search_tool - -from .scrape_webpage import create_scrape_webpage_tool - -__all__ = [ - "create_scrape_webpage_tool", - "create_web_search_tool", -] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/index.py deleted file mode 100644 index b3ea5a6b0..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/index.py +++ /dev/null @@ -1,29 +0,0 @@ -"""``research`` native tools and (empty) permission ruleset.""" - -from __future__ import annotations - -from typing import Any - -from langchain_core.tools import BaseTool - -from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset -from app.agents.chat.shared.tools.web_search import create_web_search_tool - -from .scrape_webpage import create_scrape_webpage_tool - -NAME = "research" - -RULESET = Ruleset(origin=NAME, rules=[]) - - -def load_tools( - *, dependencies: dict[str, Any] | None = None, **kwargs: Any -) -> list[BaseTool]: - d = {**(dependencies or {}), **kwargs} - return [ - create_web_search_tool( - workspace_id=d.get("workspace_id"), - available_connectors=d.get("available_connectors"), - ), - create_scrape_webpage_tool(), - ] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py deleted file mode 100644 index af7ce44e7..000000000 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py +++ /dev/null @@ -1,347 +0,0 @@ -"""Scrape pages via WebCrawlerConnector; YouTube URLs use the transcript API instead of HTML crawl.""" - -import hashlib -import logging -import time -from typing import Any -from urllib.parse import urlparse - -from fake_useragent import UserAgent -from langchain_core.tools import tool -from requests import Session -from scrapling.fetchers import AsyncFetcher -from youtube_transcript_api import YouTubeTranscriptApi - -from app.proprietary.web_crawler import ( - CrawlOutcomeStatus, - WebCrawlerConnector, -) -from app.tasks.document_processors.youtube_processor import get_youtube_video_id -from app.utils.proxy import get_proxy_url, get_requests_proxies - -logger = logging.getLogger(__name__) - - -def _bill_successful_scrape() -> None: - """Fold one successful crawl into the current chat turn's bill (Phase 3c). - - The cost rides the turn accumulator and settles at the premium - ``finalize_credit`` step — no separate wallet hit. Free / BYOK / anonymous - turns (which never reserve/finalize) still record the line in the - breakdown but are never debited. No-op when crawl billing is disabled or - there is no active turn (e.g. non-chat callers). - """ - from app.services.token_tracking_service import get_current_accumulator - from app.services.web_crawl_credit_service import WebCrawlCreditService - - if not WebCrawlCreditService.billing_enabled(): - return - acc = get_current_accumulator() - if acc is None: - return - acc.add( - model="web_crawl", - prompt_tokens=0, - completion_tokens=0, - total_tokens=0, - cost_micros=WebCrawlCreditService.successes_to_micros(1), - call_kind="web_crawl", - ) - - -def _bill_captcha_attempts(outcome) -> None: - """Fold captcha solve attempts (Phase 3d) into the current chat turn's bill. - - Per *attempt*, not per success: a solve that didn't rescue the crawl still - cost real solver money, so this runs before the success/failure branch. - Mirrors :func:`_bill_successful_scrape` (rides the turn accumulator, settles - at ``finalize_credit``; record-only on free/anonymous turns). No-op when - captcha billing is off, there were no attempts, or no turn is active. - """ - from app.services.token_tracking_service import get_current_accumulator - from app.services.web_crawl_credit_service import WebCrawlCreditService - - if not WebCrawlCreditService.captcha_billing_enabled(): - return - attempts = getattr(outcome, "captcha_attempts", 0) or 0 - if attempts <= 0: - return - acc = get_current_accumulator() - if acc is None: - return - acc.add( - model="web_crawl_captcha", - prompt_tokens=0, - completion_tokens=0, - total_tokens=0, - cost_micros=WebCrawlCreditService.captcha_solves_to_micros(attempts), - call_kind="web_crawl_captcha", - ) - - -def extract_domain(url: str) -> str: - """Extract the domain from a URL.""" - try: - parsed = urlparse(url) - domain = parsed.netloc - if domain.startswith("www."): - domain = domain[4:] - return domain - except Exception: - return "" - - -def generate_scrape_id(url: str) -> str: - """Generate a unique ID for a scraped webpage.""" - hash_val = hashlib.md5(url.encode()).hexdigest()[:12] - return f"scrape-{hash_val}" - - -def truncate_content(content: str, max_length: int = 50000) -> tuple[str, bool]: - """ - Truncate content to a maximum length. - - Returns: - Tuple of (truncated_content, was_truncated) - """ - if len(content) <= max_length: - return content, False - - # Prefer truncating at a sentence/paragraph boundary. - truncated = content[:max_length] - last_period = truncated.rfind(".") - last_newline = truncated.rfind("\n\n") - - boundary = max(last_period, last_newline) - if boundary > max_length * 0.8: # only if the boundary isn't too far back - truncated = content[: boundary + 1] - - return truncated + "\n\n[Content truncated...]", True - - -async def _scrape_youtube_video( - url: str, video_id: str, max_length: int -) -> dict[str, Any]: - """ - Fetch YouTube video metadata and transcript via the YouTubeTranscriptApi. - - Returns a result dict in the same shape as the regular scrape_webpage output. - """ - scrape_id = generate_scrape_id(url) - domain = "youtube.com" - - # --- Video metadata via oEmbed --- - residential_proxies = get_requests_proxies() - - params = { - "format": "json", - "url": f"https://www.youtube.com/watch?v={video_id}", - } - oembed_url = "https://www.youtube.com/oembed" - - try: - oembed_fetch_start = time.perf_counter() - oembed_page = await AsyncFetcher.get( - oembed_url, - params=params, - proxy=get_proxy_url(), - stealthy_headers=True, - ) - logger.info( - "[scrape_webpage][perf] source=oembed video=%s status=%s fetch_ms=%.1f", - video_id, - getattr(oembed_page, "status", None), - (time.perf_counter() - oembed_fetch_start) * 1000, - ) - video_data = oembed_page.json() - except Exception: - video_data = {} - - title = video_data.get("title", "YouTube Video") - author = video_data.get("author_name", "Unknown") - - # --- Transcript via YouTubeTranscriptApi --- - try: - transcript_fetch_start = time.perf_counter() - ua = UserAgent() - http_client = Session() - http_client.headers.update({"User-Agent": ua.random}) - if residential_proxies: - http_client.proxies.update(residential_proxies) - ytt_api = YouTubeTranscriptApi(http_client=http_client) - - # Pick the first transcript (video's primary language) rather than - # defaulting to English. - transcript_list = ytt_api.list(video_id) - transcript = next(iter(transcript_list)) - captions = transcript.fetch() - - logger.info( - "[scrape_webpage][perf] source=transcript video=%s fetch_ms=%.1f", - video_id, - (time.perf_counter() - transcript_fetch_start) * 1000, - ) - logger.info( - f"[scrape_webpage] Fetched transcript for {video_id} " - f"in {transcript.language} ({transcript.language_code})" - ) - - transcript_segments = [] - for line in captions: - start_time = line.start - duration = line.duration - text = line.text - timestamp = f"[{start_time:.2f}s-{start_time + duration:.2f}s]" - transcript_segments.append(f"{timestamp} {text}") - transcript_text = "\n".join(transcript_segments) - except Exception as e: - logger.warning(f"[scrape_webpage] No transcript for video {video_id}: {e}") - transcript_text = f"No captions available for this video. Error: {e!s}" - - content = f"# {title}\n\n**Author:** {author}\n**Video ID:** {video_id}\n\n## Transcript\n\n{transcript_text}" - - content, was_truncated = truncate_content(content, max_length) - word_count = len(content.split()) - - description = f"YouTube video by {author}" - - return { - "id": scrape_id, - "assetId": url, - "kind": "article", - "href": url, - "title": title, - "description": description, - "content": content, - "domain": domain, - "word_count": word_count, - "was_truncated": was_truncated, - "crawler_type": "youtube_transcript", - "author": author, - } - - -def create_scrape_webpage_tool(): - """ - Factory function to create the scrape_webpage tool. - - Returns: - A configured tool function for scraping webpages. - """ - - @tool - async def scrape_webpage( - url: str, - max_length: int = 50000, - ) -> dict[str, Any]: - """ - Scrape and extract the main content from a webpage. - - Use this tool when the user wants you to read, summarize, or answer - questions about a specific webpage's content. This tool actually - fetches and reads the full page content. For YouTube video URLs it - fetches the transcript directly instead of crawling the page. - - Common triggers: - - "Read this article and summarize it" - - "What does this page say about X?" - - "Summarize this blog post for me" - - "Tell me the key points from this article" - - "What's in this webpage?" - - Args: - url: The URL of the webpage to scrape (must be HTTP/HTTPS) - max_length: Maximum content length to return (default: 50000 chars) - - Returns: - A dictionary containing: - - id: Unique identifier for this scrape - - assetId: The URL (for deduplication) - - kind: "article" (type of content) - - href: The URL to open when clicked - - title: Page title - - description: Brief description or excerpt - - content: The extracted main content (markdown format) - - domain: The domain name - - word_count: Approximate word count - - was_truncated: Whether content was truncated - - error: Error message (if scraping failed) - """ - scrape_id = generate_scrape_id(url) - domain = extract_domain(url) - - if not url.startswith(("http://", "https://")): - url = f"https://{url}" - - try: - # YouTube URLs use the transcript API instead of crawling. - video_id = get_youtube_video_id(url) - if video_id: - return await _scrape_youtube_video(url, video_id, max_length) - - connector = WebCrawlerConnector() - outcome = await connector.crawl_url(url) - - # 03d: bill any captcha attempts (even if the crawl ultimately failed). - _bill_captcha_attempts(outcome) - - if outcome.status is not CrawlOutcomeStatus.SUCCESS or not outcome.result: - return { - "id": scrape_id, - "assetId": url, - "kind": "article", - "href": url, - "title": domain or "Webpage", - "domain": domain, - "error": outcome.error or "No content returned from crawler", - } - - result = outcome.result - _bill_successful_scrape() - content = result.get("content", "") - metadata = result.get("metadata", {}) - - title = metadata.get("title", "") - if not title: - title = domain or url.split("/")[-1] or "Webpage" - - description = metadata.get("description", "") - if not description and content: - first_para = content.split("\n\n")[0] if content else "" - description = ( - first_para[:300] + "..." if len(first_para) > 300 else first_para - ) - - content, was_truncated = truncate_content(content, max_length) - word_count = len(content.split()) - - return { - "id": scrape_id, - "assetId": url, - "kind": "article", - "href": url, - "title": title, - "description": description, - "content": content, - "domain": domain, - "word_count": word_count, - "was_truncated": was_truncated, - "crawler_type": result.get("crawler_type", "unknown"), - "author": metadata.get("author"), - "date": metadata.get("date"), - } - - except Exception as e: - error_message = str(e) - logger.error(f"[scrape_webpage] Error scraping {url}: {error_message}") - return { - "id": scrape_id, - "assetId": url, - "kind": "article", - "href": url, - "title": domain or "Webpage", - "domain": domain, - "error": f"Failed to scrape: {error_message[:100]}", - } - - return scrape_webpage diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index 02c196086..8fbd866dc 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -21,9 +21,6 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.knowledge_base.agent im from app.agents.chat.multi_agent_chat.subagents.builtins.memory.agent import ( build_subagent as build_memory_subagent, ) -from app.agents.chat.multi_agent_chat.subagents.builtins.research.agent import ( - build_subagent as build_research_subagent, -) from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import ( build_subagent as build_web_crawler_subagent, ) @@ -112,7 +109,6 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { "memory": build_memory_subagent, "notion": build_notion_subagent, "onedrive": build_onedrive_subagent, - "research": build_research_subagent, "slack": build_slack_subagent, "teams": build_teams_subagent, "web_crawler": build_web_crawler_subagent, @@ -127,7 +123,7 @@ def _route_resource_package(builder: SubagentBuilder) -> str: def main_prompt_registry_subagent_lines(exclude: list[str]) -> list[tuple[str, str]]: """(name, description) for registry specialists included for **task** (same rules as ``build_subagents``).""" - banned = frozenset(("memory", "research")) | frozenset(exclude) + banned = frozenset(("memory",)) | frozenset(exclude) rows: list[tuple[str, str]] = [] for name in sorted(SUBAGENT_BUILDERS_BY_NAME): if name in banned: @@ -197,10 +193,10 @@ def build_subagents( disabled_tools: list[str] | None = None, ask_kb_tool: BaseTool | None = None, ) -> list[SubAgent]: - """Build registry subagents; skip memory/research; skip names in exclude.""" + """Build registry subagents; skip memory; skip names in exclude.""" mcp = mcp_tools_by_agent or {} specs: list[SubAgent] = [] - excluded = ["memory", "research"] + excluded = ["memory"] if exclude: excluded.extend(exclude) disabled_names = frozenset(disabled_tools or ()) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py index dc0f35994..873fe26c5 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py @@ -19,7 +19,7 @@ from app.agents.chat.multi_agent_chat.subagents.registry import ( pytestmark = pytest.mark.unit -# The full specialist roster the main agent composes from: 6 builtins + 15 +# The full specialist roster the main agent composes from: 5 builtins + 15 # connector routes. Adding/removing a specialist is a deliberate product change # and must be reflected here. _EXPECTED_SUBAGENTS = frozenset( @@ -40,7 +40,6 @@ _EXPECTED_SUBAGENTS = frozenset( "memory", "notion", "onedrive", - "research", "slack", "teams", "web_crawler", @@ -50,7 +49,7 @@ _EXPECTED_SUBAGENTS = frozenset( # Specialists that are always available regardless of connected sources, so they # carry no required-connector entry. -_CONNECTORLESS = frozenset({"memory", "research"}) +_CONNECTORLESS = frozenset({"memory"}) def test_registry_contains_exactly_expected_subagents():