refactor(subagents): remove dormant research subagent

This commit is contained in:
CREDO23 2026-07-03 11:43:26 +02:00
parent 62cb0efb44
commit e3ed3b2be3
9 changed files with 5 additions and 512 deletions

View file

@ -1,52 +0,0 @@
"""``research`` route: ``SurfSenseSubagentSpec`` builder for deepagents."""
from __future__ import annotations
from typing import Any
from langchain_core.language_models import BaseChatModel
from langchain_core.tools import BaseTool
from app.agents.chat.multi_agent_chat.shared.middleware.citation_state import (
build_citation_state_mw,
)
from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import (
read_md_file,
)
from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec
from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import (
pack_subagent,
)
from .tools.index import NAME, RULESET, load_tools
def build_subagent(
*,
dependencies: dict[str, Any],
model: BaseChatModel | None = None,
middleware_stack: dict[str, Any] | None = None,
mcp_tools: list[BaseTool] | None = None,
) -> SurfSenseSubagentSpec:
tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])]
description = (
read_md_file(__package__, "description").strip()
or "Handles research tasks for this workspace."
)
system_prompt = read_md_file(__package__, "system_prompt").strip()
# web_search registers WEB_RESULT citations via Command(update=...); the
# citation-state middleware declares the channel so those [n] merge back up.
middleware_with_citations = {
**(middleware_stack or {}),
"citation_state": build_citation_state_mw(),
}
return pack_subagent(
name=NAME,
description=description,
system_prompt=system_prompt,
tools=tools,
ruleset=RULESET,
dependencies=dependencies,
model=model,
middleware_stack=middleware_with_citations,
)

View file

@ -1,2 +0,0 @@
Specialist for external research.
Use whenever a task requires finding sources on the web and extracting evidence to answer documentation questions.

View file

@ -1,62 +0,0 @@
You are the SurfSense research operations sub-agent.
You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis.
<goal>
Gather and synthesize evidence using SurfSense research tools with clear citations and uncertainty reporting.
</goal>
<available_tools>
- `web_search`
- `scrape_webpage`
</available_tools>
<tool_policy>
- Use only tools in `<available_tools>`.
- Prefer primary and recent sources when recency matters.
- If the delegated request is underspecified, return `status=blocked` with the missing research constraints.
- Never fabricate facts, citations, URLs, or quote text.
</tool_policy>
<citations>
`web_search` returns a `<web_results>` block whose results are each prefixed with a bracketed label — `[1]`, `[2]`, `[3]`. That `[n]` is the citation label. When a finding came from a specific result, append its `[n]` to that finding, copying the label **exactly** as shown. The caller relays these labels verbatim and the server resolves each one, so a wrong number silently breaks the citation.
- Use the exact `[n]` shown next to the result you actually used; never renumber, guess, or invent a label.
- Before emitting an `[n]`, confirm that bracketed label appears in the `web_search` output this turn. If you can't see it, omit it.
- Write the bare label `[n]` only — no `[citation:…]` wrapper, no markdown links.
- Several results behind one finding → each in its own brackets with nothing between: `[1][2]`.
- `scrape_webpage` returns raw page text with no `[n]` labels; a fact drawn only from a scrape carries no citation (report the URL in `evidence.sources` instead).
</citations>
<out_of_scope>
- Do not execute connector mutations (email/calendar/docs/chat writes) or deliverable generation.
</out_of_scope>
<safety>
- Report uncertainty explicitly when evidence is incomplete or conflicting.
- Never present unverified claims as facts.
</safety>
<failure_policy>
- On tool failure, return `status=error` with a concise recovery `next_step`.
- On no useful evidence, return `status=blocked` with recommended narrower filters.
</failure_policy>
<output_contract>
Return **only** one JSON object (no markdown/prose):
{
"status": "success" | "partial" | "blocked" | "error",
"action_summary": string,
"evidence": {
"findings": string[],
"sources": string[],
"confidence": "high" | "medium" | "low"
},
"next_step": string | null,
"missing_fields": string[] | null,
"assumptions": string[] | null
}
<include snippet="output_contract_base"/>
Route-specific rules:
- `evidence.findings`: max 10 entries, each a single sentence stating one distinct fact. Append the supporting `[n]` to each finding drawn from a `web_search` result. Do not paste raw paragraphs, scraped pages, or quote blocks.
- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once. (Citations travel as `[n]`; `sources` is for transparency and for scrape-only facts that carry no `[n]`.)
</output_contract>

View file

@ -1,10 +0,0 @@
"""Research-stage tools: web search (shared) and scrape."""
from app.agents.chat.shared.tools.web_search import create_web_search_tool
from .scrape_webpage import create_scrape_webpage_tool
__all__ = [
"create_scrape_webpage_tool",
"create_web_search_tool",
]

View file

@ -1,29 +0,0 @@
"""``research`` native tools and (empty) permission ruleset."""
from __future__ import annotations
from typing import Any
from langchain_core.tools import BaseTool
from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
from app.agents.chat.shared.tools.web_search import create_web_search_tool
from .scrape_webpage import create_scrape_webpage_tool
NAME = "research"
RULESET = Ruleset(origin=NAME, rules=[])
def load_tools(
*, dependencies: dict[str, Any] | None = None, **kwargs: Any
) -> list[BaseTool]:
d = {**(dependencies or {}), **kwargs}
return [
create_web_search_tool(
workspace_id=d.get("workspace_id"),
available_connectors=d.get("available_connectors"),
),
create_scrape_webpage_tool(),
]

View file

@ -1,347 +0,0 @@
"""Scrape pages via WebCrawlerConnector; YouTube URLs use the transcript API instead of HTML crawl."""
import hashlib
import logging
import time
from typing import Any
from urllib.parse import urlparse
from fake_useragent import UserAgent
from langchain_core.tools import tool
from requests import Session
from scrapling.fetchers import AsyncFetcher
from youtube_transcript_api import YouTubeTranscriptApi
from app.proprietary.web_crawler import (
CrawlOutcomeStatus,
WebCrawlerConnector,
)
from app.tasks.document_processors.youtube_processor import get_youtube_video_id
from app.utils.proxy import get_proxy_url, get_requests_proxies
logger = logging.getLogger(__name__)
def _bill_successful_scrape() -> None:
"""Fold one successful crawl into the current chat turn's bill (Phase 3c).
The cost rides the turn accumulator and settles at the premium
``finalize_credit`` step no separate wallet hit. Free / BYOK / anonymous
turns (which never reserve/finalize) still record the line in the
breakdown but are never debited. No-op when crawl billing is disabled or
there is no active turn (e.g. non-chat callers).
"""
from app.services.token_tracking_service import get_current_accumulator
from app.services.web_crawl_credit_service import WebCrawlCreditService
if not WebCrawlCreditService.billing_enabled():
return
acc = get_current_accumulator()
if acc is None:
return
acc.add(
model="web_crawl",
prompt_tokens=0,
completion_tokens=0,
total_tokens=0,
cost_micros=WebCrawlCreditService.successes_to_micros(1),
call_kind="web_crawl",
)
def _bill_captcha_attempts(outcome) -> None:
"""Fold captcha solve attempts (Phase 3d) into the current chat turn's bill.
Per *attempt*, not per success: a solve that didn't rescue the crawl still
cost real solver money, so this runs before the success/failure branch.
Mirrors :func:`_bill_successful_scrape` (rides the turn accumulator, settles
at ``finalize_credit``; record-only on free/anonymous turns). No-op when
captcha billing is off, there were no attempts, or no turn is active.
"""
from app.services.token_tracking_service import get_current_accumulator
from app.services.web_crawl_credit_service import WebCrawlCreditService
if not WebCrawlCreditService.captcha_billing_enabled():
return
attempts = getattr(outcome, "captcha_attempts", 0) or 0
if attempts <= 0:
return
acc = get_current_accumulator()
if acc is None:
return
acc.add(
model="web_crawl_captcha",
prompt_tokens=0,
completion_tokens=0,
total_tokens=0,
cost_micros=WebCrawlCreditService.captcha_solves_to_micros(attempts),
call_kind="web_crawl_captcha",
)
def extract_domain(url: str) -> str:
"""Extract the domain from a URL."""
try:
parsed = urlparse(url)
domain = parsed.netloc
if domain.startswith("www."):
domain = domain[4:]
return domain
except Exception:
return ""
def generate_scrape_id(url: str) -> str:
"""Generate a unique ID for a scraped webpage."""
hash_val = hashlib.md5(url.encode()).hexdigest()[:12]
return f"scrape-{hash_val}"
def truncate_content(content: str, max_length: int = 50000) -> tuple[str, bool]:
"""
Truncate content to a maximum length.
Returns:
Tuple of (truncated_content, was_truncated)
"""
if len(content) <= max_length:
return content, False
# Prefer truncating at a sentence/paragraph boundary.
truncated = content[:max_length]
last_period = truncated.rfind(".")
last_newline = truncated.rfind("\n\n")
boundary = max(last_period, last_newline)
if boundary > max_length * 0.8: # only if the boundary isn't too far back
truncated = content[: boundary + 1]
return truncated + "\n\n[Content truncated...]", True
async def _scrape_youtube_video(
url: str, video_id: str, max_length: int
) -> dict[str, Any]:
"""
Fetch YouTube video metadata and transcript via the YouTubeTranscriptApi.
Returns a result dict in the same shape as the regular scrape_webpage output.
"""
scrape_id = generate_scrape_id(url)
domain = "youtube.com"
# --- Video metadata via oEmbed ---
residential_proxies = get_requests_proxies()
params = {
"format": "json",
"url": f"https://www.youtube.com/watch?v={video_id}",
}
oembed_url = "https://www.youtube.com/oembed"
try:
oembed_fetch_start = time.perf_counter()
oembed_page = await AsyncFetcher.get(
oembed_url,
params=params,
proxy=get_proxy_url(),
stealthy_headers=True,
)
logger.info(
"[scrape_webpage][perf] source=oembed video=%s status=%s fetch_ms=%.1f",
video_id,
getattr(oembed_page, "status", None),
(time.perf_counter() - oembed_fetch_start) * 1000,
)
video_data = oembed_page.json()
except Exception:
video_data = {}
title = video_data.get("title", "YouTube Video")
author = video_data.get("author_name", "Unknown")
# --- Transcript via YouTubeTranscriptApi ---
try:
transcript_fetch_start = time.perf_counter()
ua = UserAgent()
http_client = Session()
http_client.headers.update({"User-Agent": ua.random})
if residential_proxies:
http_client.proxies.update(residential_proxies)
ytt_api = YouTubeTranscriptApi(http_client=http_client)
# Pick the first transcript (video's primary language) rather than
# defaulting to English.
transcript_list = ytt_api.list(video_id)
transcript = next(iter(transcript_list))
captions = transcript.fetch()
logger.info(
"[scrape_webpage][perf] source=transcript video=%s fetch_ms=%.1f",
video_id,
(time.perf_counter() - transcript_fetch_start) * 1000,
)
logger.info(
f"[scrape_webpage] Fetched transcript for {video_id} "
f"in {transcript.language} ({transcript.language_code})"
)
transcript_segments = []
for line in captions:
start_time = line.start
duration = line.duration
text = line.text
timestamp = f"[{start_time:.2f}s-{start_time + duration:.2f}s]"
transcript_segments.append(f"{timestamp} {text}")
transcript_text = "\n".join(transcript_segments)
except Exception as e:
logger.warning(f"[scrape_webpage] No transcript for video {video_id}: {e}")
transcript_text = f"No captions available for this video. Error: {e!s}"
content = f"# {title}\n\n**Author:** {author}\n**Video ID:** {video_id}\n\n## Transcript\n\n{transcript_text}"
content, was_truncated = truncate_content(content, max_length)
word_count = len(content.split())
description = f"YouTube video by {author}"
return {
"id": scrape_id,
"assetId": url,
"kind": "article",
"href": url,
"title": title,
"description": description,
"content": content,
"domain": domain,
"word_count": word_count,
"was_truncated": was_truncated,
"crawler_type": "youtube_transcript",
"author": author,
}
def create_scrape_webpage_tool():
"""
Factory function to create the scrape_webpage tool.
Returns:
A configured tool function for scraping webpages.
"""
@tool
async def scrape_webpage(
url: str,
max_length: int = 50000,
) -> dict[str, Any]:
"""
Scrape and extract the main content from a webpage.
Use this tool when the user wants you to read, summarize, or answer
questions about a specific webpage's content. This tool actually
fetches and reads the full page content. For YouTube video URLs it
fetches the transcript directly instead of crawling the page.
Common triggers:
- "Read this article and summarize it"
- "What does this page say about X?"
- "Summarize this blog post for me"
- "Tell me the key points from this article"
- "What's in this webpage?"
Args:
url: The URL of the webpage to scrape (must be HTTP/HTTPS)
max_length: Maximum content length to return (default: 50000 chars)
Returns:
A dictionary containing:
- id: Unique identifier for this scrape
- assetId: The URL (for deduplication)
- kind: "article" (type of content)
- href: The URL to open when clicked
- title: Page title
- description: Brief description or excerpt
- content: The extracted main content (markdown format)
- domain: The domain name
- word_count: Approximate word count
- was_truncated: Whether content was truncated
- error: Error message (if scraping failed)
"""
scrape_id = generate_scrape_id(url)
domain = extract_domain(url)
if not url.startswith(("http://", "https://")):
url = f"https://{url}"
try:
# YouTube URLs use the transcript API instead of crawling.
video_id = get_youtube_video_id(url)
if video_id:
return await _scrape_youtube_video(url, video_id, max_length)
connector = WebCrawlerConnector()
outcome = await connector.crawl_url(url)
# 03d: bill any captcha attempts (even if the crawl ultimately failed).
_bill_captcha_attempts(outcome)
if outcome.status is not CrawlOutcomeStatus.SUCCESS or not outcome.result:
return {
"id": scrape_id,
"assetId": url,
"kind": "article",
"href": url,
"title": domain or "Webpage",
"domain": domain,
"error": outcome.error or "No content returned from crawler",
}
result = outcome.result
_bill_successful_scrape()
content = result.get("content", "")
metadata = result.get("metadata", {})
title = metadata.get("title", "")
if not title:
title = domain or url.split("/")[-1] or "Webpage"
description = metadata.get("description", "")
if not description and content:
first_para = content.split("\n\n")[0] if content else ""
description = (
first_para[:300] + "..." if len(first_para) > 300 else first_para
)
content, was_truncated = truncate_content(content, max_length)
word_count = len(content.split())
return {
"id": scrape_id,
"assetId": url,
"kind": "article",
"href": url,
"title": title,
"description": description,
"content": content,
"domain": domain,
"word_count": word_count,
"was_truncated": was_truncated,
"crawler_type": result.get("crawler_type", "unknown"),
"author": metadata.get("author"),
"date": metadata.get("date"),
}
except Exception as e:
error_message = str(e)
logger.error(f"[scrape_webpage] Error scraping {url}: {error_message}")
return {
"id": scrape_id,
"assetId": url,
"kind": "article",
"href": url,
"title": domain or "Webpage",
"domain": domain,
"error": f"Failed to scrape: {error_message[:100]}",
}
return scrape_webpage

View file

@ -21,9 +21,6 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.knowledge_base.agent im
from app.agents.chat.multi_agent_chat.subagents.builtins.memory.agent import (
build_subagent as build_memory_subagent,
)
from app.agents.chat.multi_agent_chat.subagents.builtins.research.agent import (
build_subagent as build_research_subagent,
)
from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import (
build_subagent as build_web_crawler_subagent,
)
@ -112,7 +109,6 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = {
"memory": build_memory_subagent,
"notion": build_notion_subagent,
"onedrive": build_onedrive_subagent,
"research": build_research_subagent,
"slack": build_slack_subagent,
"teams": build_teams_subagent,
"web_crawler": build_web_crawler_subagent,
@ -127,7 +123,7 @@ def _route_resource_package(builder: SubagentBuilder) -> str:
def main_prompt_registry_subagent_lines(exclude: list[str]) -> list[tuple[str, str]]:
"""(name, description) for registry specialists included for **task** (same rules as ``build_subagents``)."""
banned = frozenset(("memory", "research")) | frozenset(exclude)
banned = frozenset(("memory",)) | frozenset(exclude)
rows: list[tuple[str, str]] = []
for name in sorted(SUBAGENT_BUILDERS_BY_NAME):
if name in banned:
@ -197,10 +193,10 @@ def build_subagents(
disabled_tools: list[str] | None = None,
ask_kb_tool: BaseTool | None = None,
) -> list[SubAgent]:
"""Build registry subagents; skip memory/research; skip names in exclude."""
"""Build registry subagents; skip memory; skip names in exclude."""
mcp = mcp_tools_by_agent or {}
specs: list[SubAgent] = []
excluded = ["memory", "research"]
excluded = ["memory"]
if exclude:
excluded.extend(exclude)
disabled_names = frozenset(disabled_tools or ())

View file

@ -19,7 +19,7 @@ from app.agents.chat.multi_agent_chat.subagents.registry import (
pytestmark = pytest.mark.unit
# The full specialist roster the main agent composes from: 6 builtins + 15
# The full specialist roster the main agent composes from: 5 builtins + 15
# connector routes. Adding/removing a specialist is a deliberate product change
# and must be reflected here.
_EXPECTED_SUBAGENTS = frozenset(
@ -40,7 +40,6 @@ _EXPECTED_SUBAGENTS = frozenset(
"memory",
"notion",
"onedrive",
"research",
"slack",
"teams",
"web_crawler",
@ -50,7 +49,7 @@ _EXPECTED_SUBAGENTS = frozenset(
# Specialists that are always available regardless of connected sources, so they
# carry no required-connector entry.
_CONNECTORLESS = frozenset({"memory", "research"})
_CONNECTORLESS = frozenset({"memory"})
def test_registry_contains_exactly_expected_subagents():