mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-14 22:52:15 +02:00
refactor(subagents): split scraping into web_crawler + youtube builtins
This commit is contained in:
CREDO23
parent
3312a442f3
commit
5da624399d
25 changed files with 239 additions and 808 deletions
|
|
@ -26,7 +26,8 @@ CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS: dict[str, str] = {
|
|||
SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = {
|
||||
"deliverables": frozenset(),
|
||||
"knowledge_base": frozenset(),
|
||||
"scraping": frozenset(),
|
||||
"web_crawler": frozenset(),
|
||||
"youtube": frozenset(),
|
||||
"airtable": frozenset({"AIRTABLE_CONNECTOR"}),
|
||||
"calendar": frozenset({"GOOGLE_CALENDAR_CONNECTOR"}),
|
||||
"clickup": frozenset({"CLICKUP_CONNECTOR"}),
|
||||
|
|
|
|||
|
|
@ -1,2 +0,0 @@
|
|||
Scraping specialist: fetches live public web pages, searches the web to find pages, pulls structured data from YouTube (videos, channels, playlists, search, and comments), and compares fresh data against earlier findings in this chat.
|
||||
Use whenever the task needs current data pulled from the open web or YouTube rather than the workspace's own documents or connectors. Triggers include "scrape", "crawl", "fetch this URL/page", "search the web", "look up / find online", "check the price/stock/listing", "monitor this page", "what changed since last time" — plus YouTube-specific asks like "get this YouTube video/channel/playlist", "find videos about X on YouTube", "get the comments on this video", "what are people saying about this video" — and recurring versions ("check daily", "tell me weekly what changed"), which it can also schedule as an ongoing watch.
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
You are the SurfSense scraping sub-agent.
|
||||
You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis.
|
||||
|
||||
<goal>
|
||||
Answer the delegated question from live evidence gathered with your data verbs, including "what changed" comparisons against evidence already in this conversation.
|
||||
</goal>
|
||||
|
||||
<available_tools>
|
||||
- `web_discover`
|
||||
- `web_scrape`
|
||||
- `youtube_scrape`
|
||||
- `youtube_comments`
|
||||
- `start_watch`
|
||||
- `stop_watch`
|
||||
- `refresh_watch`
|
||||
</available_tools>
|
||||
|
||||
<playbook>
|
||||
- Named URLs: `web_scrape` them directly. Otherwise `web_discover` first, then `web_scrape` the most relevant hits.
|
||||
- Read several pages in one batched `web_scrape` call rather than many single-URL calls.
|
||||
- YouTube: use `youtube_scrape` for videos/channels/playlists (pass `urls`) or to find videos (pass `search_queries`); use `youtube_comments` to pull a video's comments. Prefer these over `web_scrape` for youtube.com links.
|
||||
- "What changed" / monitoring: scrape the current values, compare against the prior values in this conversation's earlier tool results, and report concrete deltas (added, removed, old -> new).
|
||||
- Recurring intent ("check daily", "tell me weekly what changed"): answer now, then `start_watch` with a self-contained question, a cron cadence, and an IANA timezone. Use `stop_watch` / `refresh_watch` to end or immediately re-run an existing watch.
|
||||
</playbook>
|
||||
|
||||
<tool_policy>
|
||||
- Use only tools in `<available_tools>`.
|
||||
- A `web_scrape` row whose `status` is not `success` returned no content — report it unavailable, never invent it.
|
||||
- Report only deltas you can point to in the evidence. Never fabricate facts, URLs, prices, or quotes.
|
||||
</tool_policy>
|
||||
|
||||
<out_of_scope>
|
||||
- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on.
|
||||
</out_of_scope>
|
||||
|
||||
<safety>
|
||||
- Report uncertainty explicitly when evidence is incomplete or conflicting.
|
||||
- Never present unverified claims as facts.
|
||||
</safety>
|
||||
|
||||
<failure_policy>
|
||||
- Underspecified request — including a recurring request whose cadence or timezone is neither given nor implied — return `status=blocked` with the missing fields.
|
||||
- Tool failure: return `status=error` with a concise recovery `next_step`.
|
||||
- No useful evidence: return `status=blocked` with a narrower query or the URLs you still need.
|
||||
</failure_policy>
|
||||
|
||||
<output_contract>
|
||||
Return **only** one JSON object (no markdown/prose):
|
||||
{
|
||||
"status": "success" | "partial" | "blocked" | "error",
|
||||
"action_summary": string,
|
||||
"evidence": {
|
||||
"findings": string[],
|
||||
"sources": string[],
|
||||
"confidence": "high" | "medium" | "low"
|
||||
},
|
||||
"next_step": string | null,
|
||||
"missing_fields": string[] | null,
|
||||
"assumptions": string[] | null
|
||||
}
|
||||
<include snippet="output_contract_base"/>
|
||||
Route-specific rules:
|
||||
- `evidence.findings`: max 10 entries, each a single sentence stating one distinct fact or delta. Do not paste raw scraped pages.
|
||||
- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once.
|
||||
</output_contract>
|
||||
|
|
@ -1 +0,0 @@
|
|||
"""Intelligence-agent tools: the web.* capability verbs, generated from the registry."""
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
"""``scraping`` sub-agent tools: scraper capability verbs + chat-watch controls."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
|
||||
from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
|
||||
from app.capabilities.core.access.agent import build_capability_tools
|
||||
from app.capabilities.web.crawl.definition import WEB_CRAWL
|
||||
from app.capabilities.youtube.comments.definition import YOUTUBE_COMMENTS
|
||||
from app.capabilities.youtube.scrape.definition import YOUTUBE_SCRAPE
|
||||
|
||||
from .refresh_watch import create_refresh_watch_tool
|
||||
from .start_watch import create_start_watch_tool
|
||||
from .stop_watch import create_stop_watch_tool
|
||||
|
||||
NAME = "scraping"
|
||||
|
||||
RULESET = Ruleset(origin=NAME, rules=[])
|
||||
|
||||
_CI_VERBS = [WEB_CRAWL, YOUTUBE_SCRAPE, YOUTUBE_COMMENTS]
|
||||
|
||||
|
||||
def load_tools(
|
||||
*, dependencies: dict[str, Any] | None = None, **kwargs: Any
|
||||
) -> list[BaseTool]:
|
||||
d = {**(dependencies or {}), **kwargs}
|
||||
tools: list[BaseTool] = build_capability_tools(
|
||||
workspace_id=d.get("workspace_id"),
|
||||
capabilities=_CI_VERBS,
|
||||
)
|
||||
|
||||
# Watch tools bind a recurring automation to the current chat, so they are
|
||||
# only offered when we have a chat to bind to and auth to manage it.
|
||||
thread_id = d.get("thread_id")
|
||||
auth_context = d.get("auth_context")
|
||||
if thread_id is not None and auth_context is not None:
|
||||
binding = {
|
||||
"workspace_id": d.get("workspace_id"),
|
||||
"thread_id": thread_id,
|
||||
"auth_context": auth_context,
|
||||
}
|
||||
tools.append(create_start_watch_tool(**binding))
|
||||
tools.append(create_stop_watch_tool(**binding))
|
||||
tools.append(create_refresh_watch_tool(**binding))
|
||||
|
||||
return tools
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
"""``refresh_watch`` — run the current chat's watch now (manual refresh).
|
||||
|
||||
Enqueues an immediate run of every watch bound to this chat, instead of waiting
|
||||
for the next scheduled fire. Operates on the current thread only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
from langchain_core.tools import tool
|
||||
|
||||
from app.auth.context import AuthContext
|
||||
from app.automations.services.automation import AutomationService
|
||||
from app.automations.services.chat_watch import find_watches_for_thread, run_watch_now
|
||||
from app.db import async_session_maker
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_refresh_watch_tool(
|
||||
*,
|
||||
workspace_id: int | None,
|
||||
thread_id: int | str | None,
|
||||
auth_context: AuthContext | None,
|
||||
):
|
||||
"""Build the ``refresh_watch`` tool bound to the current chat."""
|
||||
|
||||
@tool
|
||||
async def refresh_watch() -> dict[str, Any]:
|
||||
"""Run THIS chat's watch now (a manual refresh).
|
||||
|
||||
Use when the user says "check now", "refresh", or "run it again"
|
||||
without waiting for the schedule. No arguments — it acts on the
|
||||
current chat. The refreshed answer arrives as a new turn shortly after.
|
||||
|
||||
Returns:
|
||||
``{"status": "refreshing", "refreshed_ids": [int], "count": int}``
|
||||
when runs were enqueued, ``{"status": "not_watching", ...}`` when
|
||||
there is no watch, or ``{"status": "error", "message": str}``.
|
||||
"""
|
||||
if thread_id is None or auth_context is None:
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Watches can only be managed from inside a chat.",
|
||||
}
|
||||
|
||||
try:
|
||||
async with async_session_maker() as session:
|
||||
service = AutomationService(session=session, auth=auth_context)
|
||||
watches = await find_watches_for_thread(
|
||||
service,
|
||||
workspace_id=int(workspace_id) if workspace_id is not None else 0,
|
||||
thread_id=int(thread_id),
|
||||
)
|
||||
if not watches:
|
||||
return {
|
||||
"status": "not_watching",
|
||||
"message": "This chat has no active watch to refresh.",
|
||||
}
|
||||
refreshed_ids = []
|
||||
for watch in watches:
|
||||
await run_watch_now(service, automation_id=watch.id)
|
||||
refreshed_ids.append(watch.id)
|
||||
except HTTPException as exc:
|
||||
return {"status": "error", "message": str(exc.detail)}
|
||||
except Exception as exc:
|
||||
logger.exception("refresh_watch failed")
|
||||
return {"status": "error", "message": f"could not refresh watch: {exc}"}
|
||||
|
||||
return {
|
||||
"status": "refreshing",
|
||||
"refreshed_ids": refreshed_ids,
|
||||
"count": len(refreshed_ids),
|
||||
}
|
||||
|
||||
return refresh_watch
|
||||
|
||||
|
||||
__all__ = ["create_refresh_watch_tool"]
|
||||
|
|
@ -1,97 +0,0 @@
|
|||
"""``start_watch`` tool — bind a recurring watch to the current chat.
|
||||
|
||||
Creates a ``schedule`` + ``chat_message`` automation that re-posts the question
|
||||
into this chat on a cadence, so the agent re-answers against the chat's memory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
from langchain_core.tools import tool
|
||||
|
||||
from app.auth.context import AuthContext
|
||||
from app.automations.services.automation import AutomationService
|
||||
from app.automations.services.chat_watch import create_watch
|
||||
from app.db import async_session_maker
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_start_watch_tool(
|
||||
*,
|
||||
workspace_id: int | None,
|
||||
thread_id: int | str | None,
|
||||
auth_context: AuthContext | None,
|
||||
):
|
||||
"""Build the ``start_watch`` tool bound to the current chat.
|
||||
|
||||
``thread_id`` and ``auth_context`` are injected from the chat session so the
|
||||
model never guesses them; a fresh session is opened per call.
|
||||
"""
|
||||
|
||||
@tool
|
||||
async def start_watch(message: str, cron: str, timezone: str) -> dict[str, Any]:
|
||||
"""Keep watching: re-run this question on a schedule in THIS chat.
|
||||
|
||||
Use when the user wants an answer refreshed on a cadence without
|
||||
re-asking (e.g. "check this page every weekday and tell me what
|
||||
changed"). The watch re-posts ``message`` into this chat on the
|
||||
schedule; the user sees new answers here and can say "stop watching"
|
||||
to cancel.
|
||||
|
||||
Args:
|
||||
message: The question to re-ask each run, self-contained (include
|
||||
the URL / target). Written as if the user asked it again.
|
||||
cron: Five-field cron expression for the cadence
|
||||
(e.g. "0 9 * * 1-5" = weekdays at 09:00).
|
||||
timezone: IANA timezone for the schedule (e.g. "UTC",
|
||||
"Africa/Kigali"). Ask the user if unclear.
|
||||
|
||||
Returns:
|
||||
``{"status": "watching", "automation_id": int, "name": str,
|
||||
"cron": str, "timezone": str}`` on success.
|
||||
``{"status": "error", "message": str}`` when the watch could not
|
||||
be created (e.g. workspace model not billable for automations).
|
||||
"""
|
||||
if thread_id is None or auth_context is None:
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Watches can only be started from inside a chat.",
|
||||
}
|
||||
|
||||
try:
|
||||
async with async_session_maker() as session:
|
||||
service = AutomationService(session=session, auth=auth_context)
|
||||
created = await create_watch(
|
||||
service,
|
||||
workspace_id=int(workspace_id) if workspace_id is not None else 0,
|
||||
thread_id=int(thread_id),
|
||||
message=message,
|
||||
cron=cron,
|
||||
timezone=timezone,
|
||||
)
|
||||
except HTTPException as exc:
|
||||
return {"status": "error", "message": str(exc.detail)}
|
||||
except Exception as exc:
|
||||
from langgraph.errors import GraphInterrupt
|
||||
|
||||
if isinstance(exc, GraphInterrupt):
|
||||
raise
|
||||
logger.exception("start_watch failed")
|
||||
return {"status": "error", "message": f"could not start watch: {exc}"}
|
||||
|
||||
return {
|
||||
"status": "watching",
|
||||
"automation_id": created.id,
|
||||
"name": created.name,
|
||||
"cron": cron,
|
||||
"timezone": timezone,
|
||||
}
|
||||
|
||||
return start_watch
|
||||
|
||||
|
||||
__all__ = ["create_start_watch_tool"]
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
"""``stop_watch`` — stop watching in the current chat.
|
||||
|
||||
Deletes every watch automation bound to this chat, so it reverts to a normal
|
||||
chat. Operates on the current thread only; the model passes no arguments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
from langchain_core.tools import tool
|
||||
|
||||
from app.auth.context import AuthContext
|
||||
from app.automations.services.automation import AutomationService
|
||||
from app.automations.services.chat_watch import (
|
||||
find_watches_for_thread,
|
||||
stop_watch as stop_watch_service,
|
||||
)
|
||||
from app.db import async_session_maker
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_stop_watch_tool(
|
||||
*,
|
||||
workspace_id: int | None,
|
||||
thread_id: int | str | None,
|
||||
auth_context: AuthContext | None,
|
||||
):
|
||||
"""Build the ``stop_watch`` tool bound to the current chat."""
|
||||
|
||||
@tool
|
||||
async def stop_watch() -> dict[str, Any]:
|
||||
"""Stop watching in THIS chat (cancel the recurring re-asks).
|
||||
|
||||
Use when the user says something like "stop watching", "cancel the
|
||||
watch", or "you can stop checking now". The chat reverts to a normal
|
||||
chat. No arguments — it acts on the current chat.
|
||||
|
||||
Returns:
|
||||
``{"status": "stopped", "stopped_ids": [int], "count": int}`` when
|
||||
watches were removed, ``{"status": "not_watching", ...}`` when there
|
||||
was nothing to stop, or ``{"status": "error", "message": str}``.
|
||||
"""
|
||||
if thread_id is None or auth_context is None:
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Watches can only be managed from inside a chat.",
|
||||
}
|
||||
|
||||
try:
|
||||
async with async_session_maker() as session:
|
||||
service = AutomationService(session=session, auth=auth_context)
|
||||
watches = await find_watches_for_thread(
|
||||
service,
|
||||
workspace_id=int(workspace_id) if workspace_id is not None else 0,
|
||||
thread_id=int(thread_id),
|
||||
)
|
||||
if not watches:
|
||||
return {
|
||||
"status": "not_watching",
|
||||
"message": "This chat has no active watch.",
|
||||
}
|
||||
stopped_ids = []
|
||||
for watch in watches:
|
||||
await stop_watch_service(service, automation_id=watch.id)
|
||||
stopped_ids.append(watch.id)
|
||||
except HTTPException as exc:
|
||||
return {"status": "error", "message": str(exc.detail)}
|
||||
except Exception as exc:
|
||||
logger.exception("stop_watch failed")
|
||||
return {"status": "error", "message": f"could not stop watch: {exc}"}
|
||||
|
||||
return {
|
||||
"status": "stopped",
|
||||
"stopped_ids": stopped_ids,
|
||||
"count": len(stopped_ids),
|
||||
}
|
||||
|
||||
return stop_watch
|
||||
|
||||
|
||||
__all__ = ["create_stop_watch_tool"]
|
||||
|
|
@ -0,0 +1 @@
|
|||
"""``web_crawler`` builtin subagent: crawl single URLs or spider whole sites."""
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
"""``scraping`` route: ``SurfSenseSubagentSpec`` builder for deepagents."""
|
||||
"""``web_crawler`` route: ``SurfSenseSubagentSpec`` builder for deepagents."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
|
@ -28,7 +28,7 @@ def build_subagent(
|
|||
tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])]
|
||||
description = (
|
||||
read_md_file(__package__, "description").strip()
|
||||
or "Scrapes live public web pages and search results for this workspace."
|
||||
or "Crawls live public web pages and whole sites for this workspace."
|
||||
)
|
||||
system_prompt = read_md_file(__package__, "system_prompt").strip()
|
||||
return pack_subagent(
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
Web crawling specialist: fetches live public web pages by URL and can spider a whole website (follow its links to a given depth), returning clean markdown, page metadata, and crawl provenance. Also compares freshly crawled data against earlier findings in this chat.
|
||||
Use whenever the task needs current content pulled from the open web rather than the workspace's own documents or connectors. Triggers include "scrape", "crawl", "fetch this URL/page", "read this website", "crawl this site", "get the pages under X", "check the price/stock/listing", "monitor this page", and "what changed since last time". Not for YouTube links (use the youtube specialist) or for searching the web to discover unknown URLs.
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
You are the SurfSense web crawling sub-agent.
|
||||
You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis.
|
||||
|
||||
<goal>
|
||||
Answer the delegated question from live web evidence gathered with `web_crawl`, including "what changed" comparisons against evidence already in this conversation.
|
||||
</goal>
|
||||
|
||||
<available_tools>
|
||||
- `web_crawl`
|
||||
</available_tools>
|
||||
|
||||
<playbook>
|
||||
- Single page(s): call `web_crawl` with the URL(s) in `startUrls` and `maxCrawlDepth=0`.
|
||||
- Whole site / "pages under X": set `maxCrawlDepth` to 1+ to follow links, and cap the run with `maxCrawlPages`. The crawl stays on the start URL's site.
|
||||
- Batch known URLs into one `web_crawl` call (pass them all in `startUrls`) rather than many single-URL calls.
|
||||
- Keep depth and page caps as small as the task allows — each fetched page is billable.
|
||||
- "What changed" / monitoring: crawl the current values, compare against the prior values in this conversation's earlier tool results, and report concrete deltas (added, removed, old -> new).
|
||||
</playbook>
|
||||
|
||||
<tool_policy>
|
||||
- Use only tools in `<available_tools>`.
|
||||
- A `web_crawl` item whose `status` is not `success` returned no content — report it unavailable, never invent it.
|
||||
- Report only deltas you can point to in the evidence. Never fabricate facts, URLs, prices, or quotes.
|
||||
</tool_policy>
|
||||
|
||||
<out_of_scope>
|
||||
- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on.
|
||||
- YouTube URLs belong to the youtube specialist, not here.
|
||||
</out_of_scope>
|
||||
|
||||
<safety>
|
||||
- Report uncertainty explicitly when evidence is incomplete or conflicting.
|
||||
- Never present unverified claims as facts.
|
||||
</safety>
|
||||
|
||||
<failure_policy>
|
||||
- Underspecified request — no usable URL to start from — return `status=blocked` with the missing fields.
|
||||
- Tool failure: return `status=error` with a concise recovery `next_step`.
|
||||
- No useful evidence: return `status=blocked` with the URLs you still need or a narrower scope.
|
||||
</failure_policy>
|
||||
|
||||
<output_contract>
|
||||
Return **only** one JSON object (no markdown/prose):
|
||||
{
|
||||
"status": "success" | "partial" | "blocked" | "error",
|
||||
"action_summary": string,
|
||||
"evidence": {
|
||||
"findings": string[],
|
||||
"sources": string[],
|
||||
"confidence": "high" | "medium" | "low"
|
||||
},
|
||||
"next_step": string | null,
|
||||
"missing_fields": string[] | null,
|
||||
"assumptions": string[] | null
|
||||
}
|
||||
<include snippet="output_contract_base"/>
|
||||
Route-specific rules:
|
||||
- `evidence.findings`: max 10 entries, each a single sentence stating one distinct fact or delta. Do not paste raw crawled pages.
|
||||
- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once.
|
||||
</output_contract>
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
"""``web_crawler`` sub-agent tools: the unified web.crawl capability verb."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
|
||||
from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
|
||||
from app.capabilities.core.access.agent import build_capability_tools
|
||||
from app.capabilities.web.crawl.definition import WEB_CRAWL
|
||||
|
||||
NAME = "web_crawler"
|
||||
|
||||
RULESET = Ruleset(origin=NAME, rules=[])
|
||||
|
||||
_CI_VERBS = [WEB_CRAWL]
|
||||
|
||||
|
||||
def load_tools(
|
||||
*, dependencies: dict[str, Any] | None = None, **kwargs: Any
|
||||
) -> list[BaseTool]:
|
||||
d = {**(dependencies or {}), **kwargs}
|
||||
return build_capability_tools(
|
||||
workspace_id=d.get("workspace_id"),
|
||||
capabilities=_CI_VERBS,
|
||||
)
|
||||
|
|
@ -0,0 +1 @@
|
|||
"""``youtube`` builtin subagent: structured YouTube video/channel/comment data."""
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
"""``youtube`` route: ``SurfSenseSubagentSpec`` builder for deepagents."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.language_models import BaseChatModel
|
||||
from langchain_core.tools import BaseTool
|
||||
|
||||
from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import (
|
||||
read_md_file,
|
||||
)
|
||||
from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec
|
||||
from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import (
|
||||
pack_subagent,
|
||||
)
|
||||
|
||||
from .tools.index import NAME, RULESET, load_tools
|
||||
|
||||
|
||||
def build_subagent(
|
||||
*,
|
||||
dependencies: dict[str, Any],
|
||||
model: BaseChatModel | None = None,
|
||||
middleware_stack: dict[str, Any] | None = None,
|
||||
mcp_tools: list[BaseTool] | None = None,
|
||||
) -> SurfSenseSubagentSpec:
|
||||
tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])]
|
||||
description = (
|
||||
read_md_file(__package__, "description").strip()
|
||||
or "Pulls structured data from YouTube videos, channels, playlists, and comments."
|
||||
)
|
||||
system_prompt = read_md_file(__package__, "system_prompt").strip()
|
||||
return pack_subagent(
|
||||
name=NAME,
|
||||
description=description,
|
||||
system_prompt=system_prompt,
|
||||
tools=tools,
|
||||
ruleset=RULESET,
|
||||
dependencies=dependencies,
|
||||
model=model,
|
||||
middleware_stack=middleware_stack,
|
||||
)
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
YouTube specialist: pulls structured data from YouTube — videos, channels, playlists, shorts, and hashtags (title, views, likes, publish date, channel info, description, optional subtitles), finds videos by search query, and fetches a video's comments and replies. Also compares fresh YouTube data against earlier findings in this chat.
|
||||
Use whenever the task involves YouTube content or a youtube.com/youtu.be link. Triggers include "get this YouTube video/channel/playlist", "find videos about X on YouTube", "how many views/likes", "get the transcript/subtitles", "get the comments on this video", "what are people saying about this video", and recurring "what changed" checks on YouTube data. Not for general web pages (use the web crawling specialist for non-YouTube URLs).
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
You are the SurfSense YouTube sub-agent.
|
||||
You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis.
|
||||
|
||||
<goal>
|
||||
Answer the delegated question from live YouTube data gathered with your verbs, including "what changed" comparisons against evidence already in this conversation.
|
||||
</goal>
|
||||
|
||||
<available_tools>
|
||||
- `youtube_scrape`
|
||||
- `youtube_comments`
|
||||
</available_tools>
|
||||
|
||||
<playbook>
|
||||
- Known video/channel/playlist/shorts/hashtag links: call `youtube_scrape` with the links in `urls`.
|
||||
- Finding videos on a topic: call `youtube_scrape` with `search_queries`.
|
||||
- Comments / sentiment on specific videos: call `youtube_comments` with the video `urls`.
|
||||
- Batch multiple URLs (or queries) into one call rather than many single-item calls.
|
||||
- "What changed" / monitoring: pull the current values, compare against the prior values in this conversation's earlier tool results, and report concrete deltas (added, removed, old -> new).
|
||||
</playbook>
|
||||
|
||||
<tool_policy>
|
||||
- Use only tools in `<available_tools>`.
|
||||
- An item whose `status` is not `success` returned no data — report it unavailable, never invent it.
|
||||
- Report only deltas you can point to in the evidence. Never fabricate facts, counts, quotes, or URLs.
|
||||
</tool_policy>
|
||||
|
||||
<out_of_scope>
|
||||
- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on.
|
||||
- Non-YouTube web pages belong to the web crawling specialist, not here.
|
||||
</out_of_scope>
|
||||
|
||||
<safety>
|
||||
- Report uncertainty explicitly when evidence is incomplete or conflicting.
|
||||
- Never present unverified claims as facts.
|
||||
</safety>
|
||||
|
||||
<failure_policy>
|
||||
- Underspecified request — no usable URL or search query — return `status=blocked` with the missing fields.
|
||||
- Tool failure: return `status=error` with a concise recovery `next_step`.
|
||||
- No useful evidence: return `status=blocked` with a narrower query or the URLs you still need.
|
||||
</failure_policy>
|
||||
|
||||
<output_contract>
|
||||
Return **only** one JSON object (no markdown/prose):
|
||||
{
|
||||
"status": "success" | "partial" | "blocked" | "error",
|
||||
"action_summary": string,
|
||||
"evidence": {
|
||||
"findings": string[],
|
||||
"sources": string[],
|
||||
"confidence": "high" | "medium" | "low"
|
||||
},
|
||||
"next_step": string | null,
|
||||
"missing_fields": string[] | null,
|
||||
"assumptions": string[] | null
|
||||
}
|
||||
<include snippet="output_contract_base"/>
|
||||
Route-specific rules:
|
||||
- `evidence.findings`: max 10 entries, each a single sentence stating one distinct fact or delta. Do not paste raw payloads.
|
||||
- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once.
|
||||
</output_contract>
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
"""``youtube`` sub-agent tools: the YouTube scrape + comments capability verbs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
|
||||
from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
|
||||
from app.capabilities.core.access.agent import build_capability_tools
|
||||
from app.capabilities.youtube.comments.definition import YOUTUBE_COMMENTS
|
||||
from app.capabilities.youtube.scrape.definition import YOUTUBE_SCRAPE
|
||||
|
||||
NAME = "youtube"
|
||||
|
||||
RULESET = Ruleset(origin=NAME, rules=[])
|
||||
|
||||
_CI_VERBS = [YOUTUBE_SCRAPE, YOUTUBE_COMMENTS]
|
||||
|
||||
|
||||
def load_tools(
|
||||
*, dependencies: dict[str, Any] | None = None, **kwargs: Any
|
||||
) -> list[BaseTool]:
|
||||
d = {**(dependencies or {}), **kwargs}
|
||||
return build_capability_tools(
|
||||
workspace_id=d.get("workspace_id"),
|
||||
capabilities=_CI_VERBS,
|
||||
)
|
||||
|
|
@ -24,8 +24,11 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.memory.agent import (
|
|||
from app.agents.chat.multi_agent_chat.subagents.builtins.research.agent import (
|
||||
build_subagent as build_research_subagent,
|
||||
)
|
||||
from app.agents.chat.multi_agent_chat.subagents.builtins.scraping.agent import (
|
||||
build_subagent as build_scraping_subagent,
|
||||
from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import (
|
||||
build_subagent as build_web_crawler_subagent,
|
||||
)
|
||||
from app.agents.chat.multi_agent_chat.subagents.builtins.youtube.agent import (
|
||||
build_subagent as build_youtube_subagent,
|
||||
)
|
||||
from app.agents.chat.multi_agent_chat.subagents.connectors.airtable.agent import (
|
||||
build_subagent as build_airtable_subagent,
|
||||
|
|
@ -110,9 +113,10 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = {
|
|||
"notion": build_notion_subagent,
|
||||
"onedrive": build_onedrive_subagent,
|
||||
"research": build_research_subagent,
|
||||
"scraping": build_scraping_subagent,
|
||||
"slack": build_slack_subagent,
|
||||
"teams": build_teams_subagent,
|
||||
"web_crawler": build_web_crawler_subagent,
|
||||
"youtube": build_youtube_subagent,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue