apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-10 22:32:16 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(capabilities): replace web.scrape/web.discover with unified web.crawl

Browse source 
web.crawl scrapes a single URL (maxCrawlDepth=0) or spiders a whole site,
backed by the proprietary site_crawler engine. Rewires the scraping subagent
tools and capability tests onto the new verb.
This commit is contained in:
CREDO23 2026-07-03 10:51:05 +02:00
parent f82fae3973
commit 9fe9c5b71d
31 changed files with 411 additions and 950 deletions
Download patch file Download diff file Expand all files Collapse all files

5
surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/tools/index.py
View file

@ -8,8 +8,7 @@ from langchain_core.tools import BaseTool
from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
from app.capabilities.core.access.agent import build_capability_tools
from app.capabilities.web.discover.definition import WEB_DISCOVER
from app.capabilities.web.scrape.definition import WEB_SCRAPE
from app.capabilities.web.crawl.definition import WEB_CRAWL
from app.capabilities.youtube.comments.definition import YOUTUBE_COMMENTS
from app.capabilities.youtube.scrape.definition import YOUTUBE_SCRAPE
@ -21,7 +20,7 @@ NAME = "scraping"
RULESET = Ruleset(origin=NAME, rules=[])
_CI_VERBS = [WEB_DISCOVER, WEB_SCRAPE, YOUTUBE_SCRAPE, YOUTUBE_COMMENTS]
_CI_VERBS = [WEB_CRAWL, YOUTUBE_SCRAPE, YOUTUBE_COMMENTS]
def load_tools(