mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
c600a2920b
Crawler engine: escalate thin JS-shell pages past static fetch, repair currency-lossy extractions, emit categorized link records with anchor provenance, and decode percent-encoded mailto:/tel: contacts; site crawls reuse the connector ladder via Scrapling's spider engine with URL pattern filters. Agent layer: read_run gains char_offset paging, search_run gains match excerpts, new export_run turns stored runs into CSV workspace docs; reddit search fair-shares the item budget across queries and dedupes cross-query hits. Subagent prompts and routing teach crawl-after-search, full-run coverage before summarizing, and executing own-tool next steps instead of returning partial.
37 lines
1.9 KiB
Python
37 lines
1.9 KiB
Python
"""``web.crawl`` capability registration."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from app.capabilities.core import BillingUnit, Capability, register_capability
|
|
from app.capabilities.web.crawl.executor import build_crawl_executor
|
|
from app.capabilities.web.crawl.schemas import CrawlInput, CrawlOutput
|
|
|
|
WEB_CRAWL = Capability(
|
|
name="web.crawl",
|
|
description=(
|
|
"Scrape a single web page or crawl a whole website. Give it one or more "
|
|
"startUrls. Set maxCrawlDepth=0 to fetch just those URLs, or higher to "
|
|
"also follow the links on each page (depth 1 = the start pages plus the "
|
|
"pages they link to, and so on) — staying on the same site and stopping "
|
|
"at maxCrawlPages. On a deeper crawl, narrow which links are followed with "
|
|
"includeUrlPatterns / excludeUrlPatterns (regexes). Returns one item per "
|
|
"fetched page with clean markdown content, metadata (title, description), "
|
|
"crawl provenance, every link with its anchor text and kind "
|
|
"(internal/external/social/email/tel — use the text/context to tie a "
|
|
"profile URL to a person or company), and contact signals (emails, phone "
|
|
"numbers, social profiles). The site-wide contacts summary deduplicates "
|
|
"them with provenance: siteWide=true marks footer/header values (the "
|
|
"company's own contacts) vs page-local finds (e.g. team members' "
|
|
"profiles). Useful for lead generation and competitive intelligence; "
|
|
"contact details often live on about/contact/privacy pages, so crawl "
|
|
"with maxCrawlDepth >= 1 to surface them. JS-rendered pages are loaded "
|
|
"in a real browser and auto-scrolled, so lazy-loaded listings "
|
|
"(directories, infinite-scroll feeds) are captured too."
|
|
),
|
|
input_schema=CrawlInput,
|
|
output_schema=CrawlOutput,
|
|
executor=build_crawl_executor(),
|
|
billing_unit=BillingUnit.WEB_CRAWL,
|
|
)
|
|
|
|
register_capability(WEB_CRAWL)
|