feat(capabilities): replace web.scrape/web.discover with unified web.crawl

web.crawl scrapes a single URL (maxCrawlDepth=0) or spiders a whole site,
backed by the proprietary site_crawler engine. Rewires the scraping subagent
tools and capability tests onto the new verb.

surfsense_backend/app/capabilities/web/__init__.py

@@ -1,6 +1,5 @@
-"""``web.*`` namespace: the generic web scraping + discovery verbs."""
+"""``web.*`` namespace: the unified web content crawler verb."""
 
 from __future__ import annotations
 
-from app.capabilities.web.discover import definition as _discover  # noqa: F401
-from app.capabilities.web.scrape import definition as _scrape  # noqa: F401
+from app.capabilities.web.crawl import definition as _crawl  # noqa: F401

surfsense_backend/app/capabilities/web/crawl/__init__.py

@@ -0,0 +1,3 @@
+"""``web.crawl`` verb: single-URL scrape or same-site spider → one row per page."""
+
+from __future__ import annotations

surfsense_backend/app/capabilities/web/crawl/definition.py

@@ -0,0 +1,25 @@
+"""``web.crawl`` capability registration."""
+
+from __future__ import annotations
+
+from app.capabilities.core import BillingUnit, Capability, register_capability
+from app.capabilities.web.crawl.executor import build_crawl_executor
+from app.capabilities.web.crawl.schemas import CrawlInput, CrawlOutput
+
+WEB_CRAWL = Capability(
+    name="web.crawl",
+    description=(
+        "Scrape a single web page or crawl a whole website. Give it one or more "
+        "startUrls. Set maxCrawlDepth=0 to fetch just those URLs, or higher to "
+        "also follow the links on each page (depth 1 = the start pages plus the "
+        "pages they link to, and so on) — staying on the same site and stopping "
+        "at maxCrawlPages. Returns one item per fetched page with clean markdown "
+        "content, metadata (title, description), and crawl provenance."
+    ),
+    input_schema=CrawlInput,
+    output_schema=CrawlOutput,
+    executor=build_crawl_executor(),
+    billing_unit=BillingUnit.WEB_CRAWL,
+)
+
+register_capability(WEB_CRAWL)

surfsense_backend/app/capabilities/web/crawl/executor.py

@@ -0,0 +1,65 @@
+"""``web.crawl`` executor: seeds → site spider → one cleaned item per fetched page.
+
+Boundary owned elsewhere: the crawl frontier/fetch live in the proprietary engine
+(``app.proprietary.web_crawler``). This executor only maps the engine's
+``CrawlPage`` list onto the typed ``CrawlOutput`` (status labels, truncation, and
+the captcha telemetry the billing seam reads).
+"""
+
+from __future__ import annotations
+
+from app.capabilities.core import Executor
+from app.capabilities.web.crawl.schemas import (
+    CrawlInput,
+    CrawlItem,
+    CrawlMeta,
+    CrawlOutput,
+)
+from app.proprietary.web_crawler import (
+    CrawlOutcomeStatus,
+    CrawlPage,
+    WebCrawlerConnector,
+    crawl_site,
+)
+
+_STATUS_LABEL: dict[CrawlOutcomeStatus, str] = {
+    CrawlOutcomeStatus.SUCCESS: "success",
+    CrawlOutcomeStatus.EMPTY: "empty",
+    CrawlOutcomeStatus.FAILED: "failed",
+}
+
+
+def build_crawl_executor(engine: WebCrawlerConnector | None = None) -> Executor:
+    """Build the ``web.crawl`` executor, optionally over an injected engine (tests)."""
+    crawler = engine or WebCrawlerConnector()
+
+    async def execute(payload: CrawlInput) -> CrawlOutput:
+        pages = await crawl_site(
+            crawler,
+            payload.startUrls,
+            max_crawl_depth=payload.maxCrawlDepth,
+            max_crawl_pages=payload.maxCrawlPages,
+        )
+        return CrawlOutput(
+            items=[_to_item(page, payload.maxLength) for page in pages],
+            captcha_attempts=sum(page.captcha_attempts for page in pages),
+            captcha_solved=sum(1 for page in pages if page.captcha_solved),
+        )
+
+    return execute
+
+
+def _to_item(page: CrawlPage, max_length: int) -> CrawlItem:
+    content = page.content[:max_length] if page.content is not None else None
+    return CrawlItem(
+        url=page.url,
+        status=_STATUS_LABEL[page.status],
+        crawl=CrawlMeta(
+            loadedUrl=page.loaded_url or page.url,
+            depth=page.depth,
+            referrerUrl=page.referrer,
+        ),
+        markdown=content,
+        metadata=page.metadata,
+        error=page.error,
+    )

surfsense_backend/app/capabilities/web/crawl/schemas.py

@@ -0,0 +1,115 @@
+# ruff: noqa: N815 - public field names intentionally mirror the Apify camelCase spec
+"""``web.crawl`` I/O contracts.
+
+An Apify *Website Content Crawler*-style surface (see apify.com/apify/website-
+content-crawler and docs.firecrawl.dev): one verb that either scrapes the given
+URLs (``maxCrawlDepth == 0``) or spiders their site (``maxCrawlDepth > 0``),
+bounded by ``maxCrawlPages`` and kept on the seed's site.
+
+Fields are trimmed to what the proprietary engine honors today. Apify knobs the
+engine handles automatically (crawler type, proxy, dynamic-render waits) are
+intentionally omitted, as are features we haven't built (URL globs, output
+formats, click actions, PII handling).
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+MAX_START_URLS = 20
+"""Per-call cap on seed URLs: bounds a synchronous request's fan-out (05)."""
+
+MAX_CRAWL_DEPTH = 5
+"""Deepest link distance a spider will follow from a start URL."""
+
+MAX_CRAWL_PAGES = 200
+"""Hard ceiling on pages fetched per call (protects the wallet and the run)."""
+
+
+class CrawlInput(BaseModel):
+    startUrls: list[str] = Field(
+        min_length=1,
+        max_length=MAX_START_URLS,
+        description=(
+            "Seed URLs to crawl. With maxCrawlDepth=0 only these are fetched; "
+            "with a higher depth they are also the entry points for the spider."
+        ),
+    )
+    maxCrawlDepth: int = Field(
+        default=0,
+        ge=0,
+        le=MAX_CRAWL_DEPTH,
+        description=(
+            "How many link-hops to follow from each start URL. 0 = scrape only "
+            "the start URLs (no spidering); 1 = also their linked pages; etc. "
+            "The spider stays on the start URL's site."
+        ),
+    )
+    maxCrawlPages: int = Field(
+        default=10,
+        ge=1,
+        le=MAX_CRAWL_PAGES,
+        description=(
+            "Maximum number of pages to fetch in total (start URLs included). "
+            "The crawl stops once this many pages have been fetched."
+        ),
+    )
+    maxLength: int = Field(
+        default=50_000,
+        ge=1,
+        description="Maximum characters of cleaned markdown kept per page (truncates beyond).",
+    )
+
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable pages for the pre-flight gate (03c)."""
+        if self.maxCrawlDepth == 0:
+            return len(self.startUrls)
+        return self.maxCrawlPages
+
+
+class CrawlMeta(BaseModel):
+    loadedUrl: str = Field(description="The URL actually fetched for this page.")
+    depth: int = Field(
+        description="Link distance from a start URL (0 for a start URL itself)."
+    )
+    referrerUrl: str | None = Field(
+        default=None,
+        description="The page this URL was discovered on (null for start URLs).",
+    )
+
+
+class CrawlItem(BaseModel):
+    url: str = Field(description="The requested URL for this page.")
+    status: Literal["success", "empty", "failed"] = Field(
+        description="success = content returned; empty = fetched but no content; failed = could not fetch."
+    )
+    crawl: CrawlMeta | None = Field(
+        default=None, description="Crawl provenance (loaded URL, depth, referrer)."
+    )
+    markdown: str | None = Field(
+        default=None, description="Cleaned page content as markdown (null unless success)."
+    )
+    metadata: dict[str, str] | None = Field(
+        default=None, description="Page metadata such as title and description."
+    )
+    error: str | None = Field(
+        default=None, description="Failure reason when status is not success."
+    )
+
+
+class CrawlOutput(BaseModel):
+    items: list[CrawlItem] = Field(
+        default_factory=list,
+        description="One item per fetched page, in crawl (BFS) order.",
+    )
+    # Billing-only telemetry; excluded from the wire shape (mirrors web.scrape).
+    captcha_attempts: int = Field(default=0, exclude=True)
+    captcha_solved: int = Field(default=0, exclude=True)
+
+    @property
+    def billable_units(self) -> int:
+        """Successful pages are the metered unit (03c)."""
+        return sum(1 for item in self.items if item.status == "success")

surfsense_backend/app/capabilities/web/discover/__init__.py

@@ -1,3 +0,0 @@
-"""``web.discover`` verb: a query → candidate URLs the agent can feed to ``web.scrape``."""
-
-from __future__ import annotations

surfsense_backend/app/capabilities/web/discover/definition.py

@@ -1,23 +0,0 @@
-"""``web.discover`` capability registration (free — see 04-capabilities open item)."""
-
-from __future__ import annotations
-
-from app.capabilities.core import Capability, register_capability
-from app.capabilities.web.discover.executor import build_discover_executor
-from app.capabilities.web.discover.schemas import DiscoverInput, DiscoverOutput
-
-WEB_DISCOVER = Capability(
-    name="web.discover",
-    description=(
-        "Search the web for a query and return ranked results. Use it to find "
-        "pages when you don't already have exact URLs. Returns a list of hits "
-        "(url, title, snippet, provider); pass the chosen url(s) to web.scrape "
-        "to read their full content."
-    ),
-    input_schema=DiscoverInput,
-    output_schema=DiscoverOutput,
-    executor=build_discover_executor(),
-    billing_unit=None,
-)
-
-register_capability(WEB_DISCOVER)

surfsense_backend/app/capabilities/web/discover/executor.py

@@ -1,42 +0,0 @@
-"""``web.discover`` executor: route a query to the first configured search provider."""
-
-from __future__ import annotations
-
-from collections.abc import Sequence
-
-from app.capabilities.core import Executor
-from app.capabilities.web.discover.providers.base import DiscoverProvider
-from app.capabilities.web.discover.schemas import DiscoverInput, DiscoverOutput
-
-
-class NoDiscoverProviderError(RuntimeError):
-    """Raised when no search provider is configured (no platform key/host set)."""
-
-
-def build_discover_executor(
-    providers: Sequence[DiscoverProvider] | None = None,
-) -> Executor:
-    """Bind the executor to a provider set (defaults to the real env-keyed providers)."""
-    registry = list(providers) if providers is not None else _default_providers()
-
-    async def execute(payload: DiscoverInput) -> DiscoverOutput:
-        provider = next((p for p in registry if p.is_available()), None)
-        if provider is None:
-            raise NoDiscoverProviderError(
-                "web.discover has no configured search provider "
-                "(set a SearXNG host or a Linkup/Baidu key)."
-            )
-        hits = await provider.search(payload.query, payload.top_k)
-        # Enforce the verb's documented cap here, once, for every provider:
-        # some backends (e.g. SearXNG) treat `top_k` as a hint and over-return.
-        return DiscoverOutput(hits=hits[: payload.top_k])
-
-    return execute
-
-
-def _default_providers() -> list[DiscoverProvider]:
-    from app.capabilities.web.discover.providers.baidu import BaiduProvider
-    from app.capabilities.web.discover.providers.linkup import LinkupProvider
-    from app.capabilities.web.discover.providers.searxng import SearxngProvider
-
-    return [SearxngProvider(), LinkupProvider(), BaiduProvider()]

surfsense_backend/app/capabilities/web/discover/providers/__init__.py

@@ -1,3 +0,0 @@
-"""``web.discover`` search providers (env-keyed: SearXNG, Linkup, Baidu)."""
-
-from __future__ import annotations

surfsense_backend/app/capabilities/web/discover/providers/baidu.py

@@ -1,53 +0,0 @@
-"""Baidu AI Search discover provider (env-keyed via ``BAIDU_API_KEY``)."""
-
-from __future__ import annotations
-
-import httpx
-
-from app.capabilities.web.discover.schemas import DiscoverHit
-from app.config import config
-
-_ENDPOINT = "https://qianfan.baidubce.com/v2/ai_search/chat/completions"
-_SNIPPET_MAX = 300
-
-
-class BaiduProvider:
-    name = "baidu"
-
-    def is_available(self) -> bool:
-        return bool(config.BAIDU_API_KEY)
-
-    async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
-        max_per_type = min(top_k, 20)
-        payload = {
-            "messages": [{"role": "user", "content": query}],
-            "model": config.BAIDU_MODEL,
-            "search_source": config.BAIDU_SEARCH_SOURCE,
-            "resource_type_filter": [{"type": "web", "top_k": max_per_type}],
-            "stream": False,
-        }
-        headers = {
-            "X-Appbuilder-Authorization": f"Bearer {config.BAIDU_API_KEY}",
-            "Content-Type": "application/json",
-        }
-        async with httpx.AsyncClient(timeout=90.0) as client:
-            response = await client.post(_ENDPOINT, headers=headers, json=payload)
-            response.raise_for_status()
-
-        hits: list[DiscoverHit] = []
-        for reference in response.json().get("references", []):
-            url = reference.get("url", "")
-            if not url:
-                continue
-            content = reference.get("content", "") or ""
-            hits.append(
-                DiscoverHit(
-                    url=url,
-                    title=reference.get("title") or url,
-                    snippet=content[:_SNIPPET_MAX] or None,
-                    provider=self.name,
-                )
-            )
-            if len(hits) >= top_k:
-                break
-        return hits

surfsense_backend/app/capabilities/web/discover/providers/base.py

@@ -1,22 +0,0 @@
-"""The seam every ``web.discover`` provider implements."""
-
-from __future__ import annotations
-
-from typing import Protocol, runtime_checkable
-
-from app.capabilities.web.discover.schemas import DiscoverHit
-
-
-@runtime_checkable
-class DiscoverProvider(Protocol):
-    """An env-keyed search backend that suggests candidate URLs for a query."""
-
-    name: str
-
-    def is_available(self) -> bool:
-        """True when this provider's platform key/host is configured."""
-        ...
-
-    async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
-        """Return up to ``top_k`` candidate hits for ``query``."""
-        ...

surfsense_backend/app/capabilities/web/discover/providers/linkup.py

@@ -1,40 +0,0 @@
-"""Linkup discover provider (env-keyed via ``LINKUP_API_KEY``)."""
-
-from __future__ import annotations
-
-import asyncio
-
-from linkup import LinkupClient
-
-from app.capabilities.web.discover.schemas import DiscoverHit
-from app.config import config
-
-
-class LinkupProvider:
-    name = "linkup"
-
-    def is_available(self) -> bool:
-        return bool(config.LINKUP_API_KEY)
-
-    async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
-        client = LinkupClient(api_key=config.LINKUP_API_KEY)
-        response = await asyncio.to_thread(
-            client.search, query=query, depth="standard", output_type="searchResults"
-        )
-        hits: list[DiscoverHit] = []
-        for result in getattr(response, "results", None) or []:
-            url = getattr(result, "url", "") or ""
-            if not url:
-                continue
-            content = getattr(result, "content", "") or ""
-            hits.append(
-                DiscoverHit(
-                    url=url,
-                    title=getattr(result, "name", "") or url,
-                    snippet=content or None,
-                    provider=self.name,
-                )
-            )
-            if len(hits) >= top_k:
-                break
-        return hits

surfsense_backend/app/capabilities/web/discover/providers/searxng.py

@@ -1,28 +0,0 @@
-"""SearXNG discover provider, wrapping the platform-env SearXNG search service."""
-
-from __future__ import annotations
-
-from app.capabilities.web.discover.schemas import DiscoverHit
-from app.services import web_search_service
-
-
-class SearxngProvider:
-    """Env-keyed via ``SEARXNG_DEFAULT_HOST`` (the platform SearXNG instance)."""
-
-    name = "searxng"
-
-    def is_available(self) -> bool:
-        return web_search_service.is_available()
-
-    async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
-        result_object, _documents = await web_search_service.search(query, top_k)
-        return [
-            DiscoverHit(
-                url=source["url"],
-                title=source.get("title") or source["url"],
-                snippet=source.get("description") or None,
-                provider=self.name,
-            )
-            for source in result_object.get("sources", [])
-            if source.get("url")
-        ]

surfsense_backend/app/capabilities/web/discover/schemas.py

@@ -1,29 +0,0 @@
-"""``web.discover`` I/O contracts."""
-
-from __future__ import annotations
-
-from pydantic import BaseModel, Field
-
-
-class DiscoverInput(BaseModel):
-    query: str = Field(
-        description="What to search the web for, phrased in natural language."
-    )
-    top_k: int = Field(
-        default=10, ge=1, le=50, description="Maximum number of results to return (1-50)."
-    )
-
-
-class DiscoverHit(BaseModel):
-    url: str = Field(
-        description="The result's page URL; pass it to web.scrape to read it."
-    )
-    title: str = Field(description="The result's page title.")
-    snippet: str | None = Field(
-        default=None, description="A short extract summarizing the page."
-    )
-    provider: str = Field(description="Which search engine returned this hit.")
-
-
-class DiscoverOutput(BaseModel):
-    hits: list[DiscoverHit] = Field(description="Ranked search results, best first.")

surfsense_backend/app/capabilities/web/scrape/__init__.py

@@ -1,3 +0,0 @@
-"""``web.scrape`` verb: a URL array → one cleaned row per URL."""
-
-from __future__ import annotations

surfsense_backend/app/capabilities/web/scrape/definition.py

@@ -1,23 +0,0 @@
-"""``web.scrape`` capability registration."""
-
-from __future__ import annotations
-
-from app.capabilities.core import BillingUnit, Capability, register_capability
-from app.capabilities.web.scrape.executor import build_scrape_executor
-from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput
-
-WEB_SCRAPE = Capability(
-    name="web.scrape",
-    description=(
-        "Fetch one or more web pages and return their clean, readable content. "
-        "Give it the exact URLs to read (use web.discover first if you don't have "
-        "them). Returns one row per URL with a status (success/empty/failed), the "
-        "page content, and metadata such as title and description."
-    ),
-    input_schema=ScrapeInput,
-    output_schema=ScrapeOutput,
-    executor=build_scrape_executor(),
-    billing_unit=BillingUnit.WEB_CRAWL,
-)
-
-register_capability(WEB_SCRAPE)

surfsense_backend/app/capabilities/web/scrape/executor.py

@@ -1,45 +0,0 @@
-"""``web.scrape`` executor: fetch each URL in the array → cleaned rows."""
-
-from __future__ import annotations
-
-from app.capabilities.core import Executor
-from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput, ScrapeRow
-from app.proprietary.web_crawler import (
-    CrawlOutcome,
-    CrawlOutcomeStatus,
-    WebCrawlerConnector,
-)
-
-
-def build_scrape_executor(engine: WebCrawlerConnector | None = None) -> Executor:
-    """Bind the executor to a fetch engine (defaults to the proprietary crawler)."""
-    engine = engine or WebCrawlerConnector()
-
-    async def execute(payload: ScrapeInput) -> ScrapeOutput:
-        outcomes = [await engine.crawl_url(url) for url in payload.urls]
-        rows = [
-            _to_row(url, outcome, payload.max_length)
-            for url, outcome in zip(payload.urls, outcomes, strict=True)
-        ]
-        return ScrapeOutput(
-            rows=rows,
-            captcha_attempts=sum(o.captcha_attempts for o in outcomes),
-            captcha_solved=sum(1 for o in outcomes if o.captcha_solved),
-        )
-
-    return execute
-
-
-def _to_row(url: str, outcome: CrawlOutcome, max_length: int) -> ScrapeRow:
-    if outcome.status is CrawlOutcomeStatus.SUCCESS and outcome.result:
-        content = outcome.result.get("content")
-        if content is not None:
-            content = content[:max_length]
-        return ScrapeRow(
-            url=url,
-            status="success",
-            content=content,
-            metadata=outcome.result.get("metadata"),
-        )
-    status = "empty" if outcome.status is CrawlOutcomeStatus.EMPTY else "failed"
-    return ScrapeRow(url=url, status=status, error=outcome.error)

surfsense_backend/app/capabilities/web/scrape/schemas.py

@@ -1,67 +0,0 @@
-"""``web.scrape`` I/O contracts."""
-
-from __future__ import annotations
-
-from typing import Literal
-
-from pydantic import BaseModel, Field
-
-MAX_SCRAPE_URLS = 20
-"""Per-call batch cap: bounds a synchronous request's crawl fan-out (05)."""
-
-
-class ScrapeInput(BaseModel):
-    urls: list[str] = Field(
-        min_length=1,
-        max_length=MAX_SCRAPE_URLS,
-        description=(
-            "Full page URLs to fetch and read (1-20), each starting with "
-            "http:// or https://. Pass the exact URLs you want the content of."
-        ),
-    )
-    max_length: int = Field(
-        default=50_000,
-        description=(
-            "Maximum characters of cleaned content returned per page; "
-            "content longer than this is truncated."
-        ),
-    )
-
-    @property
-    def estimated_units(self) -> int:
-        """Worst-case billable crawls for pre-flight: one per requested URL."""
-        return len(self.urls)
-
-
-class ScrapeRow(BaseModel):
-    url: str = Field(description="The requested URL this result is for.")
-    status: Literal["success", "empty", "failed"] = Field(
-        description=(
-            "'success' = content returned; 'empty' = page reached but no "
-            "readable content; 'failed' = could not be fetched (see error)."
-        )
-    )
-    content: str | None = Field(
-        default=None, description="Cleaned, readable page text (present on success)."
-    )
-    metadata: dict[str, str] | None = Field(
-        default=None, description="Page metadata such as title and description."
-    )
-    error: str | None = Field(
-        default=None, description="Why the fetch failed (present on 'failed')."
-    )
-
-
-class ScrapeOutput(BaseModel):
-    rows: list[ScrapeRow] = Field(
-        description="One result per requested URL, in the same order."
-    )
-    # Billing-only telemetry (Phase 3d), excluded from the client-facing result:
-    # captcha solves are metered per attempt, independent of crawl success.
-    captcha_attempts: int = Field(default=0, exclude=True)
-    captcha_solved: int = Field(default=0, exclude=True)
-
-    @property
-    def billable_units(self) -> int:
-        """One billable unit per successful scrape."""
-        return sum(1 for row in self.rows if row.status == "success")