feat(capabilities): replace web.scrape/web.discover with unified web.crawl

web.crawl scrapes a single URL (maxCrawlDepth=0) or spiders a whole site,
backed by the proprietary site_crawler engine. Rewires the scraping subagent
tools and capability tests onto the new verb.

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/scraping/tools/index.py

@@ -8,8 +8,7 @@ from langchain_core.tools import BaseTool
 
 from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
 from app.capabilities.core.access.agent import build_capability_tools
-from app.capabilities.web.discover.definition import WEB_DISCOVER
+from app.capabilities.web.crawl.definition import WEB_CRAWL
-from app.capabilities.web.scrape.definition import WEB_SCRAPE
 from app.capabilities.youtube.comments.definition import YOUTUBE_COMMENTS
 from app.capabilities.youtube.scrape.definition import YOUTUBE_SCRAPE
 
@@ -21,7 +20,7 @@ NAME = "scraping"
 
 RULESET = Ruleset(origin=NAME, rules=[])
 
-_CI_VERBS = [WEB_DISCOVER, WEB_SCRAPE, YOUTUBE_SCRAPE, YOUTUBE_COMMENTS]
+_CI_VERBS = [WEB_CRAWL, YOUTUBE_SCRAPE, YOUTUBE_COMMENTS]
 
 
 def load_tools(

surfsense_backend/app/capabilities/web/__init__.py

@@ -1,6 +1,5 @@
-"""``web.*`` namespace: the generic web scraping + discovery verbs."""
+"""``web.*`` namespace: the unified web content crawler verb."""
 
 from __future__ import annotations
 
-from app.capabilities.web.discover import definition as _discover  # noqa: F401
+from app.capabilities.web.crawl import definition as _crawl  # noqa: F401
-from app.capabilities.web.scrape import definition as _scrape  # noqa: F401

surfsense_backend/app/capabilities/web/crawl/__init__.py

@@ -0,0 +1,3 @@
+"""``web.crawl`` verb: single-URL scrape or same-site spider → one row per page."""
+
+from __future__ import annotations

surfsense_backend/app/capabilities/web/crawl/definition.py

@@ -0,0 +1,25 @@
+"""``web.crawl`` capability registration."""
+
+from __future__ import annotations
+
+from app.capabilities.core import BillingUnit, Capability, register_capability
+from app.capabilities.web.crawl.executor import build_crawl_executor
+from app.capabilities.web.crawl.schemas import CrawlInput, CrawlOutput
+
+WEB_CRAWL = Capability(
+    name="web.crawl",
+    description=(
+        "Scrape a single web page or crawl a whole website. Give it one or more "
+        "startUrls. Set maxCrawlDepth=0 to fetch just those URLs, or higher to "
+        "also follow the links on each page (depth 1 = the start pages plus the "
+        "pages they link to, and so on) — staying on the same site and stopping "
+        "at maxCrawlPages. Returns one item per fetched page with clean markdown "
+        "content, metadata (title, description), and crawl provenance."
+    ),
+    input_schema=CrawlInput,
+    output_schema=CrawlOutput,
+    executor=build_crawl_executor(),
+    billing_unit=BillingUnit.WEB_CRAWL,
+)
+
+register_capability(WEB_CRAWL)

surfsense_backend/app/capabilities/web/crawl/executor.py

@@ -0,0 +1,65 @@
+"""``web.crawl`` executor: seeds → site spider → one cleaned item per fetched page.
+
+Boundary owned elsewhere: the crawl frontier/fetch live in the proprietary engine
+(``app.proprietary.web_crawler``). This executor only maps the engine's
+``CrawlPage`` list onto the typed ``CrawlOutput`` (status labels, truncation, and
+the captcha telemetry the billing seam reads).
+"""
+
+from __future__ import annotations
+
+from app.capabilities.core import Executor
+from app.capabilities.web.crawl.schemas import (
+    CrawlInput,
+    CrawlItem,
+    CrawlMeta,
+    CrawlOutput,
+)
+from app.proprietary.web_crawler import (
+    CrawlOutcomeStatus,
+    CrawlPage,
+    WebCrawlerConnector,
+    crawl_site,
+)
+
+_STATUS_LABEL: dict[CrawlOutcomeStatus, str] = {
+    CrawlOutcomeStatus.SUCCESS: "success",
+    CrawlOutcomeStatus.EMPTY: "empty",
+    CrawlOutcomeStatus.FAILED: "failed",
+}
+
+
+def build_crawl_executor(engine: WebCrawlerConnector | None = None) -> Executor:
+    """Build the ``web.crawl`` executor, optionally over an injected engine (tests)."""
+    crawler = engine or WebCrawlerConnector()
+
+    async def execute(payload: CrawlInput) -> CrawlOutput:
+        pages = await crawl_site(
+            crawler,
+            payload.startUrls,
+            max_crawl_depth=payload.maxCrawlDepth,
+            max_crawl_pages=payload.maxCrawlPages,
+        )
+        return CrawlOutput(
+            items=[_to_item(page, payload.maxLength) for page in pages],
+            captcha_attempts=sum(page.captcha_attempts for page in pages),
+            captcha_solved=sum(1 for page in pages if page.captcha_solved),
+        )
+
+    return execute
+
+
+def _to_item(page: CrawlPage, max_length: int) -> CrawlItem:
+    content = page.content[:max_length] if page.content is not None else None
+    return CrawlItem(
+        url=page.url,
+        status=_STATUS_LABEL[page.status],
+        crawl=CrawlMeta(
+            loadedUrl=page.loaded_url or page.url,
+            depth=page.depth,
+            referrerUrl=page.referrer,
+        ),
+        markdown=content,
+        metadata=page.metadata,
+        error=page.error,
+    )

surfsense_backend/app/capabilities/web/crawl/schemas.py

@@ -0,0 +1,115 @@
+# ruff: noqa: N815 - public field names intentionally mirror the Apify camelCase spec
+"""``web.crawl`` I/O contracts.
+
+An Apify *Website Content Crawler*-style surface (see apify.com/apify/website-
+content-crawler and docs.firecrawl.dev): one verb that either scrapes the given
+URLs (``maxCrawlDepth == 0``) or spiders their site (``maxCrawlDepth > 0``),
+bounded by ``maxCrawlPages`` and kept on the seed's site.
+
+Fields are trimmed to what the proprietary engine honors today. Apify knobs the
+engine handles automatically (crawler type, proxy, dynamic-render waits) are
+intentionally omitted, as are features we haven't built (URL globs, output
+formats, click actions, PII handling).
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+MAX_START_URLS = 20
+"""Per-call cap on seed URLs: bounds a synchronous request's fan-out (05)."""
+
+MAX_CRAWL_DEPTH = 5
+"""Deepest link distance a spider will follow from a start URL."""
+
+MAX_CRAWL_PAGES = 200
+"""Hard ceiling on pages fetched per call (protects the wallet and the run)."""
+
+
+class CrawlInput(BaseModel):
+    startUrls: list[str] = Field(
+        min_length=1,
+        max_length=MAX_START_URLS,
+        description=(
+            "Seed URLs to crawl. With maxCrawlDepth=0 only these are fetched; "
+            "with a higher depth they are also the entry points for the spider."
+        ),
+    )
+    maxCrawlDepth: int = Field(
+        default=0,
+        ge=0,
+        le=MAX_CRAWL_DEPTH,
+        description=(
+            "How many link-hops to follow from each start URL. 0 = scrape only "
+            "the start URLs (no spidering); 1 = also their linked pages; etc. "
+            "The spider stays on the start URL's site."
+        ),
+    )
+    maxCrawlPages: int = Field(
+        default=10,
+        ge=1,
+        le=MAX_CRAWL_PAGES,
+        description=(
+            "Maximum number of pages to fetch in total (start URLs included). "
+            "The crawl stops once this many pages have been fetched."
+        ),
+    )
+    maxLength: int = Field(
+        default=50_000,
+        ge=1,
+        description="Maximum characters of cleaned markdown kept per page (truncates beyond).",
+    )
+
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable pages for the pre-flight gate (03c)."""
+        if self.maxCrawlDepth == 0:
+            return len(self.startUrls)
+        return self.maxCrawlPages
+
+
+class CrawlMeta(BaseModel):
+    loadedUrl: str = Field(description="The URL actually fetched for this page.")
+    depth: int = Field(
+        description="Link distance from a start URL (0 for a start URL itself)."
+    )
+    referrerUrl: str | None = Field(
+        default=None,
+        description="The page this URL was discovered on (null for start URLs).",
+    )
+
+
+class CrawlItem(BaseModel):
+    url: str = Field(description="The requested URL for this page.")
+    status: Literal["success", "empty", "failed"] = Field(
+        description="success = content returned; empty = fetched but no content; failed = could not fetch."
+    )
+    crawl: CrawlMeta | None = Field(
+        default=None, description="Crawl provenance (loaded URL, depth, referrer)."
+    )
+    markdown: str | None = Field(
+        default=None, description="Cleaned page content as markdown (null unless success)."
+    )
+    metadata: dict[str, str] | None = Field(
+        default=None, description="Page metadata such as title and description."
+    )
+    error: str | None = Field(
+        default=None, description="Failure reason when status is not success."
+    )
+
+
+class CrawlOutput(BaseModel):
+    items: list[CrawlItem] = Field(
+        default_factory=list,
+        description="One item per fetched page, in crawl (BFS) order.",
+    )
+    # Billing-only telemetry; excluded from the wire shape (mirrors web.scrape).
+    captcha_attempts: int = Field(default=0, exclude=True)
+    captcha_solved: int = Field(default=0, exclude=True)
+
+    @property
+    def billable_units(self) -> int:
+        """Successful pages are the metered unit (03c)."""
+        return sum(1 for item in self.items if item.status == "success")

surfsense_backend/app/capabilities/web/discover/__init__.py

@@ -1,3 +0,0 @@
-"""``web.discover`` verb: a query → candidate URLs the agent can feed to ``web.scrape``."""
-
-from __future__ import annotations

surfsense_backend/app/capabilities/web/discover/definition.py

@@ -1,23 +0,0 @@
-"""``web.discover`` capability registration (free — see 04-capabilities open item)."""
-
-from __future__ import annotations
-
-from app.capabilities.core import Capability, register_capability
-from app.capabilities.web.discover.executor import build_discover_executor
-from app.capabilities.web.discover.schemas import DiscoverInput, DiscoverOutput
-
-WEB_DISCOVER = Capability(
-    name="web.discover",
-    description=(
-        "Search the web for a query and return ranked results. Use it to find "
-        "pages when you don't already have exact URLs. Returns a list of hits "
-        "(url, title, snippet, provider); pass the chosen url(s) to web.scrape "
-        "to read their full content."
-    ),
-    input_schema=DiscoverInput,
-    output_schema=DiscoverOutput,
-    executor=build_discover_executor(),
-    billing_unit=None,
-)
-
-register_capability(WEB_DISCOVER)

surfsense_backend/app/capabilities/web/discover/executor.py

@@ -1,42 +0,0 @@
-"""``web.discover`` executor: route a query to the first configured search provider."""
-
-from __future__ import annotations
-
-from collections.abc import Sequence
-
-from app.capabilities.core import Executor
-from app.capabilities.web.discover.providers.base import DiscoverProvider
-from app.capabilities.web.discover.schemas import DiscoverInput, DiscoverOutput
-
-
-class NoDiscoverProviderError(RuntimeError):
-    """Raised when no search provider is configured (no platform key/host set)."""
-
-
-def build_discover_executor(
-    providers: Sequence[DiscoverProvider] | None = None,
-) -> Executor:
-    """Bind the executor to a provider set (defaults to the real env-keyed providers)."""
-    registry = list(providers) if providers is not None else _default_providers()
-
-    async def execute(payload: DiscoverInput) -> DiscoverOutput:
-        provider = next((p for p in registry if p.is_available()), None)
-        if provider is None:
-            raise NoDiscoverProviderError(
-                "web.discover has no configured search provider "
-                "(set a SearXNG host or a Linkup/Baidu key)."
-            )
-        hits = await provider.search(payload.query, payload.top_k)
-        # Enforce the verb's documented cap here, once, for every provider:
-        # some backends (e.g. SearXNG) treat `top_k` as a hint and over-return.
-        return DiscoverOutput(hits=hits[: payload.top_k])
-
-    return execute
-
-
-def _default_providers() -> list[DiscoverProvider]:
-    from app.capabilities.web.discover.providers.baidu import BaiduProvider
-    from app.capabilities.web.discover.providers.linkup import LinkupProvider
-    from app.capabilities.web.discover.providers.searxng import SearxngProvider
-
-    return [SearxngProvider(), LinkupProvider(), BaiduProvider()]

surfsense_backend/app/capabilities/web/discover/providers/__init__.py

@@ -1,3 +0,0 @@
-"""``web.discover`` search providers (env-keyed: SearXNG, Linkup, Baidu)."""
-
-from __future__ import annotations

surfsense_backend/app/capabilities/web/discover/providers/baidu.py

@@ -1,53 +0,0 @@
-"""Baidu AI Search discover provider (env-keyed via ``BAIDU_API_KEY``)."""
-
-from __future__ import annotations
-
-import httpx
-
-from app.capabilities.web.discover.schemas import DiscoverHit
-from app.config import config
-
-_ENDPOINT = "https://qianfan.baidubce.com/v2/ai_search/chat/completions"
-_SNIPPET_MAX = 300
-
-
-class BaiduProvider:
-    name = "baidu"
-
-    def is_available(self) -> bool:
-        return bool(config.BAIDU_API_KEY)
-
-    async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
-        max_per_type = min(top_k, 20)
-        payload = {
-            "messages": [{"role": "user", "content": query}],
-            "model": config.BAIDU_MODEL,
-            "search_source": config.BAIDU_SEARCH_SOURCE,
-            "resource_type_filter": [{"type": "web", "top_k": max_per_type}],
-            "stream": False,
-        }
-        headers = {
-            "X-Appbuilder-Authorization": f"Bearer {config.BAIDU_API_KEY}",
-            "Content-Type": "application/json",
-        }
-        async with httpx.AsyncClient(timeout=90.0) as client:
-            response = await client.post(_ENDPOINT, headers=headers, json=payload)
-            response.raise_for_status()
-
-        hits: list[DiscoverHit] = []
-        for reference in response.json().get("references", []):
-            url = reference.get("url", "")
-            if not url:
-                continue
-            content = reference.get("content", "") or ""
-            hits.append(
-                DiscoverHit(
-                    url=url,
-                    title=reference.get("title") or url,
-                    snippet=content[:_SNIPPET_MAX] or None,
-                    provider=self.name,
-                )
-            )
-            if len(hits) >= top_k:
-                break
-        return hits

surfsense_backend/app/capabilities/web/discover/providers/base.py

@@ -1,22 +0,0 @@
-"""The seam every ``web.discover`` provider implements."""
-
-from __future__ import annotations
-
-from typing import Protocol, runtime_checkable
-
-from app.capabilities.web.discover.schemas import DiscoverHit
-
-
-@runtime_checkable
-class DiscoverProvider(Protocol):
-    """An env-keyed search backend that suggests candidate URLs for a query."""
-
-    name: str
-
-    def is_available(self) -> bool:
-        """True when this provider's platform key/host is configured."""
-        ...
-
-    async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
-        """Return up to ``top_k`` candidate hits for ``query``."""
-        ...

surfsense_backend/app/capabilities/web/discover/providers/linkup.py

@@ -1,40 +0,0 @@
-"""Linkup discover provider (env-keyed via ``LINKUP_API_KEY``)."""
-
-from __future__ import annotations
-
-import asyncio
-
-from linkup import LinkupClient
-
-from app.capabilities.web.discover.schemas import DiscoverHit
-from app.config import config
-
-
-class LinkupProvider:
-    name = "linkup"
-
-    def is_available(self) -> bool:
-        return bool(config.LINKUP_API_KEY)
-
-    async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
-        client = LinkupClient(api_key=config.LINKUP_API_KEY)
-        response = await asyncio.to_thread(
-            client.search, query=query, depth="standard", output_type="searchResults"
-        )
-        hits: list[DiscoverHit] = []
-        for result in getattr(response, "results", None) or []:
-            url = getattr(result, "url", "") or ""
-            if not url:
-                continue
-            content = getattr(result, "content", "") or ""
-            hits.append(
-                DiscoverHit(
-                    url=url,
-                    title=getattr(result, "name", "") or url,
-                    snippet=content or None,
-                    provider=self.name,
-                )
-            )
-            if len(hits) >= top_k:
-                break
-        return hits

surfsense_backend/app/capabilities/web/discover/providers/searxng.py

@@ -1,28 +0,0 @@
-"""SearXNG discover provider, wrapping the platform-env SearXNG search service."""
-
-from __future__ import annotations
-
-from app.capabilities.web.discover.schemas import DiscoverHit
-from app.services import web_search_service
-
-
-class SearxngProvider:
-    """Env-keyed via ``SEARXNG_DEFAULT_HOST`` (the platform SearXNG instance)."""
-
-    name = "searxng"
-
-    def is_available(self) -> bool:
-        return web_search_service.is_available()
-
-    async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
-        result_object, _documents = await web_search_service.search(query, top_k)
-        return [
-            DiscoverHit(
-                url=source["url"],
-                title=source.get("title") or source["url"],
-                snippet=source.get("description") or None,
-                provider=self.name,
-            )
-            for source in result_object.get("sources", [])
-            if source.get("url")
-        ]

surfsense_backend/app/capabilities/web/discover/schemas.py

@@ -1,29 +0,0 @@
-"""``web.discover`` I/O contracts."""
-
-from __future__ import annotations
-
-from pydantic import BaseModel, Field
-
-
-class DiscoverInput(BaseModel):
-    query: str = Field(
-        description="What to search the web for, phrased in natural language."
-    )
-    top_k: int = Field(
-        default=10, ge=1, le=50, description="Maximum number of results to return (1-50)."
-    )
-
-
-class DiscoverHit(BaseModel):
-    url: str = Field(
-        description="The result's page URL; pass it to web.scrape to read it."
-    )
-    title: str = Field(description="The result's page title.")
-    snippet: str | None = Field(
-        default=None, description="A short extract summarizing the page."
-    )
-    provider: str = Field(description="Which search engine returned this hit.")
-
-
-class DiscoverOutput(BaseModel):
-    hits: list[DiscoverHit] = Field(description="Ranked search results, best first.")

surfsense_backend/app/capabilities/web/scrape/__init__.py

@@ -1,3 +0,0 @@
-"""``web.scrape`` verb: a URL array → one cleaned row per URL."""
-
-from __future__ import annotations

surfsense_backend/app/capabilities/web/scrape/definition.py

@@ -1,23 +0,0 @@
-"""``web.scrape`` capability registration."""
-
-from __future__ import annotations
-
-from app.capabilities.core import BillingUnit, Capability, register_capability
-from app.capabilities.web.scrape.executor import build_scrape_executor
-from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput
-
-WEB_SCRAPE = Capability(
-    name="web.scrape",
-    description=(
-        "Fetch one or more web pages and return their clean, readable content. "
-        "Give it the exact URLs to read (use web.discover first if you don't have "
-        "them). Returns one row per URL with a status (success/empty/failed), the "
-        "page content, and metadata such as title and description."
-    ),
-    input_schema=ScrapeInput,
-    output_schema=ScrapeOutput,
-    executor=build_scrape_executor(),
-    billing_unit=BillingUnit.WEB_CRAWL,
-)
-
-register_capability(WEB_SCRAPE)

surfsense_backend/app/capabilities/web/scrape/executor.py

@@ -1,45 +0,0 @@
-"""``web.scrape`` executor: fetch each URL in the array → cleaned rows."""
-
-from __future__ import annotations
-
-from app.capabilities.core import Executor
-from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput, ScrapeRow
-from app.proprietary.web_crawler import (
-    CrawlOutcome,
-    CrawlOutcomeStatus,
-    WebCrawlerConnector,
-)
-
-
-def build_scrape_executor(engine: WebCrawlerConnector | None = None) -> Executor:
-    """Bind the executor to a fetch engine (defaults to the proprietary crawler)."""
-    engine = engine or WebCrawlerConnector()
-
-    async def execute(payload: ScrapeInput) -> ScrapeOutput:
-        outcomes = [await engine.crawl_url(url) for url in payload.urls]
-        rows = [
-            _to_row(url, outcome, payload.max_length)
-            for url, outcome in zip(payload.urls, outcomes, strict=True)
-        ]
-        return ScrapeOutput(
-            rows=rows,
-            captcha_attempts=sum(o.captcha_attempts for o in outcomes),
-            captcha_solved=sum(1 for o in outcomes if o.captcha_solved),
-        )
-
-    return execute
-
-
-def _to_row(url: str, outcome: CrawlOutcome, max_length: int) -> ScrapeRow:
-    if outcome.status is CrawlOutcomeStatus.SUCCESS and outcome.result:
-        content = outcome.result.get("content")
-        if content is not None:
-            content = content[:max_length]
-        return ScrapeRow(
-            url=url,
-            status="success",
-            content=content,
-            metadata=outcome.result.get("metadata"),
-        )
-    status = "empty" if outcome.status is CrawlOutcomeStatus.EMPTY else "failed"
-    return ScrapeRow(url=url, status=status, error=outcome.error)

surfsense_backend/app/capabilities/web/scrape/schemas.py

@@ -1,67 +0,0 @@
-"""``web.scrape`` I/O contracts."""
-
-from __future__ import annotations
-
-from typing import Literal
-
-from pydantic import BaseModel, Field
-
-MAX_SCRAPE_URLS = 20
-"""Per-call batch cap: bounds a synchronous request's crawl fan-out (05)."""
-
-
-class ScrapeInput(BaseModel):
-    urls: list[str] = Field(
-        min_length=1,
-        max_length=MAX_SCRAPE_URLS,
-        description=(
-            "Full page URLs to fetch and read (1-20), each starting with "
-            "http:// or https://. Pass the exact URLs you want the content of."
-        ),
-    )
-    max_length: int = Field(
-        default=50_000,
-        description=(
-            "Maximum characters of cleaned content returned per page; "
-            "content longer than this is truncated."
-        ),
-    )
-
-    @property
-    def estimated_units(self) -> int:
-        """Worst-case billable crawls for pre-flight: one per requested URL."""
-        return len(self.urls)
-
-
-class ScrapeRow(BaseModel):
-    url: str = Field(description="The requested URL this result is for.")
-    status: Literal["success", "empty", "failed"] = Field(
-        description=(
-            "'success' = content returned; 'empty' = page reached but no "
-            "readable content; 'failed' = could not be fetched (see error)."
-        )
-    )
-    content: str | None = Field(
-        default=None, description="Cleaned, readable page text (present on success)."
-    )
-    metadata: dict[str, str] | None = Field(
-        default=None, description="Page metadata such as title and description."
-    )
-    error: str | None = Field(
-        default=None, description="Why the fetch failed (present on 'failed')."
-    )
-
-
-class ScrapeOutput(BaseModel):
-    rows: list[ScrapeRow] = Field(
-        description="One result per requested URL, in the same order."
-    )
-    # Billing-only telemetry (Phase 3d), excluded from the client-facing result:
-    # captcha solves are metered per attempt, independent of crawl success.
-    captcha_attempts: int = Field(default=0, exclude=True)
-    captcha_solved: int = Field(default=0, exclude=True)
-
-    @property
-    def billable_units(self) -> int:
-        """One billable unit per successful scrape."""
-        return sum(1 for row in self.rows if row.status == "success")

surfsense_backend/tests/unit/capabilities/access/test_rest_router.py

@@ -95,8 +95,7 @@ def test_registered_verbs_appear_on_rest():
 
     router = rest.build_capabilities_router()
     paths = {route.path for route in router.routes}
-    assert "/workspaces/{workspace_id}/capabilities/web.scrape" in paths
+    assert "/workspaces/{workspace_id}/capabilities/web.crawl" in paths
-    assert "/workspaces/{workspace_id}/capabilities/web.discover" in paths
 
 
 @pytest.mark.asyncio

surfsense_backend/tests/unit/capabilities/test_billing.py

@@ -14,7 +14,7 @@ import pytest
 import app.capabilities.core.billing as billing
 from app.capabilities.core.billing import charge_capability, gate_capability
 from app.capabilities.core.types import BillingUnit, CapabilityContext
-from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput, ScrapeRow
+from app.capabilities.web.crawl.schemas import CrawlInput, CrawlItem, CrawlOutput
 from app.config import config
 from app.services.web_crawl_credit_service import InsufficientCreditsError
 
@@ -46,10 +46,10 @@ def _make_session(owner_id, balance_micros):
     return session, fake_user
 
 
-def _output(*statuses: str) -> ScrapeOutput:
+def _output(*statuses: str) -> CrawlOutput:
-    return ScrapeOutput(
+    return CrawlOutput(
-        rows=[
+        items=[
-            ScrapeRow(url=f"https://{i}.com", status=status)
+            CrawlItem(url=f"https://{i}.com", status=status)
             for i, status in enumerate(statuses)
         ]
     )
@@ -92,7 +92,7 @@ async def test_charges_workspace_owner_per_successful_crawl(monkeypatch, record_
     assert kwargs["cost_micros"] == 2000
 
 
-def _output_with_captcha(*statuses: str, attempts: int, solved: int) -> ScrapeOutput:
+def _output_with_captcha(*statuses: str, attempts: int, solved: int) -> CrawlOutput:
     out = _output(*statuses)
     out.captcha_attempts = attempts
     out.captcha_solved = solved
@@ -196,7 +196,7 @@ async def test_gate_blocks_when_worst_case_exceeds_balance(monkeypatch):
 
     with pytest.raises(InsufficientCreditsError):
         await gate_capability(
-            ScrapeInput(urls=["https://a.com", "https://b.com"]),
+            CrawlInput(startUrls=["https://a.com", "https://b.com"]),
             BillingUnit.WEB_CRAWL,
             _ctx(session),
         )
@@ -208,7 +208,7 @@ async def test_gate_passes_when_balance_covers_worst_case(monkeypatch):
     session = _gate_session(_OWNER, balance_micros=100_000)
 
     await gate_capability(
-        ScrapeInput(urls=["https://a.com", "https://b.com"]),
+        CrawlInput(startUrls=["https://a.com", "https://b.com"]),
         BillingUnit.WEB_CRAWL,
         _ctx(session),
     )
@@ -219,7 +219,7 @@ async def test_gate_is_noop_when_disabled(monkeypatch):
     session = _gate_session(_OWNER, balance_micros=0)
 
     await gate_capability(
-        ScrapeInput(urls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
+        CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
     )
 
 
@@ -233,7 +233,7 @@ async def test_gate_reserves_worst_case_captcha_when_solving_enabled(monkeypatch
 
     with pytest.raises(InsufficientCreditsError):
         await gate_capability(
-            ScrapeInput(urls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
+            CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
         )
 
 
@@ -245,7 +245,7 @@ async def test_gate_does_not_reserve_captcha_when_solving_disabled(monkeypatch):
 
     # Solving off → attempts can never happen → nothing to reserve → passes.
     await gate_capability(
-        ScrapeInput(urls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
+        CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
     )
 
 
@@ -253,6 +253,6 @@ async def test_gate_is_noop_for_free_verb(monkeypatch):
     monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
     session = _gate_session(_OWNER, balance_micros=0)
 
-    await gate_capability(ScrapeInput(urls=["https://a.com"]), None, _ctx(session))
+    await gate_capability(CrawlInput(startUrls=["https://a.com"]), None, _ctx(session))
 
     session.execute.assert_not_called()

surfsense_backend/tests/unit/capabilities/test_registry.py

@@ -9,25 +9,15 @@ from app.capabilities import (
 )
 from app.capabilities.core.store import get_capability
 from app.capabilities.core.types import BillingUnit
-from app.capabilities.web.discover.schemas import DiscoverInput, DiscoverOutput
+from app.capabilities.web.crawl.schemas import CrawlInput, CrawlOutput
-from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput
 
 pytestmark = pytest.mark.unit
 
 
-def test_web_scrape_is_registered_with_its_schemas_and_billing_unit():
+def test_web_crawl_is_registered_with_its_schemas_and_billing_unit():
-    cap = get_capability("web.scrape")
+    cap = get_capability("web.crawl")
 
-    assert cap.name == "web.scrape"
+    assert cap.name == "web.crawl"
-    assert cap.input_schema is ScrapeInput
+    assert cap.input_schema is CrawlInput
-    assert cap.output_schema is ScrapeOutput
+    assert cap.output_schema is CrawlOutput
     assert cap.billing_unit is BillingUnit.WEB_CRAWL
-
-
-def test_web_discover_is_registered_and_free():
-    cap = get_capability("web.discover")
-
-    assert cap.name == "web.discover"
-    assert cap.input_schema is DiscoverInput
-    assert cap.output_schema is DiscoverOutput
-    assert cap.billing_unit is None

surfsense_backend/tests/unit/capabilities/web/crawl/test_executor.py

@@ -0,0 +1,113 @@
+"""``web.crawl`` executor behavior: CrawlPage list → typed CrawlOutput items.
+
+Boundary mocked: the crawler engine (fake ``crawl_url`` + link graph). NOT
+mocked: the executor's page→item mapping, truncation, and captcha rollup.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.web.crawl.executor import build_crawl_executor
+from app.capabilities.web.crawl.schemas import CrawlInput, CrawlOutput
+from app.proprietary.web_crawler import CrawlOutcome, CrawlOutcomeStatus
+
+pytestmark = pytest.mark.unit
+
+_SUCCESS = CrawlOutcomeStatus.SUCCESS
+
+
+class _FakeEngine:
+    def __init__(self, graph: dict[str, tuple[CrawlOutcomeStatus, list[str]]]):
+        self._graph = graph
+        self.calls: list[str] = []
+
+    async def crawl_url(self, url: str) -> CrawlOutcome:
+        self.calls.append(url)
+        status, links = self._graph[url]
+        if status is _SUCCESS:
+            return CrawlOutcome(
+                status=_SUCCESS,
+                result={
+                    "content": f"C:{url}",
+                    "metadata": {"title": url},
+                    "links": links,
+                },
+            )
+        return CrawlOutcome(status=status, error="boom")
+
+
+async def test_single_url_depth_zero_returns_one_item() -> None:
+    engine = _FakeEngine({"https://e.com/": (_SUCCESS, ["https://e.com/a"])})
+    execute = build_crawl_executor(engine=engine)
+
+    out = await execute(CrawlInput(startUrls=["https://e.com/"]))
+
+    assert isinstance(out, CrawlOutput)
+    assert len(out.items) == 1
+    item = out.items[0]
+    assert item.url == "https://e.com/"
+    assert item.status == "success"
+    assert item.markdown == "C:https://e.com/"
+    assert item.metadata == {"title": "https://e.com/"}
+    assert item.crawl is not None
+    assert item.crawl.depth == 0
+    assert item.crawl.referrerUrl is None
+
+
+async def test_spider_collects_multiple_pages_with_provenance() -> None:
+    engine = _FakeEngine(
+        {
+            "https://e.com/": (_SUCCESS, ["https://e.com/a"]),
+            "https://e.com/a": (_SUCCESS, []),
+        }
+    )
+    execute = build_crawl_executor(engine=engine)
+
+    out = await execute(
+        CrawlInput(startUrls=["https://e.com/"], maxCrawlDepth=1, maxCrawlPages=10)
+    )
+
+    by_url = {item.url: item for item in out.items}
+    assert set(by_url) == {"https://e.com/", "https://e.com/a"}
+    assert by_url["https://e.com/a"].crawl.referrerUrl == "https://e.com/"
+
+
+async def test_content_is_truncated_to_max_length() -> None:
+    engine = _FakeEngine({"https://e.com/": (_SUCCESS, [])})
+    execute = build_crawl_executor(engine=engine)
+
+    out = await execute(CrawlInput(startUrls=["https://e.com/"], maxLength=3))
+
+    assert out.items[0].markdown == "C:h"
+
+
+async def test_failed_page_has_no_markdown_but_keeps_error() -> None:
+    engine = _FakeEngine({"https://e.com/": (CrawlOutcomeStatus.FAILED, [])})
+    execute = build_crawl_executor(engine=engine)
+
+    out = await execute(CrawlInput(startUrls=["https://e.com/"]))
+
+    item = out.items[0]
+    assert item.status == "failed"
+    assert item.markdown is None
+    assert item.error == "boom"
+
+
+async def test_captcha_telemetry_is_rolled_up_for_billing() -> None:
+    class _CaptchaEngine:
+        async def crawl_url(self, url: str) -> CrawlOutcome:
+            return CrawlOutcome(
+                status=_SUCCESS,
+                result={"content": "ok", "metadata": {}, "links": []},
+                captcha_attempts=2,
+                captcha_solved=True,
+            )
+
+    execute = build_crawl_executor(engine=_CaptchaEngine())
+
+    out = await execute(CrawlInput(startUrls=["https://e.com/"]))
+
+    assert out.captcha_attempts == 2
+    assert out.captcha_solved == 1
+    assert out.billable_units == 1

surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py

@@ -0,0 +1,67 @@
+"""``web.crawl`` I/O contract: camelCase surface, bounds, and billing counters."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.web.crawl.schemas import (
+    CrawlInput,
+    CrawlItem,
+    CrawlMeta,
+    CrawlOutput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_requires_at_least_one_start_url() -> None:
+    with pytest.raises(ValidationError):
+        CrawlInput(startUrls=[])
+
+
+def test_camelcase_fields_and_defaults() -> None:
+    model = CrawlInput(startUrls=["https://e.com"])
+    assert model.startUrls == ["https://e.com"]
+    assert model.maxCrawlDepth == 0
+    assert model.maxCrawlPages == 10
+    assert model.maxLength == 50_000
+
+
+def test_depth_and_page_bounds_are_enforced() -> None:
+    with pytest.raises(ValidationError):
+        CrawlInput(startUrls=["https://e.com"], maxCrawlDepth=-1)
+    with pytest.raises(ValidationError):
+        CrawlInput(startUrls=["https://e.com"], maxCrawlDepth=99)
+    with pytest.raises(ValidationError):
+        CrawlInput(startUrls=["https://e.com"], maxCrawlPages=0)
+
+
+def test_estimated_units_for_single_url_is_seed_count() -> None:
+    model = CrawlInput(startUrls=["https://a.com", "https://b.com"], maxCrawlDepth=0)
+    assert model.estimated_units == 2
+
+
+def test_estimated_units_for_spider_is_max_pages() -> None:
+    model = CrawlInput(
+        startUrls=["https://a.com"], maxCrawlDepth=2, maxCrawlPages=25
+    )
+    assert model.estimated_units == 25
+
+
+def test_billable_units_counts_only_successes() -> None:
+    out = CrawlOutput(
+        items=[
+            CrawlItem(url="a", status="success", crawl=CrawlMeta(loadedUrl="a", depth=0)),
+            CrawlItem(url="b", status="empty", crawl=CrawlMeta(loadedUrl="b", depth=1)),
+            CrawlItem(url="c", status="failed", crawl=CrawlMeta(loadedUrl="c", depth=1)),
+        ]
+    )
+    assert out.billable_units == 1
+
+
+def test_captcha_counters_are_excluded_from_the_wire_shape() -> None:
+    out = CrawlOutput(items=[], captcha_attempts=3, captcha_solved=1)
+    dumped = out.model_dump()
+    assert "captcha_attempts" not in dumped
+    assert "captcha_solved" not in dumped

surfsense_backend/tests/unit/capabilities/web/discover/test_baidu_provider.py

@@ -1,91 +0,0 @@
-"""BaiduProvider maps the Baidu AI Search references to DiscoverHits, env-keyed.
-
-Boundary mocked: httpx.AsyncClient + config key. NOT mocked: reference→hit mapping.
-"""
-
-from __future__ import annotations
-
-import pytest
-
-import app.capabilities.web.discover.providers.baidu as baidu_module
-from app.capabilities.web.discover.providers.baidu import BaiduProvider
-from app.config import config
-
-pytestmark = pytest.mark.unit
-
-
-class _FakeResponse:
-    def __init__(self, payload):
-        self._payload = payload
-
-    def raise_for_status(self):
-        return None
-
-    def json(self):
-        return self._payload
-
-
-class _FakeAsyncClient:
-    """Minimal async-context httpx stand-in returning a canned payload."""
-
-    payload: dict = {}
-
-    def __init__(self, *args, **kwargs):
-        pass
-
-    async def __aenter__(self):
-        return self
-
-    async def __aexit__(self, *exc):
-        return False
-
-    async def post(self, url, headers=None, json=None):
-        return _FakeResponse(type(self).payload)
-
-
-def _install(monkeypatch, payload):
-    monkeypatch.setattr(config, "BAIDU_API_KEY", "k")
-    _FakeAsyncClient.payload = payload
-    monkeypatch.setattr(baidu_module.httpx, "AsyncClient", _FakeAsyncClient)
-
-
-def test_is_available_reflects_the_env_key(monkeypatch):
-    monkeypatch.setattr(config, "BAIDU_API_KEY", "k")
-    assert BaiduProvider().is_available() is True
-    monkeypatch.setattr(config, "BAIDU_API_KEY", None)
-    assert BaiduProvider().is_available() is False
-
-
-async def test_maps_references_to_hits(monkeypatch):
-    _install(
-        monkeypatch,
-        {
-            "references": [
-                {"title": "Acme", "url": "https://acme.cn", "content": "hello"},
-                {"title": "No URL", "url": "", "content": "skip"},
-            ]
-        },
-    )
-
-    hits = await BaiduProvider().search("acme", top_k=10)
-
-    assert [h.url for h in hits] == ["https://acme.cn"]
-    assert hits[0].title == "Acme"
-    assert hits[0].snippet == "hello"
-    assert hits[0].provider == "baidu"
-
-
-async def test_respects_top_k(monkeypatch):
-    _install(
-        monkeypatch,
-        {
-            "references": [
-                {"title": f"n{i}", "url": f"https://{i}.cn", "content": "c"}
-                for i in range(5)
-            ]
-        },
-    )
-
-    hits = await BaiduProvider().search("q", top_k=2)
-
-    assert len(hits) == 2

surfsense_backend/tests/unit/capabilities/web/discover/test_executor.py

@@ -1,99 +0,0 @@
-"""`web.discover` executor: pick the first configured provider; self-disable when none.
-
-Boundary mocked: the providers (injected fakes). NOT mocked: the executor's
-provider-selection and self-disable behavior.
-"""
-
-from __future__ import annotations
-
-import pytest
-
-from app.capabilities.web.discover.executor import (
-    NoDiscoverProviderError,
-    build_discover_executor,
-)
-from app.capabilities.web.discover.schemas import (
-    DiscoverHit,
-    DiscoverInput,
-    DiscoverOutput,
-)
-
-pytestmark = pytest.mark.unit
-
-
-class _FakeProvider:
-    def __init__(
-        self, name: str, available: bool, hits: list[DiscoverHit] | None = None
-    ):
-        self.name = name
-        self._available = available
-        self._hits = hits or []
-        self.calls: list[tuple[str, int]] = []
-
-    def is_available(self) -> bool:
-        return self._available
-
-    async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
-        self.calls.append((query, top_k))
-        return self._hits
-
-
-def _hit(url: str, provider: str) -> DiscoverHit:
-    return DiscoverHit(url=url, title=url, snippet="s", provider=provider)
-
-
-async def test_uses_the_first_available_provider():
-    first = _FakeProvider(
-        "searxng", available=True, hits=[_hit("https://a.com", "searxng")]
-    )
-    second = _FakeProvider(
-        "linkup", available=True, hits=[_hit("https://b.com", "linkup")]
-    )
-    execute = build_discover_executor(providers=[first, second])
-
-    out = await execute(DiscoverInput(query="acme pricing", top_k=5))
-
-    assert isinstance(out, DiscoverOutput)
-    assert [h.url for h in out.hits] == ["https://a.com"]
-    assert first.calls == [("acme pricing", 5)]
-    assert second.calls == []  # first available short-circuits
-
-
-async def test_skips_unavailable_providers():
-    off = _FakeProvider("searxng", available=False)
-    on = _FakeProvider("linkup", available=True, hits=[_hit("https://b.com", "linkup")])
-    execute = build_discover_executor(providers=[off, on])
-
-    out = await execute(DiscoverInput(query="q"))
-
-    assert [h.provider for h in out.hits] == ["linkup"]
-    assert off.calls == []
-
-
-async def test_self_disables_when_no_provider_is_configured():
-    execute = build_discover_executor(
-        providers=[_FakeProvider("searxng", available=False)]
-    )
-
-    with pytest.raises(NoDiscoverProviderError):
-        await execute(DiscoverInput(query="q"))
-
-
-async def test_caps_hits_to_top_k_when_provider_over_returns():
-    # SearXNG treats `limit` as a hint and can return more rows than asked; the
-    # verb must honor its own documented `top_k` cap regardless of the provider.
-    over = _FakeProvider(
-        "searxng",
-        available=True,
-        hits=[_hit(f"https://{i}.com", "searxng") for i in range(5)],
-    )
-    execute = build_discover_executor(providers=[over])
-
-    out = await execute(DiscoverInput(query="q", top_k=3))
-
-    assert len(out.hits) == 3
-    assert [h.url for h in out.hits] == [
-        "https://0.com",
-        "https://1.com",
-        "https://2.com",
-    ]

surfsense_backend/tests/unit/capabilities/web/discover/test_linkup_provider.py

@@ -1,75 +0,0 @@
-"""LinkupProvider maps the Linkup SDK results to DiscoverHits, env-keyed.
-
-Boundary mocked: the LinkupClient SDK + config key. NOT mocked: result→hit mapping.
-"""
-
-from __future__ import annotations
-
-import pytest
-
-import app.capabilities.web.discover.providers.linkup as linkup_module
-from app.capabilities.web.discover.providers.linkup import LinkupProvider
-from app.config import config
-
-pytestmark = pytest.mark.unit
-
-
-class _Result:
-    def __init__(self, name, url, content):
-        self.name = name
-        self.url = url
-        self.content = content
-        self.type = "text"
-
-
-class _Response:
-    def __init__(self, results):
-        self.results = results
-
-
-class _FakeClient:
-    def __init__(self, api_key):
-        self.api_key = api_key
-
-    def search(self, *, query, depth, output_type):
-        return _Response(
-            [
-                _Result("Acme", "https://acme.com", "acme home"),
-                _Result("No URL", "", "skip"),
-            ]
-        )
-
-
-def test_is_available_reflects_the_env_key(monkeypatch):
-    monkeypatch.setattr(config, "LINKUP_API_KEY", "k")
-    assert LinkupProvider().is_available() is True
-    monkeypatch.setattr(config, "LINKUP_API_KEY", None)
-    assert LinkupProvider().is_available() is False
-
-
-async def test_maps_results_to_hits(monkeypatch):
-    monkeypatch.setattr(config, "LINKUP_API_KEY", "k")
-    monkeypatch.setattr(linkup_module, "LinkupClient", _FakeClient)
-
-    hits = await LinkupProvider().search("acme", top_k=10)
-
-    assert [h.url for h in hits] == ["https://acme.com"]
-    assert hits[0].title == "Acme"
-    assert hits[0].snippet == "acme home"
-    assert hits[0].provider == "linkup"
-
-
-async def test_respects_top_k(monkeypatch):
-    monkeypatch.setattr(config, "LINKUP_API_KEY", "k")
-
-    class _ManyClient(_FakeClient):
-        def search(self, *, query, depth, output_type):
-            return _Response(
-                [_Result(f"n{i}", f"https://{i}.com", "c") for i in range(5)]
-            )
-
-    monkeypatch.setattr(linkup_module, "LinkupClient", _ManyClient)
-
-    hits = await LinkupProvider().search("q", top_k=2)
-
-    assert len(hits) == 2

surfsense_backend/tests/unit/capabilities/web/discover/test_schemas.py

@@ -1,22 +0,0 @@
-"""``web.discover`` I/O contract bounds."""
-
-from __future__ import annotations
-
-import pytest
-from pydantic import ValidationError
-
-from app.capabilities.web.discover.schemas import DiscoverInput
-
-pytestmark = pytest.mark.unit
-
-
-def test_top_k_defaults_and_accepts_the_valid_range():
-    assert DiscoverInput(query="q").top_k == 10
-    assert DiscoverInput(query="q", top_k=1).top_k == 1
-    assert DiscoverInput(query="q", top_k=50).top_k == 50
-
-
-@pytest.mark.parametrize("bad", [0, -1, 51, 100])
-def test_top_k_outside_1_to_50_is_rejected(bad):
-    with pytest.raises(ValidationError):
-        DiscoverInput(query="q", top_k=bad)

surfsense_backend/tests/unit/capabilities/web/discover/test_searxng_provider.py

@@ -1,58 +0,0 @@
-"""SearxngProvider maps the SearXNG service's sources to DiscoverHits.
-
-Boundary mocked: the web_search_service module. NOT mocked: the source→hit mapping.
-"""
-
-from __future__ import annotations
-
-from unittest.mock import AsyncMock
-
-import pytest
-
-import app.capabilities.web.discover.providers.searxng as searxng_module
-from app.capabilities.web.discover.providers.searxng import SearxngProvider
-
-pytestmark = pytest.mark.unit
-
-
-def _result(sources):
-    return ({"sources": sources}, [])
-
-
-async def test_maps_sources_to_hits(monkeypatch):
-    provider = SearxngProvider()
-    monkeypatch.setattr(
-        searxng_module.web_search_service,
-        "search",
-        AsyncMock(
-            return_value=_result(
-                [
-                    {"title": "Acme", "url": "https://acme.com", "description": "home"},
-                    {
-                        "title": "Docs",
-                        "url": "https://acme.com/docs",
-                        "description": "",
-                    },
-                    {"title": "no url", "url": "", "description": "skip me"},
-                ]
-            )
-        ),
-    )
-
-    hits = await provider.search("acme", top_k=5)
-
-    assert [h.url for h in hits] == ["https://acme.com", "https://acme.com/docs"]
-    assert hits[0].title == "Acme"
-    assert hits[0].snippet == "home"
-    assert hits[1].snippet is None  # empty description normalizes to None
-    assert all(h.provider == "searxng" for h in hits)
-
-
-def test_is_available_reflects_the_service(monkeypatch):
-    provider = SearxngProvider()
-    monkeypatch.setattr(searxng_module.web_search_service, "is_available", lambda: True)
-    assert provider.is_available() is True
-    monkeypatch.setattr(
-        searxng_module.web_search_service, "is_available", lambda: False
-    )
-    assert provider.is_available() is False

surfsense_backend/tests/unit/capabilities/web/scrape/test_executor.py

@@ -1,141 +0,0 @@
-"""`web.scrape` executor behavior: URLs in → cleaned rows out.
-
-Boundary mocked: the crawler (injected fake). NOT mocked: the executor's own
-CrawlOutcome → ScrapeRow mapping.
-"""
-
-from __future__ import annotations
-
-import pytest
-
-from app.capabilities.web.scrape.executor import build_scrape_executor
-from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput
-from app.proprietary.web_crawler import CrawlOutcome, CrawlOutcomeStatus
-
-pytestmark = pytest.mark.unit
-
-
-class _FakeCrawler:
-    """Stand-in for WebCrawlerConnector: serves a canned outcome per URL."""
-
-    def __init__(self, outcomes: dict[str, CrawlOutcome]):
-        self._outcomes = outcomes
-        self.calls: list[str] = []
-
-    async def crawl_url(self, url: str) -> CrawlOutcome:
-        self.calls.append(url)
-        return self._outcomes[url]
-
-
-def _success(content: str, metadata: dict[str, str]) -> CrawlOutcome:
-    return CrawlOutcome(
-        status=CrawlOutcomeStatus.SUCCESS,
-        result={
-            "content": content,
-            "metadata": metadata,
-            "crawler_type": "scrapling-static",
-        },
-    )
-
-
-async def test_scrape_returns_one_cleaned_row_for_a_successful_url():
-    url = "https://example.com"
-    crawler = _FakeCrawler({url: _success("# Hello", {"title": "Hello"})})
-    execute = build_scrape_executor(engine=crawler)
-
-    out = await execute(ScrapeInput(urls=[url]))
-
-    assert isinstance(out, ScrapeOutput)
-    assert len(out.rows) == 1
-    row = out.rows[0]
-    assert row.url == url
-    assert row.status == "success"
-    assert row.content == "# Hello"
-    assert row.metadata == {"title": "Hello"}
-
-
-async def test_scrape_returns_one_row_per_url_in_input_order():
-    a, b, c = "https://a.com", "https://b.com", "https://c.com"
-    crawler = _FakeCrawler(
-        {
-            a: _success("A", {"title": "A"}),
-            b: _success("B", {"title": "B"}),
-            c: _success("C", {"title": "C"}),
-        }
-    )
-    execute = build_scrape_executor(engine=crawler)
-
-    out = await execute(ScrapeInput(urls=[a, b, c]))
-
-    assert [row.url for row in out.rows] == [a, b, c]
-    assert [row.content for row in out.rows] == ["A", "B", "C"]
-
-
-async def test_content_longer_than_max_length_is_truncated():
-    url = "https://long.com"
-    crawler = _FakeCrawler({url: _success("A" * 100, {"title": "Long"})})
-    execute = build_scrape_executor(engine=crawler)
-
-    out = await execute(ScrapeInput(urls=[url], max_length=10))
-
-    assert out.rows[0].content == "A" * 10
-
-
-async def test_content_within_max_length_is_untouched():
-    url = "https://short.com"
-    crawler = _FakeCrawler({url: _success("hello", {"title": "Short"})})
-    execute = build_scrape_executor(engine=crawler)
-
-    out = await execute(ScrapeInput(urls=[url], max_length=10))
-
-    assert out.rows[0].content == "hello"
-
-
-async def test_scrape_surfaces_total_captcha_attempts_for_billing():
-    ok, blocked = "https://ok.com", "https://blocked.com"
-    crawler = _FakeCrawler(
-        {
-            ok: CrawlOutcome(
-                status=CrawlOutcomeStatus.SUCCESS,
-                result={"content": "OK", "metadata": {}},
-                captcha_attempts=2,
-                captcha_solved=True,
-            ),
-            blocked: CrawlOutcome(
-                status=CrawlOutcomeStatus.FAILED,
-                error="blocked",
-                captcha_attempts=1,
-                captcha_solved=False,
-            ),
-        }
-    )
-    execute = build_scrape_executor(engine=crawler)
-
-    out = await execute(ScrapeInput(urls=[ok, blocked]))
-
-    # Attempts bill even when the crawl ultimately failed (Phase 3d).
-    assert out.captcha_attempts == 3
-    assert out.captcha_solved == 1
-
-
-async def test_partial_failure_keeps_the_batch_and_labels_each_url():
-    ok, empty, failed = "https://ok.com", "https://empty.com", "https://failed.com"
-    crawler = _FakeCrawler(
-        {
-            ok: _success("OK", {"title": "OK"}),
-            empty: CrawlOutcome(status=CrawlOutcomeStatus.EMPTY, error="no content"),
-            failed: CrawlOutcome(status=CrawlOutcomeStatus.FAILED, error="blocked"),
-        }
-    )
-    execute = build_scrape_executor(engine=crawler)
-
-    out = await execute(ScrapeInput(urls=[ok, empty, failed]))
-
-    by_url = {row.url: row for row in out.rows}
-    assert {u: r.status for u, r in by_url.items()} == {
-        ok: "success",
-        empty: "empty",
-        failed: "failed",
-    }
-    assert by_url[failed].content is None
-    assert by_url[failed].error == "blocked"

surfsense_backend/tests/unit/capabilities/web/scrape/test_schemas.py

@@ -1,47 +0,0 @@
-"""ScrapeOutput reports its own billable count; ScrapeInput bounds its batch size."""
-
-from __future__ import annotations
-
-import pytest
-from pydantic import ValidationError
-
-from app.capabilities.web.scrape.schemas import (
-    MAX_SCRAPE_URLS,
-    ScrapeInput,
-    ScrapeOutput,
-    ScrapeRow,
-)
-
-pytestmark = pytest.mark.unit
-
-
-def _output(*statuses: str) -> ScrapeOutput:
-    return ScrapeOutput(
-        rows=[
-            ScrapeRow(url=f"https://{i}.com", status=status)
-            for i, status in enumerate(statuses)
-        ]
-    )
-
-
-def test_billable_units_counts_successful_rows():
-    assert _output("success", "empty", "success", "failed").billable_units == 2
-
-
-def test_billable_units_is_zero_without_successes():
-    assert _output("empty", "failed").billable_units == 0
-
-
-def test_rejects_empty_url_batch():
-    with pytest.raises(ValidationError):
-        ScrapeInput(urls=[])
-
-
-def test_rejects_batch_over_the_cap():
-    with pytest.raises(ValidationError):
-        ScrapeInput(urls=[f"https://{i}.com" for i in range(MAX_SCRAPE_URLS + 1)])
-
-
-def test_accepts_batch_at_the_cap():
-    payload = ScrapeInput(urls=[f"https://{i}.com" for i in range(MAX_SCRAPE_URLS)])
-    assert payload.estimated_units == MAX_SCRAPE_URLS