feat(capabilities): replace web.scrape/web.discover with unified web.crawl

web.crawl scrapes a single URL (maxCrawlDepth=0) or spiders a whole site,
backed by the proprietary site_crawler engine. Rewires the scraping subagent
tools and capability tests onto the new verb.

surfsense_backend/tests/unit/capabilities/access/test_rest_router.py

@@ -95,8 +95,7 @@ def test_registered_verbs_appear_on_rest():
 
     router = rest.build_capabilities_router()
     paths = {route.path for route in router.routes}
-    assert "/workspaces/{workspace_id}/capabilities/web.scrape" in paths
-    assert "/workspaces/{workspace_id}/capabilities/web.discover" in paths
+    assert "/workspaces/{workspace_id}/capabilities/web.crawl" in paths
 
 
 @pytest.mark.asyncio

surfsense_backend/tests/unit/capabilities/test_billing.py

@@ -14,7 +14,7 @@ import pytest
 import app.capabilities.core.billing as billing
 from app.capabilities.core.billing import charge_capability, gate_capability
 from app.capabilities.core.types import BillingUnit, CapabilityContext
-from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput, ScrapeRow
+from app.capabilities.web.crawl.schemas import CrawlInput, CrawlItem, CrawlOutput
 from app.config import config
 from app.services.web_crawl_credit_service import InsufficientCreditsError
 
@@ -46,10 +46,10 @@ def _make_session(owner_id, balance_micros):
     return session, fake_user
 
 
-def _output(*statuses: str) -> ScrapeOutput:
-    return ScrapeOutput(
-        rows=[
-            ScrapeRow(url=f"https://{i}.com", status=status)
+def _output(*statuses: str) -> CrawlOutput:
+    return CrawlOutput(
+        items=[
+            CrawlItem(url=f"https://{i}.com", status=status)
             for i, status in enumerate(statuses)
         ]
     )
@@ -92,7 +92,7 @@ async def test_charges_workspace_owner_per_successful_crawl(monkeypatch, record_
     assert kwargs["cost_micros"] == 2000
 
 
-def _output_with_captcha(*statuses: str, attempts: int, solved: int) -> ScrapeOutput:
+def _output_with_captcha(*statuses: str, attempts: int, solved: int) -> CrawlOutput:
     out = _output(*statuses)
     out.captcha_attempts = attempts
     out.captcha_solved = solved
@@ -196,7 +196,7 @@ async def test_gate_blocks_when_worst_case_exceeds_balance(monkeypatch):
 
     with pytest.raises(InsufficientCreditsError):
         await gate_capability(
-            ScrapeInput(urls=["https://a.com", "https://b.com"]),
+            CrawlInput(startUrls=["https://a.com", "https://b.com"]),
             BillingUnit.WEB_CRAWL,
             _ctx(session),
         )
@@ -208,7 +208,7 @@ async def test_gate_passes_when_balance_covers_worst_case(monkeypatch):
     session = _gate_session(_OWNER, balance_micros=100_000)
 
     await gate_capability(
-        ScrapeInput(urls=["https://a.com", "https://b.com"]),
+        CrawlInput(startUrls=["https://a.com", "https://b.com"]),
         BillingUnit.WEB_CRAWL,
         _ctx(session),
     )
@@ -219,7 +219,7 @@ async def test_gate_is_noop_when_disabled(monkeypatch):
     session = _gate_session(_OWNER, balance_micros=0)
 
     await gate_capability(
-        ScrapeInput(urls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
+        CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
     )
 
 
@@ -233,7 +233,7 @@ async def test_gate_reserves_worst_case_captcha_when_solving_enabled(monkeypatch
 
     with pytest.raises(InsufficientCreditsError):
         await gate_capability(
-            ScrapeInput(urls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
+            CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
         )
 
 
@@ -245,7 +245,7 @@ async def test_gate_does_not_reserve_captcha_when_solving_disabled(monkeypatch):
 
     # Solving off → attempts can never happen → nothing to reserve → passes.
     await gate_capability(
-        ScrapeInput(urls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
+        CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
     )
 
 
@@ -253,6 +253,6 @@ async def test_gate_is_noop_for_free_verb(monkeypatch):
     monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
     session = _gate_session(_OWNER, balance_micros=0)
 
-    await gate_capability(ScrapeInput(urls=["https://a.com"]), None, _ctx(session))
+    await gate_capability(CrawlInput(startUrls=["https://a.com"]), None, _ctx(session))
 
     session.execute.assert_not_called()

surfsense_backend/tests/unit/capabilities/test_registry.py

@@ -9,25 +9,15 @@ from app.capabilities import (
 )
 from app.capabilities.core.store import get_capability
 from app.capabilities.core.types import BillingUnit
-from app.capabilities.web.discover.schemas import DiscoverInput, DiscoverOutput
-from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.capabilities.web.crawl.schemas import CrawlInput, CrawlOutput
 
 pytestmark = pytest.mark.unit
 
 
-def test_web_scrape_is_registered_with_its_schemas_and_billing_unit():
-    cap = get_capability("web.scrape")
+def test_web_crawl_is_registered_with_its_schemas_and_billing_unit():
+    cap = get_capability("web.crawl")
 
-    assert cap.name == "web.scrape"
-    assert cap.input_schema is ScrapeInput
-    assert cap.output_schema is ScrapeOutput
+    assert cap.name == "web.crawl"
+    assert cap.input_schema is CrawlInput
+    assert cap.output_schema is CrawlOutput
     assert cap.billing_unit is BillingUnit.WEB_CRAWL
-
-
-def test_web_discover_is_registered_and_free():
-    cap = get_capability("web.discover")
-
-    assert cap.name == "web.discover"
-    assert cap.input_schema is DiscoverInput
-    assert cap.output_schema is DiscoverOutput
-    assert cap.billing_unit is None

surfsense_backend/tests/unit/capabilities/web/crawl/test_executor.py

@@ -0,0 +1,113 @@
+"""``web.crawl`` executor behavior: CrawlPage list → typed CrawlOutput items.
+
+Boundary mocked: the crawler engine (fake ``crawl_url`` + link graph). NOT
+mocked: the executor's page→item mapping, truncation, and captcha rollup.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.web.crawl.executor import build_crawl_executor
+from app.capabilities.web.crawl.schemas import CrawlInput, CrawlOutput
+from app.proprietary.web_crawler import CrawlOutcome, CrawlOutcomeStatus
+
+pytestmark = pytest.mark.unit
+
+_SUCCESS = CrawlOutcomeStatus.SUCCESS
+
+
+class _FakeEngine:
+    def __init__(self, graph: dict[str, tuple[CrawlOutcomeStatus, list[str]]]):
+        self._graph = graph
+        self.calls: list[str] = []
+
+    async def crawl_url(self, url: str) -> CrawlOutcome:
+        self.calls.append(url)
+        status, links = self._graph[url]
+        if status is _SUCCESS:
+            return CrawlOutcome(
+                status=_SUCCESS,
+                result={
+                    "content": f"C:{url}",
+                    "metadata": {"title": url},
+                    "links": links,
+                },
+            )
+        return CrawlOutcome(status=status, error="boom")
+
+
+async def test_single_url_depth_zero_returns_one_item() -> None:
+    engine = _FakeEngine({"https://e.com/": (_SUCCESS, ["https://e.com/a"])})
+    execute = build_crawl_executor(engine=engine)
+
+    out = await execute(CrawlInput(startUrls=["https://e.com/"]))
+
+    assert isinstance(out, CrawlOutput)
+    assert len(out.items) == 1
+    item = out.items[0]
+    assert item.url == "https://e.com/"
+    assert item.status == "success"
+    assert item.markdown == "C:https://e.com/"
+    assert item.metadata == {"title": "https://e.com/"}
+    assert item.crawl is not None
+    assert item.crawl.depth == 0
+    assert item.crawl.referrerUrl is None
+
+
+async def test_spider_collects_multiple_pages_with_provenance() -> None:
+    engine = _FakeEngine(
+        {
+            "https://e.com/": (_SUCCESS, ["https://e.com/a"]),
+            "https://e.com/a": (_SUCCESS, []),
+        }
+    )
+    execute = build_crawl_executor(engine=engine)
+
+    out = await execute(
+        CrawlInput(startUrls=["https://e.com/"], maxCrawlDepth=1, maxCrawlPages=10)
+    )
+
+    by_url = {item.url: item for item in out.items}
+    assert set(by_url) == {"https://e.com/", "https://e.com/a"}
+    assert by_url["https://e.com/a"].crawl.referrerUrl == "https://e.com/"
+
+
+async def test_content_is_truncated_to_max_length() -> None:
+    engine = _FakeEngine({"https://e.com/": (_SUCCESS, [])})
+    execute = build_crawl_executor(engine=engine)
+
+    out = await execute(CrawlInput(startUrls=["https://e.com/"], maxLength=3))
+
+    assert out.items[0].markdown == "C:h"
+
+
+async def test_failed_page_has_no_markdown_but_keeps_error() -> None:
+    engine = _FakeEngine({"https://e.com/": (CrawlOutcomeStatus.FAILED, [])})
+    execute = build_crawl_executor(engine=engine)
+
+    out = await execute(CrawlInput(startUrls=["https://e.com/"]))
+
+    item = out.items[0]
+    assert item.status == "failed"
+    assert item.markdown is None
+    assert item.error == "boom"
+
+
+async def test_captcha_telemetry_is_rolled_up_for_billing() -> None:
+    class _CaptchaEngine:
+        async def crawl_url(self, url: str) -> CrawlOutcome:
+            return CrawlOutcome(
+                status=_SUCCESS,
+                result={"content": "ok", "metadata": {}, "links": []},
+                captcha_attempts=2,
+                captcha_solved=True,
+            )
+
+    execute = build_crawl_executor(engine=_CaptchaEngine())
+
+    out = await execute(CrawlInput(startUrls=["https://e.com/"]))
+
+    assert out.captcha_attempts == 2
+    assert out.captcha_solved == 1
+    assert out.billable_units == 1

surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py

@@ -0,0 +1,67 @@
+"""``web.crawl`` I/O contract: camelCase surface, bounds, and billing counters."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.web.crawl.schemas import (
+    CrawlInput,
+    CrawlItem,
+    CrawlMeta,
+    CrawlOutput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_requires_at_least_one_start_url() -> None:
+    with pytest.raises(ValidationError):
+        CrawlInput(startUrls=[])
+
+
+def test_camelcase_fields_and_defaults() -> None:
+    model = CrawlInput(startUrls=["https://e.com"])
+    assert model.startUrls == ["https://e.com"]
+    assert model.maxCrawlDepth == 0
+    assert model.maxCrawlPages == 10
+    assert model.maxLength == 50_000
+
+
+def test_depth_and_page_bounds_are_enforced() -> None:
+    with pytest.raises(ValidationError):
+        CrawlInput(startUrls=["https://e.com"], maxCrawlDepth=-1)
+    with pytest.raises(ValidationError):
+        CrawlInput(startUrls=["https://e.com"], maxCrawlDepth=99)
+    with pytest.raises(ValidationError):
+        CrawlInput(startUrls=["https://e.com"], maxCrawlPages=0)
+
+
+def test_estimated_units_for_single_url_is_seed_count() -> None:
+    model = CrawlInput(startUrls=["https://a.com", "https://b.com"], maxCrawlDepth=0)
+    assert model.estimated_units == 2
+
+
+def test_estimated_units_for_spider_is_max_pages() -> None:
+    model = CrawlInput(
+        startUrls=["https://a.com"], maxCrawlDepth=2, maxCrawlPages=25
+    )
+    assert model.estimated_units == 25
+
+
+def test_billable_units_counts_only_successes() -> None:
+    out = CrawlOutput(
+        items=[
+            CrawlItem(url="a", status="success", crawl=CrawlMeta(loadedUrl="a", depth=0)),
+            CrawlItem(url="b", status="empty", crawl=CrawlMeta(loadedUrl="b", depth=1)),
+            CrawlItem(url="c", status="failed", crawl=CrawlMeta(loadedUrl="c", depth=1)),
+        ]
+    )
+    assert out.billable_units == 1
+
+
+def test_captcha_counters_are_excluded_from_the_wire_shape() -> None:
+    out = CrawlOutput(items=[], captcha_attempts=3, captcha_solved=1)
+    dumped = out.model_dump()
+    assert "captcha_attempts" not in dumped
+    assert "captcha_solved" not in dumped

surfsense_backend/tests/unit/capabilities/web/discover/test_baidu_provider.py

@@ -1,91 +0,0 @@
-"""BaiduProvider maps the Baidu AI Search references to DiscoverHits, env-keyed.
-
-Boundary mocked: httpx.AsyncClient + config key. NOT mocked: reference→hit mapping.
-"""
-
-from __future__ import annotations
-
-import pytest
-
-import app.capabilities.web.discover.providers.baidu as baidu_module
-from app.capabilities.web.discover.providers.baidu import BaiduProvider
-from app.config import config
-
-pytestmark = pytest.mark.unit
-
-
-class _FakeResponse:
-    def __init__(self, payload):
-        self._payload = payload
-
-    def raise_for_status(self):
-        return None
-
-    def json(self):
-        return self._payload
-
-
-class _FakeAsyncClient:
-    """Minimal async-context httpx stand-in returning a canned payload."""
-
-    payload: dict = {}
-
-    def __init__(self, *args, **kwargs):
-        pass
-
-    async def __aenter__(self):
-        return self
-
-    async def __aexit__(self, *exc):
-        return False
-
-    async def post(self, url, headers=None, json=None):
-        return _FakeResponse(type(self).payload)
-
-
-def _install(monkeypatch, payload):
-    monkeypatch.setattr(config, "BAIDU_API_KEY", "k")
-    _FakeAsyncClient.payload = payload
-    monkeypatch.setattr(baidu_module.httpx, "AsyncClient", _FakeAsyncClient)
-
-
-def test_is_available_reflects_the_env_key(monkeypatch):
-    monkeypatch.setattr(config, "BAIDU_API_KEY", "k")
-    assert BaiduProvider().is_available() is True
-    monkeypatch.setattr(config, "BAIDU_API_KEY", None)
-    assert BaiduProvider().is_available() is False
-
-
-async def test_maps_references_to_hits(monkeypatch):
-    _install(
-        monkeypatch,
-        {
-            "references": [
-                {"title": "Acme", "url": "https://acme.cn", "content": "hello"},
-                {"title": "No URL", "url": "", "content": "skip"},
-            ]
-        },
-    )
-
-    hits = await BaiduProvider().search("acme", top_k=10)
-
-    assert [h.url for h in hits] == ["https://acme.cn"]
-    assert hits[0].title == "Acme"
-    assert hits[0].snippet == "hello"
-    assert hits[0].provider == "baidu"
-
-
-async def test_respects_top_k(monkeypatch):
-    _install(
-        monkeypatch,
-        {
-            "references": [
-                {"title": f"n{i}", "url": f"https://{i}.cn", "content": "c"}
-                for i in range(5)
-            ]
-        },
-    )
-
-    hits = await BaiduProvider().search("q", top_k=2)
-
-    assert len(hits) == 2

surfsense_backend/tests/unit/capabilities/web/discover/test_executor.py

@@ -1,99 +0,0 @@
-"""`web.discover` executor: pick the first configured provider; self-disable when none.
-
-Boundary mocked: the providers (injected fakes). NOT mocked: the executor's
-provider-selection and self-disable behavior.
-"""
-
-from __future__ import annotations
-
-import pytest
-
-from app.capabilities.web.discover.executor import (
-    NoDiscoverProviderError,
-    build_discover_executor,
-)
-from app.capabilities.web.discover.schemas import (
-    DiscoverHit,
-    DiscoverInput,
-    DiscoverOutput,
-)
-
-pytestmark = pytest.mark.unit
-
-
-class _FakeProvider:
-    def __init__(
-        self, name: str, available: bool, hits: list[DiscoverHit] | None = None
-    ):
-        self.name = name
-        self._available = available
-        self._hits = hits or []
-        self.calls: list[tuple[str, int]] = []
-
-    def is_available(self) -> bool:
-        return self._available
-
-    async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
-        self.calls.append((query, top_k))
-        return self._hits
-
-
-def _hit(url: str, provider: str) -> DiscoverHit:
-    return DiscoverHit(url=url, title=url, snippet="s", provider=provider)
-
-
-async def test_uses_the_first_available_provider():
-    first = _FakeProvider(
-        "searxng", available=True, hits=[_hit("https://a.com", "searxng")]
-    )
-    second = _FakeProvider(
-        "linkup", available=True, hits=[_hit("https://b.com", "linkup")]
-    )
-    execute = build_discover_executor(providers=[first, second])
-
-    out = await execute(DiscoverInput(query="acme pricing", top_k=5))
-
-    assert isinstance(out, DiscoverOutput)
-    assert [h.url for h in out.hits] == ["https://a.com"]
-    assert first.calls == [("acme pricing", 5)]
-    assert second.calls == []  # first available short-circuits
-
-
-async def test_skips_unavailable_providers():
-    off = _FakeProvider("searxng", available=False)
-    on = _FakeProvider("linkup", available=True, hits=[_hit("https://b.com", "linkup")])
-    execute = build_discover_executor(providers=[off, on])
-
-    out = await execute(DiscoverInput(query="q"))
-
-    assert [h.provider for h in out.hits] == ["linkup"]
-    assert off.calls == []
-
-
-async def test_self_disables_when_no_provider_is_configured():
-    execute = build_discover_executor(
-        providers=[_FakeProvider("searxng", available=False)]
-    )
-
-    with pytest.raises(NoDiscoverProviderError):
-        await execute(DiscoverInput(query="q"))
-
-
-async def test_caps_hits_to_top_k_when_provider_over_returns():
-    # SearXNG treats `limit` as a hint and can return more rows than asked; the
-    # verb must honor its own documented `top_k` cap regardless of the provider.
-    over = _FakeProvider(
-        "searxng",
-        available=True,
-        hits=[_hit(f"https://{i}.com", "searxng") for i in range(5)],
-    )
-    execute = build_discover_executor(providers=[over])
-
-    out = await execute(DiscoverInput(query="q", top_k=3))
-
-    assert len(out.hits) == 3
-    assert [h.url for h in out.hits] == [
-        "https://0.com",
-        "https://1.com",
-        "https://2.com",
-    ]

surfsense_backend/tests/unit/capabilities/web/discover/test_linkup_provider.py

@@ -1,75 +0,0 @@
-"""LinkupProvider maps the Linkup SDK results to DiscoverHits, env-keyed.
-
-Boundary mocked: the LinkupClient SDK + config key. NOT mocked: result→hit mapping.
-"""
-
-from __future__ import annotations
-
-import pytest
-
-import app.capabilities.web.discover.providers.linkup as linkup_module
-from app.capabilities.web.discover.providers.linkup import LinkupProvider
-from app.config import config
-
-pytestmark = pytest.mark.unit
-
-
-class _Result:
-    def __init__(self, name, url, content):
-        self.name = name
-        self.url = url
-        self.content = content
-        self.type = "text"
-
-
-class _Response:
-    def __init__(self, results):
-        self.results = results
-
-
-class _FakeClient:
-    def __init__(self, api_key):
-        self.api_key = api_key
-
-    def search(self, *, query, depth, output_type):
-        return _Response(
-            [
-                _Result("Acme", "https://acme.com", "acme home"),
-                _Result("No URL", "", "skip"),
-            ]
-        )
-
-
-def test_is_available_reflects_the_env_key(monkeypatch):
-    monkeypatch.setattr(config, "LINKUP_API_KEY", "k")
-    assert LinkupProvider().is_available() is True
-    monkeypatch.setattr(config, "LINKUP_API_KEY", None)
-    assert LinkupProvider().is_available() is False
-
-
-async def test_maps_results_to_hits(monkeypatch):
-    monkeypatch.setattr(config, "LINKUP_API_KEY", "k")
-    monkeypatch.setattr(linkup_module, "LinkupClient", _FakeClient)
-
-    hits = await LinkupProvider().search("acme", top_k=10)
-
-    assert [h.url for h in hits] == ["https://acme.com"]
-    assert hits[0].title == "Acme"
-    assert hits[0].snippet == "acme home"
-    assert hits[0].provider == "linkup"
-
-
-async def test_respects_top_k(monkeypatch):
-    monkeypatch.setattr(config, "LINKUP_API_KEY", "k")
-
-    class _ManyClient(_FakeClient):
-        def search(self, *, query, depth, output_type):
-            return _Response(
-                [_Result(f"n{i}", f"https://{i}.com", "c") for i in range(5)]
-            )
-
-    monkeypatch.setattr(linkup_module, "LinkupClient", _ManyClient)
-
-    hits = await LinkupProvider().search("q", top_k=2)
-
-    assert len(hits) == 2

surfsense_backend/tests/unit/capabilities/web/discover/test_schemas.py

@@ -1,22 +0,0 @@
-"""``web.discover`` I/O contract bounds."""
-
-from __future__ import annotations
-
-import pytest
-from pydantic import ValidationError
-
-from app.capabilities.web.discover.schemas import DiscoverInput
-
-pytestmark = pytest.mark.unit
-
-
-def test_top_k_defaults_and_accepts_the_valid_range():
-    assert DiscoverInput(query="q").top_k == 10
-    assert DiscoverInput(query="q", top_k=1).top_k == 1
-    assert DiscoverInput(query="q", top_k=50).top_k == 50
-
-
-@pytest.mark.parametrize("bad", [0, -1, 51, 100])
-def test_top_k_outside_1_to_50_is_rejected(bad):
-    with pytest.raises(ValidationError):
-        DiscoverInput(query="q", top_k=bad)

surfsense_backend/tests/unit/capabilities/web/discover/test_searxng_provider.py

@@ -1,58 +0,0 @@
-"""SearxngProvider maps the SearXNG service's sources to DiscoverHits.
-
-Boundary mocked: the web_search_service module. NOT mocked: the source→hit mapping.
-"""
-
-from __future__ import annotations
-
-from unittest.mock import AsyncMock
-
-import pytest
-
-import app.capabilities.web.discover.providers.searxng as searxng_module
-from app.capabilities.web.discover.providers.searxng import SearxngProvider
-
-pytestmark = pytest.mark.unit
-
-
-def _result(sources):
-    return ({"sources": sources}, [])
-
-
-async def test_maps_sources_to_hits(monkeypatch):
-    provider = SearxngProvider()
-    monkeypatch.setattr(
-        searxng_module.web_search_service,
-        "search",
-        AsyncMock(
-            return_value=_result(
-                [
-                    {"title": "Acme", "url": "https://acme.com", "description": "home"},
-                    {
-                        "title": "Docs",
-                        "url": "https://acme.com/docs",
-                        "description": "",
-                    },
-                    {"title": "no url", "url": "", "description": "skip me"},
-                ]
-            )
-        ),
-    )
-
-    hits = await provider.search("acme", top_k=5)
-
-    assert [h.url for h in hits] == ["https://acme.com", "https://acme.com/docs"]
-    assert hits[0].title == "Acme"
-    assert hits[0].snippet == "home"
-    assert hits[1].snippet is None  # empty description normalizes to None
-    assert all(h.provider == "searxng" for h in hits)
-
-
-def test_is_available_reflects_the_service(monkeypatch):
-    provider = SearxngProvider()
-    monkeypatch.setattr(searxng_module.web_search_service, "is_available", lambda: True)
-    assert provider.is_available() is True
-    monkeypatch.setattr(
-        searxng_module.web_search_service, "is_available", lambda: False
-    )
-    assert provider.is_available() is False

surfsense_backend/tests/unit/capabilities/web/scrape/test_executor.py

@@ -1,141 +0,0 @@
-"""`web.scrape` executor behavior: URLs in → cleaned rows out.
-
-Boundary mocked: the crawler (injected fake). NOT mocked: the executor's own
-CrawlOutcome → ScrapeRow mapping.
-"""
-
-from __future__ import annotations
-
-import pytest
-
-from app.capabilities.web.scrape.executor import build_scrape_executor
-from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput
-from app.proprietary.web_crawler import CrawlOutcome, CrawlOutcomeStatus
-
-pytestmark = pytest.mark.unit
-
-
-class _FakeCrawler:
-    """Stand-in for WebCrawlerConnector: serves a canned outcome per URL."""
-
-    def __init__(self, outcomes: dict[str, CrawlOutcome]):
-        self._outcomes = outcomes
-        self.calls: list[str] = []
-
-    async def crawl_url(self, url: str) -> CrawlOutcome:
-        self.calls.append(url)
-        return self._outcomes[url]
-
-
-def _success(content: str, metadata: dict[str, str]) -> CrawlOutcome:
-    return CrawlOutcome(
-        status=CrawlOutcomeStatus.SUCCESS,
-        result={
-            "content": content,
-            "metadata": metadata,
-            "crawler_type": "scrapling-static",
-        },
-    )
-
-
-async def test_scrape_returns_one_cleaned_row_for_a_successful_url():
-    url = "https://example.com"
-    crawler = _FakeCrawler({url: _success("# Hello", {"title": "Hello"})})
-    execute = build_scrape_executor(engine=crawler)
-
-    out = await execute(ScrapeInput(urls=[url]))
-
-    assert isinstance(out, ScrapeOutput)
-    assert len(out.rows) == 1
-    row = out.rows[0]
-    assert row.url == url
-    assert row.status == "success"
-    assert row.content == "# Hello"
-    assert row.metadata == {"title": "Hello"}
-
-
-async def test_scrape_returns_one_row_per_url_in_input_order():
-    a, b, c = "https://a.com", "https://b.com", "https://c.com"
-    crawler = _FakeCrawler(
-        {
-            a: _success("A", {"title": "A"}),
-            b: _success("B", {"title": "B"}),
-            c: _success("C", {"title": "C"}),
-        }
-    )
-    execute = build_scrape_executor(engine=crawler)
-
-    out = await execute(ScrapeInput(urls=[a, b, c]))
-
-    assert [row.url for row in out.rows] == [a, b, c]
-    assert [row.content for row in out.rows] == ["A", "B", "C"]
-
-
-async def test_content_longer_than_max_length_is_truncated():
-    url = "https://long.com"
-    crawler = _FakeCrawler({url: _success("A" * 100, {"title": "Long"})})
-    execute = build_scrape_executor(engine=crawler)
-
-    out = await execute(ScrapeInput(urls=[url], max_length=10))
-
-    assert out.rows[0].content == "A" * 10
-
-
-async def test_content_within_max_length_is_untouched():
-    url = "https://short.com"
-    crawler = _FakeCrawler({url: _success("hello", {"title": "Short"})})
-    execute = build_scrape_executor(engine=crawler)
-
-    out = await execute(ScrapeInput(urls=[url], max_length=10))
-
-    assert out.rows[0].content == "hello"
-
-
-async def test_scrape_surfaces_total_captcha_attempts_for_billing():
-    ok, blocked = "https://ok.com", "https://blocked.com"
-    crawler = _FakeCrawler(
-        {
-            ok: CrawlOutcome(
-                status=CrawlOutcomeStatus.SUCCESS,
-                result={"content": "OK", "metadata": {}},
-                captcha_attempts=2,
-                captcha_solved=True,
-            ),
-            blocked: CrawlOutcome(
-                status=CrawlOutcomeStatus.FAILED,
-                error="blocked",
-                captcha_attempts=1,
-                captcha_solved=False,
-            ),
-        }
-    )
-    execute = build_scrape_executor(engine=crawler)
-
-    out = await execute(ScrapeInput(urls=[ok, blocked]))
-
-    # Attempts bill even when the crawl ultimately failed (Phase 3d).
-    assert out.captcha_attempts == 3
-    assert out.captcha_solved == 1
-
-
-async def test_partial_failure_keeps_the_batch_and_labels_each_url():
-    ok, empty, failed = "https://ok.com", "https://empty.com", "https://failed.com"
-    crawler = _FakeCrawler(
-        {
-            ok: _success("OK", {"title": "OK"}),
-            empty: CrawlOutcome(status=CrawlOutcomeStatus.EMPTY, error="no content"),
-            failed: CrawlOutcome(status=CrawlOutcomeStatus.FAILED, error="blocked"),
-        }
-    )
-    execute = build_scrape_executor(engine=crawler)
-
-    out = await execute(ScrapeInput(urls=[ok, empty, failed]))
-
-    by_url = {row.url: row for row in out.rows}
-    assert {u: r.status for u, r in by_url.items()} == {
-        ok: "success",
-        empty: "empty",
-        failed: "failed",
-    }
-    assert by_url[failed].content is None
-    assert by_url[failed].error == "blocked"

surfsense_backend/tests/unit/capabilities/web/scrape/test_schemas.py

@@ -1,47 +0,0 @@
-"""ScrapeOutput reports its own billable count; ScrapeInput bounds its batch size."""
-
-from __future__ import annotations
-
-import pytest
-from pydantic import ValidationError
-
-from app.capabilities.web.scrape.schemas import (
-    MAX_SCRAPE_URLS,
-    ScrapeInput,
-    ScrapeOutput,
-    ScrapeRow,
-)
-
-pytestmark = pytest.mark.unit
-
-
-def _output(*statuses: str) -> ScrapeOutput:
-    return ScrapeOutput(
-        rows=[
-            ScrapeRow(url=f"https://{i}.com", status=status)
-            for i, status in enumerate(statuses)
-        ]
-    )
-
-
-def test_billable_units_counts_successful_rows():
-    assert _output("success", "empty", "success", "failed").billable_units == 2
-
-
-def test_billable_units_is_zero_without_successes():
-    assert _output("empty", "failed").billable_units == 0
-
-
-def test_rejects_empty_url_batch():
-    with pytest.raises(ValidationError):
-        ScrapeInput(urls=[])
-
-
-def test_rejects_batch_over_the_cap():
-    with pytest.raises(ValidationError):
-        ScrapeInput(urls=[f"https://{i}.com" for i in range(MAX_SCRAPE_URLS + 1)])
-
-
-def test_accepts_batch_at_the_cap():
-    payload = ScrapeInput(urls=[f"https://{i}.com" for i in range(MAX_SCRAPE_URLS)])
-    assert payload.estimated_units == MAX_SCRAPE_URLS