feat(capabilities): meter captcha, truncate scrape, bound discover

surfsense_backend/tests/unit/capabilities/test_billing.py

@@ -92,6 +92,53 @@ async def test_charges_workspace_owner_per_successful_crawl(monkeypatch, record_
     assert kwargs["cost_micros"] == 2000
 
 
+def _output_with_captcha(*statuses: str, attempts: int, solved: int) -> ScrapeOutput:
+    out = _output(*statuses)
+    out.captcha_attempts = attempts
+    out.captcha_solved = solved
+    return out
+
+
+async def test_charges_workspace_owner_per_captcha_attempt_even_when_crawl_failed(
+    monkeypatch, record_usage
+):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
+    monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", True)
+    monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE", 3000)
+    session, user = _make_session(_OWNER, balance_micros=100_000)
+
+    # Crawl failed (no billable success) but the solver ran twice — attempts bill.
+    await charge_capability(
+        _output_with_captcha("failed", attempts=2, solved=1),
+        BillingUnit.WEB_CRAWL,
+        _ctx(session),
+    )
+
+    assert user.credit_micros_balance == 100_000 - 2 * 3000
+    record_usage.assert_awaited_once()
+    kwargs = record_usage.await_args.kwargs
+    assert kwargs["usage_type"] == "web_crawl_captcha"
+    assert kwargs["user_id"] == _OWNER
+    assert kwargs["cost_micros"] == 6000
+
+
+async def test_captcha_billing_disabled_does_not_charge_attempts(
+    monkeypatch, record_usage
+):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
+    monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", False)
+    session, user = _make_session(_OWNER, balance_micros=100_000)
+
+    await charge_capability(
+        _output_with_captcha("failed", attempts=2, solved=1),
+        BillingUnit.WEB_CRAWL,
+        _ctx(session),
+    )
+
+    record_usage.assert_not_awaited()
+    assert user.credit_micros_balance == 100_000
+
+
 async def test_no_successful_rows_is_free(monkeypatch, record_usage):
     monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
     session, user = _make_session(_OWNER, balance_micros=100_000)
@@ -176,6 +223,32 @@ async def test_gate_is_noop_when_disabled(monkeypatch):
     )
 
 
+async def test_gate_reserves_worst_case_captcha_when_solving_enabled(monkeypatch):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", False)
+    monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", True)
+    monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE", 3000)
+    monkeypatch.setattr(config, "CAPTCHA_MAX_ATTEMPTS_PER_URL", 3)
+    monkeypatch.setattr(billing, "captcha_enabled", lambda: True)
+    session = _gate_session(_OWNER, balance_micros=5000)  # < 1 url * 3 * 3000
+
+    with pytest.raises(InsufficientCreditsError):
+        await gate_capability(
+            ScrapeInput(urls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
+        )
+
+
+async def test_gate_does_not_reserve_captcha_when_solving_disabled(monkeypatch):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", False)
+    monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", True)
+    monkeypatch.setattr(billing, "captcha_enabled", lambda: False)
+    session = _gate_session(_OWNER, balance_micros=0)
+
+    # Solving off → attempts can never happen → nothing to reserve → passes.
+    await gate_capability(
+        ScrapeInput(urls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
+    )
+
+
 async def test_gate_is_noop_for_free_verb(monkeypatch):
     monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
     session = _gate_session(_OWNER, balance_micros=0)

surfsense_backend/tests/unit/capabilities/web/discover/test_schemas.py

@@ -0,0 +1,22 @@
+"""``web.discover`` I/O contract bounds."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.web.discover.schemas import DiscoverInput
+
+pytestmark = pytest.mark.unit
+
+
+def test_top_k_defaults_and_accepts_the_valid_range():
+    assert DiscoverInput(query="q").top_k == 10
+    assert DiscoverInput(query="q", top_k=1).top_k == 1
+    assert DiscoverInput(query="q", top_k=50).top_k == 50
+
+
+@pytest.mark.parametrize("bad", [0, -1, 51, 100])
+def test_top_k_outside_1_to_50_is_rejected(bad):
+    with pytest.raises(ValidationError):
+        DiscoverInput(query="q", top_k=bad)

surfsense_backend/tests/unit/capabilities/web/scrape/test_executor.py

@@ -71,6 +71,53 @@ async def test_scrape_returns_one_row_per_url_in_input_order():
     assert [row.content for row in out.rows] == ["A", "B", "C"]
 
 
+async def test_content_longer_than_max_length_is_truncated():
+    url = "https://long.com"
+    crawler = _FakeCrawler({url: _success("A" * 100, {"title": "Long"})})
+    execute = build_scrape_executor(engine=crawler)
+
+    out = await execute(ScrapeInput(urls=[url], max_length=10))
+
+    assert out.rows[0].content == "A" * 10
+
+
+async def test_content_within_max_length_is_untouched():
+    url = "https://short.com"
+    crawler = _FakeCrawler({url: _success("hello", {"title": "Short"})})
+    execute = build_scrape_executor(engine=crawler)
+
+    out = await execute(ScrapeInput(urls=[url], max_length=10))
+
+    assert out.rows[0].content == "hello"
+
+
+async def test_scrape_surfaces_total_captcha_attempts_for_billing():
+    ok, blocked = "https://ok.com", "https://blocked.com"
+    crawler = _FakeCrawler(
+        {
+            ok: CrawlOutcome(
+                status=CrawlOutcomeStatus.SUCCESS,
+                result={"content": "OK", "metadata": {}},
+                captcha_attempts=2,
+                captcha_solved=True,
+            ),
+            blocked: CrawlOutcome(
+                status=CrawlOutcomeStatus.FAILED,
+                error="blocked",
+                captcha_attempts=1,
+                captcha_solved=False,
+            ),
+        }
+    )
+    execute = build_scrape_executor(engine=crawler)
+
+    out = await execute(ScrapeInput(urls=[ok, blocked]))
+
+    # Attempts bill even when the crawl ultimately failed (Phase 3d).
+    assert out.captcha_attempts == 3
+    assert out.captcha_solved == 1
+
+
 async def test_partial_failure_keeps_the_batch_and_labels_each_url():
     ok, empty, failed = "https://ok.com", "https://empty.com", "https://failed.com"
     crawler = _FakeCrawler(