feat(billing): meter platform scrapers per item; consolidate web scraping onto web.crawl

Add per-item, per-platform billing for the platform-native connectors (Reddit, Google Search, Google Maps places/reviews, YouTube videos/comments) through the capability gate/charge seam. Rates are config-driven with a shared wallet-credit module (wallet_credit) and a dedicated PlatformScrapeCreditService; agent and REST capability runs now record cost_micros. Google Maps scrape dual-meters places and attached reviews.

Remove the main-agent scrape_webpage tool now that the web.crawl capability covers single-page (maxCrawlDepth=0) and site crawling. The main agent now reaches crawling via task(web_crawler, ...). Update prompts, tool catalog, receipts, skills, proprietary docs, and tests; drop the obsolete chat-turn crawl fold path.

Co-authored-by: Cursor <cursoragent@cursor.com>

surfsense_backend/tests/unit/agents/multi_agent_chat/test_prompt_resources.py

@@ -45,7 +45,7 @@ def test_every_subagent_has_description_md(name: str):
     [
         "core_behavior.md",
         "routing.md",
-        "tools/scrape_webpage/description.md",
+        "tools/task/description.md",
     ],
 )
 def test_main_agent_prompt_fragments_resolve(filename: str):

surfsense_backend/tests/unit/agents/new_chat/test_default_permissions_layering.py

@@ -9,7 +9,7 @@ earliest -> latest:
 3. (future) user-defined rules from the Agent Permissions UI
 
 Without #1 every read-only built-in (``ls``, ``read_file``, ``grep``,
-``glob``, ``scrape_webpage`` …) defaulted to ``ask`` because
+``glob`` …) defaulted to ``ask`` because
 ``permissions.evaluate`` returns ``ask`` when no rule matches. That
 caused two production-painful behaviors:
 
@@ -58,7 +58,6 @@ class TestReadOnlyToolsAllowed:
             "read_file",
             "grep",
             "glob",
-            "scrape_webpage",
             "get_connected_accounts",
             "write_todos",
             "task",

surfsense_backend/tests/unit/capabilities/test_billing.py

@@ -256,3 +256,172 @@ async def test_gate_is_noop_for_free_verb(monkeypatch):
     await gate_capability(CrawlInput(startUrls=["https://a.com"]), None, _ctx(session))
 
     session.execute.assert_not_called()
+
+
+# ===================================================================
+# Platform scraper per-item billing (Reddit / Search / Maps / YouTube)
+# ===================================================================
+
+
+class _FakePlatformOutput:
+    """Stand-in for a verb output: only the billing-read properties matter."""
+
+    def __init__(self, items: int, attached_review_count: int = 0):
+        self._items = items
+        self._reviews = attached_review_count
+
+    @property
+    def billable_units(self) -> int:
+        return self._items
+
+    @property
+    def attached_review_count(self) -> int:
+        return self._reviews
+
+
+class _FakePlatformInput:
+    """Stand-in for a verb input reporting its worst-case unit counts."""
+
+    def __init__(self, estimated_units: int, estimated_review_units: int = 0):
+        self._units = estimated_units
+        self._review_units = estimated_review_units
+
+    @property
+    def estimated_units(self) -> int:
+        return self._units
+
+    @property
+    def estimated_review_units(self) -> int:
+        return self._review_units
+
+
+@pytest.fixture
+def _enable_platform_billing(monkeypatch):
+    monkeypatch.setattr(config, "PLATFORM_SCRAPE_BILLING_ENABLED", True)
+
+
+async def test_platform_charges_owner_per_item(
+    monkeypatch, record_usage, _enable_platform_billing
+):
+    monkeypatch.setattr(config, "REDDIT_SCRAPE_MICROS_PER_ITEM", 3500)
+    session, user = _make_session(_OWNER, balance_micros=1_000_000)
+
+    charged = await charge_capability(
+        _FakePlatformOutput(3), BillingUnit.REDDIT_ITEM, _ctx(session)
+    )
+
+    assert charged == 3 * 3500
+    assert user.credit_micros_balance == 1_000_000 - 3 * 3500
+    record_usage.assert_awaited_once()
+    kwargs = record_usage.await_args.kwargs
+    assert kwargs["usage_type"] == "reddit_item"
+    assert kwargs["user_id"] == _OWNER
+    assert kwargs["workspace_id"] == _WORKSPACE_ID
+    assert kwargs["cost_micros"] == 3 * 3500
+
+
+async def test_platform_maps_scrape_dual_meters_places_and_reviews(
+    monkeypatch, record_usage, _enable_platform_billing
+):
+    monkeypatch.setattr(config, "GOOGLE_MAPS_MICROS_PER_PLACE", 5000)
+    monkeypatch.setattr(config, "GOOGLE_MAPS_MICROS_PER_REVIEW", 2000)
+    session, user = _make_session(_OWNER, balance_micros=1_000_000)
+
+    # 2 places + 10 attached reviews -> 2*5000 + 10*2000 = 30000.
+    charged = await charge_capability(
+        _FakePlatformOutput(2, attached_review_count=10),
+        BillingUnit.GOOGLE_MAPS_PLACE,
+        _ctx(session),
+    )
+
+    assert charged == 2 * 5000 + 10 * 2000
+    assert user.credit_micros_balance == 1_000_000 - 30_000
+    assert record_usage.await_count == 2
+    usage_types = {c.kwargs["usage_type"] for c in record_usage.await_args_list}
+    assert usage_types == {"google_maps_place", "google_maps_review"}
+
+
+async def test_platform_charge_disabled_is_noop(monkeypatch, record_usage):
+    monkeypatch.setattr(config, "PLATFORM_SCRAPE_BILLING_ENABLED", False)
+    monkeypatch.setattr(config, "REDDIT_SCRAPE_MICROS_PER_ITEM", 3500)
+    session, user = _make_session(_OWNER, balance_micros=1_000_000)
+
+    charged = await charge_capability(
+        _FakePlatformOutput(3), BillingUnit.REDDIT_ITEM, _ctx(session)
+    )
+
+    assert charged == 0
+    record_usage.assert_not_awaited()
+    session.execute.assert_not_called()
+    assert user.credit_micros_balance == 1_000_000
+
+
+async def test_platform_no_items_is_free(
+    monkeypatch, record_usage, _enable_platform_billing
+):
+    monkeypatch.setattr(config, "YOUTUBE_MICROS_PER_COMMENT", 3500)
+    session, user = _make_session(_OWNER, balance_micros=1_000_000)
+
+    charged = await charge_capability(
+        _FakePlatformOutput(0), BillingUnit.YOUTUBE_COMMENT, _ctx(session)
+    )
+
+    assert charged == 0
+    record_usage.assert_not_awaited()
+    assert user.credit_micros_balance == 1_000_000
+
+
+async def test_platform_gate_blocks_when_worst_case_exceeds_balance(
+    monkeypatch, _enable_platform_billing
+):
+    monkeypatch.setattr(config, "GOOGLE_SEARCH_MICROS_PER_SERP", 5500)
+    session = _gate_session(_OWNER, balance_micros=6000)  # affords 1 SERP, not 2
+
+    with pytest.raises(InsufficientCreditsError):
+        await gate_capability(
+            _FakePlatformInput(estimated_units=2),
+            BillingUnit.GOOGLE_SEARCH_SERP,
+            _ctx(session),
+        )
+
+
+async def test_platform_gate_maps_reserves_places_plus_reviews(
+    monkeypatch, _enable_platform_billing
+):
+    monkeypatch.setattr(config, "GOOGLE_MAPS_MICROS_PER_PLACE", 5000)
+    monkeypatch.setattr(config, "GOOGLE_MAPS_MICROS_PER_REVIEW", 2000)
+    # 1 place (5000) + 10 worst-case reviews (20000) = 25000 required.
+    session = _gate_session(_OWNER, balance_micros=20_000)
+
+    with pytest.raises(InsufficientCreditsError):
+        await gate_capability(
+            _FakePlatformInput(estimated_units=1, estimated_review_units=10),
+            BillingUnit.GOOGLE_MAPS_PLACE,
+            _ctx(session),
+        )
+
+
+async def test_platform_gate_passes_when_affordable(
+    monkeypatch, _enable_platform_billing
+):
+    monkeypatch.setattr(config, "GOOGLE_SEARCH_MICROS_PER_SERP", 5500)
+    session = _gate_session(_OWNER, balance_micros=1_000_000)
+
+    await gate_capability(
+        _FakePlatformInput(estimated_units=2),
+        BillingUnit.GOOGLE_SEARCH_SERP,
+        _ctx(session),
+    )
+
+
+async def test_platform_gate_disabled_is_noop(monkeypatch):
+    monkeypatch.setattr(config, "PLATFORM_SCRAPE_BILLING_ENABLED", False)
+    session = _gate_session(_OWNER, balance_micros=0)
+
+    await gate_capability(
+        _FakePlatformInput(estimated_units=1000),
+        BillingUnit.REDDIT_ITEM,
+        _ctx(session),
+    )
+
+    session.execute.assert_not_called()

surfsense_backend/tests/unit/services/test_web_crawl_credit_service.py

@@ -175,48 +175,7 @@ class TestChargeCredits:
 
 
 # ===================================================================
-# E) Chat-scrape fold helper
-# ===================================================================
-
-
-class TestChatScrapeFold:
-    """The scrape_webpage tools fold one success into the live turn accumulator."""
-
-    def _import_helper(self):
-        from app.agents.chat.multi_agent_chat.main_agent.tools.scrape_webpage import (
-            _bill_successful_scrape,
-        )
-
-        return _bill_successful_scrape
-
-    def test_folds_into_active_turn_when_enabled(self):
-        from app.services.token_tracking_service import start_turn
-
-        acc = start_turn()
-        self._import_helper()()
-        assert acc.total_cost_micros == 1000
-        assert len(acc.calls) == 1
-        assert acc.calls[0].call_kind == "web_crawl"
-
-    def test_noop_when_billing_disabled(self, monkeypatch):
-        from app.services.token_tracking_service import start_turn
-
-        monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", False)
-        acc = start_turn()
-        self._import_helper()()
-        assert acc.calls == []
-        assert acc.total_cost_micros == 0
-
-    def test_noop_when_no_active_turn(self):
-        import contextvars
-
-        # Run in a fresh context so the turn ContextVar resolves to its default
-        # (None) regardless of other tests — must not raise.
-        contextvars.Context().run(self._import_helper())
-
-
-# ===================================================================
-# F) Captcha per-attempt unit (Phase 3d)
+# E) Captcha per-attempt unit (Phase 3d)
 # ===================================================================
 
 
@@ -284,41 +243,3 @@ class TestCheckBalance:
         session.execute.assert_not_called()
 
 
-class TestCaptchaChatFold:
-    """The scrape tools fold captcha attempts into the live turn accumulator."""
-
-    def _import_helper(self):
-        from app.agents.chat.multi_agent_chat.main_agent.tools.scrape_webpage import (
-            _bill_captcha_attempts,
-        )
-
-        return _bill_captcha_attempts
-
-    def _outcome(self, attempts):
-        o = MagicMock()
-        o.captcha_attempts = attempts
-        return o
-
-    def test_folds_attempts_when_enabled(self, _enable_captcha_billing):
-        from app.services.token_tracking_service import start_turn
-
-        acc = start_turn()
-        self._import_helper()(self._outcome(2))
-        assert acc.total_cost_micros == 6000
-        assert len(acc.calls) == 1
-        assert acc.calls[0].call_kind == "web_crawl_captcha"
-
-    def test_noop_when_zero_attempts(self, _enable_captcha_billing):
-        from app.services.token_tracking_service import start_turn
-
-        acc = start_turn()
-        self._import_helper()(self._outcome(0))
-        assert acc.calls == []
-
-    def test_noop_when_billing_disabled(self, monkeypatch):
-        from app.services.token_tracking_service import start_turn
-
-        monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", False)
-        acc = start_turn()
-        self._import_helper()(self._outcome(3))
-        assert acc.calls == []