feat(billing): meter platform scrapers per item; consolidate web scraping onto web.crawl

Add per-item, per-platform billing for the platform-native connectors (Reddit, Google Search, Google Maps places/reviews, YouTube videos/comments) through the capability gate/charge seam. Rates are config-driven with a shared wallet-credit module (wallet_credit) and a dedicated PlatformScrapeCreditService; agent and REST capability runs now record cost_micros. Google Maps scrape dual-meters places and attached reviews.

Remove the main-agent scrape_webpage tool now that the web.crawl capability covers single-page (maxCrawlDepth=0) and site crawling. The main agent now reaches crawling via task(web_crawler, ...). Update prompts, tool catalog, receipts, skills, proprietary docs, and tests; drop the obsolete chat-turn crawl fold path.

Co-authored-by: Cursor <cursoragent@cursor.com>

surfsense_backend/app/services/platform_scrape_credit_service.py

@@ -0,0 +1,77 @@
+"""Charge the credit wallet per *item returned* by a platform-native scraper.
+
+Deliberately mirrors :class:`app.services.web_crawl_credit_service.WebCrawlCreditService`:
+a simple **gate -> pre-check -> post-charge** model (no reserve/finalize) — a
+scrape has no LLM token accumulator to settle against. The billable unit is one
+returned item (a Reddit post/comment, a SERP page, a Maps place/review, a
+YouTube video/comment); the per-item rate is passed in by the caller from the
+verb's config knob so this one service serves every platform meter.
+
+The price is **fully config-driven** — there is no hardcoded rate here. When
+``config.PLATFORM_SCRAPE_BILLING_ENABLED`` is False (the default for
+self-hosted / OSS installs) every check/charge is a no-op, preserving the prior
+effectively-free scraping behaviour.
+
+Wallet math (spendable / check / debit) is shared with the crawl biller via
+:mod:`app.services.wallet_credit`.
+"""
+
+from uuid import UUID
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import config
+
+# One "out of credit" type across every per-unit biller; the capability doors
+# already catch exactly this one.
+from app.services.etl_credit_service import InsufficientCreditsError
+from app.services import wallet_credit
+
+__all__ = ["InsufficientCreditsError", "PlatformScrapeCreditService"]
+
+
+class PlatformScrapeCreditService:
+    """Checks and charges the credit wallet for platform scraper items."""
+
+    def __init__(self, session: AsyncSession):
+        self.session = session
+
+    @staticmethod
+    def billing_enabled() -> bool:
+        return config.PLATFORM_SCRAPE_BILLING_ENABLED
+
+    @staticmethod
+    def items_to_micros(items: int, rate_micros: int) -> int:
+        """Convert an item count to USD micro-credits at ``rate_micros``/item."""
+        return int(items) * int(rate_micros)
+
+    async def check_credits(
+        self, user_id: str | UUID, estimated_items: int, rate_micros: int
+    ) -> None:
+        """Raise :class:`InsufficientCreditsError` if the wallet can't afford
+        ``estimated_items`` at ``rate_micros`` each.
+
+        No-op when platform billing is disabled. ``estimated_items`` is a safe
+        upper bound (the request's worst-case fan-out) so a passing pre-flight
+        guarantees the wallet can never go negative.
+        """
+        if not config.PLATFORM_SCRAPE_BILLING_ENABLED:
+            return
+        required = self.items_to_micros(estimated_items, rate_micros)
+        await wallet_credit.check_balance(self.session, user_id, required)
+
+    async def charge(
+        self, user_id: str | UUID, items: int, rate_micros: int
+    ) -> int | None:
+        """Debit the wallet for ``items`` returned at ``rate_micros`` each.
+
+        No-op when platform billing is disabled or ``items <= 0``. Returns the
+        new balance in micros, or ``None`` when nothing was charged.
+        """
+        if not config.PLATFORM_SCRAPE_BILLING_ENABLED:
+            return None
+        if items <= 0:
+            return None
+        return await wallet_credit.apply_debit(
+            self.session, user_id, self.items_to_micros(items, rate_micros)
+        )

surfsense_backend/app/services/wallet_credit.py

@@ -0,0 +1,99 @@
+"""Shared credit-wallet primitives for the flat-rate per-unit billers.
+
+Both :class:`app.services.web_crawl_credit_service.WebCrawlCreditService` and
+:class:`app.services.platform_scrape_credit_service.PlatformScrapeCreditService`
+follow the same gate -> pre-check -> post-charge model against the unified
+``User.credit_micros_*`` wallet. The wallet math lives here once instead of
+being copied per service:
+
+- :func:`spendable_micros` — ``balance - reserved`` (ungated by any flag)
+- :func:`check_balance` — raise :class:`InsufficientCreditsError` if short
+- :func:`apply_debit` — debit + commit + best-effort auto-reload
+
+``InsufficientCreditsError`` is re-exported from ``etl_credit_service`` so every
+per-unit biller (ETL, crawl, platform scrape) shares one "out of credit" type —
+the capability doors already catch exactly that one.
+"""
+
+from uuid import UUID
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.services.etl_credit_service import InsufficientCreditsError
+
+__all__ = ["InsufficientCreditsError", "apply_debit", "check_balance", "spendable_micros"]
+
+
+async def spendable_micros(session: AsyncSession, user_id: str | UUID) -> int:
+    """Raw ``balance - reserved`` read, **ungated** by any billing flag."""
+    from app.db import User
+
+    result = await session.execute(
+        select(User.credit_micros_balance, User.credit_micros_reserved).where(
+            User.id == user_id
+        )
+    )
+    row = result.first()
+    if not row:
+        raise ValueError(f"User with ID {user_id} not found")
+
+    balance, reserved = row
+    return balance - reserved
+
+
+async def check_balance(
+    session: AsyncSession, user_id: str | UUID, required_micros: int
+) -> None:
+    """Raise :class:`InsufficientCreditsError` if the wallet can't cover
+    ``required_micros``. Generic and **ungated** — the caller decides when at
+    least one relevant biller is enabled. No-op for a non-positive requirement.
+    """
+    if required_micros <= 0:
+        return
+    available = await spendable_micros(session, user_id)
+    if required_micros > available:
+        raise InsufficientCreditsError(
+            message=(
+                "This run would exceed your available credit. "
+                f"Available: ${available / 1_000_000:.2f}, "
+                f"estimated need: ${required_micros / 1_000_000:.2f}. "
+                "Add more credits to continue."
+            ),
+            balance_micros=available,
+            required_micros=required_micros,
+        )
+
+
+async def apply_debit(
+    session: AsyncSession, user_id: str | UUID, cost_micros: int
+) -> int | None:
+    """Debit ``cost_micros`` from the wallet and commit.
+
+    Flushes any audit row the caller staged before this, then fires a
+    best-effort auto-reload check. No-op for a non-positive cost; returns the
+    new balance in micros, or ``None`` when nothing was charged.
+    """
+    if cost_micros <= 0:
+        return None
+
+    from app.db import User
+
+    result = await session.execute(select(User).where(User.id == user_id))
+    user = result.unique().scalar_one_or_none()
+    if not user:
+        raise ValueError(f"User with ID {user_id} not found")
+
+    user.credit_micros_balance -= cost_micros
+    await session.commit()
+    await session.refresh(user)
+
+    # Best-effort: fire an auto-reload check if the balance dropped low.
+    try:
+        from app.services.auto_reload_service import maybe_trigger_auto_reload
+
+        await maybe_trigger_auto_reload(user_id)
+    except Exception:
+        pass
+
+    return user.credit_micros_balance

surfsense_backend/app/services/web_crawl_credit_service.py

@@ -13,14 +13,13 @@ for self-hosted / OSS installs) every check/charge is a no-op, preserving the
 prior effectively-free crawl behaviour.
 
 ``billing_enabled()`` and ``successes_to_micros()`` are exposed as static
-helpers so the chat ``scrape_webpage`` tools can share the flag/price math:
-they fold a single success into the current chat turn's existing bill (via the
-turn accumulator) instead of debiting the wallet directly.
+helpers so the capability biller (``app/capabilities/core/billing.py``, the
+``web.crawl`` verb) can share the flag/price math when gating and charging a
+crawl run.
 """
 
 from uuid import UUID
 
-from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.config import config
@@ -29,6 +28,10 @@ from app.config import config
 # to catch for "out of credit" across every per-unit wallet biller.
 from app.services.etl_credit_service import InsufficientCreditsError
 
+# Wallet math (spendable / check / debit) is shared with the platform-scrape
+# biller — see app/services/wallet_credit.py.
+from app.services import wallet_credit
+
 __all__ = ["InsufficientCreditsError", "WebCrawlCreditService"]
 
 
@@ -76,19 +79,7 @@ class WebCrawlCreditService:
         Used by :meth:`check_balance` for combined (crawl + captcha) pre-flight,
         where the relevant gate is decided by the caller, not by a single flag.
         """
-        from app.db import User
-
-        result = await self.session.execute(
-            select(User.credit_micros_balance, User.credit_micros_reserved).where(
-                User.id == user_id
-            )
-        )
-        row = result.first()
-        if not row:
-            raise ValueError(f"User with ID {user_id} not found")
-
-        balance, reserved = row
-        return balance - reserved
+        return await wallet_credit.spendable_micros(self.session, user_id)
 
     async def get_available_micros(self, user_id: str | UUID) -> int | None:
         """Return spendable credit in micro-USD (``balance - reserved``).
@@ -108,20 +99,7 @@ class WebCrawlCreditService:
         whichever billers are enabled and only calls this when at least one is.
         No-op for a non-positive requirement.
         """
-        if required_micros <= 0:
-            return
-        available = await self._spendable_micros(user_id)
-        if required_micros > available:
-            raise InsufficientCreditsError(
-                message=(
-                    "This run would exceed your available credit. "
-                    f"Available: ${available / 1_000_000:.2f}, "
-                    f"estimated need: ${required_micros / 1_000_000:.2f}. "
-                    "Add more credits to continue."
-                ),
-                balance_micros=available,
-                required_micros=required_micros,
-            )
+        await wallet_credit.check_balance(self.session, user_id, required_micros)
 
     async def check_credits(
         self, user_id: str | UUID, estimated_successes: int = 1
@@ -160,29 +138,7 @@ class WebCrawlCreditService:
         Mirrors ``EtlCreditService.charge_credits``' commit-then-refresh +
         best-effort auto-reload. No-op for a non-positive cost.
         """
-        if cost_micros <= 0:
-            return None
-
-        from app.db import User
-
-        result = await self.session.execute(select(User).where(User.id == user_id))
-        user = result.unique().scalar_one_or_none()
-        if not user:
-            raise ValueError(f"User with ID {user_id} not found")
-
-        user.credit_micros_balance -= cost_micros
-        await self.session.commit()
-        await self.session.refresh(user)
-
-        # Best-effort: fire an auto-reload check if the balance dropped low.
-        try:
-            from app.services.auto_reload_service import maybe_trigger_auto_reload
-
-            await maybe_trigger_auto_reload(user_id)
-        except Exception:
-            pass
-
-        return user.credit_micros_balance
+        return await wallet_credit.apply_debit(self.session, user_id, cost_micros)
 
     async def charge_credits(self, user_id: str | UUID, successes: int) -> int | None:
         """Debit the wallet for ``successes`` successful crawls.