feat(billing): meter platform scrapers per item; consolidate web scraping onto web.crawl

Add per-item, per-platform billing for the platform-native connectors (Reddit, Google Search, Google Maps places/reviews, YouTube videos/comments) through the capability gate/charge seam. Rates are config-driven with a shared wallet-credit module (wallet_credit) and a dedicated PlatformScrapeCreditService; agent and REST capability runs now record cost_micros. Google Maps scrape dual-meters places and attached reviews.

Remove the main-agent scrape_webpage tool now that the web.crawl capability covers single-page (maxCrawlDepth=0) and site crawling. The main agent now reaches crawling via task(web_crawler, ...). Update prompts, tool catalog, receipts, skills, proprietary docs, and tests; drop the obsolete chat-turn crawl fold path.

Co-authored-by: Cursor <cursoragent@cursor.com>

surfsense_backend/app/capabilities/core/access/agent.py

@@ -95,7 +95,7 @@ def _capability_tool(capability: Capability, workspace_id: int) -> BaseTool:
                 raise
 
             duration_ms = int((time.perf_counter() - started) * 1000)
-            await charge_capability(output, unit, ctx)
+            cost_micros = await charge_capability(output, unit, ctx)
 
         serialized = serialize_output(output)
         async with async_session_maker() as rec_session:
@@ -109,6 +109,7 @@ def _capability_tool(capability: Capability, workspace_id: int) -> BaseTool:
                 input=input_dump,
                 thread_id=thread_id,
                 duration_ms=duration_ms,
+                cost_micros=cost_micros,
             )
 
         if serialized.char_count <= RUN_OUTPUT_CHAR_CAP:

surfsense_backend/app/capabilities/core/access/rest.py

@@ -140,7 +140,7 @@ def _register_verb(router: APIRouter, capability: Capability) -> None:
             ) from exc
 
         duration_ms = int((time.perf_counter() - started) * 1000)
-        await charge_capability(output, unit, ctx)
+        cost_micros = await charge_capability(output, unit, ctx)
 
         serialized = serialize_output(output)
         run_id = await _record_rest_run(
@@ -152,6 +152,7 @@ def _register_verb(router: APIRouter, capability: Capability) -> None:
             input=input_dump,
             user_id=user_id,
             duration_ms=duration_ms,
+            cost_micros=cost_micros,
         )
         if run_id is not None:
             response.headers["X-Run-Id"] = f"run_{run_id}"

surfsense_backend/app/capabilities/core/billing.py

@@ -13,6 +13,8 @@ from app.capabilities.core.types import (
     CapabilityContext,
 )
 from app.config import config
+from app.services import wallet_credit
+from app.services.platform_scrape_credit_service import PlatformScrapeCreditService
 from app.services.token_tracking_service import record_token_usage
 from app.services.web_crawl_credit_service import WebCrawlCreditService
 from app.utils.captcha import captcha_enabled
@@ -23,6 +25,24 @@ if TYPE_CHECKING:
     from sqlalchemy.ext.asyncio import AsyncSession
 
 
+# Each platform meter -> the config knob holding its micro-USD per-item rate.
+# The rate is looked up live (not cached) so an env retune + restart takes
+# effect without a code change, mirroring the crawl biller.
+_PLATFORM_RATE_KEYS: dict[BillingUnit, str] = {
+    BillingUnit.REDDIT_ITEM: "REDDIT_SCRAPE_MICROS_PER_ITEM",
+    BillingUnit.GOOGLE_SEARCH_SERP: "GOOGLE_SEARCH_MICROS_PER_SERP",
+    BillingUnit.GOOGLE_MAPS_PLACE: "GOOGLE_MAPS_MICROS_PER_PLACE",
+    BillingUnit.GOOGLE_MAPS_REVIEW: "GOOGLE_MAPS_MICROS_PER_REVIEW",
+    BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO",
+    BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT",
+}
+
+
+def _platform_rate(unit: BillingUnit) -> int:
+    """Micro-USD per item for a platform meter, read live from config."""
+    return int(getattr(config, _PLATFORM_RATE_KEYS[unit]))
+
+
 async def gate_capability(
     payload: BillableInput, unit: BillingUnit | None, ctx: CapabilityContext
 ) -> None:
@@ -35,6 +55,8 @@ async def gate_capability(
         return
     if unit is BillingUnit.WEB_CRAWL:
         await _gate_web_crawl(ctx, payload.estimated_units)
+        return
+    await _gate_platform(payload, unit, ctx)
 
 
 async def _gate_web_crawl(ctx: CapabilityContext, estimated_successes: int) -> None:
@@ -62,62 +84,145 @@ async def _gate_web_crawl(ctx: CapabilityContext, estimated_successes: int) -> N
     await service.check_balance(owner_user_id, required_micros)
 
 
-async def charge_capability(
-    output: BillableOutput, unit: BillingUnit | None, ctx: CapabilityContext
+async def _gate_platform(
+    payload: BillableInput, unit: BillingUnit, ctx: CapabilityContext
 ) -> None:
-    """Bill the workspace owner for this result's billable successes (03c). ``None`` = free.
+    """Reserve the worst-case per-item cost for a platform scraper verb.
 
-    For crawl-backed verbs this also bills any captcha *attempts* (Phase 3d) as a
-    separate per-attempt unit — the solver charges per attempt even when the crawl
-    ultimately failed, so it can't ride the per-success crawl meter.
+    ``google_maps.scrape`` is dual-metered: it can attach reviews per place, so
+    its gate also reserves ``estimated_review_units`` at the review rate — same
+    two-meters-one-verb shape as crawl + captcha.
     """
-    if unit is None:
-        return
-    if unit is BillingUnit.WEB_CRAWL:
-        await _charge_web_crawl(ctx, output.billable_units)
-        await _charge_captcha(ctx, getattr(output, "captcha_attempts", 0))
-
-
-async def _charge_web_crawl(ctx: CapabilityContext, successes: int) -> None:
-    if successes <= 0:
-        return
-    service = WebCrawlCreditService(ctx.session)
+    service = PlatformScrapeCreditService(ctx.session)
     if not service.billing_enabled():
         return
     owner_user_id = await _resolve_workspace_owner(ctx.session, ctx.workspace_id)
     if owner_user_id is None:
         return
+
+    required_micros = service.items_to_micros(
+        payload.estimated_units, _platform_rate(unit)
+    )
+    if unit is BillingUnit.GOOGLE_MAPS_PLACE:
+        review_units = getattr(payload, "estimated_review_units", 0)
+        required_micros += service.items_to_micros(
+            review_units, _platform_rate(BillingUnit.GOOGLE_MAPS_REVIEW)
+        )
+    await wallet_credit.check_balance(ctx.session, owner_user_id, required_micros)
+
+
+async def charge_capability(
+    output: BillableOutput, unit: BillingUnit | None, ctx: CapabilityContext
+) -> int:
+    """Bill the workspace owner for this result and return the micros charged.
+
+    For crawl-backed verbs this also bills any captcha *attempts* (Phase 3d) as a
+    separate per-attempt unit — the solver charges per attempt even when the crawl
+    ultimately failed, so it can't ride the per-success crawl meter. Platform
+    verbs bill per item returned; ``google_maps.scrape`` additionally bills its
+    attached reviews. ``None`` unit = free = returns 0.
+
+    The returned total lets the doors persist a per-run ``cost_micros``.
+    """
+    if unit is None:
+        return 0
+    if unit is BillingUnit.WEB_CRAWL:
+        charged = await _charge_web_crawl(ctx, output.billable_units)
+        charged += await _charge_captcha(ctx, getattr(output, "captcha_attempts", 0))
+        return charged
+    return await _charge_platform(output, unit, ctx)
+
+
+async def _charge_web_crawl(ctx: CapabilityContext, successes: int) -> int:
+    if successes <= 0:
+        return 0
+    service = WebCrawlCreditService(ctx.session)
+    if not service.billing_enabled():
+        return 0
+    owner_user_id = await _resolve_workspace_owner(ctx.session, ctx.workspace_id)
+    if owner_user_id is None:
+        return 0
+    cost_micros = service.successes_to_micros(successes)
     # Stage the audit row before charge_credits' commit flushes both.
     await record_token_usage(
         ctx.session,
         usage_type="web_crawl",
         workspace_id=ctx.workspace_id,
         user_id=owner_user_id,
-        cost_micros=service.successes_to_micros(successes),
+        cost_micros=cost_micros,
         call_details={"successes": successes},
     )
     await service.charge_credits(owner_user_id, successes)
+    return cost_micros
 
 
-async def _charge_captcha(ctx: CapabilityContext, attempts: int) -> None:
+async def _charge_captcha(ctx: CapabilityContext, attempts: int) -> int:
     if attempts <= 0:
-        return
+        return 0
     service = WebCrawlCreditService(ctx.session)
     if not service.captcha_billing_enabled():
-        return
+        return 0
     owner_user_id = await _resolve_workspace_owner(ctx.session, ctx.workspace_id)
     if owner_user_id is None:
-        return
+        return 0
+    cost_micros = service.captcha_solves_to_micros(attempts)
     # Stage the audit row before charge_captcha's commit flushes both.
     await record_token_usage(
         ctx.session,
         usage_type="web_crawl_captcha",
         workspace_id=ctx.workspace_id,
         user_id=owner_user_id,
-        cost_micros=service.captcha_solves_to_micros(attempts),
+        cost_micros=cost_micros,
         call_details={"attempts": attempts},
     )
     await service.charge_captcha(owner_user_id, attempts)
+    return cost_micros
+
+
+async def _charge_platform(
+    output: BillableOutput, unit: BillingUnit, ctx: CapabilityContext
+) -> int:
+    """Charge a platform verb per item; dual-meter ``google_maps.scrape`` reviews."""
+    service = PlatformScrapeCreditService(ctx.session)
+    if not service.billing_enabled():
+        return 0
+    owner_user_id = await _resolve_workspace_owner(ctx.session, ctx.workspace_id)
+    if owner_user_id is None:
+        return 0
+
+    charged = await _charge_platform_meter(
+        service, ctx, owner_user_id, unit, output.billable_units
+    )
+    if unit is BillingUnit.GOOGLE_MAPS_PLACE:
+        reviews = getattr(output, "attached_review_count", 0)
+        charged += await _charge_platform_meter(
+            service, ctx, owner_user_id, BillingUnit.GOOGLE_MAPS_REVIEW, reviews
+        )
+    return charged
+
+
+async def _charge_platform_meter(
+    service: PlatformScrapeCreditService,
+    ctx: CapabilityContext,
+    owner_user_id: UUID,
+    unit: BillingUnit,
+    items: int,
+) -> int:
+    if items <= 0:
+        return 0
+    rate = _platform_rate(unit)
+    cost_micros = service.items_to_micros(items, rate)
+    # Stage the audit row before charge's commit flushes both.
+    await record_token_usage(
+        ctx.session,
+        usage_type=unit.value,
+        workspace_id=ctx.workspace_id,
+        user_id=owner_user_id,
+        cost_micros=cost_micros,
+        call_details={"items": items},
+    )
+    await service.charge(owner_user_id, items, rate)
+    return cost_micros
 
 
 async def _resolve_workspace_owner(

surfsense_backend/app/capabilities/core/types.py

@@ -13,9 +13,18 @@ if TYPE_CHECKING:
 
 
 class BillingUnit(StrEnum):
-    """The meter a verb charges on (priced by the billing service, 03c). ``None`` = free."""
+    """The meter a verb charges on (priced by the billing service, 03c). ``None`` = free.
+
+    Each value doubles as the ``TokenUsage.usage_type`` audit string for that meter.
+    """
 
     WEB_CRAWL = "web_crawl"
+    REDDIT_ITEM = "reddit_item"
+    GOOGLE_SEARCH_SERP = "google_search_serp"
+    GOOGLE_MAPS_PLACE = "google_maps_place"
+    GOOGLE_MAPS_REVIEW = "google_maps_review"
+    YOUTUBE_VIDEO = "youtube_video"
+    YOUTUBE_COMMENT = "youtube_comment"
 
 
 class BillableInput(Protocol):