feat(billing): meter platform scrapers per item; consolidate web scraping onto web.crawl

Add per-item, per-platform billing for the platform-native connectors (Reddit, Google Search, Google Maps places/reviews, YouTube videos/comments) through the capability gate/charge seam. Rates are config-driven with a shared wallet-credit module (wallet_credit) and a dedicated PlatformScrapeCreditService; agent and REST capability runs now record cost_micros. Google Maps scrape dual-meters places and attached reviews.

Remove the main-agent scrape_webpage tool now that the web.crawl capability covers single-page (maxCrawlDepth=0) and site crawling. The main agent now reaches crawling via task(web_crawler, ...). Update prompts, tool catalog, receipts, skills, proprietary docs, and tests; drop the obsolete chat-turn crawl fold path.

Co-authored-by: Cursor <cursoragent@cursor.com>

surfsense_backend/app/capabilities/core/access/agent.py

@@ -95,7 +95,7 @@ def _capability_tool(capability: Capability, workspace_id: int) -> BaseTool:
                 raise
 
             duration_ms = int((time.perf_counter() - started) * 1000)
-            await charge_capability(output, unit, ctx)
+            cost_micros = await charge_capability(output, unit, ctx)
 
         serialized = serialize_output(output)
         async with async_session_maker() as rec_session:
@@ -109,6 +109,7 @@ def _capability_tool(capability: Capability, workspace_id: int) -> BaseTool:
                 input=input_dump,
                 thread_id=thread_id,
                 duration_ms=duration_ms,
+                cost_micros=cost_micros,
             )
 
         if serialized.char_count <= RUN_OUTPUT_CHAR_CAP:

surfsense_backend/app/capabilities/core/access/rest.py

@@ -140,7 +140,7 @@ def _register_verb(router: APIRouter, capability: Capability) -> None:
             ) from exc
 
         duration_ms = int((time.perf_counter() - started) * 1000)
-        await charge_capability(output, unit, ctx)
+        cost_micros = await charge_capability(output, unit, ctx)
 
         serialized = serialize_output(output)
         run_id = await _record_rest_run(
@@ -152,6 +152,7 @@ def _register_verb(router: APIRouter, capability: Capability) -> None:
             input=input_dump,
             user_id=user_id,
             duration_ms=duration_ms,
+            cost_micros=cost_micros,
         )
         if run_id is not None:
             response.headers["X-Run-Id"] = f"run_{run_id}"

surfsense_backend/app/capabilities/core/billing.py

@@ -13,6 +13,8 @@ from app.capabilities.core.types import (
     CapabilityContext,
 )
 from app.config import config
+from app.services import wallet_credit
+from app.services.platform_scrape_credit_service import PlatformScrapeCreditService
 from app.services.token_tracking_service import record_token_usage
 from app.services.web_crawl_credit_service import WebCrawlCreditService
 from app.utils.captcha import captcha_enabled
@@ -23,6 +25,24 @@ if TYPE_CHECKING:
     from sqlalchemy.ext.asyncio import AsyncSession
 
 
+# Each platform meter -> the config knob holding its micro-USD per-item rate.
+# The rate is looked up live (not cached) so an env retune + restart takes
+# effect without a code change, mirroring the crawl biller.
+_PLATFORM_RATE_KEYS: dict[BillingUnit, str] = {
+    BillingUnit.REDDIT_ITEM: "REDDIT_SCRAPE_MICROS_PER_ITEM",
+    BillingUnit.GOOGLE_SEARCH_SERP: "GOOGLE_SEARCH_MICROS_PER_SERP",
+    BillingUnit.GOOGLE_MAPS_PLACE: "GOOGLE_MAPS_MICROS_PER_PLACE",
+    BillingUnit.GOOGLE_MAPS_REVIEW: "GOOGLE_MAPS_MICROS_PER_REVIEW",
+    BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO",
+    BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT",
+}
+
+
+def _platform_rate(unit: BillingUnit) -> int:
+    """Micro-USD per item for a platform meter, read live from config."""
+    return int(getattr(config, _PLATFORM_RATE_KEYS[unit]))
+
+
 async def gate_capability(
     payload: BillableInput, unit: BillingUnit | None, ctx: CapabilityContext
 ) -> None:
@@ -35,6 +55,8 @@ async def gate_capability(
         return
     if unit is BillingUnit.WEB_CRAWL:
         await _gate_web_crawl(ctx, payload.estimated_units)
+        return
+    await _gate_platform(payload, unit, ctx)
 
 
 async def _gate_web_crawl(ctx: CapabilityContext, estimated_successes: int) -> None:
@@ -62,62 +84,145 @@ async def _gate_web_crawl(ctx: CapabilityContext, estimated_successes: int) -> N
     await service.check_balance(owner_user_id, required_micros)
 
 
-async def charge_capability(
-    output: BillableOutput, unit: BillingUnit | None, ctx: CapabilityContext
+async def _gate_platform(
+    payload: BillableInput, unit: BillingUnit, ctx: CapabilityContext
 ) -> None:
-    """Bill the workspace owner for this result's billable successes (03c). ``None`` = free.
+    """Reserve the worst-case per-item cost for a platform scraper verb.
 
-    For crawl-backed verbs this also bills any captcha *attempts* (Phase 3d) as a
-    separate per-attempt unit — the solver charges per attempt even when the crawl
-    ultimately failed, so it can't ride the per-success crawl meter.
+    ``google_maps.scrape`` is dual-metered: it can attach reviews per place, so
+    its gate also reserves ``estimated_review_units`` at the review rate — same
+    two-meters-one-verb shape as crawl + captcha.
     """
-    if unit is None:
-        return
-    if unit is BillingUnit.WEB_CRAWL:
-        await _charge_web_crawl(ctx, output.billable_units)
-        await _charge_captcha(ctx, getattr(output, "captcha_attempts", 0))
-
-
-async def _charge_web_crawl(ctx: CapabilityContext, successes: int) -> None:
-    if successes <= 0:
-        return
-    service = WebCrawlCreditService(ctx.session)
+    service = PlatformScrapeCreditService(ctx.session)
     if not service.billing_enabled():
         return
     owner_user_id = await _resolve_workspace_owner(ctx.session, ctx.workspace_id)
     if owner_user_id is None:
         return
+
+    required_micros = service.items_to_micros(
+        payload.estimated_units, _platform_rate(unit)
+    )
+    if unit is BillingUnit.GOOGLE_MAPS_PLACE:
+        review_units = getattr(payload, "estimated_review_units", 0)
+        required_micros += service.items_to_micros(
+            review_units, _platform_rate(BillingUnit.GOOGLE_MAPS_REVIEW)
+        )
+    await wallet_credit.check_balance(ctx.session, owner_user_id, required_micros)
+
+
+async def charge_capability(
+    output: BillableOutput, unit: BillingUnit | None, ctx: CapabilityContext
+) -> int:
+    """Bill the workspace owner for this result and return the micros charged.
+
+    For crawl-backed verbs this also bills any captcha *attempts* (Phase 3d) as a
+    separate per-attempt unit — the solver charges per attempt even when the crawl
+    ultimately failed, so it can't ride the per-success crawl meter. Platform
+    verbs bill per item returned; ``google_maps.scrape`` additionally bills its
+    attached reviews. ``None`` unit = free = returns 0.
+
+    The returned total lets the doors persist a per-run ``cost_micros``.
+    """
+    if unit is None:
+        return 0
+    if unit is BillingUnit.WEB_CRAWL:
+        charged = await _charge_web_crawl(ctx, output.billable_units)
+        charged += await _charge_captcha(ctx, getattr(output, "captcha_attempts", 0))
+        return charged
+    return await _charge_platform(output, unit, ctx)
+
+
+async def _charge_web_crawl(ctx: CapabilityContext, successes: int) -> int:
+    if successes <= 0:
+        return 0
+    service = WebCrawlCreditService(ctx.session)
+    if not service.billing_enabled():
+        return 0
+    owner_user_id = await _resolve_workspace_owner(ctx.session, ctx.workspace_id)
+    if owner_user_id is None:
+        return 0
+    cost_micros = service.successes_to_micros(successes)
     # Stage the audit row before charge_credits' commit flushes both.
     await record_token_usage(
         ctx.session,
         usage_type="web_crawl",
         workspace_id=ctx.workspace_id,
         user_id=owner_user_id,
-        cost_micros=service.successes_to_micros(successes),
+        cost_micros=cost_micros,
         call_details={"successes": successes},
     )
     await service.charge_credits(owner_user_id, successes)
+    return cost_micros
 
 
-async def _charge_captcha(ctx: CapabilityContext, attempts: int) -> None:
+async def _charge_captcha(ctx: CapabilityContext, attempts: int) -> int:
     if attempts <= 0:
-        return
+        return 0
     service = WebCrawlCreditService(ctx.session)
     if not service.captcha_billing_enabled():
-        return
+        return 0
     owner_user_id = await _resolve_workspace_owner(ctx.session, ctx.workspace_id)
     if owner_user_id is None:
-        return
+        return 0
+    cost_micros = service.captcha_solves_to_micros(attempts)
     # Stage the audit row before charge_captcha's commit flushes both.
     await record_token_usage(
         ctx.session,
         usage_type="web_crawl_captcha",
         workspace_id=ctx.workspace_id,
         user_id=owner_user_id,
-        cost_micros=service.captcha_solves_to_micros(attempts),
+        cost_micros=cost_micros,
         call_details={"attempts": attempts},
     )
     await service.charge_captcha(owner_user_id, attempts)
+    return cost_micros
+
+
+async def _charge_platform(
+    output: BillableOutput, unit: BillingUnit, ctx: CapabilityContext
+) -> int:
+    """Charge a platform verb per item; dual-meter ``google_maps.scrape`` reviews."""
+    service = PlatformScrapeCreditService(ctx.session)
+    if not service.billing_enabled():
+        return 0
+    owner_user_id = await _resolve_workspace_owner(ctx.session, ctx.workspace_id)
+    if owner_user_id is None:
+        return 0
+
+    charged = await _charge_platform_meter(
+        service, ctx, owner_user_id, unit, output.billable_units
+    )
+    if unit is BillingUnit.GOOGLE_MAPS_PLACE:
+        reviews = getattr(output, "attached_review_count", 0)
+        charged += await _charge_platform_meter(
+            service, ctx, owner_user_id, BillingUnit.GOOGLE_MAPS_REVIEW, reviews
+        )
+    return charged
+
+
+async def _charge_platform_meter(
+    service: PlatformScrapeCreditService,
+    ctx: CapabilityContext,
+    owner_user_id: UUID,
+    unit: BillingUnit,
+    items: int,
+) -> int:
+    if items <= 0:
+        return 0
+    rate = _platform_rate(unit)
+    cost_micros = service.items_to_micros(items, rate)
+    # Stage the audit row before charge's commit flushes both.
+    await record_token_usage(
+        ctx.session,
+        usage_type=unit.value,
+        workspace_id=ctx.workspace_id,
+        user_id=owner_user_id,
+        cost_micros=cost_micros,
+        call_details={"items": items},
+    )
+    await service.charge(owner_user_id, items, rate)
+    return cost_micros
 
 
 async def _resolve_workspace_owner(

surfsense_backend/app/capabilities/core/types.py

@@ -13,9 +13,18 @@ if TYPE_CHECKING:
 
 
 class BillingUnit(StrEnum):
-    """The meter a verb charges on (priced by the billing service, 03c). ``None`` = free."""
+    """The meter a verb charges on (priced by the billing service, 03c). ``None`` = free.
+
+    Each value doubles as the ``TokenUsage.usage_type`` audit string for that meter.
+    """
 
     WEB_CRAWL = "web_crawl"
+    REDDIT_ITEM = "reddit_item"
+    GOOGLE_SEARCH_SERP = "google_search_serp"
+    GOOGLE_MAPS_PLACE = "google_maps_place"
+    GOOGLE_MAPS_REVIEW = "google_maps_review"
+    YOUTUBE_VIDEO = "youtube_video"
+    YOUTUBE_COMMENT = "youtube_comment"
 
 
 class BillableInput(Protocol):

surfsense_backend/app/capabilities/google_maps/reviews/definition.py

@@ -1,8 +1,9 @@
-"""``google_maps.reviews`` capability registration (free — see 04-capabilities open item)."""
+"""``google_maps.reviews`` capability registration (billed per review; see
+config ``GOOGLE_MAPS_MICROS_PER_REVIEW``)."""
 
 from __future__ import annotations
 
-from app.capabilities.core import Capability, register_capability
+from app.capabilities.core import BillingUnit, Capability, register_capability
 from app.capabilities.google_maps.reviews.executor import build_reviews_executor
 from app.capabilities.google_maps.reviews.schemas import ReviewsInput, ReviewsOutput
 
@@ -17,7 +18,7 @@ GOOGLE_MAPS_REVIEWS = Capability(
     input_schema=ReviewsInput,
     output_schema=ReviewsOutput,
     executor=build_reviews_executor(),
-    billing_unit=None,
+    billing_unit=BillingUnit.GOOGLE_MAPS_REVIEW,
 )
 
 register_capability(GOOGLE_MAPS_REVIEWS)

surfsense_backend/app/capabilities/google_maps/reviews/schemas.py

@@ -60,9 +60,20 @@ class ReviewsInput(BaseModel):
             raise ValueError("Provide at least one of 'urls' or 'place_ids'.")
         return self
 
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable reviews for the pre-flight gate: up to
+        ``max_reviews`` per source place."""
+        return (len(self.urls) + len(self.place_ids)) * self.max_reviews
+
 
 class ReviewsOutput(BaseModel):
     items: list[ReviewItem] = Field(
         default_factory=list,
         description="One item per review, in the scraper's emission order.",
     )
+
+    @property
+    def billable_units(self) -> int:
+        """One returned review = one billable unit."""
+        return len(self.items)

surfsense_backend/app/capabilities/google_maps/scrape/definition.py

@@ -1,8 +1,10 @@
-"""``google_maps.scrape`` capability registration (free — see 04-capabilities open item)."""
+"""``google_maps.scrape`` capability registration (dual-metered: billed per
+place via ``GOOGLE_MAPS_MICROS_PER_PLACE`` plus per attached review via
+``GOOGLE_MAPS_MICROS_PER_REVIEW``)."""
 
 from __future__ import annotations
 
-from app.capabilities.core import Capability, register_capability
+from app.capabilities.core import BillingUnit, Capability, register_capability
 from app.capabilities.google_maps.scrape.executor import build_scrape_executor
 from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutput
 
@@ -19,7 +21,7 @@ GOOGLE_MAPS_SCRAPE = Capability(
     input_schema=ScrapeInput,
     output_schema=ScrapeOutput,
     executor=build_scrape_executor(),
-    billing_unit=None,
+    billing_unit=BillingUnit.GOOGLE_MAPS_PLACE,
 )
 
 register_capability(GOOGLE_MAPS_SCRAPE)

surfsense_backend/app/capabilities/google_maps/scrape/schemas.py

@@ -83,9 +83,36 @@ class ScrapeInput(BaseModel):
             )
         return self
 
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable places: each search query yields up to
+        ``max_places``; direct URLs and place_ids yield one each."""
+        return (
+            len(self.search_queries) * self.max_places
+            + len(self.urls)
+            + len(self.place_ids)
+        )
+
+    @property
+    def estimated_review_units(self) -> int:
+        """Worst-case attached reviews for the dual-meter gate: up to
+        ``max_reviews`` per place across the worst-case place fan-out."""
+        return self.estimated_units * self.max_reviews
+
 
 class ScrapeOutput(BaseModel):
     items: list[PlaceItem] = Field(
         default_factory=list,
         description="One place item per result, in the scraper's emission order.",
     )
+
+    @property
+    def billable_units(self) -> int:
+        """One returned place = one billable place unit."""
+        return len(self.items)
+
+    @property
+    def attached_review_count(self) -> int:
+        """Reviews attached inline across all places — the second (review)
+        meter for this dual-metered verb (populated only when max_reviews > 0)."""
+        return sum(len(item.reviews) for item in self.items)

surfsense_backend/app/capabilities/google_search/scrape/definition.py

@@ -1,8 +1,9 @@
-"""``google_search.scrape`` capability registration (free — see 04-capabilities open item)."""
+"""``google_search.scrape`` capability registration (billed per SERP page; see
+config ``GOOGLE_SEARCH_MICROS_PER_SERP``)."""
 
 from __future__ import annotations
 
-from app.capabilities.core import Capability, register_capability
+from app.capabilities.core import BillingUnit, Capability, register_capability
 from app.capabilities.google_search.scrape.executor import build_scrape_executor
 from app.capabilities.google_search.scrape.schemas import ScrapeInput, ScrapeOutput
 
@@ -18,7 +19,7 @@ GOOGLE_SEARCH_SCRAPE = Capability(
     input_schema=ScrapeInput,
     output_schema=ScrapeOutput,
     executor=build_scrape_executor(),
-    billing_unit=None,
+    billing_unit=BillingUnit.GOOGLE_SEARCH_SERP,
 )
 
 register_capability(GOOGLE_SEARCH_SCRAPE)

surfsense_backend/app/capabilities/google_search/scrape/schemas.py

@@ -47,9 +47,20 @@ class ScrapeInput(BaseModel):
         description="Restrict results to a single domain, e.g. 'example.com'.",
     )
 
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable SERP pages for the pre-flight gate: one page per
+        query per requested result page."""
+        return len(self.queries) * self.max_pages_per_query
+
 
 class ScrapeOutput(BaseModel):
     items: list[SerpItem] = Field(
         default_factory=list,
         description="One item per fetched SERP page, in the scraper's emission order.",
     )
+
+    @property
+    def billable_units(self) -> int:
+        """One fetched SERP page = one billable unit."""
+        return len(self.items)

surfsense_backend/app/capabilities/reddit/scrape/definition.py

@@ -1,8 +1,9 @@
-"""``reddit.scrape`` capability registration (free — see 04-capabilities open item)."""
+"""``reddit.scrape`` capability registration (billed per item; see config
+``REDDIT_SCRAPE_MICROS_PER_ITEM``)."""
 
 from __future__ import annotations
 
-from app.capabilities.core import Capability, register_capability
+from app.capabilities.core import BillingUnit, Capability, register_capability
 from app.capabilities.reddit.scrape.executor import build_scrape_executor
 from app.capabilities.reddit.scrape.schemas import ScrapeInput, ScrapeOutput
 
@@ -18,7 +19,7 @@ REDDIT_SCRAPE = Capability(
     input_schema=ScrapeInput,
     output_schema=ScrapeOutput,
     executor=build_scrape_executor(),
-    billing_unit=None,
+    billing_unit=BillingUnit.REDDIT_ITEM,
 )
 
 register_capability(REDDIT_SCRAPE)

surfsense_backend/app/capabilities/reddit/scrape/schemas.py

@@ -94,9 +94,20 @@ class ScrapeInput(BaseModel):
             )
         return self
 
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable items for the pre-flight gate: ``max_items`` is a
+        hard cross-source ceiling (le=100), so no call can exceed it."""
+        return self.max_items
+
 
 class ScrapeOutput(BaseModel):
     items: list[RedditItem] = Field(
         default_factory=list,
         description="One item per result (post/comment/community/user), in emission order.",
     )
+
+    @property
+    def billable_units(self) -> int:
+        """One returned item = one billable unit."""
+        return len(self.items)

surfsense_backend/app/capabilities/youtube/comments/definition.py

@@ -1,8 +1,9 @@
-"""``youtube.comments`` capability registration (free — see 04-capabilities open item)."""
+"""``youtube.comments`` capability registration (billed per comment; see config
+``YOUTUBE_MICROS_PER_COMMENT``)."""
 
 from __future__ import annotations
 
-from app.capabilities.core import Capability, register_capability
+from app.capabilities.core import BillingUnit, Capability, register_capability
 from app.capabilities.youtube.comments.executor import build_comments_executor
 from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput
 
@@ -17,7 +18,7 @@ YOUTUBE_COMMENTS = Capability(
     input_schema=CommentsInput,
     output_schema=CommentsOutput,
     executor=build_comments_executor(),
-    billing_unit=None,
+    billing_unit=BillingUnit.YOUTUBE_COMMENT,
 )
 
 register_capability(YOUTUBE_COMMENTS)

surfsense_backend/app/capabilities/youtube/comments/schemas.py

@@ -36,9 +36,20 @@ class CommentsInput(BaseModel):
         description="Comment ordering: most-liked first, or most-recent first.",
     )
 
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable comments for the pre-flight gate: up to
+        ``max_comments`` per video URL."""
+        return len(self.urls) * self.max_comments
+
 
 class CommentsOutput(BaseModel):
     items: list[CommentItem] = Field(
         default_factory=list,
         description="One item per comment or reply, in the scraper's emission order.",
     )
+
+    @property
+    def billable_units(self) -> int:
+        """One returned comment or reply = one billable unit."""
+        return len(self.items)

surfsense_backend/app/capabilities/youtube/scrape/definition.py

@@ -1,8 +1,9 @@
-"""``youtube.scrape`` capability registration (free — see 04-capabilities open item)."""
+"""``youtube.scrape`` capability registration (billed per video; see config
+``YOUTUBE_MICROS_PER_VIDEO``)."""
 
 from __future__ import annotations
 
-from app.capabilities.core import Capability, register_capability
+from app.capabilities.core import BillingUnit, Capability, register_capability
 from app.capabilities.youtube.scrape.executor import build_scrape_executor
 from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput
 
@@ -18,7 +19,7 @@ YOUTUBE_SCRAPE = Capability(
     input_schema=ScrapeInput,
     output_schema=ScrapeOutput,
     executor=build_scrape_executor(),
-    billing_unit=None,
+    billing_unit=BillingUnit.YOUTUBE_VIDEO,
 )
 
 register_capability(YOUTUBE_SCRAPE)

surfsense_backend/app/capabilities/youtube/scrape/schemas.py

@@ -58,9 +58,25 @@ class ScrapeInput(BaseModel):
             raise ValueError("Provide at least one of 'urls' or 'search_queries'.")
         return self
 
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable videos for the pre-flight gate.
+
+        The x3 is load-bearing: for a channel source ``max_results`` caps
+        videos, shorts, and streams *independently*, so one source can return
+        up to 3x ``max_results`` items. Over-reserving here keeps the
+        never-go-negative invariant a passing gate promises.
+        """
+        return (len(self.urls) + len(self.search_queries)) * self.max_results * 3
+
 
 class ScrapeOutput(BaseModel):
     items: list[VideoItem] = Field(
         default_factory=list,
         description="One video item per result, in the scraper's emission order.",
     )
+
+    @property
+    def billable_units(self) -> int:
+        """One returned video/short/stream = one billable unit."""
+        return len(self.items)