feat(billing): meter platform scrapers per item; consolidate web scraping onto web.crawl

Add per-item, per-platform billing for the platform-native connectors (Reddit, Google Search, Google Maps places/reviews, YouTube videos/comments) through the capability gate/charge seam. Rates are config-driven with a shared wallet-credit module (wallet_credit) and a dedicated PlatformScrapeCreditService; agent and REST capability runs now record cost_micros. Google Maps scrape dual-meters places and attached reviews.

Remove the main-agent scrape_webpage tool now that the web.crawl capability covers single-page (maxCrawlDepth=0) and site crawling. The main agent now reaches crawling via task(web_crawler, ...). Update prompts, tool catalog, receipts, skills, proprietary docs, and tests; drop the obsolete chat-turn crawl fold path.

Co-authored-by: Cursor <cursoragent@cursor.com>

surfsense_backend/app/capabilities/google_maps/reviews/definition.py

@@ -1,8 +1,9 @@
-"""``google_maps.reviews`` capability registration (free — see 04-capabilities open item)."""
+"""``google_maps.reviews`` capability registration (billed per review; see
+config ``GOOGLE_MAPS_MICROS_PER_REVIEW``)."""
 
 from __future__ import annotations
 
-from app.capabilities.core import Capability, register_capability
+from app.capabilities.core import BillingUnit, Capability, register_capability
 from app.capabilities.google_maps.reviews.executor import build_reviews_executor
 from app.capabilities.google_maps.reviews.schemas import ReviewsInput, ReviewsOutput
 
@@ -17,7 +18,7 @@ GOOGLE_MAPS_REVIEWS = Capability(
     input_schema=ReviewsInput,
     output_schema=ReviewsOutput,
     executor=build_reviews_executor(),
-    billing_unit=None,
+    billing_unit=BillingUnit.GOOGLE_MAPS_REVIEW,
 )
 
 register_capability(GOOGLE_MAPS_REVIEWS)

surfsense_backend/app/capabilities/google_maps/reviews/schemas.py

@@ -60,9 +60,20 @@ class ReviewsInput(BaseModel):
             raise ValueError("Provide at least one of 'urls' or 'place_ids'.")
         return self
 
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable reviews for the pre-flight gate: up to
+        ``max_reviews`` per source place."""
+        return (len(self.urls) + len(self.place_ids)) * self.max_reviews
+
 
 class ReviewsOutput(BaseModel):
     items: list[ReviewItem] = Field(
         default_factory=list,
         description="One item per review, in the scraper's emission order.",
     )
+
+    @property
+    def billable_units(self) -> int:
+        """One returned review = one billable unit."""
+        return len(self.items)

surfsense_backend/app/capabilities/google_maps/scrape/definition.py

@@ -1,8 +1,10 @@
-"""``google_maps.scrape`` capability registration (free — see 04-capabilities open item)."""
+"""``google_maps.scrape`` capability registration (dual-metered: billed per
+place via ``GOOGLE_MAPS_MICROS_PER_PLACE`` plus per attached review via
+``GOOGLE_MAPS_MICROS_PER_REVIEW``)."""
 
 from __future__ import annotations
 
-from app.capabilities.core import Capability, register_capability
+from app.capabilities.core import BillingUnit, Capability, register_capability
 from app.capabilities.google_maps.scrape.executor import build_scrape_executor
 from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutput
 
@@ -19,7 +21,7 @@ GOOGLE_MAPS_SCRAPE = Capability(
     input_schema=ScrapeInput,
     output_schema=ScrapeOutput,
     executor=build_scrape_executor(),
-    billing_unit=None,
+    billing_unit=BillingUnit.GOOGLE_MAPS_PLACE,
 )
 
 register_capability(GOOGLE_MAPS_SCRAPE)

surfsense_backend/app/capabilities/google_maps/scrape/schemas.py

@@ -83,9 +83,36 @@ class ScrapeInput(BaseModel):
             )
         return self
 
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable places: each search query yields up to
+        ``max_places``; direct URLs and place_ids yield one each."""
+        return (
+            len(self.search_queries) * self.max_places
+            + len(self.urls)
+            + len(self.place_ids)
+        )
+
+    @property
+    def estimated_review_units(self) -> int:
+        """Worst-case attached reviews for the dual-meter gate: up to
+        ``max_reviews`` per place across the worst-case place fan-out."""
+        return self.estimated_units * self.max_reviews
+
 
 class ScrapeOutput(BaseModel):
     items: list[PlaceItem] = Field(
         default_factory=list,
         description="One place item per result, in the scraper's emission order.",
     )
+
+    @property
+    def billable_units(self) -> int:
+        """One returned place = one billable place unit."""
+        return len(self.items)
+
+    @property
+    def attached_review_count(self) -> int:
+        """Reviews attached inline across all places — the second (review)
+        meter for this dual-metered verb (populated only when max_reviews > 0)."""
+        return sum(len(item.reviews) for item in self.items)