feat(billing): meter platform scrapers per item; consolidate web scraping onto web.crawl

Add per-item, per-platform billing for the platform-native connectors (Reddit, Google Search, Google Maps places/reviews, YouTube videos/comments) through the capability gate/charge seam. Rates are config-driven with a shared wallet-credit module (wallet_credit) and a dedicated PlatformScrapeCreditService; agent and REST capability runs now record cost_micros. Google Maps scrape dual-meters places and attached reviews.

Remove the main-agent scrape_webpage tool now that the web.crawl capability covers single-page (maxCrawlDepth=0) and site crawling. The main agent now reaches crawling via task(web_crawler, ...). Update prompts, tool catalog, receipts, skills, proprietary docs, and tests; drop the obsolete chat-turn crawl fold path.

Co-authored-by: Cursor <cursoragent@cursor.com>

surfsense_backend/app/tasks/chat/streaming/handlers/tools/scrape_webpage/emission.py

@@ -1,43 +0,0 @@
-"""scrape_webpage: redacted payload + terminal summary."""
-
-from __future__ import annotations
-
-from collections.abc import Iterator
-
-from app.tasks.chat.streaming.handlers.tools.emission_context import (
-    ToolCompletionEmissionContext,
-)
-
-
-def iter_completion_emission_frames(
-    ctx: ToolCompletionEmissionContext,
-) -> Iterator[str]:
-    out = ctx.tool_output
-    if isinstance(out, dict):
-        display_output = {k: v for k, v in out.items() if k != "content"}
-        if "content" in out:
-            content = out.get("content", "")
-            display_output["content_preview"] = (
-                content[:500] + "..." if len(content) > 500 else content
-            )
-        yield ctx.emit_tool_output_card(display_output)
-    else:
-        yield ctx.emit_tool_output_card({"result": out})
-
-    if isinstance(out, dict) and "error" not in out:
-        title = out.get("title", "Webpage")
-        word_count = out.get("word_count", 0)
-        yield ctx.streaming_service.format_terminal_info(
-            f"Scraped: {title[:40]}{'...' if len(title) > 40 else ''} ({word_count:,} words)",
-            "success",
-        )
-    else:
-        error_msg = (
-            out.get("error", "Failed to scrape")
-            if isinstance(out, dict)
-            else "Failed to scrape"
-        )
-        yield ctx.streaming_service.format_terminal_info(
-            f"Scrape failed: {error_msg}",
-            "error",
-        )

surfsense_backend/app/tasks/chat/streaming/handlers/tools/scrape_webpage/shared/tool_input.py

@@ -1,9 +0,0 @@
-"""Tool-call args for scrape_webpage thinking."""
-
-from __future__ import annotations
-
-from typing import Any
-
-
-def as_tool_input_dict(tool_input: Any) -> dict[str, Any]:
-    return tool_input if isinstance(tool_input, dict) else {}

surfsense_backend/app/tasks/chat/streaming/handlers/tools/scrape_webpage/thinking.py

@@ -1,49 +0,0 @@
-"""scrape_webpage: thinking-step copy."""
-
-from __future__ import annotations
-
-from typing import Any
-
-from app.tasks.chat.streaming.handlers.tools.scrape_webpage.shared.tool_input import (
-    as_tool_input_dict,
-)
-from app.tasks.chat.streaming.handlers.tools.shared.model import (
-    ToolStartThinking,
-)
-
-
-def resolve_start_thinking(tool_name: str, tool_input: Any) -> ToolStartThinking:
-    del tool_name
-    d = as_tool_input_dict(tool_input)
-    url = d.get("url", "") if isinstance(tool_input, dict) else str(tool_input)
-    return ToolStartThinking(
-        title="Scraping webpage",
-        items=[f"URL: {url[:80]}{'...' if len(url) > 80 else ''}"],
-    )
-
-
-def resolve_completed_thinking(
-    tool_name: str,
-    tool_output: Any,
-    last_items: list[str],
-) -> tuple[str, list[str]]:
-    del tool_name
-    items = last_items
-    if isinstance(tool_output, dict):
-        title = tool_output.get("title", "Webpage")
-        word_count = tool_output.get("word_count", 0)
-        has_error = "error" in tool_output
-        if has_error:
-            completed = [
-                *items,
-                f"Error: {tool_output.get('error', 'Failed to scrape')[:50]}",
-            ]
-        else:
-            completed = [
-                *items,
-                f"Title: {title[:50]}{'...' if len(title) > 50 else ''}",
-                f"Extracted: {word_count:,} words",
-            ]
-    else:
-        completed = [*items, "Content extracted"]
-    return ("Scraping webpage", completed)