mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-10 22:32:16 +02:00
80927a2872
Add per-item, per-platform billing for the platform-native connectors (Reddit, Google Search, Google Maps places/reviews, YouTube videos/comments) through the capability gate/charge seam. Rates are config-driven with a shared wallet-credit module (wallet_credit) and a dedicated PlatformScrapeCreditService; agent and REST capability runs now record cost_micros. Google Maps scrape dual-meters places and attached reviews. Remove the main-agent scrape_webpage tool now that the web.crawl capability covers single-page (maxCrawlDepth=0) and site crawling. The main agent now reaches crawling via task(web_crawler, ...). Update prompts, tool catalog, receipts, skills, proprietary docs, and tests; drop the obsolete chat-turn crawl fold path. Co-authored-by: Cursor <cursoragent@cursor.com>
66 lines
2.1 KiB
Python
66 lines
2.1 KiB
Python
"""``google_search.scrape`` I/O contracts.
|
|
|
|
A lean, agent-friendly surface over ``GoogleSearchScrapeInput``
|
|
(``app/proprietary/platforms/google_search``). The executor maps this to the
|
|
full scraper input; the scraper's ``SerpItem`` is reused verbatim as the output
|
|
element.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
from app.proprietary.platforms.google_search import SerpItem
|
|
|
|
MAX_SEARCH_QUERIES = 20
|
|
"""Per-call cap on queries: bounds a synchronous request's fan-out."""
|
|
|
|
MAX_PAGES_PER_QUERY = 10
|
|
"""Deepest result-page pagination a single query will follow."""
|
|
|
|
|
|
class ScrapeInput(BaseModel):
|
|
queries: list[str] = Field(
|
|
min_length=1,
|
|
max_length=MAX_SEARCH_QUERIES,
|
|
description=(
|
|
"Search terms (e.g. 'wedding photographers denver') or full Google "
|
|
"Search URLs. Each term is searched; each URL is scraped as-is."
|
|
),
|
|
)
|
|
max_pages_per_query: int = Field(
|
|
default=1,
|
|
ge=1,
|
|
le=MAX_PAGES_PER_QUERY,
|
|
description="Result pages to fetch per query (1 = first page only).",
|
|
)
|
|
country_code: str | None = Field(
|
|
default=None,
|
|
description="Two-letter country to search from, e.g. 'us', 'fr'.",
|
|
)
|
|
language_code: str = Field(
|
|
default="",
|
|
description="Result language code, e.g. 'en', 'fr' (blank = Google default).",
|
|
)
|
|
site: str | None = Field(
|
|
default=None,
|
|
description="Restrict results to a single domain, e.g. 'example.com'.",
|
|
)
|
|
|
|
@property
|
|
def estimated_units(self) -> int:
|
|
"""Worst-case billable SERP pages for the pre-flight gate: one page per
|
|
query per requested result page."""
|
|
return len(self.queries) * self.max_pages_per_query
|
|
|
|
|
|
class ScrapeOutput(BaseModel):
|
|
items: list[SerpItem] = Field(
|
|
default_factory=list,
|
|
description="One item per fetched SERP page, in the scraper's emission order.",
|
|
)
|
|
|
|
@property
|
|
def billable_units(self) -> int:
|
|
"""One fetched SERP page = one billable unit."""
|
|
return len(self.items)
|