feat(google_maps): add scrape + reviews capability verbs

surfsense_backend/app/capabilities/google_maps/__init__.py

@@ -0,0 +1,6 @@
+"""``google_maps.*`` namespace: platform-native Google Maps data verbs."""
+
+from __future__ import annotations
+
+from app.capabilities.google_maps.reviews import definition as _reviews  # noqa: F401
+from app.capabilities.google_maps.scrape import definition as _scrape  # noqa: F401

surfsense_backend/app/capabilities/google_maps/reviews/__init__.py

@@ -0,0 +1,3 @@
+"""``google_maps.reviews`` verb: Maps place URLs / place IDs → review items."""
+
+from __future__ import annotations

surfsense_backend/app/capabilities/google_maps/reviews/definition.py

@@ -0,0 +1,23 @@
+"""``google_maps.reviews`` capability registration (free — see 04-capabilities open item)."""
+
+from __future__ import annotations
+
+from app.capabilities.core import Capability, register_capability
+from app.capabilities.google_maps.reviews.executor import build_reviews_executor
+from app.capabilities.google_maps.reviews.schemas import ReviewsInput, ReviewsOutput
+
+GOOGLE_MAPS_REVIEWS = Capability(
+    name="google_maps.reviews",
+    description=(
+        "Fetch public reviews for one or more Google Maps places. Give it place "
+        "URLs or place IDs; returns structured review items with author, text, "
+        "star rating, like count, owner response, and timestamps. Use it to "
+        "gauge sentiment or pull recent feedback on specific places."
+    ),
+    input_schema=ReviewsInput,
+    output_schema=ReviewsOutput,
+    executor=build_reviews_executor(),
+    billing_unit=None,
+)
+
+register_capability(GOOGLE_MAPS_REVIEWS)

surfsense_backend/app/capabilities/google_maps/reviews/executor.py

@@ -0,0 +1,40 @@
+"""``google_maps.reviews`` executor: verb input → scraper → review items."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+
+from app.capabilities.core import Executor
+from app.capabilities.google_maps.reviews.schemas import ReviewsInput, ReviewsOutput
+from app.exceptions import ForbiddenError
+from app.proprietary.platforms.google_maps import (
+    GoogleMapsReviewsInput,
+    scrape_reviews,
+)
+from app.proprietary.platforms.google_maps.scraper import SignInRequiredError
+
+ReviewsFn = Callable[[GoogleMapsReviewsInput], Awaitable[list[dict]]]
+
+
+def build_reviews_executor(scrape_fn: ReviewsFn | None = None) -> Executor:
+    """Bind the executor to a reviews scraper fn (defaults to the proprietary actor)."""
+    scrape_fn = scrape_fn or scrape_reviews
+
+    async def execute(payload: ReviewsInput) -> ReviewsOutput:
+        actor_input = GoogleMapsReviewsInput(
+            startUrls=[{"url": url} for url in payload.urls],
+            placeIds=payload.place_ids,
+            maxReviews=payload.max_reviews,
+            reviewsSort=payload.sort_by,
+            reviewsStartDate=payload.start_date,
+            language=payload.language,
+        )
+        try:
+            items = await scrape_fn(actor_input)
+        except SignInRequiredError as exc:
+            raise ForbiddenError(
+                f"Google sign in required: {exc}", code="GOOGLE_SIGNIN_REQUIRED"
+            ) from exc
+        return ReviewsOutput(items=items)
+
+    return execute

surfsense_backend/app/capabilities/google_maps/reviews/schemas.py

@@ -0,0 +1,68 @@
+"""``google_maps.reviews`` I/O contracts.
+
+A lean surface over ``GoogleMapsReviewsInput``; the scraper's ``ReviewItem`` is
+reused verbatim as the output element.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field, model_validator
+
+from app.proprietary.platforms.google_maps import ReviewItem
+
+MAX_MAPS_REVIEW_SOURCES = 20
+"""Per-call cap on urls + place_ids: bounds how many places one request harvests."""
+
+
+class ReviewsInput(BaseModel):
+    urls: list[str] = Field(
+        default_factory=list,
+        max_length=MAX_MAPS_REVIEW_SOURCES,
+        description=(
+            "Google Maps place URLs to fetch reviews for. Provide these OR "
+            "place_ids (at least one is required)."
+        ),
+    )
+    place_ids: list[str] = Field(
+        default_factory=list,
+        max_length=MAX_MAPS_REVIEW_SOURCES,
+        description=(
+            "Known Google place IDs (ChIJ...) to fetch reviews for. Provide "
+            "these OR urls."
+        ),
+    )
+    max_reviews: int = Field(
+        default=20,
+        ge=1,
+        le=100_000,
+        description="Max reviews to return per place.",
+    )
+    sort_by: Literal["newest", "mostRelevant", "highestRanking", "lowestRanking"] = (
+        Field(
+            default="newest",
+            description="Review ordering.",
+        )
+    )
+    language: str = Field(
+        default="en",
+        description="Review language code, e.g. 'en', 'fr'.",
+    )
+    start_date: str | None = Field(
+        default=None,
+        description="Only reviews on/after this ISO date, e.g. '2024-01-01'.",
+    )
+
+    @model_validator(mode="after")
+    def _require_a_source(self) -> ReviewsInput:
+        if not (self.urls or self.place_ids):
+            raise ValueError("Provide at least one of 'urls' or 'place_ids'.")
+        return self
+
+
+class ReviewsOutput(BaseModel):
+    items: list[ReviewItem] = Field(
+        default_factory=list,
+        description="One item per review, in the scraper's emission order.",
+    )

surfsense_backend/app/capabilities/google_maps/scrape/__init__.py

@@ -0,0 +1,3 @@
+"""``google_maps.scrape`` verb: search queries / Maps URLs / place IDs → place items."""
+
+from __future__ import annotations

surfsense_backend/app/capabilities/google_maps/scrape/definition.py

@@ -0,0 +1,25 @@
+"""``google_maps.scrape`` capability registration (free — see 04-capabilities open item)."""
+
+from __future__ import annotations
+
+from app.capabilities.core import Capability, register_capability
+from app.capabilities.google_maps.scrape.executor import build_scrape_executor
+from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutput
+
+GOOGLE_MAPS_SCRAPE = Capability(
+    name="google_maps.scrape",
+    description=(
+        "Scrape public Google Maps places. Give it search queries (optionally "
+        "scoped by location), Google Maps URLs, or place IDs, and it returns "
+        "structured place items — name, address, category, phone, website, "
+        "rating, review count, coordinates, and opening hours. Set "
+        "include_details for richer detail-page fields, or max_reviews/"
+        "max_images to attach reviews and photos per place."
+    ),
+    input_schema=ScrapeInput,
+    output_schema=ScrapeOutput,
+    executor=build_scrape_executor(),
+    billing_unit=None,
+)
+
+register_capability(GOOGLE_MAPS_SCRAPE)

surfsense_backend/app/capabilities/google_maps/scrape/executor.py

@@ -0,0 +1,43 @@
+"""``google_maps.scrape`` executor: verb input → scraper → place items."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+
+from app.capabilities.core import Executor
+from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.exceptions import ForbiddenError
+from app.proprietary.platforms.google_maps import (
+    GoogleMapsScrapeInput,
+    scrape_places,
+)
+from app.proprietary.platforms.google_maps.scraper import SignInRequiredError
+
+ScrapeFn = Callable[[GoogleMapsScrapeInput], Awaitable[list[dict]]]
+
+
+def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
+    """Bind the executor to a scraper fn (defaults to the proprietary actor)."""
+    scrape_fn = scrape_fn or scrape_places
+
+    async def execute(payload: ScrapeInput) -> ScrapeOutput:
+        actor_input = GoogleMapsScrapeInput(
+            searchStringsArray=payload.search_queries,
+            startUrls=[{"url": url} for url in payload.urls],
+            placeIds=payload.place_ids,
+            locationQuery=payload.location,
+            maxCrawledPlacesPerSearch=payload.max_places,
+            language=payload.language,
+            scrapePlaceDetailPage=payload.include_details,
+            maxReviews=payload.max_reviews,
+            maxImages=payload.max_images,
+        )
+        try:
+            items = await scrape_fn(actor_input)
+        except SignInRequiredError as exc:
+            raise ForbiddenError(
+                f"Google sign in required: {exc}", code="GOOGLE_SIGNIN_REQUIRED"
+            ) from exc
+        return ScrapeOutput(items=items)
+
+    return execute

surfsense_backend/app/capabilities/google_maps/scrape/schemas.py

@@ -0,0 +1,91 @@
+"""``google_maps.scrape`` I/O contracts.
+
+A lean, agent-friendly surface over ``GoogleMapsScrapeInput``
+(``app/proprietary/platforms/google_maps``). The executor maps this to the full
+scraper input; the scraper's ``PlaceItem`` is reused verbatim as the output
+element.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field, model_validator
+
+from app.proprietary.platforms.google_maps import PlaceItem
+
+MAX_MAPS_SOURCES = 20
+"""Per-call cap on queries + urls + place_ids: bounds a sync request's fan-out."""
+
+
+class ScrapeInput(BaseModel):
+    search_queries: list[str] = Field(
+        default_factory=list,
+        max_length=MAX_MAPS_SOURCES,
+        description=(
+            "Google Maps search terms (e.g. 'coffee shops', 'dentist'); each "
+            "returns up to max_places. Provide these OR urls OR place_ids "
+            "(at least one is required). Pair with location to scope a search."
+        ),
+    )
+    urls: list[str] = Field(
+        default_factory=list,
+        max_length=MAX_MAPS_SOURCES,
+        description=(
+            "Google Maps URLs — a place page (/maps/place/...) or a search "
+            "results URL. Provide these OR search_queries OR place_ids."
+        ),
+    )
+    place_ids: list[str] = Field(
+        default_factory=list,
+        max_length=MAX_MAPS_SOURCES,
+        description=(
+            "Known Google place IDs (ChIJ...) to fetch directly. Provide these "
+            "OR search_queries OR urls."
+        ),
+    )
+    location: str | None = Field(
+        default=None,
+        description="Location to scope search_queries to, e.g. 'New York, USA'.",
+    )
+    max_places: int = Field(
+        default=10,
+        ge=1,
+        le=1000,
+        description="Max places to return per search query.",
+    )
+    language: str = Field(
+        default="en",
+        description="Result language code, e.g. 'en', 'fr'.",
+    )
+    include_details: bool = Field(
+        default=False,
+        description=(
+            "Also fetch each place's detail page — opening hours, popular "
+            "times, extra contact info (slower; more requests)."
+        ),
+    )
+    max_reviews: int = Field(
+        default=0,
+        ge=0,
+        le=100_000,
+        description="Reviews to attach per place (0 = none).",
+    )
+    max_images: int = Field(
+        default=0,
+        ge=0,
+        description="Images to attach per place (0 = none).",
+    )
+
+    @model_validator(mode="after")
+    def _require_a_source(self) -> ScrapeInput:
+        if not (self.search_queries or self.urls or self.place_ids):
+            raise ValueError(
+                "Provide at least one of 'search_queries', 'urls', or 'place_ids'."
+            )
+        return self
+
+
+class ScrapeOutput(BaseModel):
+    items: list[PlaceItem] = Field(
+        default_factory=list,
+        description="One place item per result, in the scraper's emission order.",
+    )

surfsense_backend/app/routes/__init__.py

@@ -1,6 +1,7 @@
 from fastapi import APIRouter, Depends
 
 # Import verb namespaces for their registration side effects before the door builds.
+import app.capabilities.google_maps
 import app.capabilities.web
 import app.capabilities.youtube  # noqa: F401
 from app.automations.api import router as automations_router

surfsense_backend/tests/unit/capabilities/google_maps/reviews/test_executor.py

@@ -0,0 +1,87 @@
+"""``google_maps.reviews`` executor: verb input → actor input mapping → typed items.
+
+Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's
+own payload→GoogleMapsReviewsInput mapping and the dict→ReviewItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.google_maps.reviews.executor import build_reviews_executor
+from app.capabilities.google_maps.reviews.schemas import ReviewsInput, ReviewsOutput
+from app.exceptions import ForbiddenError
+from app.proprietary.platforms.google_maps import GoogleMapsReviewsInput
+from app.proprietary.platforms.google_maps.scraper import SignInRequiredError
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeScraper:
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[GoogleMapsReviewsInput] = []
+
+    async def __call__(self, actor_input: GoogleMapsReviewsInput) -> list[dict]:
+        self.calls.append(actor_input)
+        return self._items
+
+
+async def test_maps_urls_to_start_urls_and_wraps_items():
+    scraper = _FakeScraper([{"text": "Great place", "stars": 5.0}])
+    execute = build_reviews_executor(scrape_fn=scraper)
+
+    out = await execute(ReviewsInput(urls=["https://www.google.com/maps/place/x"]))
+
+    assert isinstance(out, ReviewsOutput)
+    assert len(out.items) == 1
+    assert out.items[0].text == "Great place"
+    assert out.items[0].stars == 5.0
+
+    (actor_input,) = scraper.calls
+    assert [u.url for u in actor_input.startUrls] == [
+        "https://www.google.com/maps/place/x"
+    ]
+
+
+async def test_forwards_place_ids_and_options():
+    scraper = _FakeScraper([])
+    execute = build_reviews_executor(scrape_fn=scraper)
+
+    await execute(
+        ReviewsInput(
+            place_ids=["ChIJx"],
+            max_reviews=50,
+            sort_by="highestRanking",
+            language="fr",
+            start_date="2024-01-01",
+        )
+    )
+
+    (actor_input,) = scraper.calls
+    assert actor_input.placeIds == ["ChIJx"]
+    assert actor_input.maxReviews == 50
+    assert actor_input.reviewsSort == "highestRanking"
+    assert actor_input.language == "fr"
+    assert actor_input.reviewsStartDate == "2024-01-01"
+
+
+async def test_sign_in_required_maps_to_forbidden_403():
+    async def _raise(_actor_input):
+        raise SignInRequiredError("wall hit")
+
+    execute = build_reviews_executor(scrape_fn=_raise)
+
+    with pytest.raises(ForbiddenError) as exc_info:
+        await execute(ReviewsInput(place_ids=["ChIJx"]))
+    assert exc_info.value.status_code == 403
+
+
+async def test_other_faults_propagate_for_the_door_to_map():
+    async def _boom(_actor_input):
+        raise RuntimeError("proxy exploded")
+
+    execute = build_reviews_executor(scrape_fn=_boom)
+
+    with pytest.raises(RuntimeError):
+        await execute(ReviewsInput(place_ids=["ChIJx"]))

surfsense_backend/tests/unit/capabilities/google_maps/reviews/test_schemas.py

@@ -0,0 +1,35 @@
+"""``google_maps.reviews`` input guards: a source is required and the batch is bounded."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.google_maps.reviews.schemas import (
+    MAX_MAPS_REVIEW_SOURCES,
+    ReviewsInput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_rejects_input_with_no_source():
+    with pytest.raises(ValidationError):
+        ReviewsInput()
+
+
+def test_accepts_urls_or_place_ids():
+    assert ReviewsInput(urls=["https://maps.google.com/x"]).place_ids == []
+    assert ReviewsInput(place_ids=["ChIJx"]).urls == []
+
+
+def test_max_reviews_defaults_and_is_bounded():
+    assert ReviewsInput(place_ids=["ChIJx"]).max_reviews == 20
+    with pytest.raises(ValidationError):
+        ReviewsInput(place_ids=["ChIJx"], max_reviews=0)
+
+
+def test_rejects_more_sources_than_the_cap():
+    too_many = [f"ChIJ{i}" for i in range(MAX_MAPS_REVIEW_SOURCES + 1)]
+    with pytest.raises(ValidationError):
+        ReviewsInput(place_ids=too_many)

surfsense_backend/tests/unit/capabilities/google_maps/scrape/test_executor.py

@@ -0,0 +1,115 @@
+"""``google_maps.scrape`` executor: verb input → actor input mapping → typed items.
+
+Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's
+own payload→GoogleMapsScrapeInput mapping and the dict→PlaceItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.google_maps.scrape.executor import build_scrape_executor
+from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.exceptions import ForbiddenError
+from app.proprietary.platforms.google_maps import GoogleMapsScrapeInput
+from app.proprietary.platforms.google_maps.scraper import SignInRequiredError
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeScraper:
+    """Records the actor input it was called with and returns canned items."""
+
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[GoogleMapsScrapeInput] = []
+
+    async def __call__(self, actor_input: GoogleMapsScrapeInput) -> list[dict]:
+        self.calls.append(actor_input)
+        return self._items
+
+
+async def test_maps_queries_and_wraps_items():
+    scraper = _FakeScraper([{"title": "Blue Bottle", "placeId": "abc"}])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    out = await execute(ScrapeInput(search_queries=["coffee"], location="Austin"))
+
+    assert isinstance(out, ScrapeOutput)
+    assert len(out.items) == 1
+    assert out.items[0].title == "Blue Bottle"
+    assert out.items[0].placeId == "abc"
+
+    (actor_input,) = scraper.calls
+    assert actor_input.searchStringsArray == ["coffee"]
+    assert actor_input.locationQuery == "Austin"
+
+
+async def test_maps_urls_and_place_ids():
+    scraper = _FakeScraper([])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    await execute(
+        ScrapeInput(
+            urls=["https://www.google.com/maps/place/x"],
+            place_ids=["ChIJxxxx"],
+        )
+    )
+
+    (actor_input,) = scraper.calls
+    assert [u.url for u in actor_input.startUrls] == [
+        "https://www.google.com/maps/place/x"
+    ]
+    assert actor_input.placeIds == ["ChIJxxxx"]
+
+
+async def test_max_places_maps_to_per_search_cap():
+    scraper = _FakeScraper([])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    await execute(ScrapeInput(search_queries=["x"], max_places=25))
+
+    (actor_input,) = scraper.calls
+    assert actor_input.maxCrawledPlacesPerSearch == 25
+
+
+async def test_forwards_detail_review_and_image_options():
+    scraper = _FakeScraper([])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    await execute(
+        ScrapeInput(
+            search_queries=["x"],
+            include_details=True,
+            max_reviews=5,
+            max_images=3,
+            language="fr",
+        )
+    )
+
+    (actor_input,) = scraper.calls
+    assert actor_input.scrapePlaceDetailPage is True
+    assert actor_input.maxReviews == 5
+    assert actor_input.maxImages == 3
+    assert actor_input.language == "fr"
+
+
+async def test_sign_in_required_maps_to_forbidden_403():
+    async def _raise(_actor_input):
+        raise SignInRequiredError("wall hit")
+
+    execute = build_scrape_executor(scrape_fn=_raise)
+
+    with pytest.raises(ForbiddenError) as exc_info:
+        await execute(ScrapeInput(search_queries=["x"]))
+    assert exc_info.value.status_code == 403
+
+
+async def test_other_faults_propagate_for_the_door_to_map():
+    async def _boom(_actor_input):
+        raise RuntimeError("proxy exploded")
+
+    execute = build_scrape_executor(scrape_fn=_boom)
+
+    with pytest.raises(RuntimeError):
+        await execute(ScrapeInput(search_queries=["x"]))

surfsense_backend/tests/unit/capabilities/google_maps/scrape/test_schemas.py

@@ -0,0 +1,36 @@
+"""``google_maps.scrape`` input guards: a source is required and the batch is bounded."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.google_maps.scrape.schemas import (
+    MAX_MAPS_SOURCES,
+    ScrapeInput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_rejects_input_with_no_source():
+    with pytest.raises(ValidationError):
+        ScrapeInput()
+
+
+def test_accepts_any_single_source():
+    assert ScrapeInput(search_queries=["coffee"]).urls == []
+    assert ScrapeInput(urls=["https://maps.google.com/x"]).place_ids == []
+    assert ScrapeInput(place_ids=["ChIJx"]).search_queries == []
+
+
+def test_max_places_defaults_and_is_bounded():
+    assert ScrapeInput(search_queries=["x"]).max_places == 10
+    with pytest.raises(ValidationError):
+        ScrapeInput(search_queries=["x"], max_places=0)
+
+
+def test_rejects_more_sources_than_the_cap():
+    too_many = [f"q{i}" for i in range(MAX_MAPS_SOURCES + 1)]
+    with pytest.raises(ValidationError):
+        ScrapeInput(search_queries=too_many)

surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py

@@ -0,0 +1,32 @@
+"""The google_maps namespace registers each verb as one Capability the doors/agent read."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities import (
+    google_maps,  # noqa: F401  — importing the namespace registers its verbs
+)
+from app.capabilities.core.store import get_capability
+from app.capabilities.google_maps.reviews.schemas import ReviewsInput, ReviewsOutput
+from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutput
+
+pytestmark = pytest.mark.unit
+
+
+def test_google_maps_scrape_is_registered_and_free():
+    cap = get_capability("google_maps.scrape")
+
+    assert cap.name == "google_maps.scrape"
+    assert cap.input_schema is ScrapeInput
+    assert cap.output_schema is ScrapeOutput
+    assert cap.billing_unit is None
+
+
+def test_google_maps_reviews_is_registered_and_free():
+    cap = get_capability("google_maps.reviews")
+
+    assert cap.name == "google_maps.reviews"
+    assert cap.input_schema is ReviewsInput
+    assert cap.output_schema is ReviewsOutput
+    assert cap.billing_unit is None