feat(native-connector): added google maps places & reviews scrapers

surfsense_backend/alembic/versions/168_harden_refresh_token_schema.py

@@ -6,6 +6,8 @@ Revises: 167
 
 from collections.abc import Sequence
 
+import sqlalchemy as sa
+
 from alembic import op
 
 revision: str = "168"
@@ -48,9 +50,7 @@ def upgrade() -> None:
             """
         )
 
-    op.execute(
+    op.execute("ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(64)")
-        "ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(64)"
-    )
     op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS is_revoked")
 
 
@@ -74,8 +74,6 @@ def downgrade() -> None:
         """
     )
     op.execute("ALTER TABLE refresh_tokens ALTER COLUMN is_revoked DROP DEFAULT")
-    op.execute(
+    op.execute("ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(256)")
-        "ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(256)"
-    )
     op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS absolute_expiry")
     op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS revoked_at")

surfsense_backend/alembic/versions/170_rename_searchspace_to_workspace.py

@@ -373,7 +373,9 @@ def _neutralize_column_list_tables(conn) -> None:
     for table in PUBLICATION_COLUMN_LIST_TABLES:
         if _is_publication_member(conn, table):
             conn.execute(
-                sa.text(f"ALTER PUBLICATION {_quote(PUBLICATION_NAME)} DROP TABLE {table}")
+                sa.text(
+                    f"ALTER PUBLICATION {_quote(PUBLICATION_NAME)} DROP TABLE {table}"
+                )
             )
 
 
@@ -440,7 +442,9 @@ def _restore_downgrade_publication(conn) -> None:
     for table in PUBLICATION_COLUMN_LIST_TABLES:
         if _is_publication_member(conn, table):
             conn.execute(
-                sa.text(f"ALTER PUBLICATION {_quote(PUBLICATION_NAME)} DROP TABLE {table}")
+                sa.text(
+                    f"ALTER PUBLICATION {_quote(PUBLICATION_NAME)} DROP TABLE {table}"
+                )
             )
 
     entries: list[str] = []

surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/middleware.py

@@ -111,9 +111,7 @@ async def _ensure_folder_hierarchy(
             sibling_query = (
                 select(Folder.position).order_by(Folder.position.desc()).limit(1)
             )
-            sibling_query = sibling_query.where(
+            sibling_query = sibling_query.where(Folder.workspace_id == workspace_id)
-                Folder.workspace_id == workspace_id
-            )
             if parent_id is None:
                 sibling_query = sibling_query.where(Folder.parent_id.is_(None))
             else:
@@ -1241,9 +1239,7 @@ async def commit_staged_filesystem_state(
 
             await session.commit()
     except Exception:  # pragma: no cover - rollback safety net
-        logger.exception(
+        logger.exception("kb_persistence: commit failed (workspace=%s)", workspace_id)
-            "kb_persistence: commit failed (workspace=%s)", workspace_id
-        )
         # Outer commit raised: everything above rolled back, so drop the
         # deferred dispatches.
         deferred_dispatches.clear()

surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/memory/middleware.py

@@ -18,7 +18,7 @@ from langgraph.runtime import Runtime
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.db import ChatVisibility, Workspace, User, shielded_async_session
+from app.db import ChatVisibility, User, Workspace, shielded_async_session
 from app.services.memory import MEMORY_HARD_LIMIT, MEMORY_SOFT_LIMIT
 from app.utils.perf import get_perf_logger
 

surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/factory.py

@@ -98,9 +98,7 @@ async def create_multi_agent_chat_deep_agent(
 
     _t0 = time.perf_counter()
     try:
-        connector_types = await connector_service.get_available_connectors(
+        connector_types = await connector_service.get_available_connectors(workspace_id)
-            workspace_id
-        )
         available_connectors = map_connectors_to_searchable_types(connector_types)
 
         available_document_types = await connector_service.get_available_document_types(
@@ -153,9 +151,7 @@ async def create_multi_agent_chat_deep_agent(
 
     _t0 = time.perf_counter()
     try:
-        mcp_tools_by_agent = await load_mcp_tools_by_connector(
+        mcp_tools_by_agent = await load_mcp_tools_by_connector(db_session, workspace_id)
-            db_session, workspace_id
-        )
     except Exception as e:
         # Degrade to builtins-only rather than aborting the turn: a transient
         # DB or MCP-server hiccup should not deny the user a response.

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/generate_image.py

@@ -116,9 +116,7 @@ def create_generate_image_tool(
                     # later workspace changes. No workspace read needed.
                     config_id = image_gen_model_id_override or IMAGE_GEN_AUTO_MODE_ID
                 else:
-                    config_id = (
+                    config_id = workspace.image_gen_model_id or IMAGE_GEN_AUTO_MODE_ID
-                        workspace.image_gen_model_id or IMAGE_GEN_AUTO_MODE_ID
-                    )
 
                 # size/quality/style are intentionally omitted: valid values
                 # differ per model, so we let each model use its own defaults.
@@ -191,10 +189,7 @@ def create_generate_image_tool(
                     ):
                         err = f"Image generation model {config_id} not found"
                         return _failed({"error": err}, error=err)
-                    if (
+                    if conn.user_id is not None and conn.user_id != workspace.user_id:
-                        conn.user_id is not None
-                        and conn.user_id != workspace.user_id
-                    ):
                         err = f"Image generation model {config_id} not found"
                         return _failed({"error": err}, error=err)
                     if not has_capability(db_model, "image_gen"):

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/create_event.py

@@ -70,9 +70,7 @@ def create_create_calendar_event_tool(
 
         try:
             metadata_service = GoogleCalendarToolMetadataService(db_session)
-            context = await metadata_service.get_creation_context(
+            context = await metadata_service.get_creation_context(workspace_id, user_id)
-                workspace_id, user_id
-            )
 
             if "error" in context:
                 logger.error(f"Failed to fetch creation context: {context['error']}")

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/create_page.py

@@ -52,9 +52,7 @@ def create_create_confluence_page_tool(
 
         try:
             metadata_service = ConfluenceToolMetadataService(db_session)
-            context = await metadata_service.get_creation_context(
+            context = await metadata_service.get_creation_context(workspace_id, user_id)
-                workspace_id, user_id
-            )
 
             if "error" in context:
                 return {"status": "error", "message": context["error"]}

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/list_channels.py

@@ -29,9 +29,7 @@ def create_list_discord_channels_tool(
             }
 
         try:
-            connector = await get_discord_connector(
+            connector = await get_discord_connector(db_session, workspace_id, user_id)
-                db_session, workspace_id, user_id
-            )
             if not connector:
                 return {"status": "error", "message": "No Discord connector found."}
 

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/read_messages.py

@@ -39,9 +39,7 @@ def create_read_discord_messages_tool(
         limit = min(limit, 50)
 
         try:
-            connector = await get_discord_connector(
+            connector = await get_discord_connector(db_session, workspace_id, user_id)
-                db_session, workspace_id, user_id
-            )
             if not connector:
                 return {"status": "error", "message": "No Discord connector found."}
 

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/send_message.py

@@ -49,9 +49,7 @@ def create_send_discord_message_tool(
             }
 
         try:
-            connector = await get_discord_connector(
+            connector = await get_discord_connector(db_session, workspace_id, user_id)
-                db_session, workspace_id, user_id
-            )
             if not connector:
                 return {"status": "error", "message": "No Discord connector found."}
 

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/create_draft.py

@@ -67,9 +67,7 @@ def create_create_gmail_draft_tool(
 
         try:
             metadata_service = GmailToolMetadataService(db_session)
-            context = await metadata_service.get_creation_context(
+            context = await metadata_service.get_creation_context(workspace_id, user_id)
-                workspace_id, user_id
-            )
 
             if "error" in context:
                 logger.error(f"Failed to fetch creation context: {context['error']}")

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/send_email.py

@@ -96,9 +96,7 @@ def create_send_gmail_email_tool(
 
         try:
             metadata_service = GmailToolMetadataService(db_session)
-            context = await metadata_service.get_creation_context(
+            context = await metadata_service.get_creation_context(workspace_id, user_id)
-                workspace_id, user_id
-            )
 
             if "error" in context:
                 logger.error(f"Failed to fetch creation context: {context['error']}")

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/create_file.py

@@ -81,9 +81,7 @@ def create_create_google_drive_file_tool(
 
         try:
             metadata_service = GoogleDriveToolMetadataService(db_session)
-            context = await metadata_service.get_creation_context(
+            context = await metadata_service.get_creation_context(workspace_id, user_id)
-                workspace_id, user_id
-            )
 
             if "error" in context:
                 logger.error(f"Failed to fetch creation context: {context['error']}")

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/create_page.py

@@ -80,9 +80,7 @@ def create_create_notion_page_tool(
 
         try:
             metadata_service = NotionToolMetadataService(db_session)
-            context = await metadata_service.get_creation_context(
+            context = await metadata_service.get_creation_context(workspace_id, user_id)
-                workspace_id, user_id
-            )
 
             if "error" in context:
                 logger.error(f"Failed to fetch creation context: {context['error']}")

surfsense_backend/app/automations/services/automation.py

@@ -43,9 +43,7 @@ class AutomationService:
 
     async def create(self, payload: AutomationCreate) -> Automation:
         """Create an automation and its initial triggers in one transaction."""
-        await self._authorize(
+        await self._authorize(payload.workspace_id, Permission.AUTOMATIONS_CREATE.value)
-            payload.workspace_id, Permission.AUTOMATIONS_CREATE.value
-        )
 
         # Capture the model profile onto the definition so runs are insulated
         # from later chat/workspace model changes. Two sources:

surfsense_backend/app/config/__init__.py

@@ -1075,9 +1075,7 @@ class Config:
     CAPTCHA_SOLVER_PROVIDER = os.getenv("CAPTCHA_SOLVER_PROVIDER", "capsolver")
     CAPTCHA_SOLVER_API_KEY = os.getenv("CAPTCHA_SOLVER_API_KEY")
     # Per-URL solve cap so one hostile page can't burn unbounded solver credit.
-    CAPTCHA_MAX_ATTEMPTS_PER_URL = int(
+    CAPTCHA_MAX_ATTEMPTS_PER_URL = int(os.getenv("CAPTCHA_MAX_ATTEMPTS_PER_URL", "1"))
-        os.getenv("CAPTCHA_MAX_ATTEMPTS_PER_URL", "1")
-    )
     # Abort a single solve after this many seconds (solves take 10-60s).
     CAPTCHA_SOLVE_TIMEOUT_S = int(os.getenv("CAPTCHA_SOLVE_TIMEOUT_S", "120"))
     # Default captcha type when detection is ambiguous: v2 | v3 | hcaptcha.
@@ -1114,9 +1112,7 @@ class Config:
     # Route DNS via Cloudflare DoH (anti DNS-leak behind proxies). Adds a DNS
     # round-trip => default FALSE to honor the "no speed regression" bar; flip on
     # when leak-safety outweighs the marginal latency.
-    CRAWL_DNS_OVER_HTTPS = (
+    CRAWL_DNS_OVER_HTTPS = os.getenv("CRAWL_DNS_OVER_HTTPS", "FALSE").upper() == "TRUE"
-        os.getenv("CRAWL_DNS_OVER_HTTPS", "FALSE").upper() == "TRUE"
-    )
 
     # Litellm TTS Configuration
     TTS_SERVICE = os.getenv("TTS_SERVICE")

surfsense_backend/app/db.py

@@ -823,9 +823,7 @@ class ExternalChatAccount(Base, TimestampMixin):
     )
 
     owner = relationship("User", foreign_keys=[owner_user_id])
-    owner_workspace = relationship(
+    owner_workspace = relationship("Workspace", foreign_keys=[owner_workspace_id])
-        "Workspace", foreign_keys=[owner_workspace_id]
-    )
     bindings = relationship(
         "ExternalChatBinding",
         back_populates="account",
@@ -984,9 +982,7 @@ class ExternalChatBinding(Base, TimestampMixin):
             postgresql_where=text("state = 'pending'"),
         ),
         Index("ix_external_chat_bindings_user_state", "user_id", "state"),
-        Index(
+        Index("ix_external_chat_bindings_workspace_state", "workspace_id", "state"),
-            "ix_external_chat_bindings_workspace_state", "workspace_id", "state"
-        ),
     )
 
 
@@ -1902,9 +1898,7 @@ class SearchSourceConnector(BaseModel, TimestampMixin):
         ForeignKey("workspaces.id", ondelete="CASCADE"),
         nullable=False,
     )
-    workspace = relationship(
+    workspace = relationship("Workspace", back_populates="search_source_connectors")
-        "Workspace", back_populates="search_source_connectors"
-    )
 
     user_id = Column(
         UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False

surfsense_backend/app/podcasts/generation/transcript/nodes.py

@@ -99,9 +99,7 @@ def finalize(state: TranscriptState, config: RunnableConfig) -> dict[str, Any]:
 async def _require_llm(state: TranscriptState, tc: TranscriptConfig):
     llm = await get_agent_llm(state.db_session, tc.workspace_id)
     if llm is None:
-        raise RuntimeError(
+        raise RuntimeError(f"no agent LLM configured for workspace {tc.workspace_id}")
-            f"no agent LLM configured for workspace {tc.workspace_id}"
-        )
     return llm
 
 

surfsense_backend/app/proprietary/scrapers/google_maps/README.md

@@ -0,0 +1,341 @@
+# Google Maps Scraper
+
+A platform-native Google Maps scraper intended as a **drop-in clone of the
+Apify "Google Maps Scraper" and "Google Maps Reviews Scraper" actors** — same
+input surface, same output item shape. Built on the same layout and
+progressive-implementation approach as the sibling `../youtube` scraper.
+
+**Current status: search + place details + reviews live.** Search terms
+(`searchStringsArray` x `locationQuery`/geolocation) are discovered via the
+public `search?tbm=map` RPC with offset paging and the Apify result filters.
+Direct place URLs and bare `placeIds` are scraped via Google's public
+`/maps/preview/place` RPC; reviews (both the standalone Reviews endpoint and
+inline `reviews[]` on places when `maxReviews > 0`) come from the public
+`GetLocalBoqProxy` review feed with full token-based pagination. No login,
+proxy-only egress.
+
+## Public vs. gated data
+
+All core Maps data is **public — no Google account needed**: search results,
+place details (phone, website, hours, price, coordinates, plus code, address
+components), and reviews (text, ratings, reviewer profiles, owner responses)
+are served by Google's internal endpoints (`/maps/preview/place`,
+`GetLocalBoqProxy`, `search?tbm=map`) which return XSSI-guarded JSON to
+anonymous requests. Only a residential proxy is required (Google blocks
+datacenter IPs) — already wired via `app/utils/proxy`.
+
+**Session-gated (but still login-free) fields:** Google trims the rich detail
+fields — `reviewsCount`, `reviewsDistribution`, popular times, image
+galleries, `reviewsTags`, and most `additionalInfo` sections — from responses
+that carry **no session cookie**. A plain GET to `/maps` mints an `NID` cookie
+that unlocks all of them (no login, no browser needed at runtime;
+`fetch.get_session_cookies` mints them into a small rotating pool with a
+30-minute TTL). The full-page `pb` selector these fields also require was
+captured once from a headless browser render of a place page and genericized
+into `_PLACE_DETAIL_PB`.
+
+> Note: the older review RPCs other scrapers documented (`listugcposts`,
+> `listentitiesreviews`) now return empty pages / 404 to anonymous callers —
+> they appear to require a signed-in session. The `GetLocalBoqProxy` feed (used
+> by Google's own search local panel) is the one that still works publicly and
+> is what `fetch.iter_reviews_pages` uses.
+
+If Google ever serves a sign-in/consent wall instead of data,
+`SignInRequiredError` is raised and the route returns **403 "Google sign in
+required"** rather than an empty result.
+
+Not sourced from Maps at all (Apify enrichment add-ons that hit third-party
+data brokers / the business's own website — out of scope for public-Maps-only):
+`maximumLeadsEnrichmentRecords`, `leadsEnrichmentDepartments`,
+`verifyLeadsEnrichmentEmails`, `scrapeSocialMediaProfiles`, `scrapeContacts`,
+`enableCompetitorAnalysis`. These stay at their schema defaults (`None`/`[]`).
+
+## Quick start
+
+```python
+from app.proprietary.scrapers.google_maps import (
+    GoogleMapsScrapeInput, scrape_places,
+    GoogleMapsReviewsInput, scrape_reviews,
+)
+
+# Places — search terms, direct URLs, and placeIds are all additive.
+# maxReviews > 0 also attaches inline reviews[] to each place item.
+places = await scrape_places(
+    GoogleMapsScrapeInput(
+        searchStringsArray=["coffee shop"],
+        locationQuery="Seattle, WA",
+        maxCrawledPlacesPerSearch=20,
+        startUrls=[{"url": "https://www.google.com/maps/place/..."}],
+        placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"],
+        maxReviews=10,
+    )
+)
+
+# Reviews — one flat item per review (review fields + place header fields)
+reviews = await scrape_reviews(
+    GoogleMapsReviewsInput(
+        placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"],
+        maxReviews=100,
+        reviewsSort="newest",
+    )
+)
+```
+
+Both have a streaming twin — `iter_places()` / `iter_reviews()`. The HTTP
+surface lives in `app/routes/google_maps_routes.py` (`POST /google-maps/scrape`
+and `POST /google-maps/reviews`).
+
+## Module map
+
+| File              | Responsibility                                                                                                                        |
+| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------ |
+| `__init__.py`     | Public exports (entry points + schemas).                                                                                                     |
+| `schemas.py`      | Pydantic input/output models mirroring the Apify camelCase specs. `extra="allow"` on outputs keeps the contract open.                        |
+| `scraper.py`      | Places orchestrator. `_place_flow` (live): fid → detail RPC → `PlaceItem` (+ inline reviews). `_search_flow` (live): tbm=map paging + filters. |
+| `reviews.py`      | Reviews orchestrator (live). Pages the BOQ feed per place, applies sort/date-cutoff/origin/personal-data rules, one flat item per review.    |
+| `fetch.py`        | Proxy-only fetch seam: `fetch_html`, `fetch_rpc_json` (XSSI-strip + HTML-wrapper tolerant), `fetch_place_darray`, `iter_reviews_pages`, `iter_search_pages`. |
+| `parsers.py`      | Pure array-path parsing: place `darray` → PlaceItem fields (paths from gosom), BOQ review arrays → ReviewFields dicts.                       |
+| `url_resolver.py` | Classify a URL into `place` / `search` / `reviews` / `cid` / `shortlink`; extract the feature ID (`0x..:0x..`).                              |
+
+## How place scraping works
+
+1. Resolve the **feature ID** (`0x<hex>:0x<hex>`): from the URL's `data=!1s…`
+   blob when present, else fetch the place page once and regex it from HTML
+   (covers `?q=place_id:ChIJ…` and `?cid=…`). Name-only URLs
+   (`/maps/place/Eiffel+Tower/`) serve a JS shell with no fid in the HTML, so
+   those fall back to a `search?tbm=map` lookup of the name
+   (`fetch.fid_from_search`) and take the top result's fid. Short links
+   (`maps.app.goo.gl` / `goo.gl`) are Firebase Dynamic Links — a plain GET only
+   returns a JS interstitial, so they're browser-rendered so the JS follows the
+   redirect to the real `/maps/place/…` URL, then the fid is read from that.
+2. GET `/maps/preview/place?…&pb=!1m13!1s{fid}…` through the proxy, with the
+   NID session cookie. The `pb` is the same selector the Maps web app sends
+   (browser-captured, genericized), so the response is the **full** payload
+   including the session-gated fields. `)]}'`-guarded JSON; `jd[6]` is the
+   place detail array ("darray") that all field paths index into.
+3. `parse_place()` maps darray positions → Apify-shaped fields (title,
+   categories, address components, phone, website, rating, plus code, hours,
+   coordinates, placeId…). Only keys whose path hit are set; the rest keep
+   schema defaults.
+
+Detail extras parsed from the same darray (no extra requests):
+
+- `kgmid` (`darray[89]`, e.g. `/g/11bw4ws2mt`) and `cid` — the CID is just the
+  decimal value of the fid's second hex half, so it's derived, not fetched.
+- `additionalInfo` — all about sections at `darray[100][1]` in Apify's shape
+  (`{"Accessibility": [{"Wheelchair accessible entrance": true}, …]}`).
+- `reviewsCount` (`[4][8]`) + `reviewsDistribution` (`[175][3]`).
+- `popularTimesHistogram` / `popularTimesLiveText` / `popularTimesLivePercent`
+  (`darray[84]`).
+- `imagesCount` (`[37][1]`), `imageCategories` (gallery tab names at
+  `[171][0]`), `imageUrls` (hero photos + tab thumbnails; emitted only when
+  `maxImages > 0`, capped at it). Full multi-thousand-photo galleries would
+  need the signed-in photo-listing RPC, so `imageUrls` tops out around a
+  dozen per place.
+- `reviewsTags` (`[153][0]` → `[{title, count}]`).
+- `tableReservationLinks` + `reserveTableUrl` (`darray[46]`), `orderBy` /
+  `googleFoodUrl` (order-online providers at `darray[75]`), `menu`
+  (`darray[38][0]`).
+
+Hotel places (detected by the star string at `darray[35][6]`) additionally
+get: `hotelStars`, `hotelDescription`, `checkInDate`/`checkOutDate` (the
+dates Google quoted prices for, `[35][0..1]`), `similarHotelsNearby`
+(`[35][29][0]` — title/fid/score/count/location/description), and `hotelAds`
+(`[35][44]` — booking-partner title/url/price). Probed live on The Plaza NYC.
+
+Search-result darrays are served **without** the session-gated fields (the
+NID cookie doesn't help `search?tbm=map` — verified live), so when
+`scrapePlaceDetailPage=true` or `maxImages > 0` the search flow makes one
+detail RPC per emitted place and merges the full payload over the search
+fields — same trade Apify makes (search-only is one request per ~20 places;
+detail adds one per place).
+
+**`allPlacesNoSearchAction`** (area scan with no search term): Apify's
+implementation OCRs / mouse-overs rendered map pins — the public RPC has no
+"list everything" query (`*` and empty return nothing). Ours approximates the
+scan with a broad category sweep (`restaurant`, `store`, `hotel`, …17 terms)
+over the requested viewport, deduped by fid, until
+`maxCrawledPlacesPerSearch`. `searchString` carries the action value on scan
+items. Verified live: 25 unique places / 14 distinct categories from a 400 m
+Times Square viewport.
+
+## How search discovery works
+
+1. Build the query: `"{search term} in {location}"` when `locationQuery` (or
+   the city/state/... geolocation fields) is set — Google localizes from the
+   query text, so no geocoding round-trip is needed. A `customGeolocation`
+   GeoJSON Point instead sets a real lat/lng/radius viewport.
+2. Page `search?tbm=map` (~20 results per page, `!8i` offset). Each result
+   entry embeds a **full place darray** at `entry[14]` — same shape as the
+   detail RPC — so `parse_place()` runs directly on it and no per-place detail
+   request is needed. Single-match queries put the one place in slot 0 of the
+   results list; multi-result pages start at slot 1 (both handled).
+3. Dedupe by fid (Google reshuffles results between pages), stamp
+   `rank`/`searchString`/`searchPageUrl`, apply the Apify filters client-side
+   (`searchMatching`, `categoryFilterWords`, `placeMinimumStars`, `website`,
+   `skipClosedPlaces`), and stop at `maxCrawledPlacesPerSearch`, an empty
+   page, or a page with no new fids. Verified live to 60 unique results
+   (3+ pages) with strictly sequential ranks, and to terminate on its own for
+   sparse queries that exhaust before the cap.
+
+Closed places: `darray[88][0]` carries a status enum (`CLOSED` verified live
+on a permanently-closed place) that maps to `permanentlyClosed`.
+
+## Performance
+
+Every RPC is a ~2s proxy round-trip, so wall-clock time is dominated by how
+many requests run in series, not by parsing. Independent requests are
+overlapped (bounded, order preserved via `gather_bounded`):
+
+- **Detail enrichment** (`scrapePlaceDetailPage` / `maxImages`) and inline
+  reviews fire concurrently across a search page instead of one place at a
+  time — the biggest lever (a 20-place enriched search went from ~50s to ~10s,
+  `_DETAIL_CONCURRENCY=8`).
+- **Search pages** are prefetched in waves sized from the result cap
+  (`_prefetch_for`) so a 60-result / 3-page search overlaps its pages.
+- **Bulk `placeIds`** are fetched in parallel.
+- **NID mint** coalesces concurrent cold callers onto one in-flight request
+  (the pool lock isn't held across the network call), so a burst of parallel
+  detail fetches doesn't serialize behind N sequential ~2s mints.
+
+Reviews within a single place stay sequential — pagination is
+continuation-token based (each token embeds the previous page's last-review
+key plus a signature, so page N+1's token can't be forged or precomputed).
+The only lever there is page size: the feed is asked for the ~60/page ceiling
+(`_REVIEWS_PAGE_SIZE`) instead of the old 10/20, cutting the sequential
+round-trips ~3x (1000 reviews: ~113s → ~40s).
+
+## How review scraping works
+
+1. Resolve the feature ID exactly as above (reviews accept the same
+   `startUrls` / `placeIds` inputs).
+2. Fetch the place detail once for the header fields (title, address,
+   location, …) that Apify stamps onto every review item.
+3. Page `GetLocalBoqProxy` (~60 reviews per page — the feed's ceiling, asked
+   via `_REVIEWS_PAGE_SIZE`; opaque continuation token in `node[6]`) until
+   `maxReviews`, the `reviewsStartDate` cutoff (newest-first), or the feed is
+   exhausted. Sort modes map to codes 1–4
+   (mostRelevant/newest/highestRanking/lowestRanking).
+4. `parse_review()` maps each ~48-slot review array → ReviewFields (author,
+   stars, text, ISO publish date, images, owner response, guided
+   context/per-aspect ratings, origin). `personalData=false` strips reviewer
+   name/id/URL/photo (reviewId stays), per Apify semantics.
+
+## API spec
+
+Mirrors the Apify "Google Maps Scraper" and "Google Maps Reviews Scraper"
+actors (camelCase, `extra="allow"`). Unknown inputs are accepted, unsourced
+outputs come back as `None`/`[]`/`{}` — parity grows without breaking
+consumers. `schemas.py` is the source of truth; the tables below list what the
+implementation actually **sources** vs. still **stubs**.
+
+### Places — input (`GoogleMapsScrapeInput`)
+
+| Field | Type / default | Status | Notes |
+|---|---|---|---|
+| `searchStringsArray` | `list[str]` `[]` | ✅ | search-term discovery; additive with the others |
+| `startUrls` | `list[{url}]` `[]` | ✅ | place / CID / short / search / reviews URLs |
+| `placeIds` | `list[str]` `[]` | ✅ | bare `ChIJ…` ids (fetched in parallel) |
+| `allPlacesNoSearchAction` | enum `""` | ✅* | area scan via broad-category sweep (approximation) |
+| `locationQuery` | `str` | ✅ | e.g. `"San Jose, CA"`, appended as `"{q} in {loc}"` |
+| `city`/`state`/`county`/`postalCode`/`countryCode` | `str` | ✅ | discrete location parts (alt. to `locationQuery`) |
+| `customGeolocation` | GeoJSON `Point`+`radiusKm` | ✅ | real lat/lng/radius viewport |
+| `maxCrawledPlacesPerSearch` | `int\|None` | ✅ | `None` = all; caps emitted places |
+| `language` | `str` `"en"` | ✅ | `hl=` on every RPC |
+| `searchMatching` | `all\|only_includes\|only_exact` | ✅ | title-match filter |
+| `categoryFilterWords` | `list[str]` `[]` | ✅ | category client-filter |
+| `placeMinimumStars` | enum `""` | ✅ | min `totalScore` filter |
+| `website` | `allPlaces\|withWebsite\|withoutWebsite` | ✅ | website-presence filter |
+| `skipClosedPlaces` | `bool` `false` | ✅ | drops permanently/temporarily closed |
+| `scrapePlaceDetailPage` | `bool` `false` | ✅ | adds detail-RPC extras (see below) |
+| `maxReviews` | `int` `0` | ✅ | `0` = none; attaches `reviews[]` |
+| `reviewsSort`/`reviewsStartDate`/`reviewsFilterString`/`reviewsOrigin`/`scrapeReviewsPersonalData` | — | ✅ | as in the Reviews scraper |
+| `maxImages` | `int` `0` | ✅ | caps `imageUrls` (URLs only, no author/date) |
+| `scrapeContacts`, `scrapeSocialMediaProfiles`, `*LeadsEnrichment*`, `enableCompetitorAnalysis`, `maxCompetitorsToAnalyze`, `scrapeTableReservationProvider`, `scrapeOrderOnline`, `includeWebResults`, `scrapeDirectories`, `maxQuestions`, `scrapeImageAuthors` | — | ⛔ stub | accepted; out of scope / login-walled (see TODO) |
+
+`✅*` = approximation, not pin-complete.
+
+### Places — output (`PlaceItem`), sourced fields
+
+- **Identity:** `title`, `description`, `price`, `categoryName`, `categories`,
+  `placeId`, `fid`, `cid`, `kgmid`
+- **Location:** `address`, `neighborhood`, `street`, `city`, `postalCode`,
+  `state`, `countryCode`, `location{lat,lng}`, `plusCode`
+- **Contact:** `website`, `phone`, `phoneUnformatted`, `menu`
+- **Ratings/status:** `totalScore`, `reviewsCount`, `reviewsDistribution`,
+  `permanentlyClosed`, `temporarilyClosed`
+- **Images:** `imageUrl`, `imagesCount`, `imageCategories`, `imageUrls`
+- **Detail-page (with `scrapePlaceDetailPage`):** `openingHours`,
+  `additionalInfo`, `reviewsTags`, `popularTimesHistogram`,
+  `popularTimesLiveText`, `popularTimesLivePercent`
+- **Hotels (hotel places):** `hotelStars`, `hotelDescription`, `checkInDate`,
+  `checkOutDate`, `similarHotelsNearby`, `hotelAds`
+- **Reviews (with `maxReviews>0`):** `reviews[]` (see review fields below)
+- **Provenance/meta:** `searchString`, `rank`, `searchPageUrl`, `url`,
+  `scrapedAt`
+- **Stubbed** (`[]`/`{}`/`None`): `peopleAlsoSearch`, `questionsAndAnswers`,
+  `images` (author/date objects), `webResults`, `bookingLinks`,
+  `tableReservationLinks`, `gasPrices`, `ownerUpdates`, leads/contact/social
+  enrichment.
+
+### Reviews — input (`GoogleMapsReviewsInput`) & output (`ReviewItem`)
+
+Input: `startUrls`, `placeIds`, `maxReviews` (default `10000000` = "all"),
+`reviewsSort`, `reviewsStartDate`, `language`, `reviewsOrigin`, `personalData`.
+
+Output is **one flat item per review** — the review fields merged with the
+parent place header. Sourced review fields: `reviewId`, `name`/`text`/
+`textTranslated`, `stars`, `publishAt` (relative) + `publishedAtDate` (ISO),
+`likesCount`, `reviewerId`/`reviewerUrl`/`reviewerPhotoUrl`/
+`reviewerNumberOfReviews`/`isLocalGuide`, `reviewOrigin`, `reviewImageUrls`,
+`reviewContext`, `reviewDetailedRating`, `responseFromOwnerText`
+(+ `responseFromOwnerDate` as relative text only — see TODO).
+
+Notable input semantics (matching Apify):
+
+- Places: `searchStringsArray`, `startUrls`, and `placeIds` are **additive**
+  (unlike YouTube where startUrls override queries).
+- `maxCrawledPlacesPerSearch=None` (unset) means "all places".
+- Reviews: `maxReviews` defaults to `10000000` ("all"); `reviewsStartDate`
+  forces newest-first and stops at the cutoff.
+- `personalData` / `scrapeReviewsPersonalData` default `true`; when off,
+  reviewer id/name/URL/photo must be stripped (reviewId always stays).
+
+## Testing
+
+Offline unit tests (no network; parser tests pin real captured fixtures):
+
+```bash
+cd surfsense_backend
+.venv/Scripts/python.exe -m pytest tests/unit/scrapers/google_maps/
+```
+
+Live end-to-end (real network + proxy; also regenerates fixtures):
+
+```bash
+.venv/Scripts/python.exe scripts/e2e_google_maps_scraper.py
+```
+
+Deep live verification (diverse places worldwide + review semantics: sort
+order, date cutoff, pagination uniqueness, personal-data stripping, French
+localization, CID URLs, name-only URLs, short links, search-filter variants
+`only_exact` / `categoryFilterWords`, and `reviewsOrigin=google`):
+
+```bash
+.venv/Scripts/python.exe scripts/e2e_google_maps_deep.py
+```
+
+## Implementation TODO (progressive, like YouTube)
+
+- Owner-response dates are only exposed as relative text ("11 months ago") in
+  the BOQ feed; `responseFromOwnerDate` carries that string, not an ISO date.
+  (The detail RPC's inline reviews don't carry reply timestamps either, and
+  `listugcposts` still returns no reviews even with the NID cookie.)
+- Full image galleries (`images` with author/date, beyond the ~dozen
+  `imageUrls`): needs the signed-in photo-listing RPC.
+- Q&A (`questionsAndAnswers`): `darray[126]` is empty everywhere we probed —
+  Google appears to have retired public Q&A. `peopleAlsoSearch`: not in the
+  detail darray; likely a separate RPC.
+- A pin-complete `allPlacesNoSearchAction` (the category sweep covers most
+  pins but not businesses outside the swept categories); would need browser
+  rendering + tile OCR like Apify's.

surfsense_backend/app/proprietary/scrapers/google_maps/__init__.py

@@ -0,0 +1,21 @@
+"""Platform-native Google Maps scraper (Apify Google Maps Scraper-compatible)."""
+
+from .reviews import iter_reviews, scrape_reviews
+from .schemas import (
+    GoogleMapsReviewsInput,
+    GoogleMapsScrapeInput,
+    PlaceItem,
+    ReviewItem,
+)
+from .scraper import iter_places, scrape_places
+
+__all__ = [
+    "GoogleMapsReviewsInput",
+    "GoogleMapsScrapeInput",
+    "PlaceItem",
+    "ReviewItem",
+    "iter_places",
+    "iter_reviews",
+    "scrape_places",
+    "scrape_reviews",
+]

surfsense_backend/app/proprietary/scrapers/google_maps/fetch.py

@@ -0,0 +1,595 @@
+"""Proxy-aware fetch seam for the Google Maps scraper.
+
+All network I/O flows through here and always egresses through the residential
+proxy (Google blocks datacenter IPs outright; a direct hit also risk-blocks the
+server IP). Mirrors the design of ``../youtube/innertube.py`` but trimmed to
+what Maps needs today: a GET that returns HTML and a GET that returns the
+XSSI-stripped JSON that Maps' internal ``/maps/preview/*`` RPC endpoints emit.
+
+Google Maps place/search/review data is **public** — no Google account is
+required. :func:`looks_like_signin_wall` flags the rare responses where Google
+serves a consent/sign-in interstitial instead of data, so callers can surface a
+clear "sign-in required" error instead of silently returning nothing.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import time
+from datetime import UTC, datetime
+from typing import Any
+from urllib.parse import quote
+
+from scrapling.fetchers import AsyncFetcher
+
+from app.utils.proxy import get_proxy_url
+
+from .parsers import brace_match_json
+from .url_resolver import ResolvedUrl, extract_fid
+
+try:  # browser tier is optional (needs patchright browsers installed)
+    from scrapling.fetchers import StealthyFetcher
+except Exception:  # pragma: no cover - import guard
+    StealthyFetcher = None  # type: ignore[assignment]
+
+logger = logging.getLogger(__name__)
+
+
+class SignInRequiredError(RuntimeError):
+    """Raised when Google serves a sign-in/consent wall instead of public data.
+
+    Public Maps data does not need a Google account, so this is rare; when it
+    happens the route surfaces it as a clear "Google sign in required" error
+    rather than returning an empty result.
+    """
+
+
+def now_iso() -> str:
+    """UTC timestamp in the millisecond ISO shape Apify stamps on items."""
+    return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
+
+
+async def gather_bounded(coro_factories, *, concurrency: int) -> list:
+    """Run zero-arg async callables with bounded concurrency, results in order.
+
+    Takes callables (``() -> awaitable``), not live coroutines, so nothing
+    starts until a semaphore slot frees up — lets callers queue far more work
+    than ``concurrency`` without opening every socket at once. Every RPC here
+    is ~2s of proxy round-trip, so overlapping independent ones is the single
+    biggest lever on wall-clock time.
+    """
+    if not coro_factories:
+        return []
+    sem = asyncio.Semaphore(concurrency)
+
+    async def _run(factory):
+        async with sem:
+            return await factory()
+
+    return await asyncio.gather(*(_run(f) for f in coro_factories))
+
+
+# XSSI guard Google prepends to its RPC JSON responses.
+_XSSI_PREFIX = ")]}'"
+
+# Consent cookie to dodge the EU consent interstitial that otherwise returns a
+# page with no APP_INITIALIZATION_STATE. Mirrors the YouTube fetcher's SOCS.
+CONSENT_COOKIES = {
+    "SOCS": "CAISNQgDEitib3FfaWRlbnRpdHlmcm9udGVuZHVpc2VydmVyXzIwMjMwODI5LjA3X3AxGgJlbiADGgYIgOa_pgY",
+}
+
+_HEADERS = {"Accept-Language": "en-US,en;q=0.9"}
+
+
+def looks_like_signin_wall(html_or_text: str | None) -> bool:
+    """Heuristic: did Google serve a sign-in / consent wall instead of data?
+
+    Public Maps data never needs login, so this should be rare — it mainly
+    fires on consent redirects (``consent.google.com``) or an account-picker
+    page. Callers turn a True here into a clear "sign-in required" error.
+    """
+    if not html_or_text:
+        return False
+    text = html_or_text[:5000].lower()
+    markers = (
+        "consent.google.com",
+        "accounts.google.com/servicelogin",
+        "sign in to continue",
+        "before you continue to google",
+    )
+    return any(m in text for m in markers)
+
+
+def strip_xssi(text: str) -> str:
+    """Return the JSON body after Google's ``)]}'`` anti-XSSI guard.
+
+    The guard is normally at position 0, but a proxy/HTML fetcher may wrap the
+    text/plain body in ``<html><body><p>…`` — so we locate the guard rather than
+    require it at the start, then return everything after its line.
+    """
+    idx = text.find(_XSSI_PREFIX)
+    if idx == -1:
+        return text
+    nl = text.find("\n", idx)
+    return text[nl + 1 :] if nl != -1 else text[idx + len(_XSSI_PREFIX) :]
+
+
+async def resolve_fid(resolved: ResolvedUrl) -> str | None:
+    """Get a place's feature ID: from the URL if present, else from its HTML.
+
+    A ``/maps/place/...`` URL usually embeds the fid in its ``data=`` blob. A
+    ``?q=place_id:...`` URL or a shortlink does not, so we fetch the page once
+    and pull the fid out of the returned HTML. Raises :class:`SignInRequiredError`
+    if Google serves a consent/sign-in wall instead of the page.
+    """
+    if resolved.fid:
+        return resolved.fid
+    # Short links (maps.app.goo.gl / goo.gl) are Firebase Dynamic Links: a plain
+    # GET only returns a JS interstitial, so render it — the JS follows the
+    # redirect to the real /maps/place/… URL, from which we read the fid.
+    # ponytail: browser render is slow (~30s) but it's the only way these
+    # links resolve; they're rare and cached downstream by fid.
+    if resolved.kind == "shortlink":
+        rendered = await _fetch_html_stealthy(resolved.url, dict(CONSENT_COOKIES))
+        return extract_fid(rendered) if rendered else None
+    html = await fetch_html(resolved.url)
+    if html is not None:
+        if looks_like_signin_wall(html):
+            raise SignInRequiredError(
+                f"Google returned a sign-in/consent wall for {resolved.url}"
+            )
+        fid = extract_fid(html)
+        if fid:
+            return fid
+    # Name-only place URLs (e.g. /maps/place/Eiffel+Tower/) serve a JS-only
+    # shell whose static HTML never contains the fid (even a headless render
+    # won't navigate to the place). The lite ``tbm=map`` search endpoint DOES
+    # embed fids in plain HTML, so look the name up there instead.
+    if resolved.kind == "place" and resolved.value and resolved.value != resolved.url:
+        return await fid_from_search(resolved.value)
+    return None
+
+
+async def fid_from_search(query: str, *, language: str = "en") -> str | None:
+    """Resolve a free-text place name to its feature ID via ``tbm=map`` search.
+
+    This is the lite map-search page (the same endpoint the search-discovery
+    flow will paginate later); its static HTML embeds every result's fid.
+    ``ponytail:`` takes the first fid in the page — that's the top-ranked
+    result. Good enough for name lookups; the full search flow will parse the
+    structured payload instead.
+    """
+    url = f"https://www.google.com/search?tbm=map&hl={language}&gl=us&q={quote(query)}"
+    html = await fetch_html(url)
+    if html is None:
+        return None
+    if looks_like_signin_wall(html):
+        raise SignInRequiredError(
+            f"Google returned a sign-in/consent wall for map search {query!r}"
+        )
+    return extract_fid(html)
+
+
+async def fetch_html(url: str, *, cookies: dict[str, str] | None = None) -> str | None:
+    """GET a Google Maps page and return raw HTML (proxy, stealthy fallback)."""
+    merged = {**CONSENT_COOKIES, **(cookies or {})}
+    try:
+        started = time.perf_counter()
+        page = await AsyncFetcher.get(
+            url,
+            headers=_HEADERS,
+            cookies=merged,
+            proxy=get_proxy_url(),
+            stealthy_headers=True,
+        )
+        fetch_ms = (time.perf_counter() - started) * 1000
+        logger.info(
+            "[google_maps][perf] source=html url=%s status=%s fetch_ms=%.1f",
+            url,
+            page.status,
+            fetch_ms,
+        )
+        if page.status == 200:
+            return page.html_content
+        logger.warning("Maps HTML GET %s returned %s", url, page.status)
+    except Exception as e:
+        logger.warning("Maps HTML GET %s failed: %s", url, e)
+    return await _fetch_html_stealthy(url, merged)
+
+
+async def _fetch_html_stealthy(url: str, cookies: dict[str, str]) -> str | None:
+    """Last-resort browser fetch for anti-bot walls (mirrors the YouTube tier)."""
+    if StealthyFetcher is None:
+        return None
+    try:
+        started = time.perf_counter()
+        page = await asyncio.to_thread(
+            StealthyFetcher.fetch,
+            url,
+            headless=True,
+            network_idle=True,
+            solve_cloudflare=True,
+            proxy=get_proxy_url(),
+        )
+        fetch_ms = (time.perf_counter() - started) * 1000
+        logger.info(
+            "[google_maps][perf] source=html tier=stealthy url=%s status=%s fetch_ms=%.1f",
+            url,
+            page.status,
+            fetch_ms,
+        )
+        if page.status == 200:
+            return page.html_content
+        logger.warning("Maps HTML GET %s tier=stealthy returned %s", url, page.status)
+    except Exception as e:
+        logger.warning("Maps HTML GET %s tier=stealthy failed: %s", url, e)
+    return None
+
+
+# Place-detail RPC field selector (protobuf-over-URL). ``{fid}`` is the place's
+# feature ID (hex:hex). This is the selector the Maps web app itself sends when
+# rendering a place page (captured live via a headless browser, then
+# genericized: the free-text ``!2s`` name and per-place ``!15m2…!4s/g/…`` kgmid
+# hint dropped with counts adjusted, coords zeroed, and the ``!14m3!1s<EI>``
+# session token replaced with a filler — verified the token value is not
+# checked). Combined with an NID session cookie it returns the FULL payload:
+# reviewsCount, reviews distribution, popular times, image galleries, review
+# tags, and every about section. The returned ``jd[6]`` is the ``darray``
+# parsers read.
+_PLACE_DETAIL_PB = (
+    "!1m13!1s{fid}"
+    "!3m8!1m3!1d5000!2d0!3d0!3m2!1i1024!2i768!4f13.1"
+    "!4m2!3d0!4d0"
+    "!12m4!2m3!1i360!2i120!4i8"
+    "!13m57!2m2!1i203!2i100!3m2!2i4!5b1"
+    "!6m6!1m2!1i86!2i86!1m2!1i408!2i240"
+    "!7m33!1m3!1e1!2b0!3e3!1m3!1e2!2b1!3e2!1m3!1e2!2b0!3e3"
+    "!1m3!1e8!2b0!3e3!1m3!1e10!2b0!3e3!1m3!1e10!2b1!3e2"
+    "!1m3!1e10!2b0!3e4!1m3!1e9!2b1!3e2!2b1!9b0"
+    "!15m8!1m7!1m2!1m1!1e2!2m2!1i195!2i195!3i20"
+    "!14m3!1s0ahUKEwixxxxxxxxxxxxxxxxxxxxxxxxx!7e81!15i10112"
+    "!15m108!1m26!13m9!2b1!3b1!4b1!6i1!8b1!9b1!14b1!20b1!25b1"
+    "!18m15!3b1!4b1!5b1!6b1!13b1!14b1!17b1!21b1!22b1!30b1!32b1!33m1!1b1!34b1!36e2"
+    "!10m1!8e3!11m1!3e1!17b1!20m2!1e3!1e6!24b1!25b1!26b1!27b1!29b1"
+    "!30m1!2b1!36b1!37b1!39m3!2m2!2i1!3i1!43b1!52b1!54m1!1b1!55b1!56m1!1b1"
+    "!61m2!1m1!1e1!65m5!3m4!1m3!1m2!1i224!2i298"
+    "!72m22!1m8!2b1!5b1!7b1!12m4!1b1!2b1!4m1!1e1!4b1"
+    "!8m10!1m6!4m1!1e1!4m1!1e3!4m1!1e4"
+    "!3sother_user_google_review_posts__and__hotel_and_vr_partner_review_posts"
+    "!6m1!1e1!9b1!89b1!90m2!1m1!1e2!98m3!1b1!2b1!3b1!103b1!113b1"
+    "!114m3!1b1!2m1!1b1!117b1!122m1!1b1!126b1!127b1!128m1!1b0"
+    "!21m0!22m1!1e81!30m8!3b1!6m2!1b1!2b1!7m2!1e3!2b1!9b1"
+    "!34m5!7b1!10b1!14b1!15m1!1b0!37i785"
+)
+
+# NID session cookie pool. Google trims the rich detail fields (counts,
+# distribution, popular times, galleries, tags, most about sections) from
+# responses that carry no session cookie; a plain GET to /maps mints an NID
+# that unlocks them — no login, no browser. A small pool is rotated
+# round-robin (mirrors the YouTube fetcher's ``_RotatingSession``) so no
+# single session accumulates the whole request volume; each expires after a
+# TTL and is re-minted lazily.
+_NID_TTL_S = 30 * 60
+_NID_POOL_SIZE = 3
+_nid_pool: list[dict[str, Any]] = [
+    {"value": None, "at": 0.0} for _ in range(_NID_POOL_SIZE)
+]
+_nid_rr = 0
+_nid_lock = asyncio.Lock()
+_nid_inflight: asyncio.Future | None = None
+
+
+async def _mint_nid() -> str | None:
+    """Mint a fresh NID session cookie with a plain GET to /maps."""
+    try:
+        page = await AsyncFetcher.get(
+            "https://www.google.com/maps?hl=en",
+            headers=_HEADERS,
+            cookies=CONSENT_COOKIES,
+            proxy=get_proxy_url(),
+            stealthy_headers=True,
+        )
+        return (page.cookies or {}).get("NID")
+    except Exception as e:
+        logger.warning("NID mint failed: %s", e)
+        return None
+
+
+async def get_session_cookies() -> dict[str, str]:
+    """Consent cookies plus an NID session cookie from the rotating pool.
+
+    Concurrent cold callers coalesce onto a single in-flight mint (the lock is
+    not held across the network call), so a burst of parallel detail fetches
+    doesn't serialize behind N sequential ~2s mints on a cold pool.
+    """
+    global _nid_rr, _nid_inflight
+    async with _nid_lock:
+        slot = _nid_pool[_nid_rr % _NID_POOL_SIZE]
+        _nid_rr += 1
+        if slot["value"] and time.time() - slot["at"] < _NID_TTL_S:
+            return {**CONSENT_COOKIES, "NID": slot["value"]}
+        if _nid_inflight is None or _nid_inflight.done():
+            _nid_inflight = asyncio.ensure_future(_mint_nid())
+        fut = _nid_inflight
+    nid = await fut
+    if nid:
+        async with _nid_lock:
+            slot["value"] = nid
+            slot["at"] = time.time()
+        return {**CONSENT_COOKIES, "NID": nid}
+    return dict(CONSENT_COOKIES)
+
+
+async def fetch_place_darray(fid: str, *, language: str = "en") -> list | None:
+    """Fetch a place's detail ``darray`` via the ``/maps/preview/place`` RPC.
+
+    ``fid`` is the feature ID (``0x..:0x..``). Returns ``jd[6]`` (the array all
+    place-field paths index into), or ``None`` on failure. Sent with an NID
+    session cookie so Google returns the full payload (see ``_PLACE_DETAIL_PB``).
+    """
+    pb = _PLACE_DETAIL_PB.format(fid=fid)
+    url = (
+        "https://www.google.com/maps/preview/place"
+        f"?authuser=0&hl={language}&gl=us&pb={quote(pb, safe='!')}"
+    )
+    jd = await fetch_rpc_json(url, cookies=await get_session_cookies())
+    darray = jd[6] if isinstance(jd, list) and len(jd) > 6 else None
+    return darray if isinstance(darray, list) else None
+
+
+# Map-search RPC (``search?tbm=map``) field selector. Viewport block first
+# (``!1d`` diameter-meters, ``!2d`` lng, ``!3d`` lat), then paging (``!7i``
+# per-page, ``!8i`` offset), then the same detail selector blocks the place RPC
+# uses so each result embeds a full place darray at ``entry[14]``.
+_SEARCH_PB = (
+    "!4m12!1m3!1d{diameter}!2d{lng}!3d{lat}"
+    "!2m3!1f0!2f0!3f0!3m2!1i1024!2i768!4f13.1"
+    "!7i{per_page}!8i{offset}!10b1"
+    "!12m6!1m1!18b1!2m1!20e3!6m1!114b1"
+    "!17m1!3e1"
+    "!20m57!2m2!1i203!2i100!3m2!2i4!5b1"
+    "!6m6!1m2!1i86!2i86!1m2!1i408!2i240"
+    "!7m33!1m3!1e1!2b0!3e3!1m3!1e2!2b1!3e2!1m3!1e2!2b0!3e3"
+    "!1m3!1e8!2b0!3e3!1m3!1e10!2b0!3e3!1m3!1e10!2b1!3e2"
+    "!1m3!1e10!2b0!3e4!1m3!1e9!2b1!3e2!2b1!9b0"
+    "!15m8!1m7!1m2!1m1!1e2!2m2!1i195!2i195!3i20"
+)
+
+# Whole-earth viewport: Google localizes from the query text itself (verified
+# live — "pizza new york" returns NYC results with this viewport), so callers
+# only need coordinates when the input provides an explicit geolocation.
+_EARTH = {"diameter": 25_000_000.0, "lat": 0.0, "lng": 0.0}
+
+
+def build_search_url(
+    query: str,
+    *,
+    offset: int = 0,
+    per_page: int = 20,
+    language: str = "en",
+    lat: float | None = None,
+    lng: float | None = None,
+    radius_m: float | None = None,
+) -> str:
+    """Build a ``search?tbm=map`` URL returning protobuf-over-JSON results."""
+    viewport = dict(_EARTH)
+    if lat is not None and lng is not None:
+        viewport = {
+            "diameter": (radius_m or 10_000) * 2,
+            "lat": lat,
+            "lng": lng,
+        }
+    pb = _SEARCH_PB.format(per_page=per_page, offset=offset, **viewport)
+    return (
+        f"https://www.google.com/search?tbm=map&authuser=0&hl={language}&gl=us"
+        f"&q={quote(query)}&pb={quote(pb, safe='!')}"
+    )
+
+
+def _search_darrays(jd: Any) -> list[list]:
+    """Extract the place darrays from a map-search response.
+
+    Results live at ``jd[0][1]``; each result entry holds its place darray at
+    ``entry[14]`` (same shape as the place detail RPC's ``jd[6]``). Multi-result
+    pages put metadata (no ``[14]``) in slot 0; single-match responses put the
+    one place directly in slot 0 — scanning every entry handles both.
+    """
+    entries = (
+        jd[0][1]
+        if isinstance(jd, list) and jd and isinstance(jd[0], list) and len(jd[0]) > 1
+        else None
+    )
+    if not isinstance(entries, list):
+        return []
+    out = []
+    for entry in entries:
+        darray = entry[14] if isinstance(entry, list) and len(entry) > 14 else None
+        if isinstance(darray, list):
+            out.append(darray)
+    return out
+
+
+async def iter_search_pages(
+    query: str,
+    *,
+    language: str = "en",
+    lat: float | None = None,
+    lng: float | None = None,
+    radius_m: float | None = None,
+    max_pages: int = 25,
+    prefetch: int = 1,
+):
+    """Yield lists of place darrays, one map-search page (~20 places) a time.
+
+    Paging is offset-based (``!8i``); Google reshuffles results between pages
+    so callers must dedupe by fid. Stops on an empty page or ``max_pages``.
+
+    ``prefetch`` pages are fetched concurrently per wave (still yielded in
+    offset order): each page is a ~2s round-trip, so a caller that knows it
+    needs several pages (large ``maxCrawledPlacesPerSearch``) gets them
+    overlapped. ``prefetch=1`` keeps the old one-at-a-time behavior for callers
+    that only need a page or two (no wasted fetches).
+    """
+    per_page = 20
+    page = 0
+    while page < max_pages:
+        wave = min(max(prefetch, 1), max_pages - page)
+        urls = [
+            build_search_url(
+                query,
+                offset=(page + i) * per_page,
+                per_page=per_page,
+                language=language,
+                lat=lat,
+                lng=lng,
+                radius_m=radius_m,
+            )
+            for i in range(wave)
+        ]
+        pages = await gather_bounded(
+            [lambda u=u: fetch_rpc_json(u) for u in urls], concurrency=wave
+        )
+        for jd in pages:
+            darrays = _search_darrays(jd)
+            if not darrays:
+                return
+            yield darrays
+        page += wave
+
+
+# Sort code the reviews RPC expects (slot 1 of the request payload).
+REVIEWS_SORT_CODES = {
+    "mostRelevant": 1,
+    "newest": 2,
+    "highestRanking": 3,
+    "lowestRanking": 4,
+}
+
+# Reviews are cursor-paged (each page's continuation token embeds the previous
+# page's last-review key + a signature), so pages CANNOT be fetched in
+# parallel — the only lever on review throughput is page size. Google honors
+# this on both the first and continuation requests but caps the actual return
+# at ~60/page (asking higher just yields the max), so 100 grabs the ceiling and
+# cuts the sequential round-trips ~3x vs the old 10/20-per-page default.
+_REVIEWS_PAGE_SIZE = 100
+
+
+def build_reviews_url(
+    fid: str, *, sort_code: int, page_token: str = "", language: str = "en"
+) -> str:
+    """Build a ``GetLocalBoqProxy`` reviews URL (public, no session token).
+
+    This is the review feed the Google search local panel uses. The older
+    ``listugcposts`` / ``listentitiesreviews`` RPCs now return empty pages for
+    anonymous callers, so this proxy is the one that still works (approach from
+    the maintained ``google-maps-review-scraper`` npm package).
+
+    ``fid`` is the feature ID (``0x..:0x..``); ``page_token`` is the opaque
+    continuation token from the previous page (empty for the first page).
+    """
+    inner: list = [None] * 12
+    inner[1] = sort_code
+    inner[9] = _REVIEWS_PAGE_SIZE
+    inner[11] = [fid]
+    if page_token:
+        inner.extend([None] * 8)
+        inner[19] = page_token
+    payload = [None, [None] * 9 + [inner]]
+    return (
+        "https://www.google.com/httpservice/web/PrivateLocalSearchUiDataService/"
+        f"GetLocalBoqProxy?msc=gwsrpc&hl={language}"
+        f"&reqpld={quote(json.dumps(payload))}"
+    )
+
+
+def _reviews_node(jd: Any) -> list | None:
+    """The reviews node of a BOQ response: ``jd[1][10]`` = [.., reviews, .., token]."""
+    node = (
+        jd[1][10]
+        if isinstance(jd, list)
+        and len(jd) > 1
+        and isinstance(jd[1], list)
+        and len(jd[1]) > 10
+        else None
+    )
+    return node if isinstance(node, list) else None
+
+
+async def iter_reviews_pages(
+    fid: str,
+    *,
+    sort: str = "newest",
+    language: str = "en",
+    max_pages: int = 10_000,
+):
+    """Yield raw review arrays, one RPC page (~10 reviews) at a time.
+
+    Each yielded item is the page's review list (``node[2]``). Pagination
+    follows ``node[6]`` (the continuation token); stops when it is missing or
+    unchanged, or ``max_pages`` is reached. ``ponytail:`` sequential paging —
+    the endpoint only hands out one continuation token at a time anyway.
+    """
+    sort_code = REVIEWS_SORT_CODES.get(sort, 2)
+    page_token = ""
+    for _ in range(max_pages):
+        url = build_reviews_url(
+            fid, sort_code=sort_code, page_token=page_token, language=language
+        )
+        node = _reviews_node(await fetch_rpc_json(url))
+        if not node or len(node) < 3 or not isinstance(node[2], list):
+            return
+        yield node[2]
+        next_token = node[6] if len(node) > 6 else None
+        if (
+            not isinstance(next_token, str)
+            or not next_token
+            or next_token == page_token
+        ):
+            return
+        page_token = next_token
+
+
+async def fetch_rpc_json(
+    url: str, *, cookies: dict[str, str] | None = None
+) -> Any | None:
+    """GET a Maps ``/maps/preview/*`` RPC URL and return parsed JSON.
+
+    These endpoints return an XSSI-guarded JSON array (not HTML). Returns the
+    decoded structure, or ``None`` on failure / non-JSON (e.g. a sign-in wall).
+    """
+    try:
+        started = time.perf_counter()
+        page = await AsyncFetcher.get(
+            url,
+            headers=_HEADERS,
+            cookies=cookies or CONSENT_COOKIES,
+            proxy=get_proxy_url(),
+            stealthy_headers=True,
+        )
+        fetch_ms = (time.perf_counter() - started) * 1000
+        logger.info(
+            "[google_maps][perf] source=rpc url=%s status=%s fetch_ms=%.1f",
+            url,
+            page.status,
+            fetch_ms,
+        )
+        if page.status != 200:
+            logger.warning("Maps RPC GET %s returned %s", url, page.status)
+            return None
+        text = page.html_content
+        if looks_like_signin_wall(text):
+            logger.warning("Maps RPC GET %s hit a sign-in/consent wall", url)
+            return None
+        body = strip_xssi(text)
+        # The proxy/HTML fetcher may wrap the JSON in <html><body><p>…</p>…,
+        # so extract just the balanced top-level array/object.
+        start = next((i for i, c in enumerate(body) if c in "[{"), -1)
+        if start == -1:
+            return None
+        blob = brace_match_json(body, start)
+        return json.loads(blob) if blob else None
+    except Exception as e:
+        logger.warning("Maps RPC GET %s failed: %s", url, e)
+        return None

surfsense_backend/app/proprietary/scrapers/google_maps/parsers.py

@@ -0,0 +1,526 @@
+"""Pure, I/O-free parsing of Google Maps' protobuf-over-JSON place data.
+
+Google Maps returns a place's data as a deeply-nested JSON array (no field
+names, just positions) from the ``/maps/preview/place`` RPC (element ``jd[6]``,
+the ``darray``). Fields are read by fixed array paths.
+
+The array paths are ported from the actively-maintained ``gosom/google-maps-
+scraper`` (Go), which tracks Google's periodic structure shifts and carries
+fallback paths (e.g. the Nov-2025 opening-hours move). Everything here is
+deterministic and offline-unit-testable against captured fixtures.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def dig(obj: Any, *path: int) -> Any:
+    """Safely walk a nested list by integer indices; ``None`` if any step misses.
+
+    The Python twin of gosom's ``getNthElementAndCast`` — Maps arrays are ragged
+    and shift between updates, so every access must tolerate a short/absent path.
+    """
+    cur = obj
+    for idx in path:
+        if not isinstance(cur, list) or idx >= len(cur) or cur[idx] is None:
+            return None
+        cur = cur[idx]
+    return cur
+
+
+def _dig_str(obj: Any, *path: int) -> str | None:
+    val = dig(obj, *path)
+    return val if isinstance(val, str) else None
+
+
+def _dig_num(obj: Any, *path: int) -> float | None:
+    val = dig(obj, *path)
+    return val if isinstance(val, int | float) else None
+
+
+def brace_match_json(text: str, start: int) -> str | None:
+    """Return the balanced ``[...]`` / ``{...}`` blob beginning at ``text[start]``."""
+    open_ch = text[start]
+    close_ch = {"[": "]", "{": "}"}.get(open_ch)
+    if close_ch is None:
+        return None
+    depth = 0
+    i = start
+    n = len(text)
+    while i < n:
+        ch = text[i]
+        if ch == open_ch:
+            depth += 1
+        elif ch == close_ch:
+            depth -= 1
+            if depth == 0:
+                return text[start : i + 1]
+        elif ch == '"':
+            i += 1
+            while i < n and text[i] != '"':
+                if text[i] == "\\":
+                    i += 1
+                i += 1
+        i += 1
+    return None
+
+
+def _opening_hours(darray: list) -> list[dict[str, str]]:
+    """Weekly hours. New layout (Nov 2025) at [203][0], old at [34][1]."""
+    items = dig(darray, 203, 0) or dig(darray, 34, 1) or []
+    hours: list[dict[str, str]] = []
+    for item in items if isinstance(items, list) else []:
+        day = _dig_str(item, 0)
+        if not day:
+            continue
+        slots = dig(item, 3)  # new format: [ [label, [[h,m],[h,m]]], ... ]
+        times: list[str] = []
+        if isinstance(slots, list) and slots:
+            times = [_dig_str(s, 0) for s in slots if _dig_str(s, 0)]
+        else:  # old format: [1] is a flat list of strings
+            old = dig(item, 1)
+            times = (
+                [t for t in old if isinstance(t, str)] if isinstance(old, list) else []
+            )
+        hours.append({"day": day, "hours": ", ".join(times) if times else "Closed"})
+    return hours
+
+
+def _categories(darray: list) -> list[str]:
+    cats = dig(darray, 13)
+    return [c for c in cats if isinstance(c, str)] if isinstance(cats, list) else []
+
+
+def _website(darray: list) -> str | None:
+    raw = _dig_str(darray, 7, 0)
+    if raw and raw.startswith("/url?q="):  # Google redirect wrapper
+        from urllib.parse import parse_qs, urlparse
+
+        q = parse_qs(urlparse(raw).query).get("q", [None])[0]
+        return q or raw
+    return raw
+
+
+def _additional_info(darray: list) -> dict[str, Any] | None:
+    """About sections (``darray[100][1]``) in Apify's ``additionalInfo`` shape:
+    ``{"Accessibility": [{"Wheelchair accessible entrance": true}, ...], ...}``.
+
+    Option paths mirror gosom's About parser; enabled state is
+    ``option[2][1][0][0] == 1``.
+    """
+    sections = dig(darray, 100, 1)
+    if not isinstance(sections, list):
+        return None
+    info: dict[str, Any] = {}
+    for section in sections:
+        name = _dig_str(section, 1)
+        options = dig(section, 2)
+        if not name or not isinstance(options, list):
+            continue
+        entries = []
+        for opt in options:
+            opt_name = _dig_str(opt, 1)
+            if opt_name:
+                entries.append({opt_name: dig(opt, 2, 1, 0, 0) == 1})
+        if entries:
+            info[name] = entries
+    return info or None
+
+
+def _link_sources(items: Any, link_path: tuple, source_path: tuple) -> list[dict]:
+    """Extract ``[{"url":…, "source":…}]`` pairs from a link-source array."""
+    out = []
+    for item in items if isinstance(items, list) else []:
+        url = _dig_str(item, *link_path)
+        source = _dig_str(item, *source_path)
+        if url and source:
+            out.append({"url": url, "source": source})
+    return out
+
+
+def _order_links(darray: list) -> list[dict]:
+    """Order-online providers (paths from gosom, two known layouts)."""
+    items = dig(darray, 75, 0, 1, 2) or dig(darray, 75, 0, 0, 2)
+    return _link_sources(items, (1, 2, 0), (0, 0))
+
+
+# Day numbers Google uses in the popular-times block (gosom's mapping),
+# rendered as Apify's two-letter histogram keys.
+_DAY_KEYS = {1: "Mo", 2: "Tu", 3: "We", 4: "Th", 5: "Fr", 6: "Sa", 7: "Su"}
+
+
+def _popular_times(darray: list) -> dict[str, Any]:
+    """Popular-times block (``darray[84]``) in Apify's field shapes.
+
+    ``[84][0]`` is a list of ``[day_num, [[hour, occupancy%, …], …]]`` days;
+    ``[84][6]`` is the live busyness text and ``[84][7][1]`` the live percent.
+    """
+    out: dict[str, Any] = {}
+    days = dig(darray, 84, 0)
+    if isinstance(days, list):
+        histogram: dict[str, list] = {}
+        for day in days:
+            key = _DAY_KEYS.get(dig(day, 0))
+            hours = dig(day, 1)
+            if not key or not isinstance(hours, list):
+                continue
+            histogram[key] = [
+                {"hour": h[0], "occupancyPercent": h[1]}
+                for h in hours
+                if isinstance(h, list) and len(h) >= 2
+            ]
+        if histogram:
+            out["popularTimesHistogram"] = histogram
+    live_text = _dig_str(darray, 84, 6)
+    if live_text:
+        out["popularTimesLiveText"] = live_text
+    live_pct = _dig_num(darray, 84, 7, 1)
+    if live_pct is not None:
+        out["popularTimesLivePercent"] = int(live_pct)
+    return out
+
+
+def _reviews_tags(darray: list) -> list[dict]:
+    """Review keyword tags (``darray[153][0]``): ``[{title, count}, …]``."""
+    entries = dig(darray, 153, 0)
+    out = []
+    for e in entries if isinstance(entries, list) else []:
+        title = _dig_str(e, 1)
+        count = _dig_num(e, 3, 4)
+        if title and count is not None:
+            out.append({"title": title, "count": int(count)})
+    return out
+
+
+def _image_fields(darray: list) -> dict[str, Any]:
+    """Image gallery fields: count, category tab names, and photo URLs.
+
+    ``darray[37]`` holds ``[hero_photos, total_count]``; ``darray[171][0]``
+    holds the gallery tabs (each with a thumbnail). ``ponytail:`` anonymous
+    sessions only expose the few hero photos + one thumbnail per tab, not the
+    full gallery — scraping thousands of photos would need the signed-in
+    photo-listing RPC. ``imageUrls`` therefore tops out around a dozen.
+    """
+    out: dict[str, Any] = {}
+    count = _dig_num(darray, 37, 1)
+    if count is not None:
+        out["imagesCount"] = int(count)
+    tabs = dig(darray, 171, 0)
+    urls: list[str] = []
+    if isinstance(tabs, list):
+        categories = [t for tab in tabs if (t := _dig_str(tab, 2))]
+        if categories:
+            out["imageCategories"] = categories
+        urls.extend(
+            u for tab in tabs if (u := _dig_str(tab, 3, 0, 6, 0)) and u not in urls
+        )
+    heroes = dig(darray, 37, 0)
+    for photo in heroes if isinstance(heroes, list) else []:
+        u = _dig_str(photo, 6, 0)
+        if u and u not in urls:
+            urls.append(u)
+    if urls:
+        out["imageUrls"] = urls
+    return out
+
+
+def _hotel_fields(darray: list) -> dict[str, Any]:
+    """Hotel block (``darray[35]`` + ``[64]``), only present for lodging.
+
+    Paths probed live on The Plaza (NYC): ``[35][6]`` star string,
+    ``[35][0]/[1]`` the check-in/out dates the price quotes are for,
+    ``[35][29][0]`` similar hotels, ``[35][44]`` booking-partner ads.
+    """
+    stars = _dig_str(darray, 35, 6) or (
+        f"{int(n)} stars" if (n := _dig_num(darray, 64, 0)) else None
+    )
+    if not stars:
+        return {}
+    out: dict[str, Any] = {"hotelStars": stars}
+    description = _dig_str(darray, 32, 1, 1)
+    if description:
+        out["hotelDescription"] = description
+    check_in = _dig_str(darray, 35, 0)
+    check_out = _dig_str(darray, 35, 1)
+    if check_in:
+        out["checkInDate"] = check_in
+    if check_out:
+        out["checkOutDate"] = check_out
+
+    similar = []
+    for entry in dig(darray, 35, 29, 0) or []:
+        info = dig(entry, 0)
+        title = _dig_str(info, 4)
+        if not title:
+            continue
+        hotel: dict[str, Any] = {"title": title, "fid": _dig_str(info, 2)}
+        if (score := _dig_num(info, 7)) is not None:
+            hotel["totalScore"] = score
+        if (count := _dig_num(info, 8)) is not None:
+            hotel["reviewsCount"] = int(count)
+        lat, lng = _dig_num(info, 3, 2), _dig_num(info, 3, 3)
+        if lat is not None and lng is not None:
+            hotel["location"] = {"lat": lat, "lng": lng}
+        if desc := _dig_str(entry, 1):
+            hotel["description"] = desc
+        similar.append(hotel)
+    if similar:
+        out["similarHotelsNearby"] = similar
+
+    ads = []
+    for ad in dig(darray, 35, 44) or []:
+        title = _dig_str(ad, 0)
+        url = _dig_str(ad, 5, 0)
+        if not title or not url:
+            continue
+        item: dict[str, Any] = {"title": title, "url": url}
+        if price := _dig_str(ad, 1):
+            item["price"] = price
+        ads.append(item)
+    if ads:
+        out["hotelAds"] = ads
+    return out
+
+
+def _cid_from_fid(fid: str | None) -> str | None:
+    """CID (a.k.a. ludocid) = the decimal value of the fid's second hex half."""
+    if not fid or ":" not in fid:
+        return None
+    try:
+        return str(int(fid.split(":")[1], 16))
+    except ValueError:
+        return None
+
+
+def parse_place(darray: list) -> dict[str, Any]:
+    """Map a place ``darray`` to PlaceItem-shaped fields (public Maps data only).
+
+    Only fields with a stable, known array path are populated; the rest stay at
+    their schema defaults. Paths mirror gosom's ``EntryFromJSON``.
+    """
+    lat = _dig_num(darray, 9, 2)
+    lng = _dig_num(darray, 9, 3)
+    dist = dig(darray, 175, 3)  # per-star review counts [1★,2★,3★,4★,5★]
+
+    result: dict[str, Any] = {
+        "title": _dig_str(darray, 11),
+        "categories": _categories(darray),
+        "address": _dig_str(darray, 18),
+        "website": _website(darray),
+        "phone": _dig_str(darray, 178, 0, 0),
+        "plusCode": _dig_str(darray, 183, 2, 2, 0),
+        "totalScore": _dig_num(darray, 4, 7),
+        "reviewsCount": (int(v) if (v := _dig_num(darray, 4, 8)) is not None else None),
+        "price": _dig_str(darray, 4, 2),
+        "description": _dig_str(darray, 32, 1, 1),
+        "placeId": _dig_str(darray, 78),
+        "fid": _dig_str(darray, 10),
+        "kgmid": _dig_str(darray, 89),
+        "menu": _dig_str(darray, 38, 0),
+        "imageUrl": _dig_str(darray, 72, 0, 1, 6, 0),
+        "additionalInfo": _additional_info(darray),
+    }
+    result["cid"] = _cid_from_fid(result["fid"])
+
+    reservations = _link_sources(dig(darray, 46), (0,), (1,))
+    if reservations:
+        result["tableReservationLinks"] = reservations
+        result["reserveTableUrl"] = reservations[0]["url"]
+    order_links = _order_links(darray)
+    if order_links:
+        result["orderBy"] = order_links  # Apify's field name for these
+        food = next(
+            (o["url"] for o in order_links if "food.google.com" in o["url"]), None
+        )
+        if food:
+            result["googleFoodUrl"] = food
+    if result["categories"]:
+        result["categoryName"] = result["categories"][0]
+    if lat is not None and lng is not None:
+        result["location"] = {"lat": lat, "lng": lng}
+    if isinstance(dist, list) and len(dist) >= 5:
+        result["reviewsDistribution"] = {
+            "oneStar": int(dist[0] or 0),
+            "twoStar": int(dist[1] or 0),
+            "threeStar": int(dist[2] or 0),
+            "fourStar": int(dist[3] or 0),
+            "fiveStar": int(dist[4] or 0),
+        }
+    hours = _opening_hours(darray)
+    if hours:
+        result["openingHours"] = hours
+
+    result.update(_popular_times(darray))
+    result.update(_image_fields(darray))
+    result.update(_hotel_fields(darray))
+    tags = _reviews_tags(darray)
+    if tags:
+        result["reviewsTags"] = tags
+
+    # Closed status: darray[88][0] carries a status enum for closed places
+    # ('CLOSED' verified live on a permanently-closed place; open places have
+    # None or an editorial snippet there, so only exact enums count).
+    status = _dig_str(darray, 88, 0)
+    if status in ("CLOSED", "CLOSED_PERMANENTLY"):
+        result["permanentlyClosed"] = True
+    elif status == "CLOSED_TEMPORARILY":
+        result["temporarilyClosed"] = True
+
+    # Complete address components (borough/street/city/postal/state/country).
+    result["neighborhood"] = _dig_str(darray, 183, 1, 0)
+    result["street"] = _dig_str(darray, 183, 1, 1)
+    result["city"] = _dig_str(darray, 183, 1, 3)
+    result["postalCode"] = _dig_str(darray, 183, 1, 4)
+    result["state"] = _dig_str(darray, 183, 1, 5)
+    result["countryCode"] = _dig_str(darray, 183, 1, 6)
+
+    return {k: v for k, v in result.items() if v is not None}
+
+
+# --- reviews (GetLocalBoqProxy) ----------------------------------------------
+#
+# The BOQ proxy returns ``jd[1][10]`` = ``[.., reviews, .., nextToken]``. Each
+# review ``r`` is a flat ~48-slot array. Index map (verified against live
+# captures, matching the ``google-maps-review-scraper`` npm package):
+#   r[1]  stars                r[2]  [relativeDate, ?, publishedMs]
+#   r[3]  [name, photoUrl, contribUrl, nReviews, nPhotos, [?, localGuide]]
+#   r[4]  owner reply [?, relativeDate, text, ...]
+#   r[5]  reviewId             r[12] reviewUrl
+#   r[13]/r[14] images [[url, caption, ...], ...]
+#   r[26] language             r[27] text        r[28] translated text
+#   r[30] guided Q&A (context + per-aspect ratings)   r[44] [origin, ...]
+
+
+_CONTRIB_ID_RE = re.compile(r"/contrib/(\d+)")
+
+
+def _ms_to_iso(ms: Any) -> str | None:
+    """Convert a millisecond epoch (Google sends it as a string) to ISO-8601."""
+    try:
+        ms_val = float(ms)
+    except (TypeError, ValueError):
+        return None
+    if not ms_val:
+        return None
+    from datetime import UTC, datetime
+
+    try:
+        dt = datetime.fromtimestamp(ms_val / 1000, tz=UTC)
+    except (ValueError, OSError, OverflowError):
+        return None
+    return dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
+
+
+def _review_images(r: Any) -> list[str]:
+    for idx in (13, 14):
+        imgs = dig(r, idx)
+        if isinstance(imgs, list):
+            urls = [_dig_str(imgs, j, 0) for j in range(len(imgs))]
+            return [u for u in urls if u]
+    return []
+
+
+def _guided_answers(r: Any) -> tuple[dict[str, Any], dict[str, Any]]:
+    """Split the guided Q&A block into (reviewContext, reviewDetailedRating).
+
+    Entries with a numeric rating (``entry[11] = [n]``) are per-aspect ratings
+    (Food/Service/Atmosphere); entries with a chosen answer are context
+    (e.g. ``Order type: Take out``).
+    """
+    context: dict[str, Any] = {}
+    detailed: dict[str, Any] = {}
+    block = dig(r, 30)
+    if not isinstance(block, list):
+        return context, detailed
+    for entry in block:
+        label = _dig_str(entry, 5) or _dig_str(entry, 1)
+        if not label:
+            continue
+        rating = _dig_num(entry, 11, 0)
+        if rating is not None:
+            detailed[label] = rating
+            continue
+        answer = _dig_str(entry, 2, 0, 0, 1)
+        if answer:
+            context[label] = answer
+    return context, detailed
+
+
+def parse_review(r: Any) -> dict[str, Any] | None:
+    """Map one BOQ review array to ReviewFields-shaped fields.
+
+    Returns ``None`` for malformed entries (no author block).
+    """
+    if not isinstance(r, list):
+        return None
+    name = _dig_str(r, 3, 0)
+    if not name:
+        return None
+
+    reviewer_url = _dig_str(r, 3, 2)
+    reviewer_id = None
+    if reviewer_url:
+        m = _CONTRIB_ID_RE.search(reviewer_url)
+        reviewer_id = m.group(1) if m else None
+
+    n_reviews = _dig_num(r, 3, 3)
+    local_guide = _dig_num(r, 3, 5, 1)
+    text = _dig_str(r, 27)
+    translated = _dig_str(r, 28)
+    context, detailed = _guided_answers(r)
+
+    fields: dict[str, Any] = {
+        "reviewId": _dig_str(r, 5),
+        "reviewUrl": _dig_str(r, 12),
+        "name": name,
+        "reviewerId": reviewer_id,
+        "reviewerUrl": reviewer_url,
+        "reviewerPhotoUrl": _dig_str(r, 3, 1),
+        "reviewerNumberOfReviews": int(n_reviews) if n_reviews is not None else None,
+        "isLocalGuide": bool(local_guide) if local_guide is not None else None,
+        "stars": _dig_num(r, 1),
+        "text": text,
+        "textTranslated": translated if translated and translated != text else None,
+        "publishAt": _dig_str(r, 2, 0),
+        "publishedAtDate": _ms_to_iso(dig(r, 2, 2)),
+        "originalLanguage": _dig_str(r, 26),
+        "reviewOrigin": _dig_str(r, 44, 0),
+        "reviewImageUrls": _review_images(r),
+        "reviewContext": context or None,
+        "reviewDetailedRating": detailed or None,
+    }
+
+    # Owner response (r[4]); only a relative date is exposed here.
+    reply_text = _dig_str(r, 4, 2)
+    if reply_text:
+        fields["responseFromOwnerText"] = reply_text
+        fields["responseFromOwnerDate"] = _dig_str(r, 4, 1)
+
+    return {k: v for k, v in fields.items() if v not in (None, [])}
+
+
+def parse_reviews_page(reviews: Any) -> list[dict[str, Any]]:
+    """Parse one BOQ page's raw review list into ReviewFields dicts."""
+    if not isinstance(reviews, list):
+        return []
+    out = []
+    for entry in reviews:
+        parsed = parse_review(entry)
+        if parsed:
+            out.append(parsed)
+    return out
+
+
+def strip_personal_data(review: dict[str, Any]) -> dict[str, Any]:
+    """Drop reviewer identity fields (Apify ``personalData=false``).
+
+    ``reviewId`` and the review content stay; name/id/url/photo are removed.
+    """
+    for key in ("name", "reviewerId", "reviewerUrl", "reviewerPhotoUrl"):
+        review.pop(key, None)
+    return review

surfsense_backend/app/proprietary/scrapers/google_maps/reviews.py

@@ -0,0 +1,210 @@
+"""Orchestrator for the Google Maps Reviews scraper (Apify-compatible).
+
+Distinct from the places scraper: one flat output item per review (the review
+fields merged with the parent place fields), per the Apify "Google Maps Reviews
+Scraper" output schema.
+
+Reviews come from Google's public ``GetLocalBoqProxy`` review feed (no login,
+~10 reviews per page, opaque continuation-token pagination). The place header
+fields stamped onto each item come from the same ``/maps/preview/place`` RPC the
+places scraper uses.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import AsyncIterator
+from datetime import datetime
+from typing import Any
+
+from .fetch import fetch_place_darray, iter_reviews_pages, now_iso, resolve_fid
+from .parsers import parse_place, parse_reviews_page, strip_personal_data
+from .schemas import GoogleMapsReviewsInput, ReviewItem
+from .url_resolver import ResolvedUrl, resolve_url
+
+logger = logging.getLogger(__name__)
+
+# Place header keys copied from a parsed place onto every ReviewItem.
+_PLACE_KEYS = (
+    "title",
+    "placeId",
+    "address",
+    "location",
+    "categories",
+    "categoryName",
+    "totalScore",
+    "reviewsCount",
+    "price",
+    "cid",
+    "fid",
+    "imageUrl",
+    "neighborhood",
+    "street",
+    "city",
+    "countryCode",
+    "postalCode",
+    "state",
+)
+
+
+def _parse_start_date(value: str | None) -> datetime | None:
+    """Parse ``reviewsStartDate`` (ISO date/datetime) into a UTC-naive cutoff."""
+    if not value:
+        return None
+    try:
+        return datetime.fromisoformat(value.replace("Z", "+00:00")).replace(tzinfo=None)
+    except ValueError:
+        logger.warning("[google_maps] bad reviewsStartDate: %r", value)
+        return None
+
+
+def _before_cutoff(review: dict[str, Any], cutoff: datetime) -> bool:
+    """True if a review predates the cutoff (newest-first stop condition)."""
+    iso = review.get("publishedAtDate")
+    if not iso:
+        return False
+    try:
+        when = datetime.fromisoformat(iso.replace("Z", "+00:00")).replace(tzinfo=None)
+    except ValueError:
+        return False
+    return when < cutoff
+
+
+def _keep(review: dict[str, Any], *, filter_string: str, origin: str) -> bool:
+    """Apply Apify's reviewsFilterString / reviewsOrigin filters client-side."""
+    if origin == "google" and (review.get("reviewOrigin") or "Google") != "Google":
+        return False
+    if filter_string:
+        text = (review.get("text") or "") + " " + (review.get("textTranslated") or "")
+        if filter_string.lower() not in text.lower():
+            return False
+    return True
+
+
+async def collect_place_reviews(
+    fid: str,
+    *,
+    max_reviews: int,
+    sort: str = "newest",
+    language: str = "en",
+    filter_string: str = "",
+    origin: str = "all",
+    personal_data: bool = True,
+    start_date: str | None = None,
+) -> tuple[list[dict[str, Any]], int | None]:
+    """Page a place's reviews via the BOQ feed.
+
+    Returns ``(reviews, exact_total_or_None)``. The feed does not expose the
+    place's total review count up front, so the total is only known when
+    pagination exhausts before ``max_reviews`` is hit (and no filters dropped
+    anything) — the common case for small/medium places.
+    """
+    cutoff = _parse_start_date(start_date)
+    reviews: list[dict[str, Any]] = []
+    seen = 0
+    exhausted = True
+    filtered = False
+
+    async for raw_page in iter_reviews_pages(fid, sort=sort, language=language):
+        stop = False
+        for review in parse_reviews_page(raw_page):
+            seen += 1
+            if cutoff and _before_cutoff(review, cutoff):
+                stop = True
+                break
+            if not _keep(review, filter_string=filter_string, origin=origin):
+                filtered = True
+                continue
+            if not personal_data:
+                strip_personal_data(review)
+            reviews.append(review)
+            if len(reviews) >= max_reviews:
+                stop = True
+                break
+        if stop:
+            exhausted = False
+            break
+
+    total = seen if exhausted and not filtered and cutoff is None else None
+    return reviews, total
+
+
+async def _reviews_for_place(
+    resolved: ResolvedUrl,
+    *,
+    input_model: GoogleMapsReviewsInput,
+    input_place_id: str | None,
+    input_start_url: str | None,
+) -> AsyncIterator[dict[str, Any]]:
+    """Page one place's reviews and yield a flat ReviewItem per review."""
+    fid = await resolve_fid(resolved)
+    if not fid:
+        logger.warning("[google_maps] reviews: no feature id for %s", resolved.url)
+        return
+
+    darray = await fetch_place_darray(fid, language=input_model.language)
+    place = parse_place(darray) if darray else {}
+    place_header = {k: place[k] for k in _PLACE_KEYS if k in place}
+    scraped_at = now_iso()
+
+    reviews, total = await collect_place_reviews(
+        place.get("fid", fid),
+        max_reviews=input_model.maxReviews,
+        sort=input_model.reviewsSort,
+        language=input_model.language,
+        origin=input_model.reviewsOrigin,
+        personal_data=input_model.personalData,
+        start_date=input_model.reviewsStartDate,
+    )
+    if total is not None and "reviewsCount" not in place_header:
+        place_header["reviewsCount"] = total
+
+    for review in reviews:
+        item = ReviewItem(**{**place_header, **review})
+        item.scrapedAt = scraped_at
+        item.language = input_model.language
+        item.inputPlaceId = input_place_id
+        item.inputStartUrl = input_start_url
+        item.url = place.get("url") or resolved.url
+        yield item.to_output()
+
+
+async def iter_reviews(
+    input_model: GoogleMapsReviewsInput,
+) -> AsyncIterator[dict[str, Any]]:
+    """Yield Apify-shaped review items for every startUrl and placeId."""
+    for entry in input_model.startUrls:
+        resolved = resolve_url(entry.url)
+        if not resolved:
+            logger.warning("Reviews: unrecognized Google Maps URL: %s", entry.url)
+            continue
+        async for item in _reviews_for_place(
+            resolved,
+            input_model=input_model,
+            input_place_id=None,
+            input_start_url=entry.url,
+        ):
+            yield item
+
+    for place_id in input_model.placeIds:
+        url = f"https://www.google.com/maps/place/?q=place_id:{place_id}"
+        resolved = ResolvedUrl("place", place_id, url)
+        async for item in _reviews_for_place(
+            resolved,
+            input_model=input_model,
+            input_place_id=place_id,
+            input_start_url=None,
+        ):
+            yield item
+
+
+async def scrape_reviews(
+    input_model: GoogleMapsReviewsInput, *, limit: int | None = None
+) -> list[dict[str, Any]]:
+    """Collect :func:`iter_reviews` into a list, honoring an optional guard."""
+    results: list[dict[str, Any]] = []
+    async for item in iter_reviews(input_model):
+        results.append(item)
+        if limit is not None and len(results) >= limit:
+            break
+    return results

surfsense_backend/app/proprietary/scrapers/google_maps/schemas.py

@@ -0,0 +1,360 @@
+# ruff: noqa: N815 - field names intentionally mirror the Apify camelCase spec
+"""Apify-compatible input/output models for the Google Maps scraper.
+
+The models mirror the public Apify "Google Maps Scraper" and "Google Maps
+Reviews Scraper" actor specs so the endpoints can be drop-ins. The skeleton
+accepts the full input surface; output fields the implementation does not
+source yet are emitted as ``None``/``[]``/``{}`` so parity is additive.
+
+Outputs use ``extra="allow"`` on purpose: it lets us grow the output shape
+without breaking existing consumers, exactly like the YouTube scraper models.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+SearchMatching = Literal["all", "only_includes", "only_exact"]
+PlaceMinimumStars = Literal[
+    "", "two", "twoAndHalf", "three", "threeAndHalf", "four", "fourAndHalf"
+]
+WebsiteFilter = Literal["allPlaces", "withWebsite", "withoutWebsite"]
+ReviewsSort = Literal["newest", "mostRelevant", "highestRanking", "lowestRanking"]
+ReviewsOrigin = Literal["all", "google"]
+AllPlacesAction = Literal["", "all_places_no_search_ocr", "all_places_no_search_mouse"]
+
+
+class StartUrl(BaseModel):
+    """A single direct URL entry (Apify passes ``{"url": ...}`` objects)."""
+
+    model_config = ConfigDict(extra="allow")
+
+    url: str
+
+
+class SocialMediaEnrichment(BaseModel):
+    """Per-platform toggles for the social media profile enrichment add-on."""
+
+    model_config = ConfigDict(extra="allow")
+
+    facebooks: bool = False
+    instagrams: bool = False
+    youtubes: bool = False
+    tiktoks: bool = False
+    twitters: bool = False
+
+
+class GoogleMapsScrapeInput(BaseModel):
+    """Full Apify "Google Maps Scraper" input surface.
+
+    Semantics follow Apify: ``maxCrawledPlacesPerSearch=None`` means "all
+    places"; add-on toggles default off; ``maxReviews``/``maxImages``/
+    ``maxQuestions`` of ``0`` mean "fetch none of that type".
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    # Discovery
+    searchStringsArray: list[str] = Field(default_factory=list)
+    locationQuery: str | None = None
+    startUrls: list[StartUrl] = Field(default_factory=list)
+    placeIds: list[str] = Field(default_factory=list)
+    allPlacesNoSearchAction: AllPlacesAction = ""
+
+    # Caps / language
+    maxCrawledPlacesPerSearch: int | None = Field(default=None, ge=1)
+    language: str = "en"
+
+    # Filters ($)
+    categoryFilterWords: list[str] = Field(default_factory=list)
+    searchMatching: SearchMatching = "all"
+    placeMinimumStars: PlaceMinimumStars = ""
+    website: WebsiteFilter = "allPlaces"
+    skipClosedPlaces: bool = False
+
+    # Place detail page ($) and its dependents
+    scrapePlaceDetailPage: bool = False
+    scrapeTableReservationProvider: bool = False
+    scrapeOrderOnline: bool = False
+    includeWebResults: bool = False
+    scrapeDirectories: bool = False
+    maxQuestions: int = Field(default=0, ge=0)
+
+    # Enrichment add-ons ($)
+    scrapeContacts: bool = False
+    scrapeSocialMediaProfiles: SocialMediaEnrichment = Field(
+        default_factory=SocialMediaEnrichment
+    )
+    maximumLeadsEnrichmentRecords: int = Field(default=0, ge=0)
+    leadsEnrichmentDepartments: list[str] = Field(default_factory=list)
+    verifyLeadsEnrichmentEmails: bool = False
+
+    # Reviews ($)
+    maxReviews: int = Field(default=0, ge=0)
+    reviewsStartDate: str | None = None
+    reviewsSort: ReviewsSort = "newest"
+    reviewsFilterString: str = ""
+    reviewsOrigin: ReviewsOrigin = "all"
+    scrapeReviewsPersonalData: bool = True
+
+    # Images ($)
+    maxImages: int = Field(default=0, ge=0)
+    scrapeImageAuthors: bool = False
+
+    # Competitor analysis add-on ($)
+    enableCompetitorAnalysis: bool = False
+    maxCompetitorsToAnalyze: int = Field(default=30, ge=0, le=100)
+
+    # Geolocation parameters (use either these or locationQuery, not both)
+    countryCode: str | None = None
+    city: str | None = None
+    state: str | None = None
+    county: str | None = None
+    postalCode: str | None = None
+    customGeolocation: dict[str, Any] | None = None
+
+
+class Location(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    lat: float | None = None
+    lng: float | None = None
+
+
+class ReviewsDistribution(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    oneStar: int | None = None
+    twoStar: int | None = None
+    threeStar: int | None = None
+    fourStar: int | None = None
+    fiveStar: int | None = None
+
+
+class OpeningHour(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    day: str | None = None
+    hours: str | None = None
+
+
+class PeopleAlsoSearchItem(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    category: str | None = None
+    title: str | None = None
+    reviewsCount: int | None = None
+    totalScore: float | None = None
+
+
+class ReviewsTag(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    title: str | None = None
+    count: int | None = None
+
+
+class ImageItem(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    imageUrl: str | None = None
+    authorName: str | None = None
+    authorUrl: str | None = None
+    uploadedAt: str | None = None
+
+
+class ReviewFields(BaseModel):
+    """Review-level fields shared by the nested place ``reviews[]`` items and
+    the standalone Reviews Scraper output items."""
+
+    model_config = ConfigDict(extra="allow")
+
+    name: str | None = None
+    text: str | None = None
+    textTranslated: str | None = None
+    publishAt: str | None = None
+    publishedAtDate: str | None = None
+    likesCount: int | None = None
+    reviewId: str | None = None
+    reviewUrl: str | None = None
+    reviewerId: str | None = None
+    reviewerUrl: str | None = None
+    reviewerPhotoUrl: str | None = None
+    reviewerNumberOfReviews: int | None = None
+    isLocalGuide: bool | None = None
+    reviewOrigin: str | None = None
+    stars: float | None = None
+    rating: str | None = None
+    responseFromOwnerDate: str | None = None
+    responseFromOwnerText: str | None = None
+    reviewImageUrls: list[str] = Field(default_factory=list)
+    reviewContext: dict[str, Any] = Field(default_factory=dict)
+    reviewDetailedRating: dict[str, Any] = Field(default_factory=dict)
+    visitedIn: str | None = None
+    originalLanguage: str | None = None
+    translatedLanguage: str | None = None
+
+
+class PlaceItem(BaseModel):
+    """Apify "Google Maps Scraper" output item (one per place).
+
+    Mirrors the actor's example JSON. Unsourced fields default to
+    ``None``/``[]``/``{}``; ``extra="allow"`` keeps the contract open.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    # Provenance
+    searchString: str | None = None
+    rank: int | None = None
+    searchPageUrl: str | None = None
+    searchPageLoadedUrl: str | None = None
+    isAdvertisement: bool = False
+
+    # Identity
+    title: str | None = None
+    subTitle: str | None = None
+    description: str | None = None
+    price: str | None = None
+    categoryName: str | None = None
+    categories: list[str] = Field(default_factory=list)
+    placeId: str | None = None
+    fid: str | None = None
+    cid: str | None = None
+    kgmid: str | None = None
+
+    # Address / location
+    address: str | None = None
+    neighborhood: str | None = None
+    street: str | None = None
+    city: str | None = None
+    postalCode: str | None = None
+    state: str | None = None
+    countryCode: str | None = None
+    location: Location | None = None
+    plusCode: str | None = None
+    locatedIn: str | None = None
+    parentPlaceUrl: str | None = None
+
+    # Contact
+    website: str | None = None
+    phone: str | None = None
+    phoneUnformatted: str | None = None
+    menu: str | None = None
+    servicesLink: str | None = None
+    claimThisBusiness: bool | None = None
+
+    # Ratings / status
+    totalScore: float | None = None
+    reviewsCount: int | None = None
+    reviewsDistribution: ReviewsDistribution | None = None
+    permanentlyClosed: bool = False
+    temporarilyClosed: bool = False
+
+    # Images
+    imageUrl: str | None = None
+    imagesCount: int | None = None
+    imageCategories: list[str] = Field(default_factory=list)
+    images: list[ImageItem] = Field(default_factory=list)
+    imageUrls: list[str] = Field(default_factory=list)
+
+    # Detail-page fields (populated only when scrapePlaceDetailPage)
+    openingHours: list[OpeningHour] = Field(default_factory=list)
+    peopleAlsoSearch: list[PeopleAlsoSearchItem] = Field(default_factory=list)
+    placesTags: list[Any] = Field(default_factory=list)
+    reviewsTags: list[ReviewsTag] = Field(default_factory=list)
+    additionalInfo: dict[str, Any] | None = None
+    questionsAndAnswers: list[Any] = Field(default_factory=list)
+    updatesFromCustomers: Any | None = None
+    ownerUpdates: list[Any] = Field(default_factory=list)
+    webResults: list[Any] = Field(default_factory=list)
+    tableReservationLinks: list[Any] = Field(default_factory=list)
+    bookingLinks: list[Any] = Field(default_factory=list)
+    reserveTableUrl: str | None = None
+    googleFoodUrl: str | None = None
+    gasPrices: list[Any] = Field(default_factory=list)
+    restaurantData: dict[str, Any] = Field(default_factory=dict)
+    userPlaceNote: str | None = None
+
+    # Hotel fields
+    hotelStars: str | None = None
+    hotelDescription: str | None = None
+    checkInDate: str | None = None
+    checkOutDate: str | None = None
+    similarHotelsNearby: Any | None = None
+    hotelReviewSummary: Any | None = None
+    hotelAds: list[Any] = Field(default_factory=list)
+
+    # Reviews (populated only when maxReviews > 0)
+    reviews: list[ReviewFields] = Field(default_factory=list)
+
+    # Meta
+    url: str | None = None
+    scrapedAt: str | None = None
+
+    def to_output(self) -> dict[str, Any]:
+        """Serialize to the flat dict shape Apify emits (keeps extras)."""
+        return self.model_dump(exclude_none=False)
+
+
+class GoogleMapsReviewsInput(BaseModel):
+    """Apify "Google Maps Reviews Scraper" input surface."""
+
+    model_config = ConfigDict(extra="allow")
+
+    startUrls: list[StartUrl] = Field(default_factory=list)
+    placeIds: list[str] = Field(default_factory=list)
+    maxReviews: int = Field(default=10_000_000, ge=1)
+    reviewsSort: ReviewsSort = "newest"
+    reviewsStartDate: str | None = None
+    language: str = "en"
+    reviewsOrigin: ReviewsOrigin = "all"
+    personalData: bool = True
+
+
+class ReviewItem(ReviewFields):
+    """Apify "Google Maps Reviews Scraper" output item.
+
+    One flat item per review: the review fields (inherited) merged with the
+    place fields, per the actor's output schema.
+    """
+
+    # Place
+    title: str | None = None
+    placeId: str | None = None
+    address: str | None = None
+    location: Location | None = None
+    categories: list[str] = Field(default_factory=list)
+    isAdvertisement: bool = False
+    categoryName: str | None = None
+    totalScore: float | None = None
+    permanentlyClosed: bool = False
+    temporarilyClosed: bool = False
+    reviewsCount: int | None = None
+    url: str | None = None
+    price: str | None = None
+    cid: str | None = None
+    fid: str | None = None
+    imageUrl: str | None = None
+    hotelStars: str | None = None
+    kgmid: str | None = None
+    neighborhood: str | None = None
+    street: str | None = None
+    city: str | None = None
+    countryCode: str | None = None
+    postalCode: str | None = None
+    state: str | None = None
+
+    # Provenance / meta
+    scrapedAt: str | None = None
+    searchPageUrl: str | None = None
+    searchString: str | None = None
+    inputPlaceId: str | None = None
+    inputStartUrl: str | None = None
+    language: str | None = None
+    rank: int | None = None
+
+    def to_output(self) -> dict[str, Any]:
+        return self.model_dump(exclude_none=False)

surfsense_backend/app/proprietary/scrapers/google_maps/scraper.py

@@ -0,0 +1,463 @@
+"""Orchestrator for the Google Maps places scraper (Apify-compatible).
+
+Skeleton mirroring the YouTube scraper layout: the core is the async generator
+:func:`iter_places` (unbounded), :func:`scrape_places` is a thin collector with
+a caller-supplied ``limit`` guard. Discovery inputs dispatch to per-kind flows
+(search / place URL / place ID) which are currently no-ops — each will be
+implemented progressively, exactly like the YouTube flows were.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import AsyncIterator
+from typing import Any
+from urllib.parse import quote
+
+from .fetch import (
+    SignInRequiredError,
+    fetch_place_darray,
+    gather_bounded,
+    iter_search_pages,
+    now_iso,
+    resolve_fid,
+)
+from .parsers import parse_place
+from .schemas import GoogleMapsScrapeInput, PlaceItem
+from .url_resolver import ResolvedUrl, resolve_url
+
+logger = logging.getLogger(__name__)
+
+# Re-exported so callers/routes can keep importing it from the orchestrator.
+__all__ = ["SignInRequiredError", "iter_places", "scrape_places"]
+
+# Max concurrent per-place detail/review fetches. Each is a ~2s proxy
+# round-trip, so overlapping them is what turns a 20-place enriched search from
+# ~50s into a handful of seconds. ``ponytail:`` a fixed ceiling — high enough to
+# hide latency, low enough not to trip Google/proxy rate limits; make it
+# configurable if a faster proxy pool ever wants more.
+_DETAIL_CONCURRENCY = 8
+
+_SEARCH_PAGE_SIZE = 20
+
+
+def _prefetch_for(cap: int | None) -> int:
+    """How many search pages to fetch per wave, from the result cap.
+
+    One page holds ~20 results, so a cap of 60 wants ~3 pages overlapped; an
+    uncapped scan overlaps a small fixed wave. Capped at 5 to bound wasted
+    fetches when dedupe or an early empty page cuts things short.
+    """
+    if cap is None:
+        return 5
+    return max(1, min(5, (cap + _SEARCH_PAGE_SIZE - 1) // _SEARCH_PAGE_SIZE))
+
+
+# Apify's placeMinimumStars options -> numeric cutoff.
+_MIN_STARS = {
+    "two": 2.0,
+    "twoAndHalf": 2.5,
+    "three": 3.0,
+    "threeAndHalf": 3.5,
+    "four": 4.0,
+    "fourAndHalf": 4.5,
+}
+
+
+def _location_text(input_model: GoogleMapsScrapeInput) -> str | None:
+    """The location to scope searches to (Apify appends it to the query)."""
+    if input_model.locationQuery:
+        return input_model.locationQuery
+    parts = [
+        input_model.city,
+        input_model.county,
+        input_model.state,
+        input_model.postalCode,
+        input_model.countryCode,
+    ]
+    joined = ", ".join(p for p in parts if p)
+    return joined or None
+
+
+def _custom_point(
+    input_model: GoogleMapsScrapeInput,
+) -> tuple[float | None, float | None, float | None]:
+    """(lat, lng, radius_m) from a GeoJSON-ish customGeolocation Point."""
+    geo = input_model.customGeolocation
+    if not isinstance(geo, dict) or geo.get("type") != "Point":
+        return None, None, None
+    coords = geo.get("coordinates")
+    if not isinstance(coords, list | tuple) or len(coords) < 2:
+        return None, None, None
+    lng, lat = coords[0], coords[1]
+    radius_km = geo.get("radiusKm") or geo.get("radius") or 10
+    return float(lat), float(lng), float(radius_km) * 1000
+
+
+def _passes_filters(
+    fields: dict[str, Any], query: str, input_model: GoogleMapsScrapeInput
+) -> bool:
+    """Apply Apify's search result filters to parsed place fields."""
+    title = (fields.get("title") or "").lower()
+    q = query.lower()
+    if input_model.searchMatching == "only_exact" and title != q:
+        return False
+    if input_model.searchMatching == "only_includes" and q not in title:
+        return False
+
+    if input_model.categoryFilterWords:
+        cats = " ".join(fields.get("categories") or []).lower()
+        if not any(w.lower() in cats for w in input_model.categoryFilterWords):
+            return False
+
+    min_stars = _MIN_STARS.get(input_model.placeMinimumStars)
+    if min_stars is not None and (fields.get("totalScore") or 0) < min_stars:
+        return False
+
+    if input_model.website == "withWebsite" and not fields.get("website"):
+        return False
+    if input_model.website == "withoutWebsite" and fields.get("website"):
+        return False
+
+    return not (
+        input_model.skipClosedPlaces
+        and (fields.get("permanentlyClosed") or fields.get("temporarilyClosed"))
+    )
+
+
+def _apply_image_cap(
+    fields: dict[str, Any], input_model: GoogleMapsScrapeInput
+) -> None:
+    """Apify semantics: gallery ``imageUrls`` only when ``maxImages > 0``."""
+    if not input_model.maxImages:
+        fields.pop("imageUrls", None)
+    elif "imageUrls" in fields:
+        fields["imageUrls"] = fields["imageUrls"][: input_model.maxImages]
+
+
+async def _enrich_from_detail(
+    fields: dict[str, Any], input_model: GoogleMapsScrapeInput
+) -> dict[str, Any]:
+    """Merge the full detail-RPC payload over search-result fields.
+
+    Search darrays are served without the session-gated extras (reviewsCount,
+    distribution, popular times, galleries, tags, full about sections); the
+    detail RPC with an NID cookie has them all. Detail values win on overlap.
+    """
+    fid = fields.get("fid")
+    if not fid:
+        return fields
+    darray = await fetch_place_darray(fid, language=input_model.language)
+    if not darray:
+        return fields
+    return {**fields, **parse_place(darray)}
+
+
+async def _build_items(
+    candidates: list[tuple[dict[str, Any], str, int]],
+    input_model: GoogleMapsScrapeInput,
+    *,
+    search_string: str,
+    search_page_url: str | None,
+    enrich: bool,
+) -> list[dict[str, Any]]:
+    """Turn dedup'd/filtered place fields into output items, in parallel.
+
+    Each candidate is ``(fields, fid, rank)``. Detail enrichment and inline
+    reviews are per-place round-trips; running the whole batch concurrently
+    (bounded, order preserved) is the main speedup over the old one-at-a-time
+    loop. Pure-CPU when neither enrichment nor reviews are requested.
+    """
+
+    async def _build(fields: dict[str, Any], fid: str, rank: int) -> dict[str, Any]:
+        if enrich:
+            fields = await _enrich_from_detail(fields, input_model)
+        _apply_image_cap(fields, input_model)
+        item = PlaceItem(**fields)
+        item.searchString = search_string
+        if search_page_url:
+            item.searchPageUrl = search_page_url
+        item.rank = rank
+        item.url = (
+            f"https://www.google.com/maps/place/?q=place_id:{fields.get('placeId')}"
+        )
+        item.scrapedAt = now_iso()
+        if input_model.maxReviews:
+            await _attach_reviews(item, fid, input_model)
+        return item.to_output()
+
+    return await gather_bounded(
+        [lambda c=c: _build(*c) for c in candidates],
+        concurrency=_DETAIL_CONCURRENCY,
+    )
+
+
+async def _search_flow(
+    query: str, *, input_model: GoogleMapsScrapeInput
+) -> AsyncIterator[dict[str, Any]]:
+    """Search-term discovery via the ``search?tbm=map`` RPC.
+
+    Pages offset-based results (~20/page), dedupes by fid (Google reshuffles
+    between pages), applies the Apify filters, and emits full place items —
+    each search hit already carries a place darray, so no per-place request is
+    needed for the core fields. When ``scrapePlaceDetailPage`` or ``maxImages``
+    is set, one detail RPC per place adds the session-gated extras; those are
+    fetched concurrently across the page rather than one at a time.
+    """
+    location = _location_text(input_model)
+    search_query = f"{query} in {location}" if location else query
+    lat, lng, radius_m = _custom_point(input_model)
+    cap = input_model.maxCrawledPlacesPerSearch
+    enrich = bool(input_model.scrapePlaceDetailPage or input_model.maxImages)
+    search_page_url = (
+        f"https://www.google.com/maps/search/{quote(search_query, safe='')}"
+    )
+
+    seen: set[str] = set()
+    emitted = 0
+    async for darrays in iter_search_pages(
+        search_query,
+        language=input_model.language,
+        lat=lat,
+        lng=lng,
+        radius_m=radius_m,
+        prefetch=_prefetch_for(cap),
+    ):
+        candidates: list[tuple[dict[str, Any], str, int]] = []
+        new_on_page = 0
+        for darray in darrays:
+            fields = parse_place(darray)
+            fid = fields.get("fid")
+            if not fid or fid in seen:
+                continue
+            seen.add(fid)
+            new_on_page += 1
+            if _passes_filters(fields, query, input_model):
+                candidates.append((fields, fid, len(seen)))
+        if cap is not None:
+            candidates = candidates[: max(0, cap - emitted)]
+        for out in await _build_items(
+            candidates,
+            input_model,
+            search_string=query,
+            search_page_url=search_page_url,
+            enrich=enrich,
+        ):
+            yield out
+            emitted += 1
+        if cap is not None and emitted >= cap:
+            return
+        if not new_on_page:  # page was all repeats -> feed is cycling, stop
+            return
+
+
+# Broad category sweep for allPlacesNoSearchAction. Apify's implementation
+# OCRs / mouse-overs the rendered map pins (hence the enum names); the public
+# search RPC has no "list everything" query (verified: '*' and '' return
+# nothing). ponytail: approximate the scan with broad category searches over
+# the same viewport, deduped by fid — covers the vast majority of pins; a true
+# pin-complete scan would need browser rendering + tile OCR.
+_ALL_PLACES_SWEEP = [
+    "restaurant",
+    "cafe",
+    "bar",
+    "store",
+    "shopping",
+    "hotel",
+    "tourist attraction",
+    "park",
+    "gym",
+    "salon",
+    "bank",
+    "gas station",
+    "pharmacy",
+    "doctor",
+    "school",
+    "church",
+    "services",
+]
+
+
+async def _all_places_flow(
+    input_model: GoogleMapsScrapeInput,
+) -> AsyncIterator[dict[str, Any]]:
+    """Area scan without a search term: sweep broad categories, dedupe by fid.
+
+    Emits the same items the search flow would; ``searchString`` carries the
+    action value so callers can tell scan hits from query hits.
+    """
+    location = _location_text(input_model)
+    lat, lng, radius_m = _custom_point(input_model)
+    cap = input_model.maxCrawledPlacesPerSearch
+    enrich = bool(input_model.scrapePlaceDetailPage or input_model.maxImages)
+    seen: set[str] = set()
+    emitted = 0
+    for term in _ALL_PLACES_SWEEP:
+        search_query = f"{term} in {location}" if location else term
+        async for darrays in iter_search_pages(
+            search_query,
+            language=input_model.language,
+            lat=lat,
+            lng=lng,
+            radius_m=radius_m,
+            prefetch=_prefetch_for(cap),
+        ):
+            candidates: list[tuple[dict[str, Any], str, int]] = []
+            new_on_page = 0
+            for darray in darrays:
+                fields = parse_place(darray)
+                fid = fields.get("fid")
+                if not fid or fid in seen:
+                    continue
+                seen.add(fid)
+                new_on_page += 1
+                if _passes_filters(fields, "", input_model):
+                    candidates.append((fields, fid, len(seen)))
+            if cap is not None:
+                candidates = candidates[: max(0, cap - emitted)]
+            for out in await _build_items(
+                candidates,
+                input_model,
+                search_string=input_model.allPlacesNoSearchAction,
+                search_page_url=None,
+                enrich=enrich,
+            ):
+                yield out
+                emitted += 1
+            if cap is not None and emitted >= cap:
+                return
+            if not new_on_page:
+                break  # this term is exhausted; move to the next one
+
+
+async def _place_flow(
+    resolved: ResolvedUrl, *, input_model: GoogleMapsScrapeInput
+) -> AsyncIterator[dict[str, Any]]:
+    """Single place from a direct Maps place URL / place ID.
+
+    Resolves the feature ID, fetches the place detail via the ``/maps/preview/
+    place`` RPC (full payload — see ``_PLACE_DETAIL_PB``), and maps the fields
+    into a ``PlaceItem``. When ``maxReviews > 0`` the place's reviews are
+    attached inline (Apify puts them on ``reviews[]``).
+    """
+    fid = await resolve_fid(resolved)
+    if not fid:
+        logger.warning("[google_maps] could not resolve feature id: %s", resolved.url)
+        return
+    darray = await fetch_place_darray(fid, language=input_model.language)
+    if not darray:
+        logger.warning(
+            "[google_maps] no place data (structure may have shifted): %s", resolved.url
+        )
+        return
+    fields = parse_place(darray)
+    _apply_image_cap(fields, input_model)
+    item = PlaceItem(**fields)
+    item.url = resolved.url
+    item.searchString = f"Direct URL: {resolved.url}"
+    item.scrapedAt = now_iso()
+    if input_model.maxReviews and fields.get("fid"):
+        await _attach_reviews(item, fields["fid"], input_model)
+    yield item.to_output()
+
+
+async def _attach_reviews(
+    item: PlaceItem, fid: str, input_model: GoogleMapsScrapeInput
+) -> None:
+    """Populate ``item.reviews`` (and total count when knowable) from the feed."""
+    from .reviews import collect_place_reviews
+
+    reviews, total = await collect_place_reviews(
+        fid,
+        max_reviews=input_model.maxReviews,
+        sort=input_model.reviewsSort,
+        language=input_model.language,
+        filter_string=input_model.reviewsFilterString,
+        origin=input_model.reviewsOrigin,
+        personal_data=input_model.scrapeReviewsPersonalData,
+        start_date=input_model.reviewsStartDate,
+    )
+    item.reviews = reviews
+    if total is not None and item.reviewsCount is None:
+        item.reviewsCount = total
+
+
+async def _place_id_flow(
+    place_id: str, *, input_model: GoogleMapsScrapeInput
+) -> AsyncIterator[dict[str, Any]]:
+    """Single place from a bare place ID (format ``ChIJ...``).
+
+    Maps resolves ``?q=place_id:<id>`` to the place page, so we build that URL
+    and reuse the place flow.
+    """
+    url = f"https://www.google.com/maps/place/?q=place_id:{place_id}"
+    resolved = ResolvedUrl("place", place_id, url)
+    async for item in _place_flow(resolved, input_model=input_model):
+        yield item
+
+
+async def _dispatch(
+    resolved: ResolvedUrl, input_model: GoogleMapsScrapeInput
+) -> AsyncIterator[dict[str, Any]]:
+    if resolved.kind == "search":
+        async for item in _search_flow(resolved.value, input_model=input_model):
+            yield item
+    else:  # place / cid / shortlink / reviews all resolve to a place page
+        async for item in _place_flow(resolved, input_model=input_model):
+            yield item
+
+
+async def iter_places(
+    input_model: GoogleMapsScrapeInput,
+) -> AsyncIterator[dict[str, Any]]:
+    """Yield Apify-shaped place items from all discovery inputs.
+
+    Apify runs searches, startUrls, and placeIds side by side (they are
+    additive, unlike the YouTube scraper where startUrls override queries).
+    """
+    for entry in input_model.startUrls:
+        resolved = resolve_url(entry.url)
+        if not resolved:
+            logger.warning("Unrecognized Google Maps URL: %s", entry.url)
+            continue
+        async for item in _dispatch(resolved, input_model):
+            yield item
+
+    # placeIds are independent single-place detail fetches (~2s each); run the
+    # batch concurrently, bounded, results in input order — a bulk list of IDs
+    # is the common case and was previously fully sequential.
+    if input_model.placeIds:
+
+        async def _collect(pid: str) -> list[dict[str, Any]]:
+            return [it async for it in _place_id_flow(pid, input_model=input_model)]
+
+        for items in await gather_bounded(
+            [lambda p=p: _collect(p) for p in input_model.placeIds],
+            concurrency=_DETAIL_CONCURRENCY,
+        ):
+            for item in items:
+                yield item
+
+    for query in input_model.searchStringsArray:
+        async for item in _search_flow(query, input_model=input_model):
+            yield item
+
+    if input_model.allPlacesNoSearchAction:
+        async for item in _all_places_flow(input_model):
+            yield item
+
+
+async def scrape_places(
+    input_model: GoogleMapsScrapeInput, *, limit: int | None = None
+) -> list[dict[str, Any]]:
+    """Collect :func:`iter_places` into a list, honoring an optional ``limit``.
+
+    ``limit`` is a request-time policy guard (used by the route), NOT a ceiling
+    in the streaming core.
+    """
+    results: list[dict[str, Any]] = []
+    async for item in iter_places(input_model):
+        results.append(item)
+        if limit is not None and len(results) >= limit:
+            break
+    return results

surfsense_backend/app/proprietary/scrapers/google_maps/url_resolver.py

@@ -0,0 +1,74 @@
+"""Classify a Google Maps URL and extract its identifier.
+
+Covers the ``startUrls`` shapes the Apify specs accept: place pages
+(``/maps/place``), search pages (``/maps/search``), review pages
+(``/maps/reviews``), CID links (``google.com/maps?cid=***``), and short links
+(``goo.gl/maps`` / ``maps.app.goo.gl``, which need a network redirect to
+resolve — classified here, resolved later in the fetch layer).
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Literal
+from urllib.parse import parse_qs, unquote, urlparse
+
+ResolvedKind = Literal["place", "search", "reviews", "cid", "shortlink"]
+
+_MAPS_HOSTS = ("google.", "www.google.")
+_SHORTLINK_HOSTS = ("goo.gl", "maps.app.goo.gl")
+
+# Feature ID (a.k.a. "fid" / data_id): two hex halves, e.g.
+# 0x89c3ca9c11f90c25:0x6cc8dba851799f09. It appears in the URL ``data=`` blob as
+# ``!1s<hex:hex>`` and is the key that links a place to its detail/review RPCs.
+_FID_RE = re.compile(r"(0x[0-9a-f]+:0x[0-9a-f]+)")
+
+
+@dataclass(frozen=True)
+class ResolvedUrl:
+    kind: ResolvedKind
+    value: str  # place slug, search query, cid, or the short URL itself
+    url: str
+    fid: str | None = None  # feature id (hex:hex) when present in the URL
+
+
+def extract_fid(url: str) -> str | None:
+    """Pull the feature ID (``0x..:0x..``) out of a Google Maps URL, if present."""
+    match = _FID_RE.search(url)
+    return match.group(1) if match else None
+
+
+def resolve_url(url: str) -> ResolvedUrl | None:
+    """Classify a Google Maps URL into a scrape job, or ``None`` if unrecognized."""
+    parsed = urlparse(url)
+    hostname = parsed.hostname or ""
+    path = parsed.path or ""
+
+    if hostname in _SHORTLINK_HOSTS or hostname.endswith(".goo.gl"):
+        return ResolvedUrl("shortlink", url, url)
+
+    if "google." not in hostname:
+        return None
+
+    # google.com/maps?cid=123... (uncommon but supported by both actors)
+    cid = parse_qs(parsed.query).get("cid", [None])[0]
+    if cid:
+        return ResolvedUrl("cid", cid, url)
+
+    for prefix, kind in (
+        ("/maps/place/", "place"),
+        ("/maps/search/", "search"),
+        ("/maps/reviews/", "reviews"),
+    ):
+        if path.startswith(prefix):
+            # Maps slugs encode spaces as "+" (e.g. /maps/place/Kim's+Island).
+            value = unquote(path[len(prefix) :].split("/")[0].replace("+", " "))
+            return ResolvedUrl(kind, value, url, extract_fid(url))  # type: ignore[arg-type]
+
+    # Bare /maps/search or /maps/reviews carrying data in the query/data blob.
+    for suffix, kind in (("/maps/search", "search"), ("/maps/reviews", "reviews")):
+        if path.rstrip("/") == suffix:
+            return ResolvedUrl(kind, url, url)  # type: ignore[arg-type]
+
+    return None

surfsense_backend/app/proprietary/scrapers/youtube/comments.py

@@ -27,8 +27,8 @@ from .parsers import (
     parse_count,
     parse_video_page,
 )
-from .scraper import _post, _published_date, fan_out
 from .schemas import CommentItem, YouTubeCommentsInput
+from .scraper import _post, _published_date, fan_out
 from .url_resolver import resolve_url
 
 logger = logging.getLogger(__name__)
@@ -39,10 +39,14 @@ _SORT_LABELS = {"TOP_COMMENTS": "Top", "NEWEST_FIRST": "Newest"}
 
 
 async def _post_next(token: str) -> dict[str, Any] | None:
-    return await _post(INNERTUBE_NEXT_URL, build_innertube_payload(continuation_token=token))
+    return await _post(
+        INNERTUBE_NEXT_URL, build_innertube_payload(continuation_token=token)
+    )
 
 
-def _finalize(partial: dict[str, Any], base: dict[str, Any], reply_to: str | None) -> dict:
+def _finalize(
+    partial: dict[str, Any], base: dict[str, Any], reply_to: str | None
+) -> dict:
     return CommentItem(**{**base, **partial, "replyToCid": reply_to}).to_output()
 
 
@@ -132,7 +136,7 @@ async def _comments_for_video(
                 for cid, tok in pending
             )
         )
-        replies_by_cid = {cid: r for (cid, _), r in zip(pending, fetched)}
+        replies_by_cid = {cid: r for (cid, _), r in zip(pending, fetched, strict=False)}
 
         for entity in entities:
             if count >= limit:

surfsense_backend/app/proprietary/scrapers/youtube/innertube.py

@@ -18,7 +18,7 @@ from __future__ import annotations
 import asyncio
 import logging
 import time
-from contextlib import asynccontextmanager
+from contextlib import asynccontextmanager, suppress
 from contextvars import ContextVar
 from typing import Any
 
@@ -78,10 +78,8 @@ class _RotatingSession:
 
     async def close(self) -> None:
         if self._cm is not None:
-            try:
+            with suppress(Exception):  # best-effort teardown
                 await self._cm.__aexit__(None, None, None)
-            except Exception:  # best-effort teardown
-                pass
         self._cm = self.session = None
 
     async def rotate(self) -> Any | None:
@@ -124,6 +122,7 @@ async def proxy_session():
     finally:
         await holder.close()
 
+
 # Consent cookies to dodge the EU consent interstitial that otherwise returns a
 # page with no ``ytInitialData``. Mirrors app/routes/youtube_routes.py.
 CONSENT_COOKIES = {
@@ -224,7 +223,9 @@ async def post_innertube(
             if page.status == 200:
                 return page.json()
             logger.warning("InnerTube POST %s returned %s", base_url, page.status)
-            if not (holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS):
+            if not (
+                holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS
+            ):
                 return None
         except Exception as e:
             logger.warning("InnerTube POST %s failed: %s", base_url, e)
@@ -270,7 +271,9 @@ async def fetch_html(url: str, *, cookies: dict[str, str] | None = None) -> str
             if page.status == 200:
                 return page.html_content
             logger.warning("HTML GET %s returned %s", url, page.status)
-            if not (holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS):
+            if not (
+                holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS
+            ):
                 break
         except Exception as e:
             logger.warning("HTML GET %s failed: %s", url, e)

surfsense_backend/app/proprietary/scrapers/youtube/parsers.py

@@ -86,7 +86,7 @@ def parse_count(value: Any) -> int | None:
     """
     if value is None:
         return None
-    if isinstance(value, (int, float)):
+    if isinstance(value, int | float):
         return int(value)
     if not isinstance(value, str):
         return None
@@ -267,7 +267,10 @@ def _comments_turned_off(initial: dict) -> bool | None:
 def _is_members_only(player: dict, initial: dict) -> bool:
     """True for members-only videos (a members badge or a members-only offer)."""
     for badge in find_all(initial, "metadataBadgeRenderer"):
-        if isinstance(badge, dict) and badge.get("style") == "BADGE_STYLE_TYPE_MEMBERS_ONLY":
+        if (
+            isinstance(badge, dict)
+            and badge.get("style") == "BADGE_STYLE_TYPE_MEMBERS_ONLY"
+        ):
             return True
     status = player.get("playabilityStatus") or {}
     return find_first(status, "offerId") == "sponsors_only_video"
@@ -421,7 +424,9 @@ def parse_translation(next_data: dict) -> tuple[str | None, str | None]:
     language. Videos without a localization return their original text.
     """
     vpir = find_first(next_data, "videoPrimaryInfoRenderer") or {}
-    title = "".join(r.get("text", "") for r in (dig(vpir, "title", "runs") or [])) or None
+    title = (
+        "".join(r.get("text", "") for r in (dig(vpir, "title", "runs") or [])) or None
+    )
 
     vsir = find_first(next_data, "videoSecondaryInfoRenderer") or {}
     description = dig(vsir, "attributedDescription", "content")
@@ -440,7 +445,9 @@ def parse_collaborators(initial: dict) -> list[dict[str, str | None]] | None:
     ``attributedTitle`` whose tap opens a dialog listing each credited channel
     (``listItemViewModel``). Returns ``None`` for ordinary single-owner videos.
     """
-    owner = dig(find_first(initial, "videoSecondaryInfoRenderer"), "owner", "videoOwnerRenderer")
+    owner = dig(
+        find_first(initial, "videoSecondaryInfoRenderer"), "owner", "videoOwnerRenderer"
+    )
     attributed = owner.get("attributedTitle") if isinstance(owner, dict) else None
     if not isinstance(attributed, dict):
         return None
@@ -460,8 +467,12 @@ def parse_collaborators(initial: dict) -> list[dict[str, str | None]] | None:
         collaborators.append(
             {
                 "name": name,
-                "username": base[2:] if isinstance(base, str) and base.startswith("/@") else None,
+                "username": base[2:]
-                "url": f"https://www.youtube.com{base}" if isinstance(base, str) and base else None,
+                if isinstance(base, str) and base.startswith("/@")
+                else None,
+                "url": f"https://www.youtube.com{base}"
+                if isinstance(base, str) and base
+                else None,
             }
         )
     return collaborators or None
@@ -537,9 +548,10 @@ def parse_video_page(html: str) -> dict[str, Any] | None:
 
         # Verified badge on the channel owner.
         badges = find_all(initial, "metadataBadgeRenderer")
-        result["isChannelVerified"] = any(
+        result["isChannelVerified"] = (
-            isinstance(b, dict) and b.get("tooltip") == "Verified" for b in badges
+            any(isinstance(b, dict) and b.get("tooltip") == "Verified" for b in badges)
-        ) or None
+            or None
+        )
 
     return result
 
@@ -632,7 +644,9 @@ def parse_channel_videos(data: dict) -> tuple[list[dict[str, Any]], str | None]:
                 "publishedTimeText": dig(parts, 1, "text", "content"),
                 "duration": _lockup_duration(lockup),
                 "thumbnailUrl": _best_thumbnail(
-                    dig(lockup, "contentImage", "thumbnailViewModel", "image", "sources")
+                    dig(
+                        lockup, "contentImage", "thumbnailViewModel", "image", "sources"
+                    )
                 ),
             }
         )
@@ -673,9 +687,7 @@ def parse_channel_shorts(data: dict) -> tuple[list[dict[str, Any]], str | None]:
 
 
 def _lockup_duration(lockup: dict) -> str | None:
-    overlays = dig(
+    overlays = dig(lockup, "contentImage", "thumbnailViewModel", "overlays") or []
-        lockup, "contentImage", "thumbnailViewModel", "overlays"
-    ) or []
     for overlay in overlays:
         text = dig(
             overlay,

surfsense_backend/app/proprietary/scrapers/youtube/scraper.py

@@ -100,9 +100,7 @@ async def fan_out(
             if holder is not None:
                 await holder.close()
 
-    tasks = [
+    tasks = [asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))]
-        asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))
-    ]
     try:
         for _ in range(len(jobs)):
             for item in await results.get():
@@ -421,7 +419,7 @@ async def _playlist_flow(
     # is the bottleneck, so fan them out (each carries its playlist position in
     # ``order``; fan_out emits as they finish, not in playlist order).
     # ponytail: nested fan_out — when many playlist URLs run at once this can
-    # stack pools (outer × inner) of proxy sessions. Fine for the common
+    # stack pools (outer x inner) of proxy sessions. Fine for the common
     # single/few-playlist case; cap inner concurrency if bulk-playlist runs trip it.
     jobs = [
         _video_flow(

surfsense_backend/app/proprietary/scrapers/youtube/subtitles.py

@@ -56,7 +56,9 @@ def _select_transcript(transcript_list, language: str, prefer_generated: bool):
     return transcript_list.find_transcript(codes)
 
 
-def _fetch_subtitles_sync(video_id: str, language: str, fmt: str, prefer_generated: bool):
+def _fetch_subtitles_sync(
+    video_id: str, language: str, fmt: str, prefer_generated: bool
+):
     api = _build_client()
     transcript_list = api.list(video_id)
     transcript = _select_transcript(transcript_list, language, prefer_generated)

surfsense_backend/app/proprietary/web_crawler/captcha.py

@@ -23,6 +23,7 @@ balance can get the IP **temporarily banned**, so ``ErrNoBalance`` /
 won't change without a restart). ``reset_solver_latch()`` clears it for tests.
 """
 
+import contextlib
 import logging
 import re
 from collections.abc import Callable
@@ -107,8 +108,10 @@ def detect_challenge(page: Any, cfg: CaptchaConfig) -> tuple[str, str] | None:
                     m = _RENDER_SITEKEY.search(src)
                     if m and m.group(1) != "explicit":
                         return "v3", m.group(1)
-    except Exception as exc:  # noqa: BLE001
+    except Exception as exc:
-        logger.debug("%s detection error (treated as no challenge): %s", _CAPTCHA_LOG, exc)
+        logger.debug(
+            "%s detection error (treated as no challenge): %s", _CAPTCHA_LOG, exc
+        )
     return None
 
 
@@ -131,7 +134,7 @@ def proxy_url_to_captchatools(proxy_url: str | None) -> str | None:
         if p.username and p.password:
             return f"{p.hostname}:{p.port}:{p.username}:{p.password}"
         return f"{p.hostname}:{p.port}"
-    except Exception:  # noqa: BLE001
+    except Exception:
         return None
 
 
@@ -220,11 +223,9 @@ _INJECT_JS = r"""
 def _inject_and_submit(page: Any, challenge_type: str, token: str) -> None:
     try:
         page.evaluate(_INJECT_JS, {"token": token, "ctype": challenge_type})
-        try:
+        with contextlib.suppress(Exception):
             page.wait_for_load_state("networkidle", timeout=15000)
-        except Exception:  # noqa: BLE001
+    except Exception as exc:
-            pass
-    except Exception as exc:  # noqa: BLE001
         logger.warning("%s injection error: %s", _CAPTCHA_LOG, exc)
 
 
@@ -264,7 +265,7 @@ def build_captcha_page_action(
         page_url = getattr(page, "url", "") or ""
         try:
             user_agent = page.evaluate("() => navigator.userAgent")
-        except Exception:  # noqa: BLE001
+        except Exception:
             user_agent = None
 
         # This counts as an attempt the moment we call the (paid) solver.
@@ -296,7 +297,7 @@ def build_captcha_page_action(
         except FuturesTimeout:
             logger.warning("%s solve timed out after %ss", _CAPTCHA_LOG, cfg.timeout_s)
             return page
-        except Exception as exc:  # noqa: BLE001
+        except Exception as exc:
             if _is_unrecoverable(exc):
                 _latch_solver(repr(exc))
             else:
@@ -310,7 +311,9 @@ def build_captcha_page_action(
 
         _inject_and_submit(page, challenge_type, token)
         state["solved"] = True
-        logger.info("%s solved type=%s site=%s", _CAPTCHA_LOG, challenge_type, sitekey[:12])
+        logger.info(
+            "%s solved type=%s site=%s", _CAPTCHA_LOG, challenge_type, sitekey[:12]
+        )
         return page
 
     return page_action

surfsense_backend/app/proprietary/web_crawler/connector.py

@@ -25,7 +25,7 @@ import logging
 import time
 from collections.abc import Awaitable, Callable
 from dataclasses import dataclass
-from enum import Enum
+from enum import StrEnum
 from typing import Any
 
 import trafilatura
@@ -48,7 +48,7 @@ logger = logging.getLogger(__name__)
 _PERF = "[webcrawler][perf]"
 
 
-class CrawlOutcomeStatus(str, Enum):
+class CrawlOutcomeStatus(StrEnum):
     """Deterministic per-URL crawl result, single-sourcing the billable signal."""
 
     SUCCESS = "success"  # a tier returned usable extracted content
@@ -202,9 +202,7 @@ class WebCrawlerConnector:
                 logger.info(f"[webcrawler] Using Scrapling StealthyFetcher for: {url}")
                 result = await self._run_tier_with_proxy_retry(
                     "scrapling-stealthy",
-                    lambda: self._crawl_with_stealthy(
+                    lambda: self._crawl_with_stealthy(url, captcha_state, block_state),
-                        url, captcha_state, block_state
-                    ),
                 )
                 if result:
                     self._log_tier_outcome(
@@ -395,9 +393,7 @@ class WebCrawlerConnector:
         Runs the sync fetch in a worker thread so it works on any event loop,
         including Windows ``SelectorEventLoop`` which cannot spawn subprocesses.
         """
-        return await asyncio.to_thread(
+        return await asyncio.to_thread(self._crawl_with_dynamic_sync, url, block_state)
-            self._crawl_with_dynamic_sync, url, block_state
-        )
 
     def _crawl_with_dynamic_sync(
         self, url: str, block_state: dict[str, Any] | None = None

surfsense_backend/app/proprietary/web_crawler/testbench/__init__.py

@@ -21,5 +21,5 @@ creds). Run it from the backend directory:
 
 See ``README.md`` for the runbook and ``plans/backend/03f-undetectability-testing.md``
 for the design. This package is dev/operator tooling, untouched by the product
-rename, and only *measures* what 03a–03e built.
+rename, and only *measures* what 03a-03e built.
 """

surfsense_backend/app/proprietary/web_crawler/testbench/core.py

@@ -14,11 +14,12 @@ change their DOM and the harness is manual.
 
 from __future__ import annotations
 
+import contextlib
 import json
 from collections.abc import Callable
 from dataclasses import asdict, dataclass
-from datetime import datetime, timezone
+from datetime import UTC, datetime
-from enum import Enum
+from enum import StrEnum
 from pathlib import Path
 from typing import Any
 from urllib.parse import urlsplit
@@ -30,7 +31,7 @@ RESULTS_DIR = _PKG_DIR / "results"
 SCREENSHOTS_DIR = RESULTS_DIR / "screenshots"
 
 
-class CheckStatus(str, Enum):
+class CheckStatus(StrEnum):
     """Outcome of a single scorecard row."""
 
     PASS = "PASS"  # met the aspirational bar
@@ -76,7 +77,7 @@ class RunMeta:
         *, suites: str, proxy: str | None, headed: bool, scrapling_version: str
     ) -> RunMeta:
         return RunMeta(
-            timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"),
+            timestamp=datetime.now(UTC).isoformat(timespec="seconds"),
             suites=suites,
             proxy=mask_proxy(proxy),
             headed=headed,
@@ -120,15 +121,13 @@ def make_page_action(
 
     def _action(page: Any) -> Any:
         if pre_wait_ms > 0:
-            try:
+            with contextlib.suppress(Exception):
                 page.wait_for_timeout(pre_wait_ms)
-            except Exception:
-                pass
         if evaluate_js is not None:
             try:
                 cell.value = page.evaluate(evaluate_js)
                 cell.captured = True
-            except Exception as exc:  # noqa: BLE001 - tolerant by design
+            except Exception as exc:
                 cell.error = f"{type(exc).__name__}: {exc}"
         if screenshot_path is not None:
             try:
@@ -137,7 +136,7 @@ def make_page_action(
                 # create it for us, so a missing dir silently drops every screenshot.
                 Path(screenshot_path).parent.mkdir(parents=True, exist_ok=True)
                 page.screenshot(path=screenshot_path, full_page=True)
-            except Exception as exc:  # noqa: BLE001 - tolerant, but surface the cause
+            except Exception as exc:
                 if cell.error is None:
                     cell.error = f"screenshot: {type(exc).__name__}: {exc}"
         return page
@@ -168,7 +167,7 @@ def ensure_dirs() -> None:
 
 def scrapling_version() -> str:
     try:
-        import scrapling  # noqa: PLC0415
+        import scrapling
 
         return getattr(scrapling, "__version__", "unknown")
     except Exception:
@@ -178,9 +177,7 @@ def scrapling_version() -> str:
 # --- scorecard I/O -----------------------------------------------------------
 
 
-def write_scorecard(
+def write_scorecard(results: list[CheckResult], meta: RunMeta) -> tuple[Path, Path]:
-    results: list[CheckResult], meta: RunMeta
-) -> tuple[Path, Path]:
     """Write the JSON snapshot (committed, baseline) + a readable markdown report.
 
     The JSON filename is timestamped so prior runs are kept as the baseline trail;
@@ -250,9 +247,7 @@ def diff_against_baseline(
             lines.append(f"+ NEW  [{r.suite}] {r.name}: {r.status.value}")
             continue
         if was["status"] != r.status.value:
-            lines.append(
+            lines.append(f"~ {r.name}: status {was['status']} -> {r.status.value}")
-                f"~ {r.name}: status {was['status']} -> {r.status.value}"
-            )
         old_n, new_n = was.get("numeric"), r.numeric
         if old_n is not None and new_n is not None and abs(old_n - new_n) > 1e-9:
             lines.append(f"~ {r.name}: numeric {old_n} -> {new_n}")
@@ -281,8 +276,10 @@ def render_console(results: list[CheckResult]) -> None:
         by_suite.setdefault(r.suite, []).append(r)
 
     for suite in sorted(by_suite):
-        label = "Suite S - stealth/anti-bot" if suite == "S" else (
+        label = (
-            "Suite E - extraction" if suite == "E" else f"Suite {suite}"
+            "Suite S - stealth/anti-bot"
+            if suite == "S"
+            else ("Suite E - extraction" if suite == "E" else f"Suite {suite}")
         )
         print(f"\n=== {label} ===")
         for r in by_suite[suite]:

surfsense_backend/app/proprietary/web_crawler/testbench/suite_extraction.py

@@ -68,7 +68,7 @@ async def _run_case(
     bar = f"SUCCESS + contains {len(case.must_contain)} marker(s)"
     try:
         outcome = await connector.crawl_url(case.url)
-    except Exception as exc:  # noqa: BLE001 - never crash the run
+    except Exception as exc:
         return CheckResult(
             suite="E",
             name=case.name,

surfsense_backend/app/proprietary/web_crawler/testbench/suite_stealth.py

@@ -22,6 +22,7 @@ parser later is a one-function change (the spec list is the extension point).
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import re
 from collections.abc import Callable
 from dataclasses import dataclass
@@ -83,10 +84,8 @@ def _full_text(page: Any, limit: int = 200_000) -> str:
 def _parse_sannysoft(page: Any, _cell: EvalCell):
     """Count failed result cells (JS sets class='failed' on red rows)."""
     html = ""
-    try:
+    with contextlib.suppress(Exception):
         html = page.html_content or ""
-    except Exception:
-        pass
     fails = len(re.findall(r'class="[^"]*\bfailed\b[^"]*"', html))
     if not html:
         return (CheckStatus.ERROR, None, "no html returned")
@@ -184,8 +183,7 @@ def _parse_incolumitas(page: Any, _cell: EvalCell):
     if not fails and dc is None:
         return (CheckStatus.INFO, None, f"unparsed: {txt[:120]!r}")
     detail = (
-        f"fpscanner FAIL={fails or 'none'} datacenter={datacenter} "
+        f"fpscanner FAIL={fails or 'none'} datacenter={datacenter} behavioral={bscore}"
-        f"behavioral={bscore}"
     )
     return (
         CheckStatus.PASS if not fails else CheckStatus.FAIL,
@@ -375,7 +373,7 @@ def _run_browser_site(
 
     try:
         page = StealthyFetcher.fetch(site.url, **kwargs)
-    except Exception as exc:  # noqa: BLE001 - never crash the run
+    except Exception as exc:
         return CheckResult(
             suite="S",
             name=site.name,
@@ -388,7 +386,7 @@ def _run_browser_site(
 
     try:
         status, numeric, detail = site.parse(page, cell)
-    except Exception as exc:  # noqa: BLE001
+    except Exception as exc:
         status, numeric, detail = (
             CheckStatus.ERROR,
             None,
@@ -429,7 +427,7 @@ async def _run_tls(proxy: str | None) -> CheckResult:
             bar="informational (diff vs real Chrome)",
             detail=f"ja3={ja3} ja4={ja4} peet={peet}",
         )
-    except Exception as exc:  # noqa: BLE001
+    except Exception as exc:
         return CheckResult(
             suite="S",
             name="tls_fingerprint",
@@ -460,7 +458,7 @@ async def _run_proxy_leak(proxy: str | None) -> CheckResult:
             bar="not your real/datacenter IP",
             detail=f"exit_ip={ip} ({note})",
         )
-    except Exception as exc:  # noqa: BLE001
+    except Exception as exc:
         return CheckResult(
             suite="S",
             name="exit_ip",

surfsense_backend/app/routes/__init__.py

@@ -39,6 +39,7 @@ from .google_drive_add_connector_route import (
 from .google_gmail_add_connector_route import (
     router as google_gmail_add_connector_router,
 )
+from .google_maps_routes import router as google_maps_router
 from .image_generation_routes import router as image_generation_router
 from .incentive_tasks_routes import router as incentive_tasks_router
 from .jira_add_connector_route import router as jira_add_connector_router
@@ -61,12 +62,12 @@ from .rbac_routes import router as rbac_router
 from .reports_routes import router as reports_router
 from .sandbox_routes import router as sandbox_router
 from .search_source_connectors_routes import router as search_source_connectors_router
-from .workspaces_routes import router as workspaces_router
 from .slack_add_connector_route import router as slack_add_connector_router
 from .stripe_routes import router as stripe_router
 from .team_memory_routes import router as team_memory_router
 from .teams_add_connector_route import router as teams_add_connector_router
 from .video_presentations_routes import router as video_presentations_router
+from .workspaces_routes import router as workspaces_router
 from .youtube_routes import router as youtube_router
 
 router = APIRouter()
@@ -133,6 +134,7 @@ router.include_router(public_chat_router)  # Public chat sharing and cloning
 router.include_router(incentive_tasks_router)  # Incentive tasks for earning free pages
 router.include_router(stripe_router)  # Stripe checkout for additional page packs
 router.include_router(youtube_router)  # YouTube playlist resolution
+router.include_router(google_maps_router)  # Google Maps places + reviews scraper
 router.include_router(prompts_router)
 router.include_router(memory_router)  # User personal memory (memory.md style)
 router.include_router(team_memory_router)  # Workspace team memory

surfsense_backend/app/routes/editor_routes.py

@@ -172,9 +172,7 @@ async def get_editor_content(
     return _build_response(markdown_content)
 
 
-@router.get(
+@router.get("/workspaces/{workspace_id}/documents/{document_id}/download-markdown")
-    "/workspaces/{workspace_id}/documents/{document_id}/download-markdown"
-)
 async def download_document_markdown(
     workspace_id: int,
     document_id: int,

surfsense_backend/app/routes/folders_routes.py

@@ -517,9 +517,7 @@ async def bulk_move_documents(
             if not target:
                 raise HTTPException(status_code=404, detail="Target folder not found")
             mismatched = [
-                doc.id
+                doc.id for doc in documents if doc.workspace_id != target.workspace_id
-                for doc in documents
-                if doc.workspace_id != target.workspace_id
             ]
             if mismatched:
                 raise HTTPException(

surfsense_backend/app/routes/gateway_webhook_routes.py

@@ -886,9 +886,7 @@ async def list_connections(
                 baileys_account_ids.add(int(account.id))
                 route_type = "account"
                 connection_id = account.id
-                workspace_id = (
+                workspace_id = account.owner_workspace_id or binding.workspace_id
-                    account.owner_workspace_id or binding.workspace_id
-                )
                 display_name = "WhatsApp Bridge"
 
         connections.append(
@@ -907,7 +905,6 @@ async def list_connections(
                     else binding.external_username
                 ),
                 "workspace_name": workspace_name,
-                "workspace_id": workspace_id,
                 "health_status": account.health_status.value,
                 "suspended_reason": binding.suspended_reason,
             }
@@ -940,7 +937,6 @@ async def list_connections(
                     "display_name": "WhatsApp Bridge",
                     "external_username": None,
                     "workspace_name": account_state.get("display_phone_number"),
-                    "workspace_id": account_state.get("phone_number_id"),
                     "health_status": account.health_status.value,
                     "suspended_reason": account.suspended_reason,
                 }

surfsense_backend/app/routes/google_maps_routes.py

@@ -0,0 +1,64 @@
+"""Google Maps scraper routes (Apify actor-compatible)."""
+
+import logging
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from app.auth.context import AuthContext
+from app.proprietary.scrapers.google_maps import (
+    GoogleMapsReviewsInput,
+    GoogleMapsScrapeInput,
+    scrape_places,
+    scrape_reviews,
+)
+from app.proprietary.scrapers.google_maps.scraper import SignInRequiredError
+from app.users import require_session_context
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+
+@router.post("/google-maps/scrape")
+async def scrape_places_route(
+    payload: GoogleMapsScrapeInput,
+    _auth: AuthContext = Depends(require_session_context),
+) -> list[dict]:
+    """Scrape Google Maps places (search terms / URLs / place IDs).
+
+    Apify "Google Maps Scraper"-compatible input/output. Runs inline and is
+    bounded only by the request's own ``maxCrawledPlacesPerSearch``.
+    """
+    try:
+        return await scrape_places(payload)
+    except SignInRequiredError as e:
+        raise HTTPException(
+            status_code=403, detail=f"Google sign in required: {e!s}"
+        ) from e
+    except Exception as e:
+        logger.error("Google Maps scrape failed: %s", e)
+        raise HTTPException(
+            status_code=502, detail=f"Google Maps scrape failed: {e!s}"
+        ) from e
+
+
+@router.post("/google-maps/reviews")
+async def scrape_reviews_route(
+    payload: GoogleMapsReviewsInput,
+    _auth: AuthContext = Depends(require_session_context),
+) -> list[dict]:
+    """Scrape Google Maps reviews for the given place URLs / place IDs.
+
+    Apify "Google Maps Reviews Scraper"-compatible. Runs inline and is bounded
+    only by the request's own ``maxReviews``.
+    """
+    try:
+        return await scrape_reviews(payload)
+    except SignInRequiredError as e:
+        raise HTTPException(
+            status_code=403, detail=f"Google sign in required: {e!s}"
+        ) from e
+    except Exception as e:
+        logger.error("Google Maps reviews scrape failed: %s", e)
+        raise HTTPException(
+            status_code=502, detail=f"Google Maps reviews scrape failed: {e!s}"
+        ) from e

surfsense_backend/app/routes/logs_routes.py

@@ -277,9 +277,7 @@ async def get_logs_summary(
         # Get logs from the time window
         result = await session.execute(
             select(Log)
-            .filter(
+            .filter(and_(Log.workspace_id == workspace_id, Log.created_at >= since))
-                and_(Log.workspace_id == workspace_id, Log.created_at >= since)
-            )
             .order_by(desc(Log.created_at))
         )
         logs = result.scalars().all()

surfsense_backend/app/routes/model_connections_routes.py

@@ -206,9 +206,7 @@ async def _resolve_role_model_id(
         return 0
 
 
-async def _clear_invalid_roles(
+async def _clear_invalid_roles(session: AsyncSession, workspace_id: int) -> Workspace:
-    session: AsyncSession, workspace_id: int
-) -> Workspace:
     workspace = await _get_workspace(session, workspace_id)
     workspace.chat_model_id = await _resolve_role_model_id(
         session,
@@ -752,9 +750,7 @@ async def test_connection_model(
     )
 
 
-@router.get(
+@router.get("/workspaces/{workspace_id}/model-roles", response_model=ModelRolesRead)
-    "/workspaces/{workspace_id}/model-roles", response_model=ModelRolesRead
-)
 async def get_model_roles(
     workspace_id: int,
     session: AsyncSession = Depends(get_async_session),
@@ -777,9 +773,7 @@ async def get_model_roles(
     )
 
 
-@router.put(
+@router.put("/workspaces/{workspace_id}/model-roles", response_model=ModelRolesRead)
-    "/workspaces/{workspace_id}/model-roles", response_model=ModelRolesRead
-)
 async def update_model_roles(
     workspace_id: int,
     data: ModelRolesUpdate,

surfsense_backend/app/routes/new_chat_routes.py

@@ -45,9 +45,9 @@ from app.db import (
     NewChatMessageRole,
     NewChatThread,
     Permission,
-    Workspace,
     TokenUsage,
     User,
+    Workspace,
     get_async_session,
     shielded_async_session,
 )
@@ -560,9 +560,7 @@ async def check_thread_access(
     # For legacy threads (created before visibility feature),
     # only the workspace owner can access
     if is_legacy:
-        workspace_query = select(Workspace).filter(
+        workspace_query = select(Workspace).filter(Workspace.id == thread.workspace_id)
-            Workspace.id == thread.workspace_id
-        )
         workspace_result = await session.execute(workspace_query)
         workspace = workspace_result.scalar_one_or_none()
         is_workspace_owner = workspace and workspace.user_id == user.id
@@ -624,9 +622,7 @@ async def list_threads(
         )
 
         # Check if user is the workspace owner (for legacy thread visibility)
-        workspace_query = select(Workspace).filter(
+        workspace_query = select(Workspace).filter(Workspace.id == workspace_id)
-            Workspace.id == workspace_id
-        )
         workspace_result = await session.execute(workspace_query)
         workspace = workspace_result.scalar_one_or_none()
         is_workspace_owner = workspace and workspace.user_id == user.id
@@ -731,9 +727,7 @@ async def search_threads(
         )
 
         # Check if user is the workspace owner (for legacy thread visibility)
-        workspace_query = select(Workspace).filter(
+        workspace_query = select(Workspace).filter(Workspace.id == workspace_id)
-            Workspace.id == workspace_id
-        )
         workspace_result = await session.execute(workspace_query)
         workspace = workspace_result.scalar_one_or_none()
         is_workspace_owner = workspace and workspace.user_id == user.id

surfsense_backend/app/routes/obsidian_plugin_routes.py

@@ -22,8 +22,8 @@ from app.db import (
     DocumentType,
     SearchSourceConnector,
     SearchSourceConnectorType,
-    Workspace,
     User,
+    Workspace,
     get_async_session,
 )
 from app.notifications.service import NotificationService

surfsense_backend/app/routes/rbac_routes.py

@@ -21,11 +21,11 @@ from sqlalchemy.orm import selectinload
 from app.auth.context import AuthContext
 from app.db import (
     Permission,
+    User,
     Workspace,
     WorkspaceInvite,
     WorkspaceMembership,
     WorkspaceRole,
-    User,
     get_async_session,
 )
 from app.schemas import (
@@ -256,9 +256,7 @@ async def list_roles(
         )
 
         result = await session.execute(
-            select(WorkspaceRole).filter(
+            select(WorkspaceRole).filter(WorkspaceRole.workspace_id == workspace_id)
-                WorkspaceRole.workspace_id == workspace_id
-            )
         )
         return result.scalars().all()
 
@@ -982,9 +980,7 @@ async def get_invite_info(
         # Check if invite is still valid
         if not invite.is_active:
             return InviteInfoResponse(
-                workspace_name=invite.workspace.name
+                workspace_name=invite.workspace.name if invite.workspace else "",
-                if invite.workspace
-                else "",
                 role_name=invite.role.name if invite.role else None,
                 is_valid=False,
                 message="This invite is no longer active",
@@ -992,9 +988,7 @@ async def get_invite_info(
 
         if invite.expires_at and invite.expires_at < datetime.now(UTC):
             return InviteInfoResponse(
-                workspace_name=invite.workspace.name
+                workspace_name=invite.workspace.name if invite.workspace else "",
-                if invite.workspace
-                else "",
                 role_name=invite.role.name if invite.role else None,
                 is_valid=False,
                 message="This invite has expired",
@@ -1002,9 +996,7 @@ async def get_invite_info(
 
         if invite.max_uses and invite.uses_count >= invite.max_uses:
             return InviteInfoResponse(
-                workspace_name=invite.workspace.name
+                workspace_name=invite.workspace.name if invite.workspace else "",
-                if invite.workspace
-                else "",
                 role_name=invite.role.name if invite.role else None,
                 is_valid=False,
                 message="This invite has reached its maximum uses",

surfsense_backend/app/routes/search_source_connectors_routes.py

@@ -489,8 +489,7 @@ async def update_search_source_connector(
         if key == "connector_type" and value != db_connector.connector_type:
             check_result = await session.execute(
                 select(SearchSourceConnector).filter(
-                    SearchSourceConnector.workspace_id
+                    SearchSourceConnector.workspace_id == db_connector.workspace_id,
-                    == db_connector.workspace_id,
                     SearchSourceConnector.connector_type == value,
                     SearchSourceConnector.id != connector_id,
                 )

surfsense_backend/app/routes/video_presentations_routes.py

@@ -19,9 +19,9 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from app.auth.context import AuthContext
 from app.db import (
     Permission,
+    VideoPresentation,
     Workspace,
     WorkspaceMembership,
-    VideoPresentation,
     get_async_session,
 )
 from app.schemas import VideoPresentationRead

surfsense_backend/app/routes/workspaces_routes.py

@@ -279,9 +279,7 @@ async def update_workspace(
         ) from e
 
 
-@router.put(
+@router.put("/workspaces/{workspace_id}/api-access", response_model=WorkspaceRead)
-    "/workspaces/{workspace_id}/api-access", response_model=WorkspaceRead
-)
 async def update_workspace_api_access(
     workspace_id: int,
     body: WorkspaceApiAccessUpdate,

surfsense_backend/app/schemas/__init__.py

@@ -103,14 +103,6 @@ from .search_source_connector import (
     SearchSourceConnectorRead,
     SearchSourceConnectorUpdate,
 )
-from .workspace import (
-    WorkspaceApiAccessUpdate,
-    WorkspaceBase,
-    WorkspaceCreate,
-    WorkspaceRead,
-    WorkspaceUpdate,
-    WorkspaceWithStats,
-)
 from .stripe import (
     CreateCreditCheckoutSessionRequest,
     CreateCreditCheckoutSessionResponse,
@@ -128,6 +120,14 @@ from .video_presentations import (
     VideoPresentationRead,
     VideoPresentationUpdate,
 )
+from .workspace import (
+    WorkspaceApiAccessUpdate,
+    WorkspaceBase,
+    WorkspaceCreate,
+    WorkspaceRead,
+    WorkspaceUpdate,
+    WorkspaceWithStats,
+)
 
 __all__ = [
     # Folder schemas
@@ -242,13 +242,6 @@ __all__ = [
     "SearchSourceConnectorCreate",
     "SearchSourceConnectorRead",
     "SearchSourceConnectorUpdate",
-    "WorkspaceApiAccessUpdate",
-    # Workspace schemas
-    "WorkspaceBase",
-    "WorkspaceCreate",
-    "WorkspaceRead",
-    "WorkspaceUpdate",
-    "WorkspaceWithStats",
     "StripeWebhookResponse",
     "ThreadHistoryLoadResponse",
     "ThreadListItem",
@@ -257,12 +250,19 @@ __all__ = [
     # User schemas
     "UserCreate",
     "UserRead",
-    "UserWorkspaceAccess",
     "UserUpdate",
+    "UserWorkspaceAccess",
     "VerifyConnectionResponse",
     # Video Presentation schemas
     "VideoPresentationBase",
     "VideoPresentationCreate",
     "VideoPresentationRead",
     "VideoPresentationUpdate",
+    "WorkspaceApiAccessUpdate",
+    # Workspace schemas
+    "WorkspaceBase",
+    "WorkspaceCreate",
+    "WorkspaceRead",
+    "WorkspaceUpdate",
+    "WorkspaceWithStats",
 ]

surfsense_backend/app/services/billable_calls.py

@@ -492,9 +492,7 @@ async def _resolve_agent_billing_for_workspace(
 
     chat_model_id = workspace.chat_model_id
     if chat_model_id is None:
-        raise ValueError(
+        raise ValueError(f"Workspace {workspace_id} has no chat_model_id configured")
-            f"Workspace {workspace_id} has no chat_model_id configured"
-        )
 
     owner_user_id: UUID = workspace.user_id
 

surfsense_backend/app/services/chat_comments_service.py

@@ -17,8 +17,8 @@ from app.db import (
     NewChatMessageRole,
     NewChatThread,
     Permission,
-    WorkspaceMembership,
     User,
+    WorkspaceMembership,
     has_permission,
 )
 from app.notifications.service import NotificationService

surfsense_backend/app/services/folder_service.py

@@ -174,9 +174,7 @@ async def ensure_folder_hierarchy_with_depth_validation(
 
         if folder is None:
             await validate_folder_depth(session, parent_id, subtree_depth=0)
-            position = await generate_folder_position(
+            position = await generate_folder_position(session, workspace_id, parent_id)
-                session, workspace_id, parent_id
-            )
             folder = Folder(
                 name=name,
                 workspace_id=workspace_id,

surfsense_backend/app/services/linear/kb_sync_service.py

@@ -213,9 +213,7 @@ class LinearKBSyncService:
 
             document.title = f"{issue_identifier}: {issue_title}"
             document.content = summary_content
-            document.content_hash = generate_content_hash(
+            document.content_hash = generate_content_hash(issue_content, workspace_id)
-                issue_content, workspace_id
-            )
             document.embedding = summary_embedding
             from sqlalchemy.orm.attributes import flag_modified
 

surfsense_backend/app/services/llm_service.py

@@ -469,9 +469,7 @@ async def get_vision_llm(
             if not candidates:
                 logger.error("No vision-capable models available for Auto mode")
                 return None
-            config_id = int(
+            config_id = int(choose_auto_model_candidate(candidates, workspace_id)["id"])
-                choose_auto_model_candidate(candidates, workspace_id)["id"]
-            )
 
         if config_id < 0:
             global_model = get_global_model(config_id)
@@ -532,9 +530,7 @@ async def get_vision_llm(
         return SanitizedChatLiteLLM(**litellm_kwargs)
 
     except Exception as e:
-        logger.error(
+        logger.error(f"Error getting vision LLM for workspace {workspace_id}: {e!s}")
-            f"Error getting vision LLM for workspace {workspace_id}: {e!s}"
-        )
         return None
 
 

surfsense_backend/app/services/memory/service.py

@@ -11,7 +11,7 @@ from uuid import UUID
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.db import Workspace, User
+from app.db import User, Workspace
 from app.services.memory.document import parse_memory_document, render_memory_document
 from app.services.memory.rewrite import forced_rewrite
 from app.services.memory.schemas import MemoryLimits

surfsense_backend/app/services/notion/tool_metadata_service.py

@@ -84,9 +84,7 @@ class NotionToolMetadataService:
                 "error": "No Notion accounts connected",
             }
 
-        parent_pages = await self._get_parent_pages_by_account(
+        parent_pages = await self._get_parent_pages_by_account(workspace_id, accounts)
-            workspace_id, accounts
-        )
 
         accounts_with_status = []
         for acc in accounts:

surfsense_backend/app/services/public_chat_service.py

@@ -31,10 +31,10 @@ from app.db import (
     PodcastStatus,
     PublicChatSnapshot,
     Report,
-    WorkspaceMembership,
     User,
     VideoPresentation,
     VideoPresentationStatus,
+    WorkspaceMembership,
 )
 from app.utils.rbac import check_permission
 

surfsense_backend/app/services/web_crawl_credit_service.py

@@ -184,9 +184,7 @@ class WebCrawlCreditService:
 
         return user.credit_micros_balance
 
-    async def charge_credits(
+    async def charge_credits(self, user_id: str | UUID, successes: int) -> int | None:
-        self, user_id: str | UUID, successes: int
-    ) -> int | None:
         """Debit the wallet for ``successes`` successful crawls.
 
         No-op when crawl billing is disabled or ``successes <= 0``. Returns the
@@ -198,9 +196,7 @@ class WebCrawlCreditService:
             return None
         return await self._apply_debit(user_id, self.successes_to_micros(successes))
 
-    async def charge_captcha(
+    async def charge_captcha(self, user_id: str | UUID, attempts: int) -> int | None:
-        self, user_id: str | UUID, attempts: int
-    ) -> int | None:
         """Debit the wallet for ``attempts`` captcha solves (Phase 3d).
 
         Per-attempt (not per-success): the solver charges for every attempt even

surfsense_backend/app/tasks/celery_tasks/document_tasks.py

@@ -212,9 +212,7 @@ async def _delete_folder_documents(
 )
 def delete_workspace_task(self, workspace_id: int):
     """Celery task to delete a workspace and heavy child rows in batches."""
-    return run_async_celery_task(
+    return run_async_celery_task(lambda: _delete_workspace_background(workspace_id))
-        lambda: _delete_workspace_background(workspace_id)
-    )
 
 
 async def _delete_workspace_background(workspace_id: int) -> None:

surfsense_backend/app/tasks/celery_tasks/video_presentation_tasks.py

@@ -127,8 +127,7 @@ async def _generate_video_presentation(
                 )
             except ValueError as resolve_err:
                 logger.error(
-                    "VideoPresentation %s: cannot resolve billing for "
+                    "VideoPresentation %s: cannot resolve billing for workspace=%s: %s",
-                    "workspace=%s: %s",
                     video_pres.id,
                     workspace_id,
                     resolve_err,

surfsense_backend/app/tasks/chat/streaming/flows/shared/llm_bundle.py

@@ -55,9 +55,7 @@ def _agent_config_from_resolved(
     )
 
 
-async def _load_workspace(
+async def _load_workspace(session: AsyncSession, workspace_id: int) -> Workspace | None:
-    session: AsyncSession, workspace_id: int
-) -> Workspace | None:
     result = await session.execute(
         select(Workspace).where(Workspace.id == workspace_id)
     )

surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py

@@ -13,12 +13,12 @@ from datetime import datetime
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app.config import config
+from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType
 from app.proprietary.web_crawler import (
     CrawlOutcomeStatus,
     WebCrawlerConnector,
 )
-from app.config import config
-from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType
 from app.services.etl_credit_service import InsufficientCreditsError
 from app.services.task_logging_service import TaskLoggingService
 from app.services.token_tracking_service import record_token_usage

surfsense_backend/app/users.py

@@ -22,10 +22,10 @@ from app.auth.session_cookies import access_expires_at, write_session
 from app.config import config
 from app.db import (
     Prompt,
+    User,
     Workspace,
     WorkspaceMembership,
     WorkspaceRole,
-    User,
     async_session_maker,
     get_async_session,
     get_default_roles_config,
@@ -207,9 +207,7 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
                     f"Created default workspace (ID: {default_workspace.id}) for user {user.id}"
                 )
         except Exception as e:
-            logger.error(
+            logger.error(f"Failed to create default workspace for user {user.id}: {e}")
-                f"Failed to create default workspace for user {user.id}: {e}"
-            )
 
     async def on_after_forgot_password(
         self, user: User, token: str, request: Request | None = None
@@ -326,7 +324,7 @@ def _token_meets_epoch(token: str) -> bool:
         return False
 
     issued_at = payload.get("iat")
-    return isinstance(issued_at, (int, float)) and int(issued_at) >= min_issued_at
+    return isinstance(issued_at, int | float) and int(issued_at) >= min_issued_at
 
 
 async def get_auth_context(

surfsense_backend/app/utils/crawl/classifier.py

@@ -12,10 +12,10 @@ our enterprise targets). Regexes are compiled once at import (hot-path hygiene).
 """
 
 import re
-from enum import Enum
+from enum import StrEnum
 
 
-class BlockType(str, Enum):
+class BlockType(StrEnum):
     """Coarse label for what a fetched page represents."""
 
     OK = "ok"  # usable content, no challenge detected

surfsense_backend/app/utils/validators.py

@@ -44,9 +44,7 @@ def validate_workspace_id(workspace_id: Any) -> int:
     if isinstance(workspace_id, str):
         # Check if it's a valid integer string
         if not workspace_id.strip():
-            raise HTTPException(
+            raise HTTPException(status_code=400, detail="workspace_id cannot be empty")
-                status_code=400, detail="workspace_id cannot be empty"
-            )
 
         # Check for valid integer format (no leading zeros, no decimal points)
         if not re.match(r"^[1-9]\d*$", workspace_id.strip()):

surfsense_backend/scripts/e2e_google_maps_deep.py

@@ -0,0 +1,824 @@
+"""Deep functional verification for the Google Maps scraper (live network).
+
+Complements the fast smoke e2e (e2e_google_maps_scraper.py) with breadth:
+diverse places (countries, scripts, categories), URL-kind coverage (name-only
+URLs, CID), and review semantics (sort order, date cutoff, pagination
+uniqueness, personal-data stripping, localization).
+
+Verification style: ground-truth invariants instead of screenshots — known
+coordinates/websites/address keywords for world-famous places, and internal
+consistency rules (newest sort is monotonically non-increasing, cutoff dates
+hold, review IDs are unique across pages, etc.).
+
+Run from the backend directory:
+    .\\.venv\\Scripts\\python.exe scripts/e2e_google_maps_deep.py
+"""
+
+import asyncio
+import itertools
+import sys
+from datetime import datetime
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8")
+
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_BACKEND_ROOT))
+for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
+    if _candidate.exists():
+        load_dotenv(_candidate)
+        break
+
+from app.proprietary.scrapers.google_maps import (  # noqa: E402
+    GoogleMapsReviewsInput,
+    GoogleMapsScrapeInput,
+    scrape_places,
+    scrape_reviews,
+)
+
+_CHECKS: list[tuple[str, bool, str]] = []
+
+
+def _check(label: str, ok: bool, detail: str = "") -> bool:
+    _CHECKS.append((label, ok, detail))
+    print(f"  [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}")
+    return ok
+
+
+def _hr(title: str) -> None:
+    print(f"\n{'=' * 70}\n{title}\n{'=' * 70}")
+
+
+def _near(actual: float | None, expected: float, tol: float = 0.05) -> bool:
+    return actual is not None and abs(actual - expected) <= tol
+
+
+def _iso(s: str | None) -> datetime | None:
+    if not s:
+        return None
+    try:
+        return datetime.fromisoformat(s.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+
+
+# Ground truth: world-famous places whose facts don't drift. Name-only URLs
+# exercise the HTML -> fid resolution path for every one of them.
+_PLACES = [
+    {
+        "label": "Eiffel Tower (FR landmark)",
+        "url": "https://www.google.com/maps/place/Eiffel+Tower/",
+        "title_contains": "eiffel",
+        "lat": 48.8584,
+        "lng": 2.2945,
+        "address_contains": ["paris", "75007", "france"],
+        "website_contains": "toureiffel",
+    },
+    {
+        "label": "Tokyo Tower (JP, non-Latin locale)",
+        "url": "https://www.google.com/maps/place/Tokyo+Tower/",
+        "title_contains": "tokyo tower",
+        "lat": 35.6586,
+        "lng": 139.7454,
+        "address_contains": ["tokyo", "japan", "minato"],
+    },
+    {
+        "label": "Sydney Opera House (AU)",
+        "url": "https://www.google.com/maps/place/Sydney+Opera+House/",
+        "title_contains": "opera house",
+        "lat": -33.8568,
+        "lng": 151.2153,
+        "address_contains": ["sydney", "nsw", "australia"],
+    },
+    {
+        "label": "The Plaza Hotel (US hotel category)",
+        "url": "https://www.google.com/maps/place/The+Plaza+Hotel+New+York/",
+        "title_contains": "plaza",
+        "lat": 40.7646,
+        "lng": -73.9744,
+        "address_contains": ["new york", "ny"],
+        "category_contains": "hotel",
+    },
+]
+
+_KIMS_CID_URL = "https://maps.google.com/?cid=7838756667406262025"  # Kim's Island
+_KIMS_PLACE_ID = "ChIJJQz5EZzKw4kRCZ95UajbyGw"
+_LOUVRE_URL = "https://www.google.com/maps/place/Louvre+Museum/"
+
+
+async def scrape_one(url: str, **kwargs) -> dict | None:
+    items = await scrape_places(
+        GoogleMapsScrapeInput(startUrls=[{"url": url}], **kwargs)
+    )
+    return items[0] if items else None
+
+
+async def step_diverse_places() -> None:
+    _hr("A — diverse places via name-only URLs (HTML -> fid path)")
+    for spec in _PLACES:
+        it = await scrape_one(spec["url"])
+        if it is None:
+            _check(spec["label"], False, "no item returned")
+            continue
+        title = (it.get("title") or "").lower()
+        loc = it.get("location") or {}
+        addr = (it.get("address") or "").lower()
+        problems = []
+        if spec["title_contains"] not in title:
+            problems.append(f"title={it.get('title')!r}")
+        if not _near(loc.get("lat"), spec["lat"]) or not _near(
+            loc.get("lng"), spec["lng"]
+        ):
+            problems.append(f"loc={loc}")
+        if not any(k in addr for k in spec["address_contains"]):
+            problems.append(f"address={it.get('address')!r}")
+        if not it.get("placeId"):
+            problems.append("no placeId")
+        if not it.get("categories"):
+            problems.append("no categories")
+        if spec.get("website_contains") and spec["website_contains"] not in (
+            it.get("website") or ""
+        ):
+            problems.append(f"website={it.get('website')!r}")
+        if (
+            spec.get("category_contains")
+            and spec["category_contains"]
+            not in (
+                (it.get("categoryName") or "") + " ".join(it.get("categories") or [])
+            ).lower()
+        ):
+            problems.append(f"categories={it.get('categories')}")
+        _check(
+            spec["label"],
+            not problems,
+            "; ".join(problems)
+            or f"{it.get('title')!r} @ ({loc.get('lat'):.4f},{loc.get('lng'):.4f}) "
+            f"cat={it.get('categoryName')!r} score={it.get('totalScore')}",
+        )
+
+
+async def step_cid_url() -> None:
+    _hr("B — CID URL dispatch")
+    it = await scrape_one(_KIMS_CID_URL)
+    ok = it is not None and it.get("title") == "Kim's Island"
+    _check(
+        "?cid=... resolves to the right place",
+        ok,
+        f"title={it.get('title')!r}" if it else "no item",
+    )
+    if it:
+        _check(
+            "cid place has full detail (phone+hours)",
+            bool(it.get("phone")) and bool(it.get("openingHours")),
+            f"phone={it.get('phone')!r}, hours={len(it.get('openingHours') or [])} days",
+        )
+
+
+async def step_review_sorts() -> None:
+    _hr("C — review sort semantics (Kim's Island)")
+    newest = await scrape_reviews(
+        GoogleMapsReviewsInput(placeIds=[_KIMS_PLACE_ID], maxReviews=10)
+    )
+    dates = [_iso(r.get("publishedAtDate")) for r in newest]
+    dated = [d for d in dates if d]
+    _check(
+        "newest: publishedAtDate non-increasing",
+        len(dated) >= 5 and all(a >= b for a, b in itertools.pairwise(dated)),
+        f"{len(newest)} reviews, first={newest[0].get('publishAt') if newest else None}",
+    )
+
+    lowest = await scrape_reviews(
+        GoogleMapsReviewsInput(
+            placeIds=[_KIMS_PLACE_ID], maxReviews=10, reviewsSort="lowestRanking"
+        )
+    )
+    highest = await scrape_reviews(
+        GoogleMapsReviewsInput(
+            placeIds=[_KIMS_PLACE_ID], maxReviews=10, reviewsSort="highestRanking"
+        )
+    )
+    lo = [r["stars"] for r in lowest if r.get("stars")]
+    hi = [r["stars"] for r in highest if r.get("stars")]
+    lo_avg = sum(lo) / len(lo) if lo else 0
+    hi_avg = sum(hi) / len(hi) if hi else 0
+    _check(
+        "lowestRanking avg < highestRanking avg",
+        bool(lo and hi) and lo_avg < hi_avg,
+        f"lowest avg={lo_avg:.2f} (first={lo[:3]}), highest avg={hi_avg:.2f} (first={hi[:3]})",
+    )
+    _check(
+        "highestRanking page is all 5 stars",
+        bool(hi) and all(s == 5 for s in hi),
+        f"stars={hi}",
+    )
+
+
+async def step_start_date_cutoff() -> None:
+    _hr("D — reviewsStartDate cutoff")
+    baseline = await scrape_reviews(
+        GoogleMapsReviewsInput(placeIds=[_KIMS_PLACE_ID], maxReviews=10)
+    )
+    dated = [(_iso(r.get("publishedAtDate")), r) for r in baseline]
+    dated = [(d, r) for d, r in dated if d]
+    if len(dated) < 6:
+        _check("cutoff test has enough dated reviews", False, f"only {len(dated)}")
+        return
+    # Cut between the 4th and 5th newest review -> expect exactly 4 back.
+    cutoff_dt = dated[4][0]
+    cutoff = (
+        dated[3][0].strftime("%Y-%m-%dT%H:%M:%S.000Z")
+        if dated[3][0] == cutoff_dt
+        else cutoff_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+    )
+    got = await scrape_reviews(
+        GoogleMapsReviewsInput(
+            placeIds=[_KIMS_PLACE_ID], maxReviews=100, reviewsStartDate=cutoff
+        )
+    )
+    got_dates = [_iso(r.get("publishedAtDate")) for r in got]
+    cutoff_parsed = _iso(cutoff)
+    _check(
+        "all returned reviews >= cutoff",
+        bool(got) and all(d is None or d >= cutoff_parsed for d in got_dates),
+        f"cutoff={cutoff}, returned={len(got)} (expected ~4)",
+    )
+    _check(
+        "cutoff actually limits the result",
+        0 < len(got) < len(baseline) + 1 and len(got) <= 6,
+        f"{len(got)} vs baseline {len(baseline)}",
+    )
+
+
+async def step_personal_data() -> None:
+    _hr("E — personalData=false stripping")
+    items = await scrape_reviews(
+        GoogleMapsReviewsInput(
+            placeIds=[_KIMS_PLACE_ID], maxReviews=3, personalData=False
+        )
+    )
+    if not items:
+        _check("reviews returned", False)
+        return
+    leaked = [
+        k
+        for k in ("name", "reviewerId", "reviewerUrl", "reviewerPhotoUrl")
+        if any(r.get(k) for r in items)
+    ]
+    _check(
+        "reviewer identity fields are absent",
+        not leaked,
+        f"leaked={leaked}" if leaked else f"{len(items)} reviews, ids+stars kept",
+    )
+    _check(
+        "non-personal fields survive",
+        all(r.get("reviewId") and r.get("stars") for r in items),
+    )
+
+
+async def step_localization() -> None:
+    _hr("F — localization (language=fr)")
+    items = await scrape_reviews(
+        GoogleMapsReviewsInput(
+            startUrls=[{"url": "https://www.google.com/maps/place/Eiffel+Tower/"}],
+            maxReviews=5,
+            language="fr",
+        )
+    )
+    if not items:
+        _check("french reviews returned", False)
+        return
+    rel = [r.get("publishAt") or "" for r in items]
+    french = [s for s in rel if "il y a" in s or "mois" in s or "semaine" in s]
+    _check(
+        "relative dates come back in French",
+        len(french) >= 3,
+        f"publishAt={rel}",
+    )
+    _check(
+        "items stamped language=fr",
+        all(r.get("language") == "fr" for r in items),
+    )
+
+
+async def step_big_place_pagination() -> None:
+    _hr("G — big place, 30 reviews across >=3 pages (Louvre)")
+    items = await scrape_reviews(
+        GoogleMapsReviewsInput(startUrls=[{"url": _LOUVRE_URL}], maxReviews=30)
+    )
+    ids = [r.get("reviewId") for r in items]
+    _check(
+        "30 reviews with unique IDs",
+        len(items) == 30 and len(set(ids)) == 30,
+        f"{len(items)} reviews, {len(set(ids))} unique",
+    )
+    ok_fields = all(
+        r.get("name") and r.get("stars") is not None and r.get("publishedAtDate")
+        for r in items
+    )
+    _check("every review has author/stars/date", ok_fields)
+    _check(
+        "place header stamped on all (Louvre)",
+        all("louvre" in (r.get("title") or "").lower() for r in items),
+        f"title={items[0].get('title')!r}" if items else "",
+    )
+
+
+async def step_search_discovery() -> None:
+    _hr("I — search discovery: paging, rank, dedupe (pizza in New York)")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["pizza"],
+            locationQuery="New York, NY",
+            maxCrawledPlacesPerSearch=25,
+        )
+    )
+    fids = [i.get("fid") for i in items]
+    _check(
+        "25 places across >1 page, all unique fids",
+        len(items) == 25 and len(set(fids)) == 25,
+        f"{len(items)} items, {len(set(fids))} unique",
+    )
+    _check(
+        "ranks are 1..25 in order",
+        [i.get("rank") for i in items] == list(range(1, 26)),
+    )
+    ny = sum(
+        1
+        for i in items
+        if i.get("location")
+        and 40.4 < (i["location"]["lat"] or 0) < 41.1
+        and -74.3 < (i["location"]["lng"] or 0) < -73.6
+    )
+    _check(
+        "locationQuery scopes results to NYC",
+        ny >= 23,
+        f"{ny}/25 within NYC bounds",
+    )
+    with_core = sum(
+        1 for i in items if i.get("title") and i.get("placeId") and i.get("address")
+    )
+    _check("all items have title/placeId/address", with_core == 25, f"{with_core}/25")
+
+
+async def step_search_filters() -> None:
+    _hr("J — search filters (stars, website, matching)")
+    starred = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["restaurant"],
+            locationQuery="Chicago, IL",
+            maxCrawledPlacesPerSearch=10,
+            placeMinimumStars="fourAndHalf",
+        )
+    )
+    scores = [i.get("totalScore") for i in starred]
+    _check(
+        "placeMinimumStars=fourAndHalf holds",
+        bool(scores) and all(s is not None and s >= 4.5 for s in scores),
+        f"scores={scores}",
+    )
+
+    no_web = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["restaurant"],
+            locationQuery="Chicago, IL",
+            maxCrawledPlacesPerSearch=5,
+            website="withoutWebsite",
+        )
+    )
+    _check(
+        "website=withoutWebsite holds",
+        bool(no_web) and all(not i.get("website") for i in no_web),
+        f"{len(no_web)} items, websites={[i.get('website') for i in no_web]}",
+    )
+
+    matching = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["pizza"],
+            locationQuery="Boston, MA",
+            maxCrawledPlacesPerSearch=5,
+            searchMatching="only_includes",
+        )
+    )
+    titles = [i.get("title") for i in matching]
+    _check(
+        "searchMatching=only_includes keeps 'pizza' in titles",
+        bool(titles) and all("pizza" in (t or "").lower() for t in titles),
+        f"titles={titles}",
+    )
+
+
+async def step_search_closed() -> None:
+    _hr("K — closed-place detection + skipClosedPlaces (Dean & DeLuca)")
+    q = "Dean DeLuca 560 Broadway New York"
+    found = await scrape_places(
+        GoogleMapsScrapeInput(searchStringsArray=[q], maxCrawledPlacesPerSearch=3)
+    )
+    dean = next((i for i in found if "DeLuca" in (i.get("title") or "")), None)
+    _check(
+        "permanently closed place flagged",
+        dean is not None and dean.get("permanentlyClosed") is True,
+        f"title={dean.get('title') if dean else None}, closed={dean.get('permanentlyClosed') if dean else None}",
+    )
+    skipped = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=[q],
+            maxCrawledPlacesPerSearch=3,
+            skipClosedPlaces=True,
+        )
+    )
+    _check(
+        "skipClosedPlaces filters it out",
+        not any("DeLuca" in (i.get("title") or "") for i in skipped),
+        f"{len(skipped)} items after skip",
+    )
+
+
+async def step_search_url_and_geo() -> None:
+    _hr("L — /maps/search/ URL routing + customGeolocation")
+    via_url = await scrape_places(
+        GoogleMapsScrapeInput(
+            startUrls=[
+                {"url": "https://www.google.com/maps/search/ramen+in+Osaka+Japan/"}
+            ],
+            maxCrawledPlacesPerSearch=5,
+        )
+    )
+    osaka = sum(
+        1
+        for i in via_url
+        if i.get("location") and 34.4 < (i["location"]["lat"] or 0) < 35.0
+    )
+    _check(
+        "/maps/search/ startUrl yields Osaka ramen places",
+        len(via_url) == 5 and osaka >= 4,
+        f"{len(via_url)} items, {osaka} in Osaka",
+    )
+
+    geo = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["museum"],
+            customGeolocation={
+                "type": "Point",
+                "coordinates": [2.3522, 48.8566],  # [lng, lat] Paris
+                "radiusKm": 10,
+            },
+            maxCrawledPlacesPerSearch=5,
+        )
+    )
+    paris = sum(
+        1
+        for i in geo
+        if i.get("location")
+        and 48.5 < (i["location"]["lat"] or 0) < 49.2
+        and 1.9 < (i["location"]["lng"] or 0) < 2.8
+    )
+    _check(
+        "customGeolocation Point scopes to Paris",
+        len(geo) >= 3 and paris >= 3,
+        f"{len(geo)} items, {paris} in Paris: {[i.get('title') for i in geo]}",
+    )
+
+
+async def step_search_pagination_stress() -> None:
+    _hr("M — search pagination stress (60 results / 3+ pages, exhaustion)")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["restaurant"],
+            locationQuery="Manhattan, New York",
+            maxCrawledPlacesPerSearch=60,
+        )
+    )
+    fids = [i.get("fid") for i in items]
+    _check(
+        "60 places, all fids unique (offset paging + dedupe)",
+        len(items) == 60 and len(set(fids)) == 60,
+        f"{len(items)} items, {len(set(fids))} unique",
+    )
+    _check(
+        "ranks strictly sequential 1..60",
+        [i.get("rank") for i in items] == list(range(1, 61)),
+    )
+    manhattan = sum(
+        1
+        for i in items
+        if i.get("location") and 40.68 < (i["location"]["lat"] or 0) < 40.9
+    )
+    _check(
+        "results stay in Manhattan across pages",
+        manhattan >= 55,
+        f"{manhattan}/60 in bounds",
+    )
+
+    # A hyper-specific query has few results: paging must terminate on its
+    # own (no infinite loop, no error) well before the requested cap.
+    sparse = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["Kim's Island Staten Island"],
+            maxCrawledPlacesPerSearch=100,
+        )
+    )
+    _check(
+        "sparse query exhausts naturally below the cap",
+        0 < len(sparse) < 25,
+        f"{len(sparse)} results",
+    )
+
+
+async def step_detail_extras() -> None:
+    _hr("N — detail-page extras (kgmid/cid/additionalInfo/links)")
+    # pin the exact place by fid — a name search returns a different Joe's
+    # branch run to run, which makes magnitude assertions flaky
+    items = await scrape_places(
+        GoogleMapsScrapeInput(
+            startUrls=[
+                {
+                    "url": "https://www.google.com/maps/place/Joe's+Pizza+Broadway/"
+                    "data=!4m2!3m1!1s0x89c259ab3c1ef289:0x3b67a41175949f55"
+                }
+            ],
+            maxImages=5,
+        )
+    )
+    if not items:
+        _check("place returned", False)
+        return
+    it = items[0]
+    dist = it.get("reviewsDistribution") or {}
+    _check(
+        "reviewsCount + distribution (NID-session fields)",
+        (it.get("reviewsCount") or 0) > 20_000
+        and (it.get("reviewsCount") == sum(dist.values())),
+        f"count={it.get('reviewsCount')}, dist={dist}",
+    )
+    hist = it.get("popularTimesHistogram") or {}
+    _check(
+        "popular times histogram covers the week",
+        set(hist) == {"Su", "Mo", "Tu", "We", "Th", "Fr", "Sa"}
+        and all(
+            {"hour", "occupancyPercent"} <= set(slot)
+            for d in hist.values()
+            for slot in d
+        ),
+        f"days={sorted(hist)}",
+    )
+    _check(
+        "image gallery fields (count/categories/urls capped at maxImages)",
+        (it.get("imagesCount") or 0) > 1000
+        and "All" in (it.get("imageCategories") or [])
+        and len(it.get("imageUrls") or []) == 5,
+        f"count={it.get('imagesCount')}, cats={len(it.get('imageCategories') or [])}, "
+        f"urls={len(it.get('imageUrls') or [])}",
+    )
+    tags = it.get("reviewsTags") or []
+    _check(
+        "reviewsTags with counts",
+        len(tags) >= 5 and all(t.get("title") and t.get("count") for t in tags),
+        f"{tags[:2]}",
+    )
+    _check(
+        "additionalInfo has full section set (not just Accessibility)",
+        len(it.get("additionalInfo") or {}) >= 8,
+        f"sections={list((it.get('additionalInfo') or {}).keys())}",
+    )
+
+    # scrapePlaceDetailPage on the SEARCH flow: search darrays lack the
+    # session-gated fields, so each hit must get enriched via a detail RPC.
+    enriched = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["ramen"],
+            locationQuery="Chicago, IL",
+            maxCrawledPlacesPerSearch=2,
+            scrapePlaceDetailPage=True,
+        )
+    )
+    _check(
+        "search flow + scrapePlaceDetailPage enriches every hit",
+        len(enriched) == 2
+        and all(
+            (e.get("reviewsCount") or 0) > 0 and e.get("reviewsDistribution")
+            for e in enriched
+        ),
+        f"counts={[e.get('reviewsCount') for e in enriched]}",
+    )
+    fid = it.get("fid") or ""
+    cid_ok = bool(fid) and it.get("cid") == str(int(fid.split(":")[1], 16))
+    _check(
+        "kgmid + cid derived from fid",
+        bool(it.get("kgmid", "").startswith("/g/")) and cid_ok,
+        f"kgmid={it.get('kgmid')}, cid={it.get('cid')}",
+    )
+    info = it.get("additionalInfo") or {}
+    _check(
+        "additionalInfo has sections with boolean options",
+        bool(info)
+        and all(
+            isinstance(v, list) and all(isinstance(e, dict) for e in v)
+            for v in info.values()
+        ),
+        f"sections={list(info.keys())}",
+    )
+
+
+async def step_hotel_fields() -> None:
+    _hr("O — hotel fields (The Plaza: stars, dates, similar, ads)")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(
+            startUrls=[
+                {
+                    "url": "https://www.google.com/maps/place/The+Plaza/"
+                    "data=!4m2!3m1!1s0x89c258f07d5da561:0x61f6aa300ba8339d"
+                }
+            ],
+        )
+    )
+    if not items:
+        _check("hotel returned", False)
+        return
+    it = items[0]
+    _check(
+        "hotelStars + check-in/out dates",
+        it.get("hotelStars") == "5 stars"
+        and (it.get("checkInDate") or "") < (it.get("checkOutDate") or ""),
+        f"stars={it.get('hotelStars')}, {it.get('checkInDate')}..{it.get('checkOutDate')}",
+    )
+    similar = it.get("similarHotelsNearby") or []
+    _check(
+        "similarHotelsNearby with fid/score",
+        len(similar) >= 3 and all(h.get("title") and h.get("fid") for h in similar),
+        f"{len(similar)} hotels, first={similar[0].get('title') if similar else None}",
+    )
+    ads = it.get("hotelAds") or []
+    _check(
+        "hotelAds with booking links",
+        bool(ads) and all(a.get("url", "").startswith("https://") for a in ads),
+        f"{len(ads)} ads",
+    )
+
+
+async def step_all_places_scan() -> None:
+    _hr("P — allPlacesNoSearchAction area scan (Times Square 400m)")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(
+            allPlacesNoSearchAction="all_places_no_search_mouse",
+            customGeolocation={
+                "type": "Point",
+                "coordinates": [-73.9855, 40.758],
+                "radiusKm": 0.4,
+            },
+            maxCrawledPlacesPerSearch=25,
+        )
+    )
+    fids = [i.get("fid") for i in items]
+    _check(
+        "25 unique places without any search term",
+        len(items) == 25 and len(set(fids)) == 25,
+        f"{len(items)} items",
+    )
+    cats = {i.get("categoryName") for i in items if i.get("categoryName")}
+    _check(
+        "multiple categories represented (sweep, not one query)",
+        len(cats) >= 5,
+        f"{len(cats)} categories",
+    )
+    in_view = sum(
+        1 for i in items if i.get("location") and 40.74 < i["location"]["lat"] < 40.78
+    )
+    _check("scan respects the viewport", in_view >= 23, f"{in_view}/25 in bounds")
+
+
+async def step_short_url() -> None:
+    _hr("Q — short link (maps.app.goo.gl Firebase redirect -> place)")
+    # A real shared short link -> "Aux Merveilleux de Fred" (NYC). These are
+    # Firebase Dynamic Links (JS interstitial), so this exercises the browser-
+    # render redirect path in resolve_fid. fid is the ground-truth invariant.
+    it = await scrape_one("https://maps.app.goo.gl/8YUvDPbQPrasqC528")
+    _check(
+        "maps.app.goo.gl short link resolves to the right place",
+        it is not None
+        and it.get("fid") == "0x89c259957da502cd:0xed3eb58a4ca08a95"
+        and "merveilleux" in (it.get("title") or "").lower(),
+        f"title={it.get('title') if it else None!r}, fid={it.get('fid') if it else None}",
+    )
+
+
+async def step_filter_variants() -> None:
+    _hr("R — search filter variants (only_exact, categoryFilterWords)")
+    # only_exact: the parsed title must equal the query exactly (Seattle
+    # Starbucks are titled "Starbucks Coffee Company", not "Starbucks").
+    exact = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["Starbucks Coffee Company"],
+            locationQuery="Seattle, WA",
+            maxCrawledPlacesPerSearch=8,
+            searchMatching="only_exact",
+        )
+    )
+    titles = [i.get("title") for i in exact]
+    _check(
+        "searchMatching=only_exact keeps only exact-title matches",
+        bool(titles)
+        and all((t or "").lower() == "starbucks coffee company" for t in titles),
+        f"{len(titles)} items, titles={titles[:3]}",
+    )
+    # categoryFilterWords: drop places whose categories don't include a word.
+    coffee = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["food"],
+            locationQuery="Seattle, WA",
+            maxCrawledPlacesPerSearch=6,
+            categoryFilterWords=["coffee"],
+        )
+    )
+    _check(
+        "categoryFilterWords keeps only matching categories",
+        bool(coffee)
+        and all(
+            any("coffee" in c.lower() for c in (i.get("categories") or []))
+            for i in coffee
+        ),
+        f"{len(coffee)} items, cats={[i.get('categories') for i in coffee][:2]}",
+    )
+
+
+async def step_reviews_origin() -> None:
+    _hr("S — reviewsOrigin=google filter")
+    # The public BOQ feed only carries Google-origin reviews (partner reviews
+    # aren't exposed anonymously), so the invariant is: nothing non-Google
+    # leaks through when origin is pinned to google.
+    items = await scrape_reviews(
+        GoogleMapsReviewsInput(
+            placeIds=[_KIMS_PLACE_ID], maxReviews=10, reviewsOrigin="google"
+        )
+    )
+    origins = {(r.get("reviewOrigin") or "Google") for r in items}
+    _check(
+        "reviewsOrigin=google -> only Google-origin reviews",
+        bool(items) and origins <= {"Google"},
+        f"{len(items)} reviews, origins={origins}",
+    )
+
+
+async def step_inline_consistency() -> None:
+    _hr("H — inline reviews[] match the standalone reviews endpoint")
+    place = await scrape_one(
+        f"https://www.google.com/maps/place/?q=place_id:{_KIMS_PLACE_ID}",
+        maxReviews=5,
+    )
+    standalone = await scrape_reviews(
+        GoogleMapsReviewsInput(placeIds=[_KIMS_PLACE_ID], maxReviews=5)
+    )
+    if not place or not standalone:
+        _check("both sources returned data", False)
+        return
+    inline_ids = {r.get("reviewId") for r in (place.get("reviews") or [])}
+    standalone_ids = {r.get("reviewId") for r in standalone}
+    overlap = len(inline_ids & standalone_ids)
+    _check(
+        "inline and standalone reviews overlap (same feed)",
+        overlap >= 3,  # feed ordering can shift slightly between calls
+        f"{overlap}/5 shared review IDs",
+    )
+
+
+async def main() -> int:
+    steps = [
+        step_diverse_places(),
+        step_cid_url(),
+        step_review_sorts(),
+        step_start_date_cutoff(),
+        step_personal_data(),
+        step_localization(),
+        step_big_place_pagination(),
+        step_inline_consistency(),
+        step_search_discovery(),
+        step_search_filters(),
+        step_search_closed(),
+        step_search_url_and_geo(),
+        step_search_pagination_stress(),
+        step_detail_extras(),
+        step_hotel_fields(),
+        step_all_places_scan(),
+        step_short_url(),
+        step_filter_variants(),
+        step_reviews_origin(),
+    ]
+    for coro in steps:
+        try:
+            await coro
+        except Exception as e:  # keep going; report the step as failed
+            _check(f"step crashed: {coro}", False, repr(e))
+
+    _hr("SUMMARY")
+    passed = sum(1 for _, ok, _ in _CHECKS if ok)
+    for label, ok, detail in _CHECKS:
+        if not ok:
+            print(f"  FAILED: {label} — {detail}")
+    print(f"  {passed}/{len(_CHECKS)} checks passed")
+    return 0 if passed == len(_CHECKS) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

surfsense_backend/scripts/e2e_google_maps_scraper.py

@@ -0,0 +1,241 @@
+"""Manual functional e2e for the Google Maps scraper (app/proprietary/scrapers/google_maps).
+
+Run from the backend directory:
+    cd surfsense_backend
+    uv run python scripts/e2e_google_maps_scraper.py
+    # or: .\\.venv\\Scripts\\python.exe scripts/e2e_google_maps_scraper.py
+
+NOT a pytest test (needs live network + optional proxy creds). It:
+  Step 1 — scrapes a known place URL and prints the core fields.
+  Step 2 — scrapes the same place by bare placeId (HTML -> fid -> RPC path).
+  Step 3 — dumps the raw place darray (jd[6] of /maps/preview/place) to
+      tests/unit/scrapers/google_maps/fixtures/ for the offline parser test.
+  Step 4 — scrapes reviews via the Reviews endpoint (BOQ feed), checks fields.
+  Step 5 — paginates past one page (maxReviews=15) and checks the count.
+  Step 6 — place scrape with maxReviews>0 attaches inline reviews[].
+  Step 7 — dumps a raw BOQ reviews page fixture for the offline parser test.
+  Step 8 — search discovery (searchStringsArray + locationQuery), checks items.
+  Step 9 — dumps a raw map-search response fixture for the offline parser test.
+"""
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+# Windows consoles default to cp1252; reviews/ratings contain non-latin chars.
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8")
+
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_BACKEND_ROOT))
+for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
+    if _candidate.exists():
+        load_dotenv(_candidate)
+        break
+
+from app.proprietary.scrapers.google_maps import (  # noqa: E402
+    GoogleMapsReviewsInput,
+    GoogleMapsScrapeInput,
+    scrape_places,
+    scrape_reviews,
+)
+from app.proprietary.scrapers.google_maps.fetch import (  # noqa: E402
+    build_search_url,
+    fetch_place_darray,
+    fetch_rpc_json,
+    iter_reviews_pages,
+)
+from app.proprietary.scrapers.google_maps.url_resolver import extract_fid  # noqa: E402
+
+# A well-known, stable place (the restaurant used in the Apify output example).
+_PLACE_URL = (
+    "https://www.google.com/maps/place/Kim's+Island/"
+    "@40.5107736,-74.2482624,17z/data=!4m6!3m5!1s0x89c3ca9c11f90c25:"
+    "0x6cc8dba851799f09!8m2!3d40.5107736!4d-74.2482624!16s%2Fg%2F1tmgdcj8?hl=en"
+)
+
+_FIXTURE_DIR = (
+    _BACKEND_ROOT / "tests" / "unit" / "scrapers" / "google_maps" / "fixtures"
+)
+
+
+def _hr(title: str) -> None:
+    print(f"\n{'=' * 70}\n{title}\n{'=' * 70}")
+
+
+def _check(label: str, ok: bool, detail: str = "") -> bool:
+    print(f"  [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}")
+    return ok
+
+
+async def step1_place() -> bool:
+    _hr("STEP 1 — scrape a known place URL")
+    items = await scrape_places(GoogleMapsScrapeInput(startUrls=[{"url": _PLACE_URL}]))
+    if not items:
+        return _check("place scraped", False, "no items returned")
+    it = items[0]
+    print(json.dumps(it, indent=2, ensure_ascii=False)[:2500])
+    ok = bool(it.get("title")) and it.get("placeId") is not None
+    return _check(
+        "place has title + placeId",
+        ok,
+        f"{it.get('title')!r} / {it.get('totalScore')}★ / {it.get('reviewsCount')} reviews",
+    )
+
+
+async def step2_place_id() -> bool:
+    _hr("STEP 2 — scrape by bare placeId (HTML -> fid -> RPC path)")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"])
+    )
+    if not items:
+        return _check("placeId scraped", False, "no items returned")
+    it = items[0]
+    return _check(
+        "placeId resolves to same place",
+        it.get("title") == "Kim's Island",
+        f"{it.get('title')!r}",
+    )
+
+
+async def step3_dump_fixture() -> bool:
+    _hr("STEP 3 — dump raw place darray fixture for offline test")
+    fid = extract_fid(_PLACE_URL)
+    if not fid:
+        return _check("extracted fid from URL", False)
+    darray = await fetch_place_darray(fid)
+    if not darray:
+        return _check("fetched place darray via RPC", False)
+    _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+    (_FIXTURE_DIR / "place_darray.json").write_text(
+        json.dumps(darray), encoding="utf-8"
+    )
+    return _check("dumped fixture", True, f"-> {_FIXTURE_DIR / 'place_darray.json'}")
+
+
+async def step4_reviews() -> bool:
+    _hr("STEP 4 — reviews endpoint (one page, newest first)")
+    items = await scrape_reviews(
+        GoogleMapsReviewsInput(startUrls=[{"url": _PLACE_URL}], maxReviews=5)
+    )
+    if not items:
+        return _check("reviews returned", False, "no items")
+    it = items[0]
+    print(json.dumps(it, indent=2, ensure_ascii=False)[:1800])
+    ok = (
+        bool(it.get("name"))
+        and it.get("stars") is not None
+        and bool(it.get("reviewId"))
+        and it.get("title") == "Kim's Island"  # place header stamped on
+        and bool(it.get("publishedAtDate"))
+    )
+    return _check(
+        "review has author/stars/id/place header",
+        ok and len(items) == 5,
+        f"{len(items)} reviews, first by {it.get('name')!r} ({it.get('stars')}★)",
+    )
+
+
+async def step5_pagination() -> bool:
+    _hr("STEP 5 — pagination past one page (maxReviews=15)")
+    items = await scrape_reviews(
+        GoogleMapsReviewsInput(placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"], maxReviews=15)
+    )
+    dates = [i.get("publishedAtDate") for i in items[:3]]
+    return _check(
+        "got 15 reviews across >1 page",
+        len(items) == 15,
+        f"{len(items)} reviews; newest: {dates}",
+    )
+
+
+async def step6_inline_reviews() -> bool:
+    _hr("STEP 6 — place scrape with maxReviews=3 attaches inline reviews[]")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(startUrls=[{"url": _PLACE_URL}], maxReviews=3)
+    )
+    if not items:
+        return _check("place scraped", False, "no items")
+    reviews = items[0].get("reviews") or []
+    return _check(
+        "place item carries 3 inline reviews",
+        len(reviews) == 3 and bool(reviews[0].get("name")),
+        f"{len(reviews)} reviews inline",
+    )
+
+
+async def step7_dump_reviews_fixture() -> bool:
+    _hr("STEP 7 — dump raw BOQ reviews page fixture for offline test")
+    fid = extract_fid(_PLACE_URL)
+    async for raw_page in iter_reviews_pages(fid, sort="newest", max_pages=1):
+        _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+        (_FIXTURE_DIR / "boq_reviews_page.json").write_text(
+            json.dumps(raw_page), encoding="utf-8"
+        )
+        return _check(
+            "dumped fixture",
+            len(raw_page) > 0,
+            f"{len(raw_page)} reviews -> {_FIXTURE_DIR / 'boq_reviews_page.json'}",
+        )
+    return _check("fetched a reviews page", False)
+
+
+async def step8_search() -> bool:
+    _hr("STEP 8 — search discovery (query + locationQuery)")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["coffee shop"],
+            locationQuery="Seattle, WA",
+            maxCrawledPlacesPerSearch=5,
+        )
+    )
+    if not items:
+        return _check("search returned items", False, "no items")
+    it = items[0]
+    print(json.dumps(it, indent=2, ensure_ascii=False)[:1500])
+    ok = (
+        len(items) == 5
+        and all(i.get("title") and i.get("placeId") and i.get("fid") for i in items)
+        and [i.get("rank") for i in items] == [1, 2, 3, 4, 5]
+        and it.get("searchString") == "coffee shop"
+    )
+    return _check(
+        "5 ranked places with title/placeId/fid",
+        ok,
+        f"first: {it.get('title')!r} ({it.get('totalScore')}★, {it.get('city')})",
+    )
+
+
+async def step9_dump_search_fixture() -> bool:
+    _hr("STEP 9 — dump raw map-search response fixture for offline test")
+    url = build_search_url("pizza new york")
+    jd = await fetch_rpc_json(url)
+    if not isinstance(jd, list):
+        return _check("fetched search response", False)
+    _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+    (_FIXTURE_DIR / "search_response.json").write_text(json.dumps(jd), encoding="utf-8")
+    return _check("dumped fixture", True, f"-> {_FIXTURE_DIR / 'search_response.json'}")
+
+
+async def main() -> int:
+    results = [
+        await step1_place(),
+        await step2_place_id(),
+        await step3_dump_fixture(),
+        await step4_reviews(),
+        await step5_pagination(),
+        await step6_inline_reviews(),
+        await step7_dump_reviews_fixture(),
+        await step8_search(),
+        await step9_dump_search_fixture(),
+    ]
+    _hr("SUMMARY")
+    print(f"  {sum(results)}/{len(results)} steps passed")
+    return 0 if all(results) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

surfsense_backend/scripts/e2e_phase3_crawl_billing.py

@@ -93,7 +93,7 @@ async def stage1_crawl_and_proxy() -> bool:
     try:
         direct = await AsyncFetcher.get(_IP_ECHO, impersonate="chrome", timeout=30)
         direct_ip = direct.json().get("ip")
-    except Exception as exc:  # noqa: BLE001
+    except Exception as exc:
         print(f"  [INFO] direct IP fetch failed: {exc}")
     if proxy_url:
         try:
@@ -101,7 +101,7 @@ async def stage1_crawl_and_proxy() -> bool:
                 _IP_ECHO, impersonate="chrome", proxy=proxy_url, timeout=45
             )
             proxied_ip = via.json().get("ip")
-        except Exception as exc:  # noqa: BLE001
+        except Exception as exc:
             print(f"  [INFO] proxied IP fetch failed: {exc}")
     print(f"  egress IP (direct)    : {direct_ip}")
     print(f"  egress IP (via proxy) : {proxied_ip}")
@@ -327,7 +327,7 @@ async def main() -> int:
     ):
         try:
             results[name] = await coro()
-        except Exception as exc:  # noqa: BLE001
+        except Exception as exc:
             import traceback
 
             traceback.print_exc()

surfsense_backend/scripts/e2e_youtube_scraper.py

@@ -125,9 +125,7 @@ async def step4_dump_fixtures() -> bool:
         (_FIXTURE_DIR / "video_initial_data.json").write_text(
             json.dumps(initial), encoding="utf-8"
         )
-    return _check(
+    return _check("dumped fixtures", bool(player), f"-> {_FIXTURE_DIR}")
-        "dumped fixtures", bool(player), f"-> {_FIXTURE_DIR}"
-    )
 
 
 async def step5_comments() -> bool:
@@ -137,11 +135,16 @@ async def step5_comments() -> bool:
     )
     items = await scrape_comments(inp)
     for it in items[:6]:
-        print(f"  - [{it.get('type')}] {it.get('author')} | {it.get('voteCount')} votes")
+        print(
+            f"  - [{it.get('type')}] {it.get('author')} | {it.get('voteCount')} votes"
+        )
     ok = bool(items) and all(it.get("cid") and it.get("videoId") for it in items)
     has_reply = any(it.get("type") == "reply" and it.get("replyToCid") for it in items)
-    return _check("comments scraped (cid+videoId, reply linkage)", ok and has_reply,
+    return _check(
-                  f"{len(items)} items")
+        "comments scraped (cid+videoId, reply linkage)",
+        ok and has_reply,
+        f"{len(items)} items",
+    )
 
 
 async def step6_location_collaborators() -> bool:
@@ -165,13 +168,17 @@ async def step6_location_collaborators() -> bool:
 async def step7_translation() -> bool:
     _hr("STEP 7 — translatedTitle/translatedText (subtitlesLanguage=es)")
     items = await scrape_youtube(
-        YouTubeScrapeInput(startUrls=[{"url": _TRANSLATED_VIDEO_URL}], subtitlesLanguage="es")
+        YouTubeScrapeInput(
+            startUrls=[{"url": _TRANSLATED_VIDEO_URL}], subtitlesLanguage="es"
+        )
     )
     it = items[0] if items else {}
     print(f"  title          : {it.get('title')}")
     print(f"  translatedTitle: {it.get('translatedTitle')}")
     # A localized video's translated title differs from the canonical English one.
-    ok = bool(it.get("translatedTitle")) and it.get("translatedTitle") != it.get("title")
+    ok = bool(it.get("translatedTitle")) and it.get("translatedTitle") != it.get(
+        "title"
+    )
     return _check("translatedTitle differs from original", ok)
 
 

surfsense_backend/tests/integration/chat/test_append_message_recovery.py

@@ -46,9 +46,9 @@ from app.db import (
     NewChatMessage,
     NewChatMessageRole,
     NewChatThread,
-    Workspace,
     TokenUsage,
     User,
+    Workspace,
 )
 from app.routes import new_chat_routes
 from app.services.token_tracking_service import TurnTokenAccumulator

surfsense_backend/tests/integration/chat/test_message_id_sse.py

@@ -48,8 +48,8 @@ from app.db import (
     NewChatMessage,
     NewChatMessageRole,
     NewChatThread,
-    Workspace,
     User,
+    Workspace,
 )
 from app.services.new_streaming_service import VercelStreamingService
 from app.tasks.chat import persistence as persistence_module

surfsense_backend/tests/integration/chat/test_persistence.py

@@ -38,9 +38,9 @@ from app.db import (
     NewChatMessage,
     NewChatMessageRole,
     NewChatThread,
-    Workspace,
     TokenUsage,
     User,
+    Workspace,
 )
 from app.services.token_tracking_service import TurnTokenAccumulator
 from app.tasks.chat import persistence as persistence_module

surfsense_backend/tests/integration/chat/test_thread_visibility.py

@@ -19,10 +19,10 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from app.auth.context import AuthContext
 from app.db import (
     ChatVisibility,
+    User,
     Workspace,
     WorkspaceMembership,
     WorkspaceRole,
-    User,
 )
 from app.routes import new_chat_routes
 from app.schemas.new_chat import (

surfsense_backend/tests/integration/composio/test_oauth_callback.py

@@ -9,8 +9,8 @@ from app.config import config
 from app.db import (
     SearchSourceConnector,
     SearchSourceConnectorType,
-    Workspace,
     User,
+    Workspace,
 )
 from app.utils.oauth_security import OAuthStateManager
 
@@ -57,8 +57,7 @@ async def test_callback_with_error_param_redirects_to_denied_page(
     assert response.status_code in {302, 303, 307}
     location = response.headers["location"]
     assert (
-        f"/dashboard/{db_workspace.id}/connectors/callback?"
+        f"/dashboard/{db_workspace.id}/connectors/callback?error=composio_oauth_denied"
-        "error=composio_oauth_denied"
     ) in location
 
     connectors = await _drive_connectors(

surfsense_backend/tests/integration/document_upload/test_document_upload.py

@@ -320,9 +320,7 @@ class TestDocumentSearchability:
         doc_ids = resp.json()["document_ids"]
         cleanup_doc_ids.extend(doc_ids)
 
-        await poll_document_status(
+        await poll_document_status(client, headers, doc_ids, workspace_id=workspace_id)
-            client, headers, doc_ids, workspace_id=workspace_id
-        )
 
         search_resp = await client.get(
             "/api/v1/documents/search",

surfsense_backend/tests/integration/document_upload/test_etl_credits.py

@@ -219,9 +219,7 @@ class TestDocumentProcessingNotification:
         doc_ids = resp.json()["document_ids"]
         cleanup_doc_ids.extend(doc_ids)
 
-        await poll_document_status(
+        await poll_document_status(client, headers, doc_ids, workspace_id=workspace_id)
-            client, headers, doc_ids, workspace_id=workspace_id
-        )
 
         notifications = await get_notifications(
             client,

surfsense_backend/tests/integration/google_unification/conftest.py

@@ -18,8 +18,8 @@ from app.db import (
     DocumentType,
     SearchSourceConnector,
     SearchSourceConnectorType,
-    Workspace,
     User,
+    Workspace,
 )
 
 EMBEDDING_DIM = app_config.embedding_model_instance.dimension
@@ -275,9 +275,7 @@ async def seed_connector(
         session.add(user)
         await session.flush()
 
-        space = Workspace(
+        space = Workspace(name=f"{name_prefix} {uuid.uuid4().hex[:6]}", user_id=user.id)
-            name=f"{name_prefix} {uuid.uuid4().hex[:6]}", user_id=user.id
-        )
         session.add(space)
         await session.flush()
         space_id = space.id

surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py

@@ -119,9 +119,7 @@ async def test_reindex_updates_content(db_session, db_workspace, db_user, mocker
 
 
 @pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
-async def test_reindex_updates_content_hash(
+async def test_reindex_updates_content_hash(db_session, db_workspace, db_user, mocker):
-    db_session, db_workspace, db_user, mocker
-):
     """Content hash is recomputed after reindexing with new source markdown."""
     adapter = UploadDocumentAdapter(db_session)
     await adapter.index(

surfsense_backend/tests/integration/indexing_pipeline/test_index_editions.py

@@ -55,9 +55,7 @@ async def test_edit_keeps_unchanged_rows_and_embeds_only_the_new_text(
     patched_embed_texts,
 ):
     service = IndexingPipelineService(session=db_session)
-    doc_v1 = make_connector_document(
+    doc_v1 = make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1)
-        workspace_id=db_workspace.id, source_markdown=_V1
-    )
     document = await _index(service, doc_v1)
 
     ids_v1 = {c.content: c.id for c in await _load_chunks(db_session, document.id)}
@@ -100,9 +98,7 @@ async def test_head_insert_shifts_positions_without_new_rows_for_old_text(
     service = IndexingPipelineService(session=db_session)
     document = await _index(
         service,
-        make_connector_document(
+        make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1),
-            workspace_id=db_workspace.id, source_markdown=_V1
-        ),
     )
     ids_v1 = {c.content: c.id for c in await _load_chunks(db_session, document.id)}
 
@@ -136,9 +132,7 @@ async def test_removed_paragraph_is_deleted_and_order_compacts(
     service = IndexingPipelineService(session=db_session)
     document = await _index(
         service,
-        make_connector_document(
+        make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1),
-            workspace_id=db_workspace.id, source_markdown=_V1
-        ),
     )
     ids_v1 = {c.content: c.id for c in await _load_chunks(db_session, document.id)}
 
@@ -171,9 +165,7 @@ async def test_kill_switch_falls_back_to_full_replace(
     service = IndexingPipelineService(session=db_session)
     document = await _index(
         service,
-        make_connector_document(
+        make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1),
-            workspace_id=db_workspace.id, source_markdown=_V1
-        ),
     )
     ids_v1 = {c.id for c in await _load_chunks(db_session, document.id)}
 

surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py

@@ -14,8 +14,8 @@ from app.db import (
     DocumentType,
     DocumentVersion,
     Folder,
-    Workspace,
     User,
+    Workspace,
 )
 
 pytestmark = pytest.mark.integration

surfsense_backend/tests/integration/indexing_pipeline/test_migrate_legacy_docs.py

@@ -59,9 +59,7 @@ async def test_legacy_composio_gmail_doc_migrated_in_db(
     assert reloaded.document_type == DocumentType.GOOGLE_GMAIL_CONNECTOR
 
 
-async def test_no_legacy_doc_is_noop(
+async def test_no_legacy_doc_is_noop(db_session, db_workspace, make_connector_document):
-    db_session, db_workspace, make_connector_document
-):
     """When no legacy document exists, migrate_legacy_docs does nothing."""
     connector_doc = make_connector_document(
         document_type=DocumentType.GOOGLE_CALENDAR_CONNECTOR,

surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py

@@ -225,9 +225,7 @@ async def test_updated_at_advances_when_title_only_changes(
     db_session, db_workspace, make_connector_document
 ):
     """updated_at advances even when only the title changes."""
-    original = make_connector_document(
+    original = make_connector_document(workspace_id=db_workspace.id, title="Old Title")
-        workspace_id=db_workspace.id, title="Old Title"
-    )
     service = IndexingPipelineService(session=db_session)
 
     first = await service.prepare_for_indexing([original])
@@ -238,9 +236,7 @@ async def test_updated_at_advances_when_title_only_changes(
     )
     updated_at_v1 = result.scalars().first().updated_at
 
-    renamed = make_connector_document(
+    renamed = make_connector_document(workspace_id=db_workspace.id, title="New Title")
-        workspace_id=db_workspace.id, title="New Title"
-    )
     await service.prepare_for_indexing([renamed])
 
     result = await db_session.execute(

surfsense_backend/tests/integration/notifications/test_base_handler.py

@@ -10,7 +10,7 @@ import pytest
 from sqlalchemy import func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.db import Workspace, User
+from app.db import User, Workspace
 from app.notifications.persistence import Notification
 from app.notifications.service import NotificationService
 

surfsense_backend/tests/integration/notifications/test_comment_reply_handler.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import pytest
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.db import Workspace, User
+from app.db import User, Workspace
 from app.notifications.service import NotificationService
 
 pytestmark = pytest.mark.integration
@@ -47,9 +47,7 @@ async def test_comment_reply_truncates_long_preview(
     db_session: AsyncSession, db_user: User, db_workspace: Workspace
 ):
     """A long comment preview is truncated in the reply message."""
-    notification = await _notify(
+    notification = await _notify(db_session, db_user, db_workspace, preview="y" * 150)
-        db_session, db_user, db_workspace, preview="y" * 150
-    )
 
     assert notification.message == "y" * 100 + "..."
 

surfsense_backend/tests/integration/notifications/test_connector_indexing_handler.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 import pytest
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.db import Workspace, User
+from app.db import User, Workspace
 from app.notifications.service import NotificationService
 
 pytestmark = pytest.mark.integration

surfsense_backend/tests/integration/notifications/test_document_processing_handler.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import pytest
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.db import Workspace, User
+from app.db import User, Workspace
 from app.notifications.service import NotificationService
 
 pytestmark = pytest.mark.integration

surfsense_backend/tests/integration/notifications/test_insufficient_credits_handler.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import pytest
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.db import Workspace, User
+from app.db import User, Workspace
 from app.notifications.service import NotificationService
 
 pytestmark = pytest.mark.integration

surfsense_backend/tests/integration/notifications/test_mention_handler.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import pytest
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.db import Workspace, User
+from app.db import User, Workspace
 from app.notifications.service import NotificationService
 
 pytestmark = pytest.mark.integration
@@ -47,9 +47,7 @@ async def test_new_mention_truncates_long_preview(
     db_session: AsyncSession, db_user: User, db_workspace: Workspace
 ):
     """A long comment preview is truncated in the mention message."""
-    notification = await _notify(
+    notification = await _notify(db_session, db_user, db_workspace, preview="x" * 150)
-        db_session, db_user, db_workspace, preview="x" * 150
-    )
 
     assert notification.message == "x" * 100 + "..."
 

surfsense_backend/tests/integration/podcasts/conftest.py

@@ -26,7 +26,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from app.app import app, limiter
 from app.auth.context import AuthContext
 from app.config import config as app_config
-from app.db import Workspace, User, get_async_session
+from app.db import User, Workspace, get_async_session
 from app.podcasts.persistence import Podcast, PodcastStatus
 from app.podcasts.schemas import (
     DurationTarget,

surfsense_backend/tests/integration/podcasts/test_cancel.py

@@ -38,9 +38,7 @@ async def test_cancel_from_a_terminal_state_conflicts(
     assert resp.status_code == 409
 
 
-async def test_cancel_of_a_regeneration_is_rejected(
+async def test_cancel_of_a_regeneration_is_rejected(client, db_workspace, make_podcast):
-    client, db_workspace, make_podcast
-):
     # Cancelling here would destroy a playable episode; reverting the
     # regeneration is the way back.
     podcast = await make_podcast(

surfsense_backend/tests/integration/retriever/conftest.py

@@ -9,7 +9,7 @@ import pytest_asyncio
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.config import config as app_config
-from app.db import Chunk, Document, DocumentType, Workspace, User
+from app.db import Chunk, Document, DocumentType, User, Workspace
 
 EMBEDDING_DIM = app_config.embedding_model_instance.dimension
 DUMMY_EMBEDDING = [0.1] * EMBEDDING_DIM

surfsense_backend/tests/integration/test_connector_index_authz.py

@@ -27,8 +27,8 @@ from app.auth.context import AuthContext
 from app.db import (
     SearchSourceConnector,
     SearchSourceConnectorType,
-    Workspace,
     User,
+    Workspace,
 )
 from app.routes.search_source_connectors_routes import index_connector_content
 from app.routes.workspaces_routes import create_default_roles_and_membership

surfsense_backend/tests/integration/test_document_versioning.py

@@ -7,7 +7,7 @@ import pytest_asyncio
 from sqlalchemy import func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.db import Document, DocumentType, DocumentVersion, Workspace, User
+from app.db import Document, DocumentType, DocumentVersion, User, Workspace
 
 pytestmark = pytest.mark.integration
 

surfsense_backend/tests/integration/test_obsidian_plugin_routes.py

@@ -32,8 +32,8 @@ from app.auth.context import AuthContext
 from app.db import (
     SearchSourceConnector,
     SearchSourceConnectorType,
-    Workspace,
     User,
+    Workspace,
 )
 from app.routes.obsidian_plugin_routes import (
     obsidian_connect,

surfsense_backend/tests/integration/test_pat_fail_closed_authz.py

@@ -7,7 +7,7 @@ from fastapi import HTTPException
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.auth.context import AuthContext
-from app.db import PersonalAccessToken, Workspace, User
+from app.db import PersonalAccessToken, User, Workspace
 from app.users import allow_any_principal, require_session_context
 from app.utils.rbac import check_workspace_access
 

surfsense_backend/tests/integration/test_zero_authz_context.py

@@ -7,7 +7,7 @@ from fastapi import HTTPException
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.auth.context import AuthContext
-from app.db import PersonalAccessToken, Workspace, User
+from app.db import PersonalAccessToken, User, Workspace
 from app.routes.workspaces_routes import create_default_roles_and_membership
 from app.utils.rbac import check_workspace_access, get_allowed_read_space_ids
 

surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py

@@ -158,9 +158,7 @@ async def test_build_dependencies_falls_back_to_workspace(
     )
 
     workspace = SimpleNamespace(chat_model_id=-7)
-    result = await build_dependencies(
+    result = await build_dependencies(session=_FakeSession(workspace), workspace_id=42)
-        session=_FakeSession(workspace), workspace_id=42
-    )
 
     assert captured == {"config_id": -7}
     assert result.llm.name == "llm"