diff --git a/surfsense_backend/alembic/versions/168_harden_refresh_token_schema.py b/surfsense_backend/alembic/versions/168_harden_refresh_token_schema.py index 75c2725c3..ebea4eaf5 100644 --- a/surfsense_backend/alembic/versions/168_harden_refresh_token_schema.py +++ b/surfsense_backend/alembic/versions/168_harden_refresh_token_schema.py @@ -6,6 +6,8 @@ Revises: 167 from collections.abc import Sequence +import sqlalchemy as sa + from alembic import op revision: str = "168" @@ -48,9 +50,7 @@ def upgrade() -> None: """ ) - op.execute( - "ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(64)" - ) + op.execute("ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(64)") op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS is_revoked") @@ -74,8 +74,6 @@ def downgrade() -> None: """ ) op.execute("ALTER TABLE refresh_tokens ALTER COLUMN is_revoked DROP DEFAULT") - op.execute( - "ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(256)" - ) + op.execute("ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(256)") op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS absolute_expiry") op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS revoked_at") diff --git a/surfsense_backend/alembic/versions/170_rename_searchspace_to_workspace.py b/surfsense_backend/alembic/versions/170_rename_searchspace_to_workspace.py index 2815f9603..265243bb9 100644 --- a/surfsense_backend/alembic/versions/170_rename_searchspace_to_workspace.py +++ b/surfsense_backend/alembic/versions/170_rename_searchspace_to_workspace.py @@ -373,7 +373,9 @@ def _neutralize_column_list_tables(conn) -> None: for table in PUBLICATION_COLUMN_LIST_TABLES: if _is_publication_member(conn, table): conn.execute( - sa.text(f"ALTER PUBLICATION {_quote(PUBLICATION_NAME)} DROP TABLE {table}") + sa.text( + f"ALTER PUBLICATION {_quote(PUBLICATION_NAME)} DROP TABLE {table}" + ) ) @@ -440,7 +442,9 @@ def _restore_downgrade_publication(conn) -> None: for table in PUBLICATION_COLUMN_LIST_TABLES: if _is_publication_member(conn, table): conn.execute( - sa.text(f"ALTER PUBLICATION {_quote(PUBLICATION_NAME)} DROP TABLE {table}") + sa.text( + f"ALTER PUBLICATION {_quote(PUBLICATION_NAME)} DROP TABLE {table}" + ) ) entries: list[str] = [] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/middleware.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/middleware.py index 0554dc747..7d6694523 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/middleware.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/middleware.py @@ -111,9 +111,7 @@ async def _ensure_folder_hierarchy( sibling_query = ( select(Folder.position).order_by(Folder.position.desc()).limit(1) ) - sibling_query = sibling_query.where( - Folder.workspace_id == workspace_id - ) + sibling_query = sibling_query.where(Folder.workspace_id == workspace_id) if parent_id is None: sibling_query = sibling_query.where(Folder.parent_id.is_(None)) else: @@ -1241,9 +1239,7 @@ async def commit_staged_filesystem_state( await session.commit() except Exception: # pragma: no cover - rollback safety net - logger.exception( - "kb_persistence: commit failed (workspace=%s)", workspace_id - ) + logger.exception("kb_persistence: commit failed (workspace=%s)", workspace_id) # Outer commit raised: everything above rolled back, so drop the # deferred dispatches. deferred_dispatches.clear() diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/memory/middleware.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/memory/middleware.py index 98ead2dd7..7b4e9d369 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/memory/middleware.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/memory/middleware.py @@ -18,7 +18,7 @@ from langgraph.runtime import Runtime from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.db import ChatVisibility, Workspace, User, shielded_async_session +from app.db import ChatVisibility, User, Workspace, shielded_async_session from app.services.memory import MEMORY_HARD_LIMIT, MEMORY_SOFT_LIMIT from app.utils.perf import get_perf_logger diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/factory.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/factory.py index 91834ff25..331ff363f 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/factory.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/factory.py @@ -98,9 +98,7 @@ async def create_multi_agent_chat_deep_agent( _t0 = time.perf_counter() try: - connector_types = await connector_service.get_available_connectors( - workspace_id - ) + connector_types = await connector_service.get_available_connectors(workspace_id) available_connectors = map_connectors_to_searchable_types(connector_types) available_document_types = await connector_service.get_available_document_types( @@ -153,9 +151,7 @@ async def create_multi_agent_chat_deep_agent( _t0 = time.perf_counter() try: - mcp_tools_by_agent = await load_mcp_tools_by_connector( - db_session, workspace_id - ) + mcp_tools_by_agent = await load_mcp_tools_by_connector(db_session, workspace_id) except Exception as e: # Degrade to builtins-only rather than aborting the turn: a transient # DB or MCP-server hiccup should not deny the user a response. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/generate_image.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/generate_image.py index f9158335b..6ab48d815 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/generate_image.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/generate_image.py @@ -116,9 +116,7 @@ def create_generate_image_tool( # later workspace changes. No workspace read needed. config_id = image_gen_model_id_override or IMAGE_GEN_AUTO_MODE_ID else: - config_id = ( - workspace.image_gen_model_id or IMAGE_GEN_AUTO_MODE_ID - ) + config_id = workspace.image_gen_model_id or IMAGE_GEN_AUTO_MODE_ID # size/quality/style are intentionally omitted: valid values # differ per model, so we let each model use its own defaults. @@ -191,10 +189,7 @@ def create_generate_image_tool( ): err = f"Image generation model {config_id} not found" return _failed({"error": err}, error=err) - if ( - conn.user_id is not None - and conn.user_id != workspace.user_id - ): + if conn.user_id is not None and conn.user_id != workspace.user_id: err = f"Image generation model {config_id} not found" return _failed({"error": err}, error=err) if not has_capability(db_model, "image_gen"): diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/create_event.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/create_event.py index d233dd16d..a4286e1cb 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/create_event.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/create_event.py @@ -70,9 +70,7 @@ def create_create_calendar_event_tool( try: metadata_service = GoogleCalendarToolMetadataService(db_session) - context = await metadata_service.get_creation_context( - workspace_id, user_id - ) + context = await metadata_service.get_creation_context(workspace_id, user_id) if "error" in context: logger.error(f"Failed to fetch creation context: {context['error']}") diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/create_page.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/create_page.py index 7108f0168..aa270f98e 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/create_page.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/create_page.py @@ -52,9 +52,7 @@ def create_create_confluence_page_tool( try: metadata_service = ConfluenceToolMetadataService(db_session) - context = await metadata_service.get_creation_context( - workspace_id, user_id - ) + context = await metadata_service.get_creation_context(workspace_id, user_id) if "error" in context: return {"status": "error", "message": context["error"]} diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/list_channels.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/list_channels.py index 68c6b07a0..1f66127fd 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/list_channels.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/list_channels.py @@ -29,9 +29,7 @@ def create_list_discord_channels_tool( } try: - connector = await get_discord_connector( - db_session, workspace_id, user_id - ) + connector = await get_discord_connector(db_session, workspace_id, user_id) if not connector: return {"status": "error", "message": "No Discord connector found."} diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/read_messages.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/read_messages.py index 200289996..e24d65d81 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/read_messages.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/read_messages.py @@ -39,9 +39,7 @@ def create_read_discord_messages_tool( limit = min(limit, 50) try: - connector = await get_discord_connector( - db_session, workspace_id, user_id - ) + connector = await get_discord_connector(db_session, workspace_id, user_id) if not connector: return {"status": "error", "message": "No Discord connector found."} diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/send_message.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/send_message.py index d6d6bf1bc..4a909c8ad 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/send_message.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/send_message.py @@ -49,9 +49,7 @@ def create_send_discord_message_tool( } try: - connector = await get_discord_connector( - db_session, workspace_id, user_id - ) + connector = await get_discord_connector(db_session, workspace_id, user_id) if not connector: return {"status": "error", "message": "No Discord connector found."} diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/create_draft.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/create_draft.py index cd3f9a67e..e736607e5 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/create_draft.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/create_draft.py @@ -67,9 +67,7 @@ def create_create_gmail_draft_tool( try: metadata_service = GmailToolMetadataService(db_session) - context = await metadata_service.get_creation_context( - workspace_id, user_id - ) + context = await metadata_service.get_creation_context(workspace_id, user_id) if "error" in context: logger.error(f"Failed to fetch creation context: {context['error']}") diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/send_email.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/send_email.py index 89af473b7..d3c4b64bb 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/send_email.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/send_email.py @@ -96,9 +96,7 @@ def create_send_gmail_email_tool( try: metadata_service = GmailToolMetadataService(db_session) - context = await metadata_service.get_creation_context( - workspace_id, user_id - ) + context = await metadata_service.get_creation_context(workspace_id, user_id) if "error" in context: logger.error(f"Failed to fetch creation context: {context['error']}") diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/create_file.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/create_file.py index 24a2d1685..ad712e5bd 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/create_file.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/create_file.py @@ -81,9 +81,7 @@ def create_create_google_drive_file_tool( try: metadata_service = GoogleDriveToolMetadataService(db_session) - context = await metadata_service.get_creation_context( - workspace_id, user_id - ) + context = await metadata_service.get_creation_context(workspace_id, user_id) if "error" in context: logger.error(f"Failed to fetch creation context: {context['error']}") diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/create_page.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/create_page.py index 57f14a849..05eff9e73 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/create_page.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/create_page.py @@ -80,9 +80,7 @@ def create_create_notion_page_tool( try: metadata_service = NotionToolMetadataService(db_session) - context = await metadata_service.get_creation_context( - workspace_id, user_id - ) + context = await metadata_service.get_creation_context(workspace_id, user_id) if "error" in context: logger.error(f"Failed to fetch creation context: {context['error']}") diff --git a/surfsense_backend/app/automations/services/automation.py b/surfsense_backend/app/automations/services/automation.py index eeb5b61bd..a419327bf 100644 --- a/surfsense_backend/app/automations/services/automation.py +++ b/surfsense_backend/app/automations/services/automation.py @@ -43,9 +43,7 @@ class AutomationService: async def create(self, payload: AutomationCreate) -> Automation: """Create an automation and its initial triggers in one transaction.""" - await self._authorize( - payload.workspace_id, Permission.AUTOMATIONS_CREATE.value - ) + await self._authorize(payload.workspace_id, Permission.AUTOMATIONS_CREATE.value) # Capture the model profile onto the definition so runs are insulated # from later chat/workspace model changes. Two sources: diff --git a/surfsense_backend/app/capabilities/youtube/scrape/schemas.py b/surfsense_backend/app/capabilities/youtube/scrape/schemas.py index 3c1f35ef1..f24e19828 100644 --- a/surfsense_backend/app/capabilities/youtube/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/youtube/scrape/schemas.py @@ -1,7 +1,7 @@ """``youtube.scrape`` I/O contracts. A lean, agent-friendly surface over the Apify-compatible ``YouTubeScrapeInput`` -(``app/proprietary/scrapers/youtube``). The executor maps this to the full actor +(``app/proprietary/platforms/youtube``). The executor maps this to the full actor input; the actor's ``VideoItem`` is reused verbatim as the output element so the verb stays parity-faithful with the underlying Apify shape. """ diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index bacfc3c38..b0d37ea39 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -1079,9 +1079,7 @@ class Config: CAPTCHA_SOLVER_PROVIDER = os.getenv("CAPTCHA_SOLVER_PROVIDER", "capsolver") CAPTCHA_SOLVER_API_KEY = os.getenv("CAPTCHA_SOLVER_API_KEY") # Per-URL solve cap so one hostile page can't burn unbounded solver credit. - CAPTCHA_MAX_ATTEMPTS_PER_URL = int( - os.getenv("CAPTCHA_MAX_ATTEMPTS_PER_URL", "1") - ) + CAPTCHA_MAX_ATTEMPTS_PER_URL = int(os.getenv("CAPTCHA_MAX_ATTEMPTS_PER_URL", "1")) # Abort a single solve after this many seconds (solves take 10-60s). CAPTCHA_SOLVE_TIMEOUT_S = int(os.getenv("CAPTCHA_SOLVE_TIMEOUT_S", "120")) # Default captcha type when detection is ambiguous: v2 | v3 | hcaptcha. @@ -1118,9 +1116,7 @@ class Config: # Route DNS via Cloudflare DoH (anti DNS-leak behind proxies). Adds a DNS # round-trip => default FALSE to honor the "no speed regression" bar; flip on # when leak-safety outweighs the marginal latency. - CRAWL_DNS_OVER_HTTPS = ( - os.getenv("CRAWL_DNS_OVER_HTTPS", "FALSE").upper() == "TRUE" - ) + CRAWL_DNS_OVER_HTTPS = os.getenv("CRAWL_DNS_OVER_HTTPS", "FALSE").upper() == "TRUE" # Litellm TTS Configuration TTS_SERVICE = os.getenv("TTS_SERVICE") diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 365e56c8e..fd64b43ea 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -823,9 +823,7 @@ class ExternalChatAccount(Base, TimestampMixin): ) owner = relationship("User", foreign_keys=[owner_user_id]) - owner_workspace = relationship( - "Workspace", foreign_keys=[owner_workspace_id] - ) + owner_workspace = relationship("Workspace", foreign_keys=[owner_workspace_id]) bindings = relationship( "ExternalChatBinding", back_populates="account", @@ -984,9 +982,7 @@ class ExternalChatBinding(Base, TimestampMixin): postgresql_where=text("state = 'pending'"), ), Index("ix_external_chat_bindings_user_state", "user_id", "state"), - Index( - "ix_external_chat_bindings_workspace_state", "workspace_id", "state" - ), + Index("ix_external_chat_bindings_workspace_state", "workspace_id", "state"), ) @@ -1902,9 +1898,7 @@ class SearchSourceConnector(BaseModel, TimestampMixin): ForeignKey("workspaces.id", ondelete="CASCADE"), nullable=False, ) - workspace = relationship( - "Workspace", back_populates="search_source_connectors" - ) + workspace = relationship("Workspace", back_populates="search_source_connectors") user_id = Column( UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False diff --git a/surfsense_backend/app/podcasts/generation/transcript/nodes.py b/surfsense_backend/app/podcasts/generation/transcript/nodes.py index 6bbbb7777..df8df7d2c 100644 --- a/surfsense_backend/app/podcasts/generation/transcript/nodes.py +++ b/surfsense_backend/app/podcasts/generation/transcript/nodes.py @@ -99,9 +99,7 @@ def finalize(state: TranscriptState, config: RunnableConfig) -> dict[str, Any]: async def _require_llm(state: TranscriptState, tc: TranscriptConfig): llm = await get_agent_llm(state.db_session, tc.workspace_id) if llm is None: - raise RuntimeError( - f"no agent LLM configured for workspace {tc.workspace_id}" - ) + raise RuntimeError(f"no agent LLM configured for workspace {tc.workspace_id}") return llm diff --git a/surfsense_backend/app/proprietary/platforms/google_maps/README.md b/surfsense_backend/app/proprietary/platforms/google_maps/README.md new file mode 100644 index 000000000..9e438181f --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_maps/README.md @@ -0,0 +1,341 @@ +# Google Maps Scraper + +A platform-native Google Maps scraper intended as a **drop-in clone of the +Apify "Google Maps Scraper" and "Google Maps Reviews Scraper" actors** — same +input surface, same output item shape. Built on the same layout and +progressive-implementation approach as the sibling `../youtube` scraper. + +**Current status: search + place details + reviews live.** Search terms +(`searchStringsArray` x `locationQuery`/geolocation) are discovered via the +public `search?tbm=map` RPC with offset paging and the Apify result filters. +Direct place URLs and bare `placeIds` are scraped via Google's public +`/maps/preview/place` RPC; reviews (both the standalone Reviews endpoint and +inline `reviews[]` on places when `maxReviews > 0`) come from the public +`GetLocalBoqProxy` review feed with full token-based pagination. No login, +proxy-only egress. + +## Public vs. gated data + +All core Maps data is **public — no Google account needed**: search results, +place details (phone, website, hours, price, coordinates, plus code, address +components), and reviews (text, ratings, reviewer profiles, owner responses) +are served by Google's internal endpoints (`/maps/preview/place`, +`GetLocalBoqProxy`, `search?tbm=map`) which return XSSI-guarded JSON to +anonymous requests. Only a residential proxy is required (Google blocks +datacenter IPs) — already wired via `app/utils/proxy`. + +**Session-gated (but still login-free) fields:** Google trims the rich detail +fields — `reviewsCount`, `reviewsDistribution`, popular times, image +galleries, `reviewsTags`, and most `additionalInfo` sections — from responses +that carry **no session cookie**. A plain GET to `/maps` mints an `NID` cookie +that unlocks all of them (no login, no browser needed at runtime; +`fetch.get_session_cookies` mints them into a small rotating pool with a +30-minute TTL). The full-page `pb` selector these fields also require was +captured once from a headless browser render of a place page and genericized +into `_PLACE_DETAIL_PB`. + +> Note: the older review RPCs other scrapers documented (`listugcposts`, +> `listentitiesreviews`) now return empty pages / 404 to anonymous callers — +> they appear to require a signed-in session. The `GetLocalBoqProxy` feed (used +> by Google's own search local panel) is the one that still works publicly and +> is what `fetch.iter_reviews_pages` uses. + +If Google ever serves a sign-in/consent wall instead of data, +`SignInRequiredError` is raised and the route returns **403 "Google sign in +required"** rather than an empty result. + +Not sourced from Maps at all (Apify enrichment add-ons that hit third-party +data brokers / the business's own website — out of scope for public-Maps-only): +`maximumLeadsEnrichmentRecords`, `leadsEnrichmentDepartments`, +`verifyLeadsEnrichmentEmails`, `scrapeSocialMediaProfiles`, `scrapeContacts`, +`enableCompetitorAnalysis`. These stay at their schema defaults (`None`/`[]`). + +## Quick start + +```python +from app.proprietary.scrapers.google_maps import ( + GoogleMapsScrapeInput, scrape_places, + GoogleMapsReviewsInput, scrape_reviews, +) + +# Places — search terms, direct URLs, and placeIds are all additive. +# maxReviews > 0 also attaches inline reviews[] to each place item. +places = await scrape_places( + GoogleMapsScrapeInput( + searchStringsArray=["coffee shop"], + locationQuery="Seattle, WA", + maxCrawledPlacesPerSearch=20, + startUrls=[{"url": "https://www.google.com/maps/place/..."}], + placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"], + maxReviews=10, + ) +) + +# Reviews — one flat item per review (review fields + place header fields) +reviews = await scrape_reviews( + GoogleMapsReviewsInput( + placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"], + maxReviews=100, + reviewsSort="newest", + ) +) +``` + +Both have a streaming twin — `iter_places()` / `iter_reviews()`. The HTTP +surface lives in `app/routes/google_maps_routes.py` (`POST /google-maps/scrape` +and `POST /google-maps/reviews`). + +## Module map + +| File | Responsibility | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------ | +| `__init__.py` | Public exports (entry points + schemas). | +| `schemas.py` | Pydantic input/output models mirroring the Apify camelCase specs. `extra="allow"` on outputs keeps the contract open. | +| `scraper.py` | Places orchestrator. `_place_flow` (live): fid → detail RPC → `PlaceItem` (+ inline reviews). `_search_flow` (live): tbm=map paging + filters. | +| `reviews.py` | Reviews orchestrator (live). Pages the BOQ feed per place, applies sort/date-cutoff/origin/personal-data rules, one flat item per review. | +| `fetch.py` | Proxy-only fetch seam: `fetch_html`, `fetch_rpc_json` (XSSI-strip + HTML-wrapper tolerant), `fetch_place_darray`, `iter_reviews_pages`, `iter_search_pages`. | +| `parsers.py` | Pure array-path parsing: place `darray` → PlaceItem fields (paths from gosom), BOQ review arrays → ReviewFields dicts. | +| `url_resolver.py` | Classify a URL into `place` / `search` / `reviews` / `cid` / `shortlink`; extract the feature ID (`0x..:0x..`). | + +## How place scraping works + +1. Resolve the **feature ID** (`0x:0x`): from the URL's `data=!1s…` + blob when present, else fetch the place page once and regex it from HTML + (covers `?q=place_id:ChIJ…` and `?cid=…`). Name-only URLs + (`/maps/place/Eiffel+Tower/`) serve a JS shell with no fid in the HTML, so + those fall back to a `search?tbm=map` lookup of the name + (`fetch.fid_from_search`) and take the top result's fid. Short links + (`maps.app.goo.gl` / `goo.gl`) are Firebase Dynamic Links — a plain GET only + returns a JS interstitial, so they're browser-rendered so the JS follows the + redirect to the real `/maps/place/…` URL, then the fid is read from that. +2. GET `/maps/preview/place?…&pb=!1m13!1s{fid}…` through the proxy, with the + NID session cookie. The `pb` is the same selector the Maps web app sends + (browser-captured, genericized), so the response is the **full** payload + including the session-gated fields. `)]}'`-guarded JSON; `jd[6]` is the + place detail array ("darray") that all field paths index into. +3. `parse_place()` maps darray positions → Apify-shaped fields (title, + categories, address components, phone, website, rating, plus code, hours, + coordinates, placeId…). Only keys whose path hit are set; the rest keep + schema defaults. + +Detail extras parsed from the same darray (no extra requests): + +- `kgmid` (`darray[89]`, e.g. `/g/11bw4ws2mt`) and `cid` — the CID is just the + decimal value of the fid's second hex half, so it's derived, not fetched. +- `additionalInfo` — all about sections at `darray[100][1]` in Apify's shape + (`{"Accessibility": [{"Wheelchair accessible entrance": true}, …]}`). +- `reviewsCount` (`[4][8]`) + `reviewsDistribution` (`[175][3]`). +- `popularTimesHistogram` / `popularTimesLiveText` / `popularTimesLivePercent` + (`darray[84]`). +- `imagesCount` (`[37][1]`), `imageCategories` (gallery tab names at + `[171][0]`), `imageUrls` (hero photos + tab thumbnails; emitted only when + `maxImages > 0`, capped at it). Full multi-thousand-photo galleries would + need the signed-in photo-listing RPC, so `imageUrls` tops out around a + dozen per place. +- `reviewsTags` (`[153][0]` → `[{title, count}]`). +- `tableReservationLinks` + `reserveTableUrl` (`darray[46]`), `orderBy` / + `googleFoodUrl` (order-online providers at `darray[75]`), `menu` + (`darray[38][0]`). + +Hotel places (detected by the star string at `darray[35][6]`) additionally +get: `hotelStars`, `hotelDescription`, `checkInDate`/`checkOutDate` (the +dates Google quoted prices for, `[35][0..1]`), `similarHotelsNearby` +(`[35][29][0]` — title/fid/score/count/location/description), and `hotelAds` +(`[35][44]` — booking-partner title/url/price). Probed live on The Plaza NYC. + +Search-result darrays are served **without** the session-gated fields (the +NID cookie doesn't help `search?tbm=map` — verified live), so when +`scrapePlaceDetailPage=true` or `maxImages > 0` the search flow makes one +detail RPC per emitted place and merges the full payload over the search +fields — same trade Apify makes (search-only is one request per ~20 places; +detail adds one per place). + +**`allPlacesNoSearchAction`** (area scan with no search term): Apify's +implementation OCRs / mouse-overs rendered map pins — the public RPC has no +"list everything" query (`*` and empty return nothing). Ours approximates the +scan with a broad category sweep (`restaurant`, `store`, `hotel`, …17 terms) +over the requested viewport, deduped by fid, until +`maxCrawledPlacesPerSearch`. `searchString` carries the action value on scan +items. Verified live: 25 unique places / 14 distinct categories from a 400 m +Times Square viewport. + +## How search discovery works + +1. Build the query: `"{search term} in {location}"` when `locationQuery` (or + the city/state/... geolocation fields) is set — Google localizes from the + query text, so no geocoding round-trip is needed. A `customGeolocation` + GeoJSON Point instead sets a real lat/lng/radius viewport. +2. Page `search?tbm=map` (~20 results per page, `!8i` offset). Each result + entry embeds a **full place darray** at `entry[14]` — same shape as the + detail RPC — so `parse_place()` runs directly on it and no per-place detail + request is needed. Single-match queries put the one place in slot 0 of the + results list; multi-result pages start at slot 1 (both handled). +3. Dedupe by fid (Google reshuffles results between pages), stamp + `rank`/`searchString`/`searchPageUrl`, apply the Apify filters client-side + (`searchMatching`, `categoryFilterWords`, `placeMinimumStars`, `website`, + `skipClosedPlaces`), and stop at `maxCrawledPlacesPerSearch`, an empty + page, or a page with no new fids. Verified live to 60 unique results + (3+ pages) with strictly sequential ranks, and to terminate on its own for + sparse queries that exhaust before the cap. + +Closed places: `darray[88][0]` carries a status enum (`CLOSED` verified live +on a permanently-closed place) that maps to `permanentlyClosed`. + +## Performance + +Every RPC is a ~2s proxy round-trip, so wall-clock time is dominated by how +many requests run in series, not by parsing. Independent requests are +overlapped (bounded, order preserved via `gather_bounded`): + +- **Detail enrichment** (`scrapePlaceDetailPage` / `maxImages`) and inline + reviews fire concurrently across a search page instead of one place at a + time — the biggest lever (a 20-place enriched search went from ~50s to ~10s, + `_DETAIL_CONCURRENCY=8`). +- **Search pages** are prefetched in waves sized from the result cap + (`_prefetch_for`) so a 60-result / 3-page search overlaps its pages. +- **Bulk `placeIds`** are fetched in parallel. +- **NID mint** coalesces concurrent cold callers onto one in-flight request + (the pool lock isn't held across the network call), so a burst of parallel + detail fetches doesn't serialize behind N sequential ~2s mints. + +Reviews within a single place stay sequential — pagination is +continuation-token based (each token embeds the previous page's last-review +key plus a signature, so page N+1's token can't be forged or precomputed). +The only lever there is page size: the feed is asked for the ~60/page ceiling +(`_REVIEWS_PAGE_SIZE`) instead of the old 10/20, cutting the sequential +round-trips ~3x (1000 reviews: ~113s → ~40s). + +## How review scraping works + +1. Resolve the feature ID exactly as above (reviews accept the same + `startUrls` / `placeIds` inputs). +2. Fetch the place detail once for the header fields (title, address, + location, …) that Apify stamps onto every review item. +3. Page `GetLocalBoqProxy` (~60 reviews per page — the feed's ceiling, asked + via `_REVIEWS_PAGE_SIZE`; opaque continuation token in `node[6]`) until + `maxReviews`, the `reviewsStartDate` cutoff (newest-first), or the feed is + exhausted. Sort modes map to codes 1–4 + (mostRelevant/newest/highestRanking/lowestRanking). +4. `parse_review()` maps each ~48-slot review array → ReviewFields (author, + stars, text, ISO publish date, images, owner response, guided + context/per-aspect ratings, origin). `personalData=false` strips reviewer + name/id/URL/photo (reviewId stays), per Apify semantics. + +## API spec + +Mirrors the Apify "Google Maps Scraper" and "Google Maps Reviews Scraper" +actors (camelCase, `extra="allow"`). Unknown inputs are accepted, unsourced +outputs come back as `None`/`[]`/`{}` — parity grows without breaking +consumers. `schemas.py` is the source of truth; the tables below list what the +implementation actually **sources** vs. still **stubs**. + +### Places — input (`GoogleMapsScrapeInput`) + +| Field | Type / default | Status | Notes | +|---|---|---|---| +| `searchStringsArray` | `list[str]` `[]` | ✅ | search-term discovery; additive with the others | +| `startUrls` | `list[{url}]` `[]` | ✅ | place / CID / short / search / reviews URLs | +| `placeIds` | `list[str]` `[]` | ✅ | bare `ChIJ…` ids (fetched in parallel) | +| `allPlacesNoSearchAction` | enum `""` | ✅* | area scan via broad-category sweep (approximation) | +| `locationQuery` | `str` | ✅ | e.g. `"San Jose, CA"`, appended as `"{q} in {loc}"` | +| `city`/`state`/`county`/`postalCode`/`countryCode` | `str` | ✅ | discrete location parts (alt. to `locationQuery`) | +| `customGeolocation` | GeoJSON `Point`+`radiusKm` | ✅ | real lat/lng/radius viewport | +| `maxCrawledPlacesPerSearch` | `int\|None` | ✅ | `None` = all; caps emitted places | +| `language` | `str` `"en"` | ✅ | `hl=` on every RPC | +| `searchMatching` | `all\|only_includes\|only_exact` | ✅ | title-match filter | +| `categoryFilterWords` | `list[str]` `[]` | ✅ | category client-filter | +| `placeMinimumStars` | enum `""` | ✅ | min `totalScore` filter | +| `website` | `allPlaces\|withWebsite\|withoutWebsite` | ✅ | website-presence filter | +| `skipClosedPlaces` | `bool` `false` | ✅ | drops permanently/temporarily closed | +| `scrapePlaceDetailPage` | `bool` `false` | ✅ | adds detail-RPC extras (see below) | +| `maxReviews` | `int` `0` | ✅ | `0` = none; attaches `reviews[]` | +| `reviewsSort`/`reviewsStartDate`/`reviewsFilterString`/`reviewsOrigin`/`scrapeReviewsPersonalData` | — | ✅ | as in the Reviews scraper | +| `maxImages` | `int` `0` | ✅ | caps `imageUrls` (URLs only, no author/date) | +| `scrapeContacts`, `scrapeSocialMediaProfiles`, `*LeadsEnrichment*`, `enableCompetitorAnalysis`, `maxCompetitorsToAnalyze`, `scrapeTableReservationProvider`, `scrapeOrderOnline`, `includeWebResults`, `scrapeDirectories`, `maxQuestions`, `scrapeImageAuthors` | — | ⛔ stub | accepted; out of scope / login-walled (see TODO) | + +`✅*` = approximation, not pin-complete. + +### Places — output (`PlaceItem`), sourced fields + +- **Identity:** `title`, `description`, `price`, `categoryName`, `categories`, + `placeId`, `fid`, `cid`, `kgmid` +- **Location:** `address`, `neighborhood`, `street`, `city`, `postalCode`, + `state`, `countryCode`, `location{lat,lng}`, `plusCode` +- **Contact:** `website`, `phone`, `phoneUnformatted`, `menu` +- **Ratings/status:** `totalScore`, `reviewsCount`, `reviewsDistribution`, + `permanentlyClosed`, `temporarilyClosed` +- **Images:** `imageUrl`, `imagesCount`, `imageCategories`, `imageUrls` +- **Detail-page (with `scrapePlaceDetailPage`):** `openingHours`, + `additionalInfo`, `reviewsTags`, `popularTimesHistogram`, + `popularTimesLiveText`, `popularTimesLivePercent` +- **Hotels (hotel places):** `hotelStars`, `hotelDescription`, `checkInDate`, + `checkOutDate`, `similarHotelsNearby`, `hotelAds` +- **Reviews (with `maxReviews>0`):** `reviews[]` (see review fields below) +- **Provenance/meta:** `searchString`, `rank`, `searchPageUrl`, `url`, + `scrapedAt` +- **Stubbed** (`[]`/`{}`/`None`): `peopleAlsoSearch`, `questionsAndAnswers`, + `images` (author/date objects), `webResults`, `bookingLinks`, + `tableReservationLinks`, `gasPrices`, `ownerUpdates`, leads/contact/social + enrichment. + +### Reviews — input (`GoogleMapsReviewsInput`) & output (`ReviewItem`) + +Input: `startUrls`, `placeIds`, `maxReviews` (default `10000000` = "all"), +`reviewsSort`, `reviewsStartDate`, `language`, `reviewsOrigin`, `personalData`. + +Output is **one flat item per review** — the review fields merged with the +parent place header. Sourced review fields: `reviewId`, `name`/`text`/ +`textTranslated`, `stars`, `publishAt` (relative) + `publishedAtDate` (ISO), +`likesCount`, `reviewerId`/`reviewerUrl`/`reviewerPhotoUrl`/ +`reviewerNumberOfReviews`/`isLocalGuide`, `reviewOrigin`, `reviewImageUrls`, +`reviewContext`, `reviewDetailedRating`, `responseFromOwnerText` +(+ `responseFromOwnerDate` as relative text only — see TODO). + +Notable input semantics (matching Apify): + +- Places: `searchStringsArray`, `startUrls`, and `placeIds` are **additive** + (unlike YouTube where startUrls override queries). +- `maxCrawledPlacesPerSearch=None` (unset) means "all places". +- Reviews: `maxReviews` defaults to `10000000` ("all"); `reviewsStartDate` + forces newest-first and stops at the cutoff. +- `personalData` / `scrapeReviewsPersonalData` default `true`; when off, + reviewer id/name/URL/photo must be stripped (reviewId always stays). + +## Testing + +Offline unit tests (no network; parser tests pin real captured fixtures): + +```bash +cd surfsense_backend +.venv/Scripts/python.exe -m pytest tests/unit/scrapers/google_maps/ +``` + +Live end-to-end (real network + proxy; also regenerates fixtures): + +```bash +.venv/Scripts/python.exe scripts/e2e_google_maps_scraper.py +``` + +Deep live verification (diverse places worldwide + review semantics: sort +order, date cutoff, pagination uniqueness, personal-data stripping, French +localization, CID URLs, name-only URLs, short links, search-filter variants +`only_exact` / `categoryFilterWords`, and `reviewsOrigin=google`): + +```bash +.venv/Scripts/python.exe scripts/e2e_google_maps_deep.py +``` + +## Implementation TODO (progressive, like YouTube) + +- Owner-response dates are only exposed as relative text ("11 months ago") in + the BOQ feed; `responseFromOwnerDate` carries that string, not an ISO date. + (The detail RPC's inline reviews don't carry reply timestamps either, and + `listugcposts` still returns no reviews even with the NID cookie.) +- Full image galleries (`images` with author/date, beyond the ~dozen + `imageUrls`): needs the signed-in photo-listing RPC. +- Q&A (`questionsAndAnswers`): `darray[126]` is empty everywhere we probed — + Google appears to have retired public Q&A. `peopleAlsoSearch`: not in the + detail darray; likely a separate RPC. +- A pin-complete `allPlacesNoSearchAction` (the category sweep covers most + pins but not businesses outside the swept categories); would need browser + rendering + tile OCR like Apify's. diff --git a/surfsense_backend/app/proprietary/platforms/google_maps/__init__.py b/surfsense_backend/app/proprietary/platforms/google_maps/__init__.py new file mode 100644 index 000000000..f3a8eec14 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_maps/__init__.py @@ -0,0 +1,21 @@ +"""Platform-native Google Maps scraper (Apify Google Maps Scraper-compatible).""" + +from .reviews import iter_reviews, scrape_reviews +from .schemas import ( + GoogleMapsReviewsInput, + GoogleMapsScrapeInput, + PlaceItem, + ReviewItem, +) +from .scraper import iter_places, scrape_places + +__all__ = [ + "GoogleMapsReviewsInput", + "GoogleMapsScrapeInput", + "PlaceItem", + "ReviewItem", + "iter_places", + "iter_reviews", + "scrape_places", + "scrape_reviews", +] diff --git a/surfsense_backend/app/proprietary/platforms/google_maps/fetch.py b/surfsense_backend/app/proprietary/platforms/google_maps/fetch.py new file mode 100644 index 000000000..8ec73ef1f --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_maps/fetch.py @@ -0,0 +1,595 @@ +"""Proxy-aware fetch seam for the Google Maps scraper. + +All network I/O flows through here and always egresses through the residential +proxy (Google blocks datacenter IPs outright; a direct hit also risk-blocks the +server IP). Mirrors the design of ``../youtube/innertube.py`` but trimmed to +what Maps needs today: a GET that returns HTML and a GET that returns the +XSSI-stripped JSON that Maps' internal ``/maps/preview/*`` RPC endpoints emit. + +Google Maps place/search/review data is **public** — no Google account is +required. :func:`looks_like_signin_wall` flags the rare responses where Google +serves a consent/sign-in interstitial instead of data, so callers can surface a +clear "sign-in required" error instead of silently returning nothing. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import time +from datetime import UTC, datetime +from typing import Any +from urllib.parse import quote + +from scrapling.fetchers import AsyncFetcher + +from app.utils.proxy import get_proxy_url + +from .parsers import brace_match_json +from .url_resolver import ResolvedUrl, extract_fid + +try: # browser tier is optional (needs patchright browsers installed) + from scrapling.fetchers import StealthyFetcher +except Exception: # pragma: no cover - import guard + StealthyFetcher = None # type: ignore[assignment] + +logger = logging.getLogger(__name__) + + +class SignInRequiredError(RuntimeError): + """Raised when Google serves a sign-in/consent wall instead of public data. + + Public Maps data does not need a Google account, so this is rare; when it + happens the route surfaces it as a clear "Google sign in required" error + rather than returning an empty result. + """ + + +def now_iso() -> str: + """UTC timestamp in the millisecond ISO shape Apify stamps on items.""" + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +async def gather_bounded(coro_factories, *, concurrency: int) -> list: + """Run zero-arg async callables with bounded concurrency, results in order. + + Takes callables (``() -> awaitable``), not live coroutines, so nothing + starts until a semaphore slot frees up — lets callers queue far more work + than ``concurrency`` without opening every socket at once. Every RPC here + is ~2s of proxy round-trip, so overlapping independent ones is the single + biggest lever on wall-clock time. + """ + if not coro_factories: + return [] + sem = asyncio.Semaphore(concurrency) + + async def _run(factory): + async with sem: + return await factory() + + return await asyncio.gather(*(_run(f) for f in coro_factories)) + + +# XSSI guard Google prepends to its RPC JSON responses. +_XSSI_PREFIX = ")]}'" + +# Consent cookie to dodge the EU consent interstitial that otherwise returns a +# page with no APP_INITIALIZATION_STATE. Mirrors the YouTube fetcher's SOCS. +CONSENT_COOKIES = { + "SOCS": "CAISNQgDEitib3FfaWRlbnRpdHlmcm9udGVuZHVpc2VydmVyXzIwMjMwODI5LjA3X3AxGgJlbiADGgYIgOa_pgY", +} + +_HEADERS = {"Accept-Language": "en-US,en;q=0.9"} + + +def looks_like_signin_wall(html_or_text: str | None) -> bool: + """Heuristic: did Google serve a sign-in / consent wall instead of data? + + Public Maps data never needs login, so this should be rare — it mainly + fires on consent redirects (``consent.google.com``) or an account-picker + page. Callers turn a True here into a clear "sign-in required" error. + """ + if not html_or_text: + return False + text = html_or_text[:5000].lower() + markers = ( + "consent.google.com", + "accounts.google.com/servicelogin", + "sign in to continue", + "before you continue to google", + ) + return any(m in text for m in markers) + + +def strip_xssi(text: str) -> str: + """Return the JSON body after Google's ``)]}'`` anti-XSSI guard. + + The guard is normally at position 0, but a proxy/HTML fetcher may wrap the + text/plain body in ``

…`` — so we locate the guard rather than + require it at the start, then return everything after its line. + """ + idx = text.find(_XSSI_PREFIX) + if idx == -1: + return text + nl = text.find("\n", idx) + return text[nl + 1 :] if nl != -1 else text[idx + len(_XSSI_PREFIX) :] + + +async def resolve_fid(resolved: ResolvedUrl) -> str | None: + """Get a place's feature ID: from the URL if present, else from its HTML. + + A ``/maps/place/...`` URL usually embeds the fid in its ``data=`` blob. A + ``?q=place_id:...`` URL or a shortlink does not, so we fetch the page once + and pull the fid out of the returned HTML. Raises :class:`SignInRequiredError` + if Google serves a consent/sign-in wall instead of the page. + """ + if resolved.fid: + return resolved.fid + # Short links (maps.app.goo.gl / goo.gl) are Firebase Dynamic Links: a plain + # GET only returns a JS interstitial, so render it — the JS follows the + # redirect to the real /maps/place/… URL, from which we read the fid. + # ponytail: browser render is slow (~30s) but it's the only way these + # links resolve; they're rare and cached downstream by fid. + if resolved.kind == "shortlink": + rendered = await _fetch_html_stealthy(resolved.url, dict(CONSENT_COOKIES)) + return extract_fid(rendered) if rendered else None + html = await fetch_html(resolved.url) + if html is not None: + if looks_like_signin_wall(html): + raise SignInRequiredError( + f"Google returned a sign-in/consent wall for {resolved.url}" + ) + fid = extract_fid(html) + if fid: + return fid + # Name-only place URLs (e.g. /maps/place/Eiffel+Tower/) serve a JS-only + # shell whose static HTML never contains the fid (even a headless render + # won't navigate to the place). The lite ``tbm=map`` search endpoint DOES + # embed fids in plain HTML, so look the name up there instead. + if resolved.kind == "place" and resolved.value and resolved.value != resolved.url: + return await fid_from_search(resolved.value) + return None + + +async def fid_from_search(query: str, *, language: str = "en") -> str | None: + """Resolve a free-text place name to its feature ID via ``tbm=map`` search. + + This is the lite map-search page (the same endpoint the search-discovery + flow will paginate later); its static HTML embeds every result's fid. + ``ponytail:`` takes the first fid in the page — that's the top-ranked + result. Good enough for name lookups; the full search flow will parse the + structured payload instead. + """ + url = f"https://www.google.com/search?tbm=map&hl={language}&gl=us&q={quote(query)}" + html = await fetch_html(url) + if html is None: + return None + if looks_like_signin_wall(html): + raise SignInRequiredError( + f"Google returned a sign-in/consent wall for map search {query!r}" + ) + return extract_fid(html) + + +async def fetch_html(url: str, *, cookies: dict[str, str] | None = None) -> str | None: + """GET a Google Maps page and return raw HTML (proxy, stealthy fallback).""" + merged = {**CONSENT_COOKIES, **(cookies or {})} + try: + started = time.perf_counter() + page = await AsyncFetcher.get( + url, + headers=_HEADERS, + cookies=merged, + proxy=get_proxy_url(), + stealthy_headers=True, + ) + fetch_ms = (time.perf_counter() - started) * 1000 + logger.info( + "[google_maps][perf] source=html url=%s status=%s fetch_ms=%.1f", + url, + page.status, + fetch_ms, + ) + if page.status == 200: + return page.html_content + logger.warning("Maps HTML GET %s returned %s", url, page.status) + except Exception as e: + logger.warning("Maps HTML GET %s failed: %s", url, e) + return await _fetch_html_stealthy(url, merged) + + +async def _fetch_html_stealthy(url: str, cookies: dict[str, str]) -> str | None: + """Last-resort browser fetch for anti-bot walls (mirrors the YouTube tier).""" + if StealthyFetcher is None: + return None + try: + started = time.perf_counter() + page = await asyncio.to_thread( + StealthyFetcher.fetch, + url, + headless=True, + network_idle=True, + solve_cloudflare=True, + proxy=get_proxy_url(), + ) + fetch_ms = (time.perf_counter() - started) * 1000 + logger.info( + "[google_maps][perf] source=html tier=stealthy url=%s status=%s fetch_ms=%.1f", + url, + page.status, + fetch_ms, + ) + if page.status == 200: + return page.html_content + logger.warning("Maps HTML GET %s tier=stealthy returned %s", url, page.status) + except Exception as e: + logger.warning("Maps HTML GET %s tier=stealthy failed: %s", url, e) + return None + + +# Place-detail RPC field selector (protobuf-over-URL). ``{fid}`` is the place's +# feature ID (hex:hex). This is the selector the Maps web app itself sends when +# rendering a place page (captured live via a headless browser, then +# genericized: the free-text ``!2s`` name and per-place ``!15m2…!4s/g/…`` kgmid +# hint dropped with counts adjusted, coords zeroed, and the ``!14m3!1s`` +# session token replaced with a filler — verified the token value is not +# checked). Combined with an NID session cookie it returns the FULL payload: +# reviewsCount, reviews distribution, popular times, image galleries, review +# tags, and every about section. The returned ``jd[6]`` is the ``darray`` +# parsers read. +_PLACE_DETAIL_PB = ( + "!1m13!1s{fid}" + "!3m8!1m3!1d5000!2d0!3d0!3m2!1i1024!2i768!4f13.1" + "!4m2!3d0!4d0" + "!12m4!2m3!1i360!2i120!4i8" + "!13m57!2m2!1i203!2i100!3m2!2i4!5b1" + "!6m6!1m2!1i86!2i86!1m2!1i408!2i240" + "!7m33!1m3!1e1!2b0!3e3!1m3!1e2!2b1!3e2!1m3!1e2!2b0!3e3" + "!1m3!1e8!2b0!3e3!1m3!1e10!2b0!3e3!1m3!1e10!2b1!3e2" + "!1m3!1e10!2b0!3e4!1m3!1e9!2b1!3e2!2b1!9b0" + "!15m8!1m7!1m2!1m1!1e2!2m2!1i195!2i195!3i20" + "!14m3!1s0ahUKEwixxxxxxxxxxxxxxxxxxxxxxxxx!7e81!15i10112" + "!15m108!1m26!13m9!2b1!3b1!4b1!6i1!8b1!9b1!14b1!20b1!25b1" + "!18m15!3b1!4b1!5b1!6b1!13b1!14b1!17b1!21b1!22b1!30b1!32b1!33m1!1b1!34b1!36e2" + "!10m1!8e3!11m1!3e1!17b1!20m2!1e3!1e6!24b1!25b1!26b1!27b1!29b1" + "!30m1!2b1!36b1!37b1!39m3!2m2!2i1!3i1!43b1!52b1!54m1!1b1!55b1!56m1!1b1" + "!61m2!1m1!1e1!65m5!3m4!1m3!1m2!1i224!2i298" + "!72m22!1m8!2b1!5b1!7b1!12m4!1b1!2b1!4m1!1e1!4b1" + "!8m10!1m6!4m1!1e1!4m1!1e3!4m1!1e4" + "!3sother_user_google_review_posts__and__hotel_and_vr_partner_review_posts" + "!6m1!1e1!9b1!89b1!90m2!1m1!1e2!98m3!1b1!2b1!3b1!103b1!113b1" + "!114m3!1b1!2m1!1b1!117b1!122m1!1b1!126b1!127b1!128m1!1b0" + "!21m0!22m1!1e81!30m8!3b1!6m2!1b1!2b1!7m2!1e3!2b1!9b1" + "!34m5!7b1!10b1!14b1!15m1!1b0!37i785" +) + +# NID session cookie pool. Google trims the rich detail fields (counts, +# distribution, popular times, galleries, tags, most about sections) from +# responses that carry no session cookie; a plain GET to /maps mints an NID +# that unlocks them — no login, no browser. A small pool is rotated +# round-robin (mirrors the YouTube fetcher's ``_RotatingSession``) so no +# single session accumulates the whole request volume; each expires after a +# TTL and is re-minted lazily. +_NID_TTL_S = 30 * 60 +_NID_POOL_SIZE = 3 +_nid_pool: list[dict[str, Any]] = [ + {"value": None, "at": 0.0} for _ in range(_NID_POOL_SIZE) +] +_nid_rr = 0 +_nid_lock = asyncio.Lock() +_nid_inflight: asyncio.Future | None = None + + +async def _mint_nid() -> str | None: + """Mint a fresh NID session cookie with a plain GET to /maps.""" + try: + page = await AsyncFetcher.get( + "https://www.google.com/maps?hl=en", + headers=_HEADERS, + cookies=CONSENT_COOKIES, + proxy=get_proxy_url(), + stealthy_headers=True, + ) + return (page.cookies or {}).get("NID") + except Exception as e: + logger.warning("NID mint failed: %s", e) + return None + + +async def get_session_cookies() -> dict[str, str]: + """Consent cookies plus an NID session cookie from the rotating pool. + + Concurrent cold callers coalesce onto a single in-flight mint (the lock is + not held across the network call), so a burst of parallel detail fetches + doesn't serialize behind N sequential ~2s mints on a cold pool. + """ + global _nid_rr, _nid_inflight + async with _nid_lock: + slot = _nid_pool[_nid_rr % _NID_POOL_SIZE] + _nid_rr += 1 + if slot["value"] and time.time() - slot["at"] < _NID_TTL_S: + return {**CONSENT_COOKIES, "NID": slot["value"]} + if _nid_inflight is None or _nid_inflight.done(): + _nid_inflight = asyncio.ensure_future(_mint_nid()) + fut = _nid_inflight + nid = await fut + if nid: + async with _nid_lock: + slot["value"] = nid + slot["at"] = time.time() + return {**CONSENT_COOKIES, "NID": nid} + return dict(CONSENT_COOKIES) + + +async def fetch_place_darray(fid: str, *, language: str = "en") -> list | None: + """Fetch a place's detail ``darray`` via the ``/maps/preview/place`` RPC. + + ``fid`` is the feature ID (``0x..:0x..``). Returns ``jd[6]`` (the array all + place-field paths index into), or ``None`` on failure. Sent with an NID + session cookie so Google returns the full payload (see ``_PLACE_DETAIL_PB``). + """ + pb = _PLACE_DETAIL_PB.format(fid=fid) + url = ( + "https://www.google.com/maps/preview/place" + f"?authuser=0&hl={language}&gl=us&pb={quote(pb, safe='!')}" + ) + jd = await fetch_rpc_json(url, cookies=await get_session_cookies()) + darray = jd[6] if isinstance(jd, list) and len(jd) > 6 else None + return darray if isinstance(darray, list) else None + + +# Map-search RPC (``search?tbm=map``) field selector. Viewport block first +# (``!1d`` diameter-meters, ``!2d`` lng, ``!3d`` lat), then paging (``!7i`` +# per-page, ``!8i`` offset), then the same detail selector blocks the place RPC +# uses so each result embeds a full place darray at ``entry[14]``. +_SEARCH_PB = ( + "!4m12!1m3!1d{diameter}!2d{lng}!3d{lat}" + "!2m3!1f0!2f0!3f0!3m2!1i1024!2i768!4f13.1" + "!7i{per_page}!8i{offset}!10b1" + "!12m6!1m1!18b1!2m1!20e3!6m1!114b1" + "!17m1!3e1" + "!20m57!2m2!1i203!2i100!3m2!2i4!5b1" + "!6m6!1m2!1i86!2i86!1m2!1i408!2i240" + "!7m33!1m3!1e1!2b0!3e3!1m3!1e2!2b1!3e2!1m3!1e2!2b0!3e3" + "!1m3!1e8!2b0!3e3!1m3!1e10!2b0!3e3!1m3!1e10!2b1!3e2" + "!1m3!1e10!2b0!3e4!1m3!1e9!2b1!3e2!2b1!9b0" + "!15m8!1m7!1m2!1m1!1e2!2m2!1i195!2i195!3i20" +) + +# Whole-earth viewport: Google localizes from the query text itself (verified +# live — "pizza new york" returns NYC results with this viewport), so callers +# only need coordinates when the input provides an explicit geolocation. +_EARTH = {"diameter": 25_000_000.0, "lat": 0.0, "lng": 0.0} + + +def build_search_url( + query: str, + *, + offset: int = 0, + per_page: int = 20, + language: str = "en", + lat: float | None = None, + lng: float | None = None, + radius_m: float | None = None, +) -> str: + """Build a ``search?tbm=map`` URL returning protobuf-over-JSON results.""" + viewport = dict(_EARTH) + if lat is not None and lng is not None: + viewport = { + "diameter": (radius_m or 10_000) * 2, + "lat": lat, + "lng": lng, + } + pb = _SEARCH_PB.format(per_page=per_page, offset=offset, **viewport) + return ( + f"https://www.google.com/search?tbm=map&authuser=0&hl={language}&gl=us" + f"&q={quote(query)}&pb={quote(pb, safe='!')}" + ) + + +def _search_darrays(jd: Any) -> list[list]: + """Extract the place darrays from a map-search response. + + Results live at ``jd[0][1]``; each result entry holds its place darray at + ``entry[14]`` (same shape as the place detail RPC's ``jd[6]``). Multi-result + pages put metadata (no ``[14]``) in slot 0; single-match responses put the + one place directly in slot 0 — scanning every entry handles both. + """ + entries = ( + jd[0][1] + if isinstance(jd, list) and jd and isinstance(jd[0], list) and len(jd[0]) > 1 + else None + ) + if not isinstance(entries, list): + return [] + out = [] + for entry in entries: + darray = entry[14] if isinstance(entry, list) and len(entry) > 14 else None + if isinstance(darray, list): + out.append(darray) + return out + + +async def iter_search_pages( + query: str, + *, + language: str = "en", + lat: float | None = None, + lng: float | None = None, + radius_m: float | None = None, + max_pages: int = 25, + prefetch: int = 1, +): + """Yield lists of place darrays, one map-search page (~20 places) a time. + + Paging is offset-based (``!8i``); Google reshuffles results between pages + so callers must dedupe by fid. Stops on an empty page or ``max_pages``. + + ``prefetch`` pages are fetched concurrently per wave (still yielded in + offset order): each page is a ~2s round-trip, so a caller that knows it + needs several pages (large ``maxCrawledPlacesPerSearch``) gets them + overlapped. ``prefetch=1`` keeps the old one-at-a-time behavior for callers + that only need a page or two (no wasted fetches). + """ + per_page = 20 + page = 0 + while page < max_pages: + wave = min(max(prefetch, 1), max_pages - page) + urls = [ + build_search_url( + query, + offset=(page + i) * per_page, + per_page=per_page, + language=language, + lat=lat, + lng=lng, + radius_m=radius_m, + ) + for i in range(wave) + ] + pages = await gather_bounded( + [lambda u=u: fetch_rpc_json(u) for u in urls], concurrency=wave + ) + for jd in pages: + darrays = _search_darrays(jd) + if not darrays: + return + yield darrays + page += wave + + +# Sort code the reviews RPC expects (slot 1 of the request payload). +REVIEWS_SORT_CODES = { + "mostRelevant": 1, + "newest": 2, + "highestRanking": 3, + "lowestRanking": 4, +} + +# Reviews are cursor-paged (each page's continuation token embeds the previous +# page's last-review key + a signature), so pages CANNOT be fetched in +# parallel — the only lever on review throughput is page size. Google honors +# this on both the first and continuation requests but caps the actual return +# at ~60/page (asking higher just yields the max), so 100 grabs the ceiling and +# cuts the sequential round-trips ~3x vs the old 10/20-per-page default. +_REVIEWS_PAGE_SIZE = 100 + + +def build_reviews_url( + fid: str, *, sort_code: int, page_token: str = "", language: str = "en" +) -> str: + """Build a ``GetLocalBoqProxy`` reviews URL (public, no session token). + + This is the review feed the Google search local panel uses. The older + ``listugcposts`` / ``listentitiesreviews`` RPCs now return empty pages for + anonymous callers, so this proxy is the one that still works (approach from + the maintained ``google-maps-review-scraper`` npm package). + + ``fid`` is the feature ID (``0x..:0x..``); ``page_token`` is the opaque + continuation token from the previous page (empty for the first page). + """ + inner: list = [None] * 12 + inner[1] = sort_code + inner[9] = _REVIEWS_PAGE_SIZE + inner[11] = [fid] + if page_token: + inner.extend([None] * 8) + inner[19] = page_token + payload = [None, [None] * 9 + [inner]] + return ( + "https://www.google.com/httpservice/web/PrivateLocalSearchUiDataService/" + f"GetLocalBoqProxy?msc=gwsrpc&hl={language}" + f"&reqpld={quote(json.dumps(payload))}" + ) + + +def _reviews_node(jd: Any) -> list | None: + """The reviews node of a BOQ response: ``jd[1][10]`` = [.., reviews, .., token].""" + node = ( + jd[1][10] + if isinstance(jd, list) + and len(jd) > 1 + and isinstance(jd[1], list) + and len(jd[1]) > 10 + else None + ) + return node if isinstance(node, list) else None + + +async def iter_reviews_pages( + fid: str, + *, + sort: str = "newest", + language: str = "en", + max_pages: int = 10_000, +): + """Yield raw review arrays, one RPC page (~10 reviews) at a time. + + Each yielded item is the page's review list (``node[2]``). Pagination + follows ``node[6]`` (the continuation token); stops when it is missing or + unchanged, or ``max_pages`` is reached. ``ponytail:`` sequential paging — + the endpoint only hands out one continuation token at a time anyway. + """ + sort_code = REVIEWS_SORT_CODES.get(sort, 2) + page_token = "" + for _ in range(max_pages): + url = build_reviews_url( + fid, sort_code=sort_code, page_token=page_token, language=language + ) + node = _reviews_node(await fetch_rpc_json(url)) + if not node or len(node) < 3 or not isinstance(node[2], list): + return + yield node[2] + next_token = node[6] if len(node) > 6 else None + if ( + not isinstance(next_token, str) + or not next_token + or next_token == page_token + ): + return + page_token = next_token + + +async def fetch_rpc_json( + url: str, *, cookies: dict[str, str] | None = None +) -> Any | None: + """GET a Maps ``/maps/preview/*`` RPC URL and return parsed JSON. + + These endpoints return an XSSI-guarded JSON array (not HTML). Returns the + decoded structure, or ``None`` on failure / non-JSON (e.g. a sign-in wall). + """ + try: + started = time.perf_counter() + page = await AsyncFetcher.get( + url, + headers=_HEADERS, + cookies=cookies or CONSENT_COOKIES, + proxy=get_proxy_url(), + stealthy_headers=True, + ) + fetch_ms = (time.perf_counter() - started) * 1000 + logger.info( + "[google_maps][perf] source=rpc url=%s status=%s fetch_ms=%.1f", + url, + page.status, + fetch_ms, + ) + if page.status != 200: + logger.warning("Maps RPC GET %s returned %s", url, page.status) + return None + text = page.html_content + if looks_like_signin_wall(text): + logger.warning("Maps RPC GET %s hit a sign-in/consent wall", url) + return None + body = strip_xssi(text) + # The proxy/HTML fetcher may wrap the JSON in

…, + # so extract just the balanced top-level array/object. + start = next((i for i, c in enumerate(body) if c in "[{"), -1) + if start == -1: + return None + blob = brace_match_json(body, start) + return json.loads(blob) if blob else None + except Exception as e: + logger.warning("Maps RPC GET %s failed: %s", url, e) + return None diff --git a/surfsense_backend/app/proprietary/platforms/google_maps/parsers.py b/surfsense_backend/app/proprietary/platforms/google_maps/parsers.py new file mode 100644 index 000000000..3ae62c653 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_maps/parsers.py @@ -0,0 +1,526 @@ +"""Pure, I/O-free parsing of Google Maps' protobuf-over-JSON place data. + +Google Maps returns a place's data as a deeply-nested JSON array (no field +names, just positions) from the ``/maps/preview/place`` RPC (element ``jd[6]``, +the ``darray``). Fields are read by fixed array paths. + +The array paths are ported from the actively-maintained ``gosom/google-maps- +scraper`` (Go), which tracks Google's periodic structure shifts and carries +fallback paths (e.g. the Nov-2025 opening-hours move). Everything here is +deterministic and offline-unit-testable against captured fixtures. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any + +logger = logging.getLogger(__name__) + + +def dig(obj: Any, *path: int) -> Any: + """Safely walk a nested list by integer indices; ``None`` if any step misses. + + The Python twin of gosom's ``getNthElementAndCast`` — Maps arrays are ragged + and shift between updates, so every access must tolerate a short/absent path. + """ + cur = obj + for idx in path: + if not isinstance(cur, list) or idx >= len(cur) or cur[idx] is None: + return None + cur = cur[idx] + return cur + + +def _dig_str(obj: Any, *path: int) -> str | None: + val = dig(obj, *path) + return val if isinstance(val, str) else None + + +def _dig_num(obj: Any, *path: int) -> float | None: + val = dig(obj, *path) + return val if isinstance(val, int | float) else None + + +def brace_match_json(text: str, start: int) -> str | None: + """Return the balanced ``[...]`` / ``{...}`` blob beginning at ``text[start]``.""" + open_ch = text[start] + close_ch = {"[": "]", "{": "}"}.get(open_ch) + if close_ch is None: + return None + depth = 0 + i = start + n = len(text) + while i < n: + ch = text[i] + if ch == open_ch: + depth += 1 + elif ch == close_ch: + depth -= 1 + if depth == 0: + return text[start : i + 1] + elif ch == '"': + i += 1 + while i < n and text[i] != '"': + if text[i] == "\\": + i += 1 + i += 1 + i += 1 + return None + + +def _opening_hours(darray: list) -> list[dict[str, str]]: + """Weekly hours. New layout (Nov 2025) at [203][0], old at [34][1].""" + items = dig(darray, 203, 0) or dig(darray, 34, 1) or [] + hours: list[dict[str, str]] = [] + for item in items if isinstance(items, list) else []: + day = _dig_str(item, 0) + if not day: + continue + slots = dig(item, 3) # new format: [ [label, [[h,m],[h,m]]], ... ] + times: list[str] = [] + if isinstance(slots, list) and slots: + times = [_dig_str(s, 0) for s in slots if _dig_str(s, 0)] + else: # old format: [1] is a flat list of strings + old = dig(item, 1) + times = ( + [t for t in old if isinstance(t, str)] if isinstance(old, list) else [] + ) + hours.append({"day": day, "hours": ", ".join(times) if times else "Closed"}) + return hours + + +def _categories(darray: list) -> list[str]: + cats = dig(darray, 13) + return [c for c in cats if isinstance(c, str)] if isinstance(cats, list) else [] + + +def _website(darray: list) -> str | None: + raw = _dig_str(darray, 7, 0) + if raw and raw.startswith("/url?q="): # Google redirect wrapper + from urllib.parse import parse_qs, urlparse + + q = parse_qs(urlparse(raw).query).get("q", [None])[0] + return q or raw + return raw + + +def _additional_info(darray: list) -> dict[str, Any] | None: + """About sections (``darray[100][1]``) in Apify's ``additionalInfo`` shape: + ``{"Accessibility": [{"Wheelchair accessible entrance": true}, ...], ...}``. + + Option paths mirror gosom's About parser; enabled state is + ``option[2][1][0][0] == 1``. + """ + sections = dig(darray, 100, 1) + if not isinstance(sections, list): + return None + info: dict[str, Any] = {} + for section in sections: + name = _dig_str(section, 1) + options = dig(section, 2) + if not name or not isinstance(options, list): + continue + entries = [] + for opt in options: + opt_name = _dig_str(opt, 1) + if opt_name: + entries.append({opt_name: dig(opt, 2, 1, 0, 0) == 1}) + if entries: + info[name] = entries + return info or None + + +def _link_sources(items: Any, link_path: tuple, source_path: tuple) -> list[dict]: + """Extract ``[{"url":…, "source":…}]`` pairs from a link-source array.""" + out = [] + for item in items if isinstance(items, list) else []: + url = _dig_str(item, *link_path) + source = _dig_str(item, *source_path) + if url and source: + out.append({"url": url, "source": source}) + return out + + +def _order_links(darray: list) -> list[dict]: + """Order-online providers (paths from gosom, two known layouts).""" + items = dig(darray, 75, 0, 1, 2) or dig(darray, 75, 0, 0, 2) + return _link_sources(items, (1, 2, 0), (0, 0)) + + +# Day numbers Google uses in the popular-times block (gosom's mapping), +# rendered as Apify's two-letter histogram keys. +_DAY_KEYS = {1: "Mo", 2: "Tu", 3: "We", 4: "Th", 5: "Fr", 6: "Sa", 7: "Su"} + + +def _popular_times(darray: list) -> dict[str, Any]: + """Popular-times block (``darray[84]``) in Apify's field shapes. + + ``[84][0]`` is a list of ``[day_num, [[hour, occupancy%, …], …]]`` days; + ``[84][6]`` is the live busyness text and ``[84][7][1]`` the live percent. + """ + out: dict[str, Any] = {} + days = dig(darray, 84, 0) + if isinstance(days, list): + histogram: dict[str, list] = {} + for day in days: + key = _DAY_KEYS.get(dig(day, 0)) + hours = dig(day, 1) + if not key or not isinstance(hours, list): + continue + histogram[key] = [ + {"hour": h[0], "occupancyPercent": h[1]} + for h in hours + if isinstance(h, list) and len(h) >= 2 + ] + if histogram: + out["popularTimesHistogram"] = histogram + live_text = _dig_str(darray, 84, 6) + if live_text: + out["popularTimesLiveText"] = live_text + live_pct = _dig_num(darray, 84, 7, 1) + if live_pct is not None: + out["popularTimesLivePercent"] = int(live_pct) + return out + + +def _reviews_tags(darray: list) -> list[dict]: + """Review keyword tags (``darray[153][0]``): ``[{title, count}, …]``.""" + entries = dig(darray, 153, 0) + out = [] + for e in entries if isinstance(entries, list) else []: + title = _dig_str(e, 1) + count = _dig_num(e, 3, 4) + if title and count is not None: + out.append({"title": title, "count": int(count)}) + return out + + +def _image_fields(darray: list) -> dict[str, Any]: + """Image gallery fields: count, category tab names, and photo URLs. + + ``darray[37]`` holds ``[hero_photos, total_count]``; ``darray[171][0]`` + holds the gallery tabs (each with a thumbnail). ``ponytail:`` anonymous + sessions only expose the few hero photos + one thumbnail per tab, not the + full gallery — scraping thousands of photos would need the signed-in + photo-listing RPC. ``imageUrls`` therefore tops out around a dozen. + """ + out: dict[str, Any] = {} + count = _dig_num(darray, 37, 1) + if count is not None: + out["imagesCount"] = int(count) + tabs = dig(darray, 171, 0) + urls: list[str] = [] + if isinstance(tabs, list): + categories = [t for tab in tabs if (t := _dig_str(tab, 2))] + if categories: + out["imageCategories"] = categories + urls.extend( + u for tab in tabs if (u := _dig_str(tab, 3, 0, 6, 0)) and u not in urls + ) + heroes = dig(darray, 37, 0) + for photo in heroes if isinstance(heroes, list) else []: + u = _dig_str(photo, 6, 0) + if u and u not in urls: + urls.append(u) + if urls: + out["imageUrls"] = urls + return out + + +def _hotel_fields(darray: list) -> dict[str, Any]: + """Hotel block (``darray[35]`` + ``[64]``), only present for lodging. + + Paths probed live on The Plaza (NYC): ``[35][6]`` star string, + ``[35][0]/[1]`` the check-in/out dates the price quotes are for, + ``[35][29][0]`` similar hotels, ``[35][44]`` booking-partner ads. + """ + stars = _dig_str(darray, 35, 6) or ( + f"{int(n)} stars" if (n := _dig_num(darray, 64, 0)) else None + ) + if not stars: + return {} + out: dict[str, Any] = {"hotelStars": stars} + description = _dig_str(darray, 32, 1, 1) + if description: + out["hotelDescription"] = description + check_in = _dig_str(darray, 35, 0) + check_out = _dig_str(darray, 35, 1) + if check_in: + out["checkInDate"] = check_in + if check_out: + out["checkOutDate"] = check_out + + similar = [] + for entry in dig(darray, 35, 29, 0) or []: + info = dig(entry, 0) + title = _dig_str(info, 4) + if not title: + continue + hotel: dict[str, Any] = {"title": title, "fid": _dig_str(info, 2)} + if (score := _dig_num(info, 7)) is not None: + hotel["totalScore"] = score + if (count := _dig_num(info, 8)) is not None: + hotel["reviewsCount"] = int(count) + lat, lng = _dig_num(info, 3, 2), _dig_num(info, 3, 3) + if lat is not None and lng is not None: + hotel["location"] = {"lat": lat, "lng": lng} + if desc := _dig_str(entry, 1): + hotel["description"] = desc + similar.append(hotel) + if similar: + out["similarHotelsNearby"] = similar + + ads = [] + for ad in dig(darray, 35, 44) or []: + title = _dig_str(ad, 0) + url = _dig_str(ad, 5, 0) + if not title or not url: + continue + item: dict[str, Any] = {"title": title, "url": url} + if price := _dig_str(ad, 1): + item["price"] = price + ads.append(item) + if ads: + out["hotelAds"] = ads + return out + + +def _cid_from_fid(fid: str | None) -> str | None: + """CID (a.k.a. ludocid) = the decimal value of the fid's second hex half.""" + if not fid or ":" not in fid: + return None + try: + return str(int(fid.split(":")[1], 16)) + except ValueError: + return None + + +def parse_place(darray: list) -> dict[str, Any]: + """Map a place ``darray`` to PlaceItem-shaped fields (public Maps data only). + + Only fields with a stable, known array path are populated; the rest stay at + their schema defaults. Paths mirror gosom's ``EntryFromJSON``. + """ + lat = _dig_num(darray, 9, 2) + lng = _dig_num(darray, 9, 3) + dist = dig(darray, 175, 3) # per-star review counts [1★,2★,3★,4★,5★] + + result: dict[str, Any] = { + "title": _dig_str(darray, 11), + "categories": _categories(darray), + "address": _dig_str(darray, 18), + "website": _website(darray), + "phone": _dig_str(darray, 178, 0, 0), + "plusCode": _dig_str(darray, 183, 2, 2, 0), + "totalScore": _dig_num(darray, 4, 7), + "reviewsCount": (int(v) if (v := _dig_num(darray, 4, 8)) is not None else None), + "price": _dig_str(darray, 4, 2), + "description": _dig_str(darray, 32, 1, 1), + "placeId": _dig_str(darray, 78), + "fid": _dig_str(darray, 10), + "kgmid": _dig_str(darray, 89), + "menu": _dig_str(darray, 38, 0), + "imageUrl": _dig_str(darray, 72, 0, 1, 6, 0), + "additionalInfo": _additional_info(darray), + } + result["cid"] = _cid_from_fid(result["fid"]) + + reservations = _link_sources(dig(darray, 46), (0,), (1,)) + if reservations: + result["tableReservationLinks"] = reservations + result["reserveTableUrl"] = reservations[0]["url"] + order_links = _order_links(darray) + if order_links: + result["orderBy"] = order_links # Apify's field name for these + food = next( + (o["url"] for o in order_links if "food.google.com" in o["url"]), None + ) + if food: + result["googleFoodUrl"] = food + if result["categories"]: + result["categoryName"] = result["categories"][0] + if lat is not None and lng is not None: + result["location"] = {"lat": lat, "lng": lng} + if isinstance(dist, list) and len(dist) >= 5: + result["reviewsDistribution"] = { + "oneStar": int(dist[0] or 0), + "twoStar": int(dist[1] or 0), + "threeStar": int(dist[2] or 0), + "fourStar": int(dist[3] or 0), + "fiveStar": int(dist[4] or 0), + } + hours = _opening_hours(darray) + if hours: + result["openingHours"] = hours + + result.update(_popular_times(darray)) + result.update(_image_fields(darray)) + result.update(_hotel_fields(darray)) + tags = _reviews_tags(darray) + if tags: + result["reviewsTags"] = tags + + # Closed status: darray[88][0] carries a status enum for closed places + # ('CLOSED' verified live on a permanently-closed place; open places have + # None or an editorial snippet there, so only exact enums count). + status = _dig_str(darray, 88, 0) + if status in ("CLOSED", "CLOSED_PERMANENTLY"): + result["permanentlyClosed"] = True + elif status == "CLOSED_TEMPORARILY": + result["temporarilyClosed"] = True + + # Complete address components (borough/street/city/postal/state/country). + result["neighborhood"] = _dig_str(darray, 183, 1, 0) + result["street"] = _dig_str(darray, 183, 1, 1) + result["city"] = _dig_str(darray, 183, 1, 3) + result["postalCode"] = _dig_str(darray, 183, 1, 4) + result["state"] = _dig_str(darray, 183, 1, 5) + result["countryCode"] = _dig_str(darray, 183, 1, 6) + + return {k: v for k, v in result.items() if v is not None} + + +# --- reviews (GetLocalBoqProxy) ---------------------------------------------- +# +# The BOQ proxy returns ``jd[1][10]`` = ``[.., reviews, .., nextToken]``. Each +# review ``r`` is a flat ~48-slot array. Index map (verified against live +# captures, matching the ``google-maps-review-scraper`` npm package): +# r[1] stars r[2] [relativeDate, ?, publishedMs] +# r[3] [name, photoUrl, contribUrl, nReviews, nPhotos, [?, localGuide]] +# r[4] owner reply [?, relativeDate, text, ...] +# r[5] reviewId r[12] reviewUrl +# r[13]/r[14] images [[url, caption, ...], ...] +# r[26] language r[27] text r[28] translated text +# r[30] guided Q&A (context + per-aspect ratings) r[44] [origin, ...] + + +_CONTRIB_ID_RE = re.compile(r"/contrib/(\d+)") + + +def _ms_to_iso(ms: Any) -> str | None: + """Convert a millisecond epoch (Google sends it as a string) to ISO-8601.""" + try: + ms_val = float(ms) + except (TypeError, ValueError): + return None + if not ms_val: + return None + from datetime import UTC, datetime + + try: + dt = datetime.fromtimestamp(ms_val / 1000, tz=UTC) + except (ValueError, OSError, OverflowError): + return None + return dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def _review_images(r: Any) -> list[str]: + for idx in (13, 14): + imgs = dig(r, idx) + if isinstance(imgs, list): + urls = [_dig_str(imgs, j, 0) for j in range(len(imgs))] + return [u for u in urls if u] + return [] + + +def _guided_answers(r: Any) -> tuple[dict[str, Any], dict[str, Any]]: + """Split the guided Q&A block into (reviewContext, reviewDetailedRating). + + Entries with a numeric rating (``entry[11] = [n]``) are per-aspect ratings + (Food/Service/Atmosphere); entries with a chosen answer are context + (e.g. ``Order type: Take out``). + """ + context: dict[str, Any] = {} + detailed: dict[str, Any] = {} + block = dig(r, 30) + if not isinstance(block, list): + return context, detailed + for entry in block: + label = _dig_str(entry, 5) or _dig_str(entry, 1) + if not label: + continue + rating = _dig_num(entry, 11, 0) + if rating is not None: + detailed[label] = rating + continue + answer = _dig_str(entry, 2, 0, 0, 1) + if answer: + context[label] = answer + return context, detailed + + +def parse_review(r: Any) -> dict[str, Any] | None: + """Map one BOQ review array to ReviewFields-shaped fields. + + Returns ``None`` for malformed entries (no author block). + """ + if not isinstance(r, list): + return None + name = _dig_str(r, 3, 0) + if not name: + return None + + reviewer_url = _dig_str(r, 3, 2) + reviewer_id = None + if reviewer_url: + m = _CONTRIB_ID_RE.search(reviewer_url) + reviewer_id = m.group(1) if m else None + + n_reviews = _dig_num(r, 3, 3) + local_guide = _dig_num(r, 3, 5, 1) + text = _dig_str(r, 27) + translated = _dig_str(r, 28) + context, detailed = _guided_answers(r) + + fields: dict[str, Any] = { + "reviewId": _dig_str(r, 5), + "reviewUrl": _dig_str(r, 12), + "name": name, + "reviewerId": reviewer_id, + "reviewerUrl": reviewer_url, + "reviewerPhotoUrl": _dig_str(r, 3, 1), + "reviewerNumberOfReviews": int(n_reviews) if n_reviews is not None else None, + "isLocalGuide": bool(local_guide) if local_guide is not None else None, + "stars": _dig_num(r, 1), + "text": text, + "textTranslated": translated if translated and translated != text else None, + "publishAt": _dig_str(r, 2, 0), + "publishedAtDate": _ms_to_iso(dig(r, 2, 2)), + "originalLanguage": _dig_str(r, 26), + "reviewOrigin": _dig_str(r, 44, 0), + "reviewImageUrls": _review_images(r), + "reviewContext": context or None, + "reviewDetailedRating": detailed or None, + } + + # Owner response (r[4]); only a relative date is exposed here. + reply_text = _dig_str(r, 4, 2) + if reply_text: + fields["responseFromOwnerText"] = reply_text + fields["responseFromOwnerDate"] = _dig_str(r, 4, 1) + + return {k: v for k, v in fields.items() if v not in (None, [])} + + +def parse_reviews_page(reviews: Any) -> list[dict[str, Any]]: + """Parse one BOQ page's raw review list into ReviewFields dicts.""" + if not isinstance(reviews, list): + return [] + out = [] + for entry in reviews: + parsed = parse_review(entry) + if parsed: + out.append(parsed) + return out + + +def strip_personal_data(review: dict[str, Any]) -> dict[str, Any]: + """Drop reviewer identity fields (Apify ``personalData=false``). + + ``reviewId`` and the review content stay; name/id/url/photo are removed. + """ + for key in ("name", "reviewerId", "reviewerUrl", "reviewerPhotoUrl"): + review.pop(key, None) + return review diff --git a/surfsense_backend/app/proprietary/platforms/google_maps/reviews.py b/surfsense_backend/app/proprietary/platforms/google_maps/reviews.py new file mode 100644 index 000000000..32c83faf6 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_maps/reviews.py @@ -0,0 +1,210 @@ +"""Orchestrator for the Google Maps Reviews scraper (Apify-compatible). + +Distinct from the places scraper: one flat output item per review (the review +fields merged with the parent place fields), per the Apify "Google Maps Reviews +Scraper" output schema. + +Reviews come from Google's public ``GetLocalBoqProxy`` review feed (no login, +~10 reviews per page, opaque continuation-token pagination). The place header +fields stamped onto each item come from the same ``/maps/preview/place`` RPC the +places scraper uses. +""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from datetime import datetime +from typing import Any + +from .fetch import fetch_place_darray, iter_reviews_pages, now_iso, resolve_fid +from .parsers import parse_place, parse_reviews_page, strip_personal_data +from .schemas import GoogleMapsReviewsInput, ReviewItem +from .url_resolver import ResolvedUrl, resolve_url + +logger = logging.getLogger(__name__) + +# Place header keys copied from a parsed place onto every ReviewItem. +_PLACE_KEYS = ( + "title", + "placeId", + "address", + "location", + "categories", + "categoryName", + "totalScore", + "reviewsCount", + "price", + "cid", + "fid", + "imageUrl", + "neighborhood", + "street", + "city", + "countryCode", + "postalCode", + "state", +) + + +def _parse_start_date(value: str | None) -> datetime | None: + """Parse ``reviewsStartDate`` (ISO date/datetime) into a UTC-naive cutoff.""" + if not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).replace(tzinfo=None) + except ValueError: + logger.warning("[google_maps] bad reviewsStartDate: %r", value) + return None + + +def _before_cutoff(review: dict[str, Any], cutoff: datetime) -> bool: + """True if a review predates the cutoff (newest-first stop condition).""" + iso = review.get("publishedAtDate") + if not iso: + return False + try: + when = datetime.fromisoformat(iso.replace("Z", "+00:00")).replace(tzinfo=None) + except ValueError: + return False + return when < cutoff + + +def _keep(review: dict[str, Any], *, filter_string: str, origin: str) -> bool: + """Apply Apify's reviewsFilterString / reviewsOrigin filters client-side.""" + if origin == "google" and (review.get("reviewOrigin") or "Google") != "Google": + return False + if filter_string: + text = (review.get("text") or "") + " " + (review.get("textTranslated") or "") + if filter_string.lower() not in text.lower(): + return False + return True + + +async def collect_place_reviews( + fid: str, + *, + max_reviews: int, + sort: str = "newest", + language: str = "en", + filter_string: str = "", + origin: str = "all", + personal_data: bool = True, + start_date: str | None = None, +) -> tuple[list[dict[str, Any]], int | None]: + """Page a place's reviews via the BOQ feed. + + Returns ``(reviews, exact_total_or_None)``. The feed does not expose the + place's total review count up front, so the total is only known when + pagination exhausts before ``max_reviews`` is hit (and no filters dropped + anything) — the common case for small/medium places. + """ + cutoff = _parse_start_date(start_date) + reviews: list[dict[str, Any]] = [] + seen = 0 + exhausted = True + filtered = False + + async for raw_page in iter_reviews_pages(fid, sort=sort, language=language): + stop = False + for review in parse_reviews_page(raw_page): + seen += 1 + if cutoff and _before_cutoff(review, cutoff): + stop = True + break + if not _keep(review, filter_string=filter_string, origin=origin): + filtered = True + continue + if not personal_data: + strip_personal_data(review) + reviews.append(review) + if len(reviews) >= max_reviews: + stop = True + break + if stop: + exhausted = False + break + + total = seen if exhausted and not filtered and cutoff is None else None + return reviews, total + + +async def _reviews_for_place( + resolved: ResolvedUrl, + *, + input_model: GoogleMapsReviewsInput, + input_place_id: str | None, + input_start_url: str | None, +) -> AsyncIterator[dict[str, Any]]: + """Page one place's reviews and yield a flat ReviewItem per review.""" + fid = await resolve_fid(resolved) + if not fid: + logger.warning("[google_maps] reviews: no feature id for %s", resolved.url) + return + + darray = await fetch_place_darray(fid, language=input_model.language) + place = parse_place(darray) if darray else {} + place_header = {k: place[k] for k in _PLACE_KEYS if k in place} + scraped_at = now_iso() + + reviews, total = await collect_place_reviews( + place.get("fid", fid), + max_reviews=input_model.maxReviews, + sort=input_model.reviewsSort, + language=input_model.language, + origin=input_model.reviewsOrigin, + personal_data=input_model.personalData, + start_date=input_model.reviewsStartDate, + ) + if total is not None and "reviewsCount" not in place_header: + place_header["reviewsCount"] = total + + for review in reviews: + item = ReviewItem(**{**place_header, **review}) + item.scrapedAt = scraped_at + item.language = input_model.language + item.inputPlaceId = input_place_id + item.inputStartUrl = input_start_url + item.url = place.get("url") or resolved.url + yield item.to_output() + + +async def iter_reviews( + input_model: GoogleMapsReviewsInput, +) -> AsyncIterator[dict[str, Any]]: + """Yield Apify-shaped review items for every startUrl and placeId.""" + for entry in input_model.startUrls: + resolved = resolve_url(entry.url) + if not resolved: + logger.warning("Reviews: unrecognized Google Maps URL: %s", entry.url) + continue + async for item in _reviews_for_place( + resolved, + input_model=input_model, + input_place_id=None, + input_start_url=entry.url, + ): + yield item + + for place_id in input_model.placeIds: + url = f"https://www.google.com/maps/place/?q=place_id:{place_id}" + resolved = ResolvedUrl("place", place_id, url) + async for item in _reviews_for_place( + resolved, + input_model=input_model, + input_place_id=place_id, + input_start_url=None, + ): + yield item + + +async def scrape_reviews( + input_model: GoogleMapsReviewsInput, *, limit: int | None = None +) -> list[dict[str, Any]]: + """Collect :func:`iter_reviews` into a list, honoring an optional guard.""" + results: list[dict[str, Any]] = [] + async for item in iter_reviews(input_model): + results.append(item) + if limit is not None and len(results) >= limit: + break + return results diff --git a/surfsense_backend/app/proprietary/platforms/google_maps/schemas.py b/surfsense_backend/app/proprietary/platforms/google_maps/schemas.py new file mode 100644 index 000000000..e47477045 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_maps/schemas.py @@ -0,0 +1,360 @@ +# ruff: noqa: N815 - field names intentionally mirror the Apify camelCase spec +"""Apify-compatible input/output models for the Google Maps scraper. + +The models mirror the public Apify "Google Maps Scraper" and "Google Maps +Reviews Scraper" actor specs so the endpoints can be drop-ins. The skeleton +accepts the full input surface; output fields the implementation does not +source yet are emitted as ``None``/``[]``/``{}`` so parity is additive. + +Outputs use ``extra="allow"`` on purpose: it lets us grow the output shape +without breaking existing consumers, exactly like the YouTube scraper models. +""" + +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + +SearchMatching = Literal["all", "only_includes", "only_exact"] +PlaceMinimumStars = Literal[ + "", "two", "twoAndHalf", "three", "threeAndHalf", "four", "fourAndHalf" +] +WebsiteFilter = Literal["allPlaces", "withWebsite", "withoutWebsite"] +ReviewsSort = Literal["newest", "mostRelevant", "highestRanking", "lowestRanking"] +ReviewsOrigin = Literal["all", "google"] +AllPlacesAction = Literal["", "all_places_no_search_ocr", "all_places_no_search_mouse"] + + +class StartUrl(BaseModel): + """A single direct URL entry (Apify passes ``{"url": ...}`` objects).""" + + model_config = ConfigDict(extra="allow") + + url: str + + +class SocialMediaEnrichment(BaseModel): + """Per-platform toggles for the social media profile enrichment add-on.""" + + model_config = ConfigDict(extra="allow") + + facebooks: bool = False + instagrams: bool = False + youtubes: bool = False + tiktoks: bool = False + twitters: bool = False + + +class GoogleMapsScrapeInput(BaseModel): + """Full Apify "Google Maps Scraper" input surface. + + Semantics follow Apify: ``maxCrawledPlacesPerSearch=None`` means "all + places"; add-on toggles default off; ``maxReviews``/``maxImages``/ + ``maxQuestions`` of ``0`` mean "fetch none of that type". + """ + + model_config = ConfigDict(extra="allow") + + # Discovery + searchStringsArray: list[str] = Field(default_factory=list) + locationQuery: str | None = None + startUrls: list[StartUrl] = Field(default_factory=list) + placeIds: list[str] = Field(default_factory=list) + allPlacesNoSearchAction: AllPlacesAction = "" + + # Caps / language + maxCrawledPlacesPerSearch: int | None = Field(default=None, ge=1) + language: str = "en" + + # Filters ($) + categoryFilterWords: list[str] = Field(default_factory=list) + searchMatching: SearchMatching = "all" + placeMinimumStars: PlaceMinimumStars = "" + website: WebsiteFilter = "allPlaces" + skipClosedPlaces: bool = False + + # Place detail page ($) and its dependents + scrapePlaceDetailPage: bool = False + scrapeTableReservationProvider: bool = False + scrapeOrderOnline: bool = False + includeWebResults: bool = False + scrapeDirectories: bool = False + maxQuestions: int = Field(default=0, ge=0) + + # Enrichment add-ons ($) + scrapeContacts: bool = False + scrapeSocialMediaProfiles: SocialMediaEnrichment = Field( + default_factory=SocialMediaEnrichment + ) + maximumLeadsEnrichmentRecords: int = Field(default=0, ge=0) + leadsEnrichmentDepartments: list[str] = Field(default_factory=list) + verifyLeadsEnrichmentEmails: bool = False + + # Reviews ($) + maxReviews: int = Field(default=0, ge=0) + reviewsStartDate: str | None = None + reviewsSort: ReviewsSort = "newest" + reviewsFilterString: str = "" + reviewsOrigin: ReviewsOrigin = "all" + scrapeReviewsPersonalData: bool = True + + # Images ($) + maxImages: int = Field(default=0, ge=0) + scrapeImageAuthors: bool = False + + # Competitor analysis add-on ($) + enableCompetitorAnalysis: bool = False + maxCompetitorsToAnalyze: int = Field(default=30, ge=0, le=100) + + # Geolocation parameters (use either these or locationQuery, not both) + countryCode: str | None = None + city: str | None = None + state: str | None = None + county: str | None = None + postalCode: str | None = None + customGeolocation: dict[str, Any] | None = None + + +class Location(BaseModel): + model_config = ConfigDict(extra="allow") + + lat: float | None = None + lng: float | None = None + + +class ReviewsDistribution(BaseModel): + model_config = ConfigDict(extra="allow") + + oneStar: int | None = None + twoStar: int | None = None + threeStar: int | None = None + fourStar: int | None = None + fiveStar: int | None = None + + +class OpeningHour(BaseModel): + model_config = ConfigDict(extra="allow") + + day: str | None = None + hours: str | None = None + + +class PeopleAlsoSearchItem(BaseModel): + model_config = ConfigDict(extra="allow") + + category: str | None = None + title: str | None = None + reviewsCount: int | None = None + totalScore: float | None = None + + +class ReviewsTag(BaseModel): + model_config = ConfigDict(extra="allow") + + title: str | None = None + count: int | None = None + + +class ImageItem(BaseModel): + model_config = ConfigDict(extra="allow") + + imageUrl: str | None = None + authorName: str | None = None + authorUrl: str | None = None + uploadedAt: str | None = None + + +class ReviewFields(BaseModel): + """Review-level fields shared by the nested place ``reviews[]`` items and + the standalone Reviews Scraper output items.""" + + model_config = ConfigDict(extra="allow") + + name: str | None = None + text: str | None = None + textTranslated: str | None = None + publishAt: str | None = None + publishedAtDate: str | None = None + likesCount: int | None = None + reviewId: str | None = None + reviewUrl: str | None = None + reviewerId: str | None = None + reviewerUrl: str | None = None + reviewerPhotoUrl: str | None = None + reviewerNumberOfReviews: int | None = None + isLocalGuide: bool | None = None + reviewOrigin: str | None = None + stars: float | None = None + rating: str | None = None + responseFromOwnerDate: str | None = None + responseFromOwnerText: str | None = None + reviewImageUrls: list[str] = Field(default_factory=list) + reviewContext: dict[str, Any] = Field(default_factory=dict) + reviewDetailedRating: dict[str, Any] = Field(default_factory=dict) + visitedIn: str | None = None + originalLanguage: str | None = None + translatedLanguage: str | None = None + + +class PlaceItem(BaseModel): + """Apify "Google Maps Scraper" output item (one per place). + + Mirrors the actor's example JSON. Unsourced fields default to + ``None``/``[]``/``{}``; ``extra="allow"`` keeps the contract open. + """ + + model_config = ConfigDict(extra="allow") + + # Provenance + searchString: str | None = None + rank: int | None = None + searchPageUrl: str | None = None + searchPageLoadedUrl: str | None = None + isAdvertisement: bool = False + + # Identity + title: str | None = None + subTitle: str | None = None + description: str | None = None + price: str | None = None + categoryName: str | None = None + categories: list[str] = Field(default_factory=list) + placeId: str | None = None + fid: str | None = None + cid: str | None = None + kgmid: str | None = None + + # Address / location + address: str | None = None + neighborhood: str | None = None + street: str | None = None + city: str | None = None + postalCode: str | None = None + state: str | None = None + countryCode: str | None = None + location: Location | None = None + plusCode: str | None = None + locatedIn: str | None = None + parentPlaceUrl: str | None = None + + # Contact + website: str | None = None + phone: str | None = None + phoneUnformatted: str | None = None + menu: str | None = None + servicesLink: str | None = None + claimThisBusiness: bool | None = None + + # Ratings / status + totalScore: float | None = None + reviewsCount: int | None = None + reviewsDistribution: ReviewsDistribution | None = None + permanentlyClosed: bool = False + temporarilyClosed: bool = False + + # Images + imageUrl: str | None = None + imagesCount: int | None = None + imageCategories: list[str] = Field(default_factory=list) + images: list[ImageItem] = Field(default_factory=list) + imageUrls: list[str] = Field(default_factory=list) + + # Detail-page fields (populated only when scrapePlaceDetailPage) + openingHours: list[OpeningHour] = Field(default_factory=list) + peopleAlsoSearch: list[PeopleAlsoSearchItem] = Field(default_factory=list) + placesTags: list[Any] = Field(default_factory=list) + reviewsTags: list[ReviewsTag] = Field(default_factory=list) + additionalInfo: dict[str, Any] | None = None + questionsAndAnswers: list[Any] = Field(default_factory=list) + updatesFromCustomers: Any | None = None + ownerUpdates: list[Any] = Field(default_factory=list) + webResults: list[Any] = Field(default_factory=list) + tableReservationLinks: list[Any] = Field(default_factory=list) + bookingLinks: list[Any] = Field(default_factory=list) + reserveTableUrl: str | None = None + googleFoodUrl: str | None = None + gasPrices: list[Any] = Field(default_factory=list) + restaurantData: dict[str, Any] = Field(default_factory=dict) + userPlaceNote: str | None = None + + # Hotel fields + hotelStars: str | None = None + hotelDescription: str | None = None + checkInDate: str | None = None + checkOutDate: str | None = None + similarHotelsNearby: Any | None = None + hotelReviewSummary: Any | None = None + hotelAds: list[Any] = Field(default_factory=list) + + # Reviews (populated only when maxReviews > 0) + reviews: list[ReviewFields] = Field(default_factory=list) + + # Meta + url: str | None = None + scrapedAt: str | None = None + + def to_output(self) -> dict[str, Any]: + """Serialize to the flat dict shape Apify emits (keeps extras).""" + return self.model_dump(exclude_none=False) + + +class GoogleMapsReviewsInput(BaseModel): + """Apify "Google Maps Reviews Scraper" input surface.""" + + model_config = ConfigDict(extra="allow") + + startUrls: list[StartUrl] = Field(default_factory=list) + placeIds: list[str] = Field(default_factory=list) + maxReviews: int = Field(default=10_000_000, ge=1) + reviewsSort: ReviewsSort = "newest" + reviewsStartDate: str | None = None + language: str = "en" + reviewsOrigin: ReviewsOrigin = "all" + personalData: bool = True + + +class ReviewItem(ReviewFields): + """Apify "Google Maps Reviews Scraper" output item. + + One flat item per review: the review fields (inherited) merged with the + place fields, per the actor's output schema. + """ + + # Place + title: str | None = None + placeId: str | None = None + address: str | None = None + location: Location | None = None + categories: list[str] = Field(default_factory=list) + isAdvertisement: bool = False + categoryName: str | None = None + totalScore: float | None = None + permanentlyClosed: bool = False + temporarilyClosed: bool = False + reviewsCount: int | None = None + url: str | None = None + price: str | None = None + cid: str | None = None + fid: str | None = None + imageUrl: str | None = None + hotelStars: str | None = None + kgmid: str | None = None + neighborhood: str | None = None + street: str | None = None + city: str | None = None + countryCode: str | None = None + postalCode: str | None = None + state: str | None = None + + # Provenance / meta + scrapedAt: str | None = None + searchPageUrl: str | None = None + searchString: str | None = None + inputPlaceId: str | None = None + inputStartUrl: str | None = None + language: str | None = None + rank: int | None = None + + def to_output(self) -> dict[str, Any]: + return self.model_dump(exclude_none=False) diff --git a/surfsense_backend/app/proprietary/platforms/google_maps/scraper.py b/surfsense_backend/app/proprietary/platforms/google_maps/scraper.py new file mode 100644 index 000000000..d7a959b0b --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_maps/scraper.py @@ -0,0 +1,463 @@ +"""Orchestrator for the Google Maps places scraper (Apify-compatible). + +Skeleton mirroring the YouTube scraper layout: the core is the async generator +:func:`iter_places` (unbounded), :func:`scrape_places` is a thin collector with +a caller-supplied ``limit`` guard. Discovery inputs dispatch to per-kind flows +(search / place URL / place ID) which are currently no-ops — each will be +implemented progressively, exactly like the YouTube flows were. +""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import quote + +from .fetch import ( + SignInRequiredError, + fetch_place_darray, + gather_bounded, + iter_search_pages, + now_iso, + resolve_fid, +) +from .parsers import parse_place +from .schemas import GoogleMapsScrapeInput, PlaceItem +from .url_resolver import ResolvedUrl, resolve_url + +logger = logging.getLogger(__name__) + +# Re-exported so callers/routes can keep importing it from the orchestrator. +__all__ = ["SignInRequiredError", "iter_places", "scrape_places"] + +# Max concurrent per-place detail/review fetches. Each is a ~2s proxy +# round-trip, so overlapping them is what turns a 20-place enriched search from +# ~50s into a handful of seconds. ``ponytail:`` a fixed ceiling — high enough to +# hide latency, low enough not to trip Google/proxy rate limits; make it +# configurable if a faster proxy pool ever wants more. +_DETAIL_CONCURRENCY = 8 + +_SEARCH_PAGE_SIZE = 20 + + +def _prefetch_for(cap: int | None) -> int: + """How many search pages to fetch per wave, from the result cap. + + One page holds ~20 results, so a cap of 60 wants ~3 pages overlapped; an + uncapped scan overlaps a small fixed wave. Capped at 5 to bound wasted + fetches when dedupe or an early empty page cuts things short. + """ + if cap is None: + return 5 + return max(1, min(5, (cap + _SEARCH_PAGE_SIZE - 1) // _SEARCH_PAGE_SIZE)) + + +# Apify's placeMinimumStars options -> numeric cutoff. +_MIN_STARS = { + "two": 2.0, + "twoAndHalf": 2.5, + "three": 3.0, + "threeAndHalf": 3.5, + "four": 4.0, + "fourAndHalf": 4.5, +} + + +def _location_text(input_model: GoogleMapsScrapeInput) -> str | None: + """The location to scope searches to (Apify appends it to the query).""" + if input_model.locationQuery: + return input_model.locationQuery + parts = [ + input_model.city, + input_model.county, + input_model.state, + input_model.postalCode, + input_model.countryCode, + ] + joined = ", ".join(p for p in parts if p) + return joined or None + + +def _custom_point( + input_model: GoogleMapsScrapeInput, +) -> tuple[float | None, float | None, float | None]: + """(lat, lng, radius_m) from a GeoJSON-ish customGeolocation Point.""" + geo = input_model.customGeolocation + if not isinstance(geo, dict) or geo.get("type") != "Point": + return None, None, None + coords = geo.get("coordinates") + if not isinstance(coords, list | tuple) or len(coords) < 2: + return None, None, None + lng, lat = coords[0], coords[1] + radius_km = geo.get("radiusKm") or geo.get("radius") or 10 + return float(lat), float(lng), float(radius_km) * 1000 + + +def _passes_filters( + fields: dict[str, Any], query: str, input_model: GoogleMapsScrapeInput +) -> bool: + """Apply Apify's search result filters to parsed place fields.""" + title = (fields.get("title") or "").lower() + q = query.lower() + if input_model.searchMatching == "only_exact" and title != q: + return False + if input_model.searchMatching == "only_includes" and q not in title: + return False + + if input_model.categoryFilterWords: + cats = " ".join(fields.get("categories") or []).lower() + if not any(w.lower() in cats for w in input_model.categoryFilterWords): + return False + + min_stars = _MIN_STARS.get(input_model.placeMinimumStars) + if min_stars is not None and (fields.get("totalScore") or 0) < min_stars: + return False + + if input_model.website == "withWebsite" and not fields.get("website"): + return False + if input_model.website == "withoutWebsite" and fields.get("website"): + return False + + return not ( + input_model.skipClosedPlaces + and (fields.get("permanentlyClosed") or fields.get("temporarilyClosed")) + ) + + +def _apply_image_cap( + fields: dict[str, Any], input_model: GoogleMapsScrapeInput +) -> None: + """Apify semantics: gallery ``imageUrls`` only when ``maxImages > 0``.""" + if not input_model.maxImages: + fields.pop("imageUrls", None) + elif "imageUrls" in fields: + fields["imageUrls"] = fields["imageUrls"][: input_model.maxImages] + + +async def _enrich_from_detail( + fields: dict[str, Any], input_model: GoogleMapsScrapeInput +) -> dict[str, Any]: + """Merge the full detail-RPC payload over search-result fields. + + Search darrays are served without the session-gated extras (reviewsCount, + distribution, popular times, galleries, tags, full about sections); the + detail RPC with an NID cookie has them all. Detail values win on overlap. + """ + fid = fields.get("fid") + if not fid: + return fields + darray = await fetch_place_darray(fid, language=input_model.language) + if not darray: + return fields + return {**fields, **parse_place(darray)} + + +async def _build_items( + candidates: list[tuple[dict[str, Any], str, int]], + input_model: GoogleMapsScrapeInput, + *, + search_string: str, + search_page_url: str | None, + enrich: bool, +) -> list[dict[str, Any]]: + """Turn dedup'd/filtered place fields into output items, in parallel. + + Each candidate is ``(fields, fid, rank)``. Detail enrichment and inline + reviews are per-place round-trips; running the whole batch concurrently + (bounded, order preserved) is the main speedup over the old one-at-a-time + loop. Pure-CPU when neither enrichment nor reviews are requested. + """ + + async def _build(fields: dict[str, Any], fid: str, rank: int) -> dict[str, Any]: + if enrich: + fields = await _enrich_from_detail(fields, input_model) + _apply_image_cap(fields, input_model) + item = PlaceItem(**fields) + item.searchString = search_string + if search_page_url: + item.searchPageUrl = search_page_url + item.rank = rank + item.url = ( + f"https://www.google.com/maps/place/?q=place_id:{fields.get('placeId')}" + ) + item.scrapedAt = now_iso() + if input_model.maxReviews: + await _attach_reviews(item, fid, input_model) + return item.to_output() + + return await gather_bounded( + [lambda c=c: _build(*c) for c in candidates], + concurrency=_DETAIL_CONCURRENCY, + ) + + +async def _search_flow( + query: str, *, input_model: GoogleMapsScrapeInput +) -> AsyncIterator[dict[str, Any]]: + """Search-term discovery via the ``search?tbm=map`` RPC. + + Pages offset-based results (~20/page), dedupes by fid (Google reshuffles + between pages), applies the Apify filters, and emits full place items — + each search hit already carries a place darray, so no per-place request is + needed for the core fields. When ``scrapePlaceDetailPage`` or ``maxImages`` + is set, one detail RPC per place adds the session-gated extras; those are + fetched concurrently across the page rather than one at a time. + """ + location = _location_text(input_model) + search_query = f"{query} in {location}" if location else query + lat, lng, radius_m = _custom_point(input_model) + cap = input_model.maxCrawledPlacesPerSearch + enrich = bool(input_model.scrapePlaceDetailPage or input_model.maxImages) + search_page_url = ( + f"https://www.google.com/maps/search/{quote(search_query, safe='')}" + ) + + seen: set[str] = set() + emitted = 0 + async for darrays in iter_search_pages( + search_query, + language=input_model.language, + lat=lat, + lng=lng, + radius_m=radius_m, + prefetch=_prefetch_for(cap), + ): + candidates: list[tuple[dict[str, Any], str, int]] = [] + new_on_page = 0 + for darray in darrays: + fields = parse_place(darray) + fid = fields.get("fid") + if not fid or fid in seen: + continue + seen.add(fid) + new_on_page += 1 + if _passes_filters(fields, query, input_model): + candidates.append((fields, fid, len(seen))) + if cap is not None: + candidates = candidates[: max(0, cap - emitted)] + for out in await _build_items( + candidates, + input_model, + search_string=query, + search_page_url=search_page_url, + enrich=enrich, + ): + yield out + emitted += 1 + if cap is not None and emitted >= cap: + return + if not new_on_page: # page was all repeats -> feed is cycling, stop + return + + +# Broad category sweep for allPlacesNoSearchAction. Apify's implementation +# OCRs / mouse-overs the rendered map pins (hence the enum names); the public +# search RPC has no "list everything" query (verified: '*' and '' return +# nothing). ponytail: approximate the scan with broad category searches over +# the same viewport, deduped by fid — covers the vast majority of pins; a true +# pin-complete scan would need browser rendering + tile OCR. +_ALL_PLACES_SWEEP = [ + "restaurant", + "cafe", + "bar", + "store", + "shopping", + "hotel", + "tourist attraction", + "park", + "gym", + "salon", + "bank", + "gas station", + "pharmacy", + "doctor", + "school", + "church", + "services", +] + + +async def _all_places_flow( + input_model: GoogleMapsScrapeInput, +) -> AsyncIterator[dict[str, Any]]: + """Area scan without a search term: sweep broad categories, dedupe by fid. + + Emits the same items the search flow would; ``searchString`` carries the + action value so callers can tell scan hits from query hits. + """ + location = _location_text(input_model) + lat, lng, radius_m = _custom_point(input_model) + cap = input_model.maxCrawledPlacesPerSearch + enrich = bool(input_model.scrapePlaceDetailPage or input_model.maxImages) + seen: set[str] = set() + emitted = 0 + for term in _ALL_PLACES_SWEEP: + search_query = f"{term} in {location}" if location else term + async for darrays in iter_search_pages( + search_query, + language=input_model.language, + lat=lat, + lng=lng, + radius_m=radius_m, + prefetch=_prefetch_for(cap), + ): + candidates: list[tuple[dict[str, Any], str, int]] = [] + new_on_page = 0 + for darray in darrays: + fields = parse_place(darray) + fid = fields.get("fid") + if not fid or fid in seen: + continue + seen.add(fid) + new_on_page += 1 + if _passes_filters(fields, "", input_model): + candidates.append((fields, fid, len(seen))) + if cap is not None: + candidates = candidates[: max(0, cap - emitted)] + for out in await _build_items( + candidates, + input_model, + search_string=input_model.allPlacesNoSearchAction, + search_page_url=None, + enrich=enrich, + ): + yield out + emitted += 1 + if cap is not None and emitted >= cap: + return + if not new_on_page: + break # this term is exhausted; move to the next one + + +async def _place_flow( + resolved: ResolvedUrl, *, input_model: GoogleMapsScrapeInput +) -> AsyncIterator[dict[str, Any]]: + """Single place from a direct Maps place URL / place ID. + + Resolves the feature ID, fetches the place detail via the ``/maps/preview/ + place`` RPC (full payload — see ``_PLACE_DETAIL_PB``), and maps the fields + into a ``PlaceItem``. When ``maxReviews > 0`` the place's reviews are + attached inline (Apify puts them on ``reviews[]``). + """ + fid = await resolve_fid(resolved) + if not fid: + logger.warning("[google_maps] could not resolve feature id: %s", resolved.url) + return + darray = await fetch_place_darray(fid, language=input_model.language) + if not darray: + logger.warning( + "[google_maps] no place data (structure may have shifted): %s", resolved.url + ) + return + fields = parse_place(darray) + _apply_image_cap(fields, input_model) + item = PlaceItem(**fields) + item.url = resolved.url + item.searchString = f"Direct URL: {resolved.url}" + item.scrapedAt = now_iso() + if input_model.maxReviews and fields.get("fid"): + await _attach_reviews(item, fields["fid"], input_model) + yield item.to_output() + + +async def _attach_reviews( + item: PlaceItem, fid: str, input_model: GoogleMapsScrapeInput +) -> None: + """Populate ``item.reviews`` (and total count when knowable) from the feed.""" + from .reviews import collect_place_reviews + + reviews, total = await collect_place_reviews( + fid, + max_reviews=input_model.maxReviews, + sort=input_model.reviewsSort, + language=input_model.language, + filter_string=input_model.reviewsFilterString, + origin=input_model.reviewsOrigin, + personal_data=input_model.scrapeReviewsPersonalData, + start_date=input_model.reviewsStartDate, + ) + item.reviews = reviews + if total is not None and item.reviewsCount is None: + item.reviewsCount = total + + +async def _place_id_flow( + place_id: str, *, input_model: GoogleMapsScrapeInput +) -> AsyncIterator[dict[str, Any]]: + """Single place from a bare place ID (format ``ChIJ...``). + + Maps resolves ``?q=place_id:`` to the place page, so we build that URL + and reuse the place flow. + """ + url = f"https://www.google.com/maps/place/?q=place_id:{place_id}" + resolved = ResolvedUrl("place", place_id, url) + async for item in _place_flow(resolved, input_model=input_model): + yield item + + +async def _dispatch( + resolved: ResolvedUrl, input_model: GoogleMapsScrapeInput +) -> AsyncIterator[dict[str, Any]]: + if resolved.kind == "search": + async for item in _search_flow(resolved.value, input_model=input_model): + yield item + else: # place / cid / shortlink / reviews all resolve to a place page + async for item in _place_flow(resolved, input_model=input_model): + yield item + + +async def iter_places( + input_model: GoogleMapsScrapeInput, +) -> AsyncIterator[dict[str, Any]]: + """Yield Apify-shaped place items from all discovery inputs. + + Apify runs searches, startUrls, and placeIds side by side (they are + additive, unlike the YouTube scraper where startUrls override queries). + """ + for entry in input_model.startUrls: + resolved = resolve_url(entry.url) + if not resolved: + logger.warning("Unrecognized Google Maps URL: %s", entry.url) + continue + async for item in _dispatch(resolved, input_model): + yield item + + # placeIds are independent single-place detail fetches (~2s each); run the + # batch concurrently, bounded, results in input order — a bulk list of IDs + # is the common case and was previously fully sequential. + if input_model.placeIds: + + async def _collect(pid: str) -> list[dict[str, Any]]: + return [it async for it in _place_id_flow(pid, input_model=input_model)] + + for items in await gather_bounded( + [lambda p=p: _collect(p) for p in input_model.placeIds], + concurrency=_DETAIL_CONCURRENCY, + ): + for item in items: + yield item + + for query in input_model.searchStringsArray: + async for item in _search_flow(query, input_model=input_model): + yield item + + if input_model.allPlacesNoSearchAction: + async for item in _all_places_flow(input_model): + yield item + + +async def scrape_places( + input_model: GoogleMapsScrapeInput, *, limit: int | None = None +) -> list[dict[str, Any]]: + """Collect :func:`iter_places` into a list, honoring an optional ``limit``. + + ``limit`` is a request-time policy guard (used by the route), NOT a ceiling + in the streaming core. + """ + results: list[dict[str, Any]] = [] + async for item in iter_places(input_model): + results.append(item) + if limit is not None and len(results) >= limit: + break + return results diff --git a/surfsense_backend/app/proprietary/platforms/google_maps/url_resolver.py b/surfsense_backend/app/proprietary/platforms/google_maps/url_resolver.py new file mode 100644 index 000000000..5f8357805 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_maps/url_resolver.py @@ -0,0 +1,74 @@ +"""Classify a Google Maps URL and extract its identifier. + +Covers the ``startUrls`` shapes the Apify specs accept: place pages +(``/maps/place``), search pages (``/maps/search``), review pages +(``/maps/reviews``), CID links (``google.com/maps?cid=***``), and short links +(``goo.gl/maps`` / ``maps.app.goo.gl``, which need a network redirect to +resolve — classified here, resolved later in the fetch layer). +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Literal +from urllib.parse import parse_qs, unquote, urlparse + +ResolvedKind = Literal["place", "search", "reviews", "cid", "shortlink"] + +_MAPS_HOSTS = ("google.", "www.google.") +_SHORTLINK_HOSTS = ("goo.gl", "maps.app.goo.gl") + +# Feature ID (a.k.a. "fid" / data_id): two hex halves, e.g. +# 0x89c3ca9c11f90c25:0x6cc8dba851799f09. It appears in the URL ``data=`` blob as +# ``!1s`` and is the key that links a place to its detail/review RPCs. +_FID_RE = re.compile(r"(0x[0-9a-f]+:0x[0-9a-f]+)") + + +@dataclass(frozen=True) +class ResolvedUrl: + kind: ResolvedKind + value: str # place slug, search query, cid, or the short URL itself + url: str + fid: str | None = None # feature id (hex:hex) when present in the URL + + +def extract_fid(url: str) -> str | None: + """Pull the feature ID (``0x..:0x..``) out of a Google Maps URL, if present.""" + match = _FID_RE.search(url) + return match.group(1) if match else None + + +def resolve_url(url: str) -> ResolvedUrl | None: + """Classify a Google Maps URL into a scrape job, or ``None`` if unrecognized.""" + parsed = urlparse(url) + hostname = parsed.hostname or "" + path = parsed.path or "" + + if hostname in _SHORTLINK_HOSTS or hostname.endswith(".goo.gl"): + return ResolvedUrl("shortlink", url, url) + + if "google." not in hostname: + return None + + # google.com/maps?cid=123... (uncommon but supported by both actors) + cid = parse_qs(parsed.query).get("cid", [None])[0] + if cid: + return ResolvedUrl("cid", cid, url) + + for prefix, kind in ( + ("/maps/place/", "place"), + ("/maps/search/", "search"), + ("/maps/reviews/", "reviews"), + ): + if path.startswith(prefix): + # Maps slugs encode spaces as "+" (e.g. /maps/place/Kim's+Island). + value = unquote(path[len(prefix) :].split("/")[0].replace("+", " ")) + return ResolvedUrl(kind, value, url, extract_fid(url)) # type: ignore[arg-type] + + # Bare /maps/search or /maps/reviews carrying data in the query/data blob. + for suffix, kind in (("/maps/search", "search"), ("/maps/reviews", "reviews")): + if path.rstrip("/") == suffix: + return ResolvedUrl(kind, url, url) # type: ignore[arg-type] + + return None diff --git a/surfsense_backend/app/proprietary/platforms/youtube/comments.py b/surfsense_backend/app/proprietary/platforms/youtube/comments.py index c02331199..5b28acd8f 100644 --- a/surfsense_backend/app/proprietary/platforms/youtube/comments.py +++ b/surfsense_backend/app/proprietary/platforms/youtube/comments.py @@ -39,10 +39,14 @@ _SORT_LABELS = {"TOP_COMMENTS": "Top", "NEWEST_FIRST": "Newest"} async def _post_next(token: str) -> dict[str, Any] | None: - return await _post(INNERTUBE_NEXT_URL, build_innertube_payload(continuation_token=token)) + return await _post( + INNERTUBE_NEXT_URL, build_innertube_payload(continuation_token=token) + ) -def _finalize(partial: dict[str, Any], base: dict[str, Any], reply_to: str | None) -> dict: +def _finalize( + partial: dict[str, Any], base: dict[str, Any], reply_to: str | None +) -> dict: return CommentItem(**{**base, **partial, "replyToCid": reply_to}).to_output() @@ -132,7 +136,7 @@ async def _comments_for_video( for cid, tok in pending ) ) - replies_by_cid = {cid: r for (cid, _), r in zip(pending, fetched)} + replies_by_cid = {cid: r for (cid, _), r in zip(pending, fetched, strict=False)} for entity in entities: if count >= limit: diff --git a/surfsense_backend/app/proprietary/platforms/youtube/innertube.py b/surfsense_backend/app/proprietary/platforms/youtube/innertube.py index a0f4e14f2..dd5b71457 100644 --- a/surfsense_backend/app/proprietary/platforms/youtube/innertube.py +++ b/surfsense_backend/app/proprietary/platforms/youtube/innertube.py @@ -18,7 +18,7 @@ from __future__ import annotations import asyncio import logging import time -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, suppress from contextvars import ContextVar from typing import Any @@ -78,10 +78,8 @@ class _RotatingSession: async def close(self) -> None: if self._cm is not None: - try: + with suppress(Exception): # best-effort teardown await self._cm.__aexit__(None, None, None) - except Exception: # best-effort teardown - pass self._cm = self.session = None async def rotate(self) -> Any | None: @@ -124,6 +122,7 @@ async def proxy_session(): finally: await holder.close() + # Consent cookies to dodge the EU consent interstitial that otherwise returns a # page with no ``ytInitialData``. Mirrors app/routes/youtube_routes.py. CONSENT_COOKIES = { @@ -224,7 +223,9 @@ async def post_innertube( if page.status == 200: return page.json() logger.warning("InnerTube POST %s returned %s", base_url, page.status) - if not (holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS): + if not ( + holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS + ): return None except Exception as e: logger.warning("InnerTube POST %s failed: %s", base_url, e) @@ -270,7 +271,9 @@ async def fetch_html(url: str, *, cookies: dict[str, str] | None = None) -> str if page.status == 200: return page.html_content logger.warning("HTML GET %s returned %s", url, page.status) - if not (holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS): + if not ( + holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS + ): break except Exception as e: logger.warning("HTML GET %s failed: %s", url, e) diff --git a/surfsense_backend/app/proprietary/platforms/youtube/parsers.py b/surfsense_backend/app/proprietary/platforms/youtube/parsers.py index e07a2b869..0f721c29e 100644 --- a/surfsense_backend/app/proprietary/platforms/youtube/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/youtube/parsers.py @@ -86,7 +86,7 @@ def parse_count(value: Any) -> int | None: """ if value is None: return None - if isinstance(value, (int, float)): + if isinstance(value, int | float): return int(value) if not isinstance(value, str): return None @@ -267,7 +267,10 @@ def _comments_turned_off(initial: dict) -> bool | None: def _is_members_only(player: dict, initial: dict) -> bool: """True for members-only videos (a members badge or a members-only offer).""" for badge in find_all(initial, "metadataBadgeRenderer"): - if isinstance(badge, dict) and badge.get("style") == "BADGE_STYLE_TYPE_MEMBERS_ONLY": + if ( + isinstance(badge, dict) + and badge.get("style") == "BADGE_STYLE_TYPE_MEMBERS_ONLY" + ): return True status = player.get("playabilityStatus") or {} return find_first(status, "offerId") == "sponsors_only_video" @@ -421,7 +424,9 @@ def parse_translation(next_data: dict) -> tuple[str | None, str | None]: language. Videos without a localization return their original text. """ vpir = find_first(next_data, "videoPrimaryInfoRenderer") or {} - title = "".join(r.get("text", "") for r in (dig(vpir, "title", "runs") or [])) or None + title = ( + "".join(r.get("text", "") for r in (dig(vpir, "title", "runs") or [])) or None + ) vsir = find_first(next_data, "videoSecondaryInfoRenderer") or {} description = dig(vsir, "attributedDescription", "content") @@ -440,7 +445,9 @@ def parse_collaborators(initial: dict) -> list[dict[str, str | None]] | None: ``attributedTitle`` whose tap opens a dialog listing each credited channel (``listItemViewModel``). Returns ``None`` for ordinary single-owner videos. """ - owner = dig(find_first(initial, "videoSecondaryInfoRenderer"), "owner", "videoOwnerRenderer") + owner = dig( + find_first(initial, "videoSecondaryInfoRenderer"), "owner", "videoOwnerRenderer" + ) attributed = owner.get("attributedTitle") if isinstance(owner, dict) else None if not isinstance(attributed, dict): return None @@ -460,8 +467,12 @@ def parse_collaborators(initial: dict) -> list[dict[str, str | None]] | None: collaborators.append( { "name": name, - "username": base[2:] if isinstance(base, str) and base.startswith("/@") else None, - "url": f"https://www.youtube.com{base}" if isinstance(base, str) and base else None, + "username": base[2:] + if isinstance(base, str) and base.startswith("/@") + else None, + "url": f"https://www.youtube.com{base}" + if isinstance(base, str) and base + else None, } ) return collaborators or None @@ -537,9 +548,10 @@ def parse_video_page(html: str) -> dict[str, Any] | None: # Verified badge on the channel owner. badges = find_all(initial, "metadataBadgeRenderer") - result["isChannelVerified"] = any( - isinstance(b, dict) and b.get("tooltip") == "Verified" for b in badges - ) or None + result["isChannelVerified"] = ( + any(isinstance(b, dict) and b.get("tooltip") == "Verified" for b in badges) + or None + ) return result @@ -632,7 +644,9 @@ def parse_channel_videos(data: dict) -> tuple[list[dict[str, Any]], str | None]: "publishedTimeText": dig(parts, 1, "text", "content"), "duration": _lockup_duration(lockup), "thumbnailUrl": _best_thumbnail( - dig(lockup, "contentImage", "thumbnailViewModel", "image", "sources") + dig( + lockup, "contentImage", "thumbnailViewModel", "image", "sources" + ) ), } ) @@ -673,9 +687,7 @@ def parse_channel_shorts(data: dict) -> tuple[list[dict[str, Any]], str | None]: def _lockup_duration(lockup: dict) -> str | None: - overlays = dig( - lockup, "contentImage", "thumbnailViewModel", "overlays" - ) or [] + overlays = dig(lockup, "contentImage", "thumbnailViewModel", "overlays") or [] for overlay in overlays: text = dig( overlay, diff --git a/surfsense_backend/app/proprietary/platforms/youtube/scraper.py b/surfsense_backend/app/proprietary/platforms/youtube/scraper.py index c72343c20..1e36ba6fa 100644 --- a/surfsense_backend/app/proprietary/platforms/youtube/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/youtube/scraper.py @@ -100,9 +100,7 @@ async def fan_out( if holder is not None: await holder.close() - tasks = [ - asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs))) - ] + tasks = [asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))] try: for _ in range(len(jobs)): for item in await results.get(): @@ -421,7 +419,7 @@ async def _playlist_flow( # is the bottleneck, so fan them out (each carries its playlist position in # ``order``; fan_out emits as they finish, not in playlist order). # ponytail: nested fan_out — when many playlist URLs run at once this can - # stack pools (outer × inner) of proxy sessions. Fine for the common + # stack pools (outer x inner) of proxy sessions. Fine for the common # single/few-playlist case; cap inner concurrency if bulk-playlist runs trip it. jobs = [ _video_flow( diff --git a/surfsense_backend/app/proprietary/platforms/youtube/subtitles.py b/surfsense_backend/app/proprietary/platforms/youtube/subtitles.py index dc5647b30..a9a103acf 100644 --- a/surfsense_backend/app/proprietary/platforms/youtube/subtitles.py +++ b/surfsense_backend/app/proprietary/platforms/youtube/subtitles.py @@ -56,7 +56,9 @@ def _select_transcript(transcript_list, language: str, prefer_generated: bool): return transcript_list.find_transcript(codes) -def _fetch_subtitles_sync(video_id: str, language: str, fmt: str, prefer_generated: bool): +def _fetch_subtitles_sync( + video_id: str, language: str, fmt: str, prefer_generated: bool +): api = _build_client() transcript_list = api.list(video_id) transcript = _select_transcript(transcript_list, language, prefer_generated) diff --git a/surfsense_backend/app/proprietary/web_crawler/captcha.py b/surfsense_backend/app/proprietary/web_crawler/captcha.py index 90b6dde81..151eac4ce 100644 --- a/surfsense_backend/app/proprietary/web_crawler/captcha.py +++ b/surfsense_backend/app/proprietary/web_crawler/captcha.py @@ -23,6 +23,7 @@ balance can get the IP **temporarily banned**, so ``ErrNoBalance`` / won't change without a restart). ``reset_solver_latch()`` clears it for tests. """ +import contextlib import logging import re from collections.abc import Callable @@ -107,8 +108,10 @@ def detect_challenge(page: Any, cfg: CaptchaConfig) -> tuple[str, str] | None: m = _RENDER_SITEKEY.search(src) if m and m.group(1) != "explicit": return "v3", m.group(1) - except Exception as exc: # noqa: BLE001 - logger.debug("%s detection error (treated as no challenge): %s", _CAPTCHA_LOG, exc) + except Exception as exc: + logger.debug( + "%s detection error (treated as no challenge): %s", _CAPTCHA_LOG, exc + ) return None @@ -131,7 +134,7 @@ def proxy_url_to_captchatools(proxy_url: str | None) -> str | None: if p.username and p.password: return f"{p.hostname}:{p.port}:{p.username}:{p.password}" return f"{p.hostname}:{p.port}" - except Exception: # noqa: BLE001 + except Exception: return None @@ -220,11 +223,9 @@ _INJECT_JS = r""" def _inject_and_submit(page: Any, challenge_type: str, token: str) -> None: try: page.evaluate(_INJECT_JS, {"token": token, "ctype": challenge_type}) - try: + with contextlib.suppress(Exception): page.wait_for_load_state("networkidle", timeout=15000) - except Exception: # noqa: BLE001 - pass - except Exception as exc: # noqa: BLE001 + except Exception as exc: logger.warning("%s injection error: %s", _CAPTCHA_LOG, exc) @@ -264,7 +265,7 @@ def build_captcha_page_action( page_url = getattr(page, "url", "") or "" try: user_agent = page.evaluate("() => navigator.userAgent") - except Exception: # noqa: BLE001 + except Exception: user_agent = None # This counts as an attempt the moment we call the (paid) solver. @@ -296,7 +297,7 @@ def build_captcha_page_action( except FuturesTimeout: logger.warning("%s solve timed out after %ss", _CAPTCHA_LOG, cfg.timeout_s) return page - except Exception as exc: # noqa: BLE001 + except Exception as exc: if _is_unrecoverable(exc): _latch_solver(repr(exc)) else: @@ -310,7 +311,9 @@ def build_captcha_page_action( _inject_and_submit(page, challenge_type, token) state["solved"] = True - logger.info("%s solved type=%s site=%s", _CAPTCHA_LOG, challenge_type, sitekey[:12]) + logger.info( + "%s solved type=%s site=%s", _CAPTCHA_LOG, challenge_type, sitekey[:12] + ) return page return page_action diff --git a/surfsense_backend/app/proprietary/web_crawler/connector.py b/surfsense_backend/app/proprietary/web_crawler/connector.py index b54bcc170..061cc1b00 100644 --- a/surfsense_backend/app/proprietary/web_crawler/connector.py +++ b/surfsense_backend/app/proprietary/web_crawler/connector.py @@ -25,7 +25,7 @@ import logging import time from collections.abc import Awaitable, Callable from dataclasses import dataclass -from enum import Enum +from enum import StrEnum from typing import Any import trafilatura @@ -34,11 +34,11 @@ from scrapling.engines.toolbelt import is_proxy_error from scrapling.fetchers import AsyncFetcher, DynamicFetcher, StealthyFetcher from app.proprietary.web_crawler.captcha import build_captcha_page_action -from app.proprietary.web_crawler.url_policy import extract_links from app.proprietary.web_crawler.stealth import ( build_stealthy_kwargs, get_stealth_config, ) +from app.proprietary.web_crawler.url_policy import extract_links from app.utils.captcha import captcha_enabled, get_captcha_config from app.utils.crawl import BlockType, classify_block from app.utils.proxy import get_proxy_url, is_pool_backed @@ -49,7 +49,7 @@ logger = logging.getLogger(__name__) _PERF = "[webcrawler][perf]" -class CrawlOutcomeStatus(str, Enum): +class CrawlOutcomeStatus(StrEnum): """Deterministic per-URL crawl result, single-sourcing the billable signal.""" SUCCESS = "success" # a tier returned usable extracted content @@ -203,9 +203,7 @@ class WebCrawlerConnector: logger.info(f"[webcrawler] Using Scrapling StealthyFetcher for: {url}") result = await self._run_tier_with_proxy_retry( "scrapling-stealthy", - lambda: self._crawl_with_stealthy( - url, captcha_state, block_state - ), + lambda: self._crawl_with_stealthy(url, captcha_state, block_state), ) if result: self._log_tier_outcome( @@ -396,9 +394,7 @@ class WebCrawlerConnector: Runs the sync fetch in a worker thread so it works on any event loop, including Windows ``SelectorEventLoop`` which cannot spawn subprocesses. """ - return await asyncio.to_thread( - self._crawl_with_dynamic_sync, url, block_state - ) + return await asyncio.to_thread(self._crawl_with_dynamic_sync, url, block_state) def _crawl_with_dynamic_sync( self, url: str, block_state: dict[str, Any] | None = None diff --git a/surfsense_backend/app/proprietary/web_crawler/testbench/__init__.py b/surfsense_backend/app/proprietary/web_crawler/testbench/__init__.py index 40c2edae1..04e4a3187 100644 --- a/surfsense_backend/app/proprietary/web_crawler/testbench/__init__.py +++ b/surfsense_backend/app/proprietary/web_crawler/testbench/__init__.py @@ -21,5 +21,5 @@ creds). Run it from the backend directory: See ``README.md`` for the runbook and ``plans/backend/03f-undetectability-testing.md`` for the design. This package is dev/operator tooling, untouched by the product -rename, and only *measures* what 03a–03e built. +rename, and only *measures* what 03a-03e built. """ diff --git a/surfsense_backend/app/proprietary/web_crawler/testbench/core.py b/surfsense_backend/app/proprietary/web_crawler/testbench/core.py index e201b74d8..6003ca035 100644 --- a/surfsense_backend/app/proprietary/web_crawler/testbench/core.py +++ b/surfsense_backend/app/proprietary/web_crawler/testbench/core.py @@ -14,11 +14,12 @@ change their DOM and the harness is manual. from __future__ import annotations +import contextlib import json from collections.abc import Callable from dataclasses import asdict, dataclass -from datetime import datetime, timezone -from enum import Enum +from datetime import UTC, datetime +from enum import StrEnum from pathlib import Path from typing import Any from urllib.parse import urlsplit @@ -30,7 +31,7 @@ RESULTS_DIR = _PKG_DIR / "results" SCREENSHOTS_DIR = RESULTS_DIR / "screenshots" -class CheckStatus(str, Enum): +class CheckStatus(StrEnum): """Outcome of a single scorecard row.""" PASS = "PASS" # met the aspirational bar @@ -76,7 +77,7 @@ class RunMeta: *, suites: str, proxy: str | None, headed: bool, scrapling_version: str ) -> RunMeta: return RunMeta( - timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"), + timestamp=datetime.now(UTC).isoformat(timespec="seconds"), suites=suites, proxy=mask_proxy(proxy), headed=headed, @@ -120,15 +121,13 @@ def make_page_action( def _action(page: Any) -> Any: if pre_wait_ms > 0: - try: + with contextlib.suppress(Exception): page.wait_for_timeout(pre_wait_ms) - except Exception: - pass if evaluate_js is not None: try: cell.value = page.evaluate(evaluate_js) cell.captured = True - except Exception as exc: # noqa: BLE001 - tolerant by design + except Exception as exc: cell.error = f"{type(exc).__name__}: {exc}" if screenshot_path is not None: try: @@ -137,7 +136,7 @@ def make_page_action( # create it for us, so a missing dir silently drops every screenshot. Path(screenshot_path).parent.mkdir(parents=True, exist_ok=True) page.screenshot(path=screenshot_path, full_page=True) - except Exception as exc: # noqa: BLE001 - tolerant, but surface the cause + except Exception as exc: if cell.error is None: cell.error = f"screenshot: {type(exc).__name__}: {exc}" return page @@ -168,7 +167,7 @@ def ensure_dirs() -> None: def scrapling_version() -> str: try: - import scrapling # noqa: PLC0415 + import scrapling return getattr(scrapling, "__version__", "unknown") except Exception: @@ -178,9 +177,7 @@ def scrapling_version() -> str: # --- scorecard I/O ----------------------------------------------------------- -def write_scorecard( - results: list[CheckResult], meta: RunMeta -) -> tuple[Path, Path]: +def write_scorecard(results: list[CheckResult], meta: RunMeta) -> tuple[Path, Path]: """Write the JSON snapshot (committed, baseline) + a readable markdown report. The JSON filename is timestamped so prior runs are kept as the baseline trail; @@ -250,9 +247,7 @@ def diff_against_baseline( lines.append(f"+ NEW [{r.suite}] {r.name}: {r.status.value}") continue if was["status"] != r.status.value: - lines.append( - f"~ {r.name}: status {was['status']} -> {r.status.value}" - ) + lines.append(f"~ {r.name}: status {was['status']} -> {r.status.value}") old_n, new_n = was.get("numeric"), r.numeric if old_n is not None and new_n is not None and abs(old_n - new_n) > 1e-9: lines.append(f"~ {r.name}: numeric {old_n} -> {new_n}") @@ -281,8 +276,10 @@ def render_console(results: list[CheckResult]) -> None: by_suite.setdefault(r.suite, []).append(r) for suite in sorted(by_suite): - label = "Suite S - stealth/anti-bot" if suite == "S" else ( - "Suite E - extraction" if suite == "E" else f"Suite {suite}" + label = ( + "Suite S - stealth/anti-bot" + if suite == "S" + else ("Suite E - extraction" if suite == "E" else f"Suite {suite}") ) print(f"\n=== {label} ===") for r in by_suite[suite]: diff --git a/surfsense_backend/app/proprietary/web_crawler/testbench/suite_extraction.py b/surfsense_backend/app/proprietary/web_crawler/testbench/suite_extraction.py index 40fc1a223..92e338eb8 100644 --- a/surfsense_backend/app/proprietary/web_crawler/testbench/suite_extraction.py +++ b/surfsense_backend/app/proprietary/web_crawler/testbench/suite_extraction.py @@ -68,7 +68,7 @@ async def _run_case( bar = f"SUCCESS + contains {len(case.must_contain)} marker(s)" try: outcome = await connector.crawl_url(case.url) - except Exception as exc: # noqa: BLE001 - never crash the run + except Exception as exc: return CheckResult( suite="E", name=case.name, diff --git a/surfsense_backend/app/proprietary/web_crawler/testbench/suite_stealth.py b/surfsense_backend/app/proprietary/web_crawler/testbench/suite_stealth.py index 83b2ce023..09f2e56d8 100644 --- a/surfsense_backend/app/proprietary/web_crawler/testbench/suite_stealth.py +++ b/surfsense_backend/app/proprietary/web_crawler/testbench/suite_stealth.py @@ -22,6 +22,7 @@ parser later is a one-function change (the spec list is the extension point). from __future__ import annotations import asyncio +import contextlib import re from collections.abc import Callable from dataclasses import dataclass @@ -83,10 +84,8 @@ def _full_text(page: Any, limit: int = 200_000) -> str: def _parse_sannysoft(page: Any, _cell: EvalCell): """Count failed result cells (JS sets class='failed' on red rows).""" html = "" - try: + with contextlib.suppress(Exception): html = page.html_content or "" - except Exception: - pass fails = len(re.findall(r'class="[^"]*\bfailed\b[^"]*"', html)) if not html: return (CheckStatus.ERROR, None, "no html returned") @@ -184,8 +183,7 @@ def _parse_incolumitas(page: Any, _cell: EvalCell): if not fails and dc is None: return (CheckStatus.INFO, None, f"unparsed: {txt[:120]!r}") detail = ( - f"fpscanner FAIL={fails or 'none'} datacenter={datacenter} " - f"behavioral={bscore}" + f"fpscanner FAIL={fails or 'none'} datacenter={datacenter} behavioral={bscore}" ) return ( CheckStatus.PASS if not fails else CheckStatus.FAIL, @@ -375,7 +373,7 @@ def _run_browser_site( try: page = StealthyFetcher.fetch(site.url, **kwargs) - except Exception as exc: # noqa: BLE001 - never crash the run + except Exception as exc: return CheckResult( suite="S", name=site.name, @@ -388,7 +386,7 @@ def _run_browser_site( try: status, numeric, detail = site.parse(page, cell) - except Exception as exc: # noqa: BLE001 + except Exception as exc: status, numeric, detail = ( CheckStatus.ERROR, None, @@ -429,7 +427,7 @@ async def _run_tls(proxy: str | None) -> CheckResult: bar="informational (diff vs real Chrome)", detail=f"ja3={ja3} ja4={ja4} peet={peet}", ) - except Exception as exc: # noqa: BLE001 + except Exception as exc: return CheckResult( suite="S", name="tls_fingerprint", @@ -460,7 +458,7 @@ async def _run_proxy_leak(proxy: str | None) -> CheckResult: bar="not your real/datacenter IP", detail=f"exit_ip={ip} ({note})", ) - except Exception as exc: # noqa: BLE001 + except Exception as exc: return CheckResult( suite="S", name="exit_ip", diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index f4e2c5e6c..b2f6a8fab 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -43,6 +43,7 @@ from .google_drive_add_connector_route import ( from .google_gmail_add_connector_route import ( router as google_gmail_add_connector_router, ) +from .google_maps_routes import router as google_maps_router from .image_generation_routes import router as image_generation_router from .incentive_tasks_routes import router as incentive_tasks_router from .jira_add_connector_route import router as jira_add_connector_router @@ -137,6 +138,7 @@ router.include_router(public_chat_router) # Public chat sharing and cloning router.include_router(incentive_tasks_router) # Incentive tasks for earning free pages router.include_router(stripe_router) # Stripe checkout for additional page packs router.include_router(youtube_router) # YouTube playlist resolution +router.include_router(google_maps_router) # Google Maps places + reviews scraper router.include_router(prompts_router) router.include_router(memory_router) # User personal memory (memory.md style) router.include_router(team_memory_router) # Workspace team memory diff --git a/surfsense_backend/app/routes/editor_routes.py b/surfsense_backend/app/routes/editor_routes.py index 189e9b433..c0776b7fc 100644 --- a/surfsense_backend/app/routes/editor_routes.py +++ b/surfsense_backend/app/routes/editor_routes.py @@ -172,9 +172,7 @@ async def get_editor_content( return _build_response(markdown_content) -@router.get( - "/workspaces/{workspace_id}/documents/{document_id}/download-markdown" -) +@router.get("/workspaces/{workspace_id}/documents/{document_id}/download-markdown") async def download_document_markdown( workspace_id: int, document_id: int, diff --git a/surfsense_backend/app/routes/folders_routes.py b/surfsense_backend/app/routes/folders_routes.py index af4dda737..3e238b7e5 100644 --- a/surfsense_backend/app/routes/folders_routes.py +++ b/surfsense_backend/app/routes/folders_routes.py @@ -517,9 +517,7 @@ async def bulk_move_documents( if not target: raise HTTPException(status_code=404, detail="Target folder not found") mismatched = [ - doc.id - for doc in documents - if doc.workspace_id != target.workspace_id + doc.id for doc in documents if doc.workspace_id != target.workspace_id ] if mismatched: raise HTTPException( diff --git a/surfsense_backend/app/routes/gateway_webhook_routes.py b/surfsense_backend/app/routes/gateway_webhook_routes.py index f09186295..a3c0609ce 100644 --- a/surfsense_backend/app/routes/gateway_webhook_routes.py +++ b/surfsense_backend/app/routes/gateway_webhook_routes.py @@ -886,9 +886,7 @@ async def list_connections( baileys_account_ids.add(int(account.id)) route_type = "account" connection_id = account.id - workspace_id = ( - account.owner_workspace_id or binding.workspace_id - ) + workspace_id = account.owner_workspace_id or binding.workspace_id display_name = "WhatsApp Bridge" connections.append( @@ -907,7 +905,6 @@ async def list_connections( else binding.external_username ), "workspace_name": workspace_name, - "workspace_id": workspace_id, "health_status": account.health_status.value, "suspended_reason": binding.suspended_reason, } @@ -940,7 +937,6 @@ async def list_connections( "display_name": "WhatsApp Bridge", "external_username": None, "workspace_name": account_state.get("display_phone_number"), - "workspace_id": account_state.get("phone_number_id"), "health_status": account.health_status.value, "suspended_reason": account.suspended_reason, } diff --git a/surfsense_backend/app/routes/google_maps_routes.py b/surfsense_backend/app/routes/google_maps_routes.py new file mode 100644 index 000000000..590dc452a --- /dev/null +++ b/surfsense_backend/app/routes/google_maps_routes.py @@ -0,0 +1,64 @@ +"""Google Maps scraper routes (Apify actor-compatible).""" + +import logging + +from fastapi import APIRouter, Depends, HTTPException + +from app.auth.context import AuthContext +from app.proprietary.platforms.google_maps import ( + GoogleMapsReviewsInput, + GoogleMapsScrapeInput, + scrape_places, + scrape_reviews, +) +from app.proprietary.platforms.google_maps.scraper import SignInRequiredError +from app.users import require_session_context + +router = APIRouter() +logger = logging.getLogger(__name__) + + +@router.post("/google-maps/scrape") +async def scrape_places_route( + payload: GoogleMapsScrapeInput, + _auth: AuthContext = Depends(require_session_context), +) -> list[dict]: + """Scrape Google Maps places (search terms / URLs / place IDs). + + Apify "Google Maps Scraper"-compatible input/output. Runs inline and is + bounded only by the request's own ``maxCrawledPlacesPerSearch``. + """ + try: + return await scrape_places(payload) + except SignInRequiredError as e: + raise HTTPException( + status_code=403, detail=f"Google sign in required: {e!s}" + ) from e + except Exception as e: + logger.error("Google Maps scrape failed: %s", e) + raise HTTPException( + status_code=502, detail=f"Google Maps scrape failed: {e!s}" + ) from e + + +@router.post("/google-maps/reviews") +async def scrape_reviews_route( + payload: GoogleMapsReviewsInput, + _auth: AuthContext = Depends(require_session_context), +) -> list[dict]: + """Scrape Google Maps reviews for the given place URLs / place IDs. + + Apify "Google Maps Reviews Scraper"-compatible. Runs inline and is bounded + only by the request's own ``maxReviews``. + """ + try: + return await scrape_reviews(payload) + except SignInRequiredError as e: + raise HTTPException( + status_code=403, detail=f"Google sign in required: {e!s}" + ) from e + except Exception as e: + logger.error("Google Maps reviews scrape failed: %s", e) + raise HTTPException( + status_code=502, detail=f"Google Maps reviews scrape failed: {e!s}" + ) from e diff --git a/surfsense_backend/app/routes/logs_routes.py b/surfsense_backend/app/routes/logs_routes.py index 355afece2..c4903327d 100644 --- a/surfsense_backend/app/routes/logs_routes.py +++ b/surfsense_backend/app/routes/logs_routes.py @@ -277,9 +277,7 @@ async def get_logs_summary( # Get logs from the time window result = await session.execute( select(Log) - .filter( - and_(Log.workspace_id == workspace_id, Log.created_at >= since) - ) + .filter(and_(Log.workspace_id == workspace_id, Log.created_at >= since)) .order_by(desc(Log.created_at)) ) logs = result.scalars().all() diff --git a/surfsense_backend/app/routes/model_connections_routes.py b/surfsense_backend/app/routes/model_connections_routes.py index 0623340b7..8f6a17cfd 100644 --- a/surfsense_backend/app/routes/model_connections_routes.py +++ b/surfsense_backend/app/routes/model_connections_routes.py @@ -206,9 +206,7 @@ async def _resolve_role_model_id( return 0 -async def _clear_invalid_roles( - session: AsyncSession, workspace_id: int -) -> Workspace: +async def _clear_invalid_roles(session: AsyncSession, workspace_id: int) -> Workspace: workspace = await _get_workspace(session, workspace_id) workspace.chat_model_id = await _resolve_role_model_id( session, @@ -752,9 +750,7 @@ async def test_connection_model( ) -@router.get( - "/workspaces/{workspace_id}/model-roles", response_model=ModelRolesRead -) +@router.get("/workspaces/{workspace_id}/model-roles", response_model=ModelRolesRead) async def get_model_roles( workspace_id: int, session: AsyncSession = Depends(get_async_session), @@ -777,9 +773,7 @@ async def get_model_roles( ) -@router.put( - "/workspaces/{workspace_id}/model-roles", response_model=ModelRolesRead -) +@router.put("/workspaces/{workspace_id}/model-roles", response_model=ModelRolesRead) async def update_model_roles( workspace_id: int, data: ModelRolesUpdate, diff --git a/surfsense_backend/app/routes/new_chat_routes.py b/surfsense_backend/app/routes/new_chat_routes.py index 6307264f9..89d1cef95 100644 --- a/surfsense_backend/app/routes/new_chat_routes.py +++ b/surfsense_backend/app/routes/new_chat_routes.py @@ -45,9 +45,9 @@ from app.db import ( NewChatMessageRole, NewChatThread, Permission, - Workspace, TokenUsage, User, + Workspace, get_async_session, shielded_async_session, ) @@ -560,9 +560,7 @@ async def check_thread_access( # For legacy threads (created before visibility feature), # only the workspace owner can access if is_legacy: - workspace_query = select(Workspace).filter( - Workspace.id == thread.workspace_id - ) + workspace_query = select(Workspace).filter(Workspace.id == thread.workspace_id) workspace_result = await session.execute(workspace_query) workspace = workspace_result.scalar_one_or_none() is_workspace_owner = workspace and workspace.user_id == user.id @@ -624,9 +622,7 @@ async def list_threads( ) # Check if user is the workspace owner (for legacy thread visibility) - workspace_query = select(Workspace).filter( - Workspace.id == workspace_id - ) + workspace_query = select(Workspace).filter(Workspace.id == workspace_id) workspace_result = await session.execute(workspace_query) workspace = workspace_result.scalar_one_or_none() is_workspace_owner = workspace and workspace.user_id == user.id @@ -731,9 +727,7 @@ async def search_threads( ) # Check if user is the workspace owner (for legacy thread visibility) - workspace_query = select(Workspace).filter( - Workspace.id == workspace_id - ) + workspace_query = select(Workspace).filter(Workspace.id == workspace_id) workspace_result = await session.execute(workspace_query) workspace = workspace_result.scalar_one_or_none() is_workspace_owner = workspace and workspace.user_id == user.id diff --git a/surfsense_backend/app/routes/obsidian_plugin_routes.py b/surfsense_backend/app/routes/obsidian_plugin_routes.py index 5aa7cf8fa..65adef33b 100644 --- a/surfsense_backend/app/routes/obsidian_plugin_routes.py +++ b/surfsense_backend/app/routes/obsidian_plugin_routes.py @@ -22,8 +22,8 @@ from app.db import ( DocumentType, SearchSourceConnector, SearchSourceConnectorType, - Workspace, User, + Workspace, get_async_session, ) from app.notifications.service import NotificationService diff --git a/surfsense_backend/app/routes/rbac_routes.py b/surfsense_backend/app/routes/rbac_routes.py index e9eaf5502..2d8f58d33 100644 --- a/surfsense_backend/app/routes/rbac_routes.py +++ b/surfsense_backend/app/routes/rbac_routes.py @@ -21,11 +21,11 @@ from sqlalchemy.orm import selectinload from app.auth.context import AuthContext from app.db import ( Permission, + User, Workspace, WorkspaceInvite, WorkspaceMembership, WorkspaceRole, - User, get_async_session, ) from app.schemas import ( @@ -256,9 +256,7 @@ async def list_roles( ) result = await session.execute( - select(WorkspaceRole).filter( - WorkspaceRole.workspace_id == workspace_id - ) + select(WorkspaceRole).filter(WorkspaceRole.workspace_id == workspace_id) ) return result.scalars().all() @@ -982,9 +980,7 @@ async def get_invite_info( # Check if invite is still valid if not invite.is_active: return InviteInfoResponse( - workspace_name=invite.workspace.name - if invite.workspace - else "", + workspace_name=invite.workspace.name if invite.workspace else "", role_name=invite.role.name if invite.role else None, is_valid=False, message="This invite is no longer active", @@ -992,9 +988,7 @@ async def get_invite_info( if invite.expires_at and invite.expires_at < datetime.now(UTC): return InviteInfoResponse( - workspace_name=invite.workspace.name - if invite.workspace - else "", + workspace_name=invite.workspace.name if invite.workspace else "", role_name=invite.role.name if invite.role else None, is_valid=False, message="This invite has expired", @@ -1002,9 +996,7 @@ async def get_invite_info( if invite.max_uses and invite.uses_count >= invite.max_uses: return InviteInfoResponse( - workspace_name=invite.workspace.name - if invite.workspace - else "", + workspace_name=invite.workspace.name if invite.workspace else "", role_name=invite.role.name if invite.role else None, is_valid=False, message="This invite has reached its maximum uses", diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 2bdf29756..1aec3f172 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -489,8 +489,7 @@ async def update_search_source_connector( if key == "connector_type" and value != db_connector.connector_type: check_result = await session.execute( select(SearchSourceConnector).filter( - SearchSourceConnector.workspace_id - == db_connector.workspace_id, + SearchSourceConnector.workspace_id == db_connector.workspace_id, SearchSourceConnector.connector_type == value, SearchSourceConnector.id != connector_id, ) diff --git a/surfsense_backend/app/routes/video_presentations_routes.py b/surfsense_backend/app/routes/video_presentations_routes.py index 2bc7a8ee1..e0d3ea78e 100644 --- a/surfsense_backend/app/routes/video_presentations_routes.py +++ b/surfsense_backend/app/routes/video_presentations_routes.py @@ -19,9 +19,9 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.auth.context import AuthContext from app.db import ( Permission, + VideoPresentation, Workspace, WorkspaceMembership, - VideoPresentation, get_async_session, ) from app.schemas import VideoPresentationRead diff --git a/surfsense_backend/app/routes/workspaces_routes.py b/surfsense_backend/app/routes/workspaces_routes.py index 721038c6f..55eaa8ea9 100644 --- a/surfsense_backend/app/routes/workspaces_routes.py +++ b/surfsense_backend/app/routes/workspaces_routes.py @@ -279,9 +279,7 @@ async def update_workspace( ) from e -@router.put( - "/workspaces/{workspace_id}/api-access", response_model=WorkspaceRead -) +@router.put("/workspaces/{workspace_id}/api-access", response_model=WorkspaceRead) async def update_workspace_api_access( workspace_id: int, body: WorkspaceApiAccessUpdate, diff --git a/surfsense_backend/app/schemas/__init__.py b/surfsense_backend/app/schemas/__init__.py index e9a62e5ec..845136cfc 100644 --- a/surfsense_backend/app/schemas/__init__.py +++ b/surfsense_backend/app/schemas/__init__.py @@ -103,14 +103,6 @@ from .search_source_connector import ( SearchSourceConnectorRead, SearchSourceConnectorUpdate, ) -from .workspace import ( - WorkspaceApiAccessUpdate, - WorkspaceBase, - WorkspaceCreate, - WorkspaceRead, - WorkspaceUpdate, - WorkspaceWithStats, -) from .stripe import ( CreateCreditCheckoutSessionRequest, CreateCreditCheckoutSessionResponse, @@ -128,6 +120,14 @@ from .video_presentations import ( VideoPresentationRead, VideoPresentationUpdate, ) +from .workspace import ( + WorkspaceApiAccessUpdate, + WorkspaceBase, + WorkspaceCreate, + WorkspaceRead, + WorkspaceUpdate, + WorkspaceWithStats, +) __all__ = [ # Folder schemas @@ -242,13 +242,6 @@ __all__ = [ "SearchSourceConnectorCreate", "SearchSourceConnectorRead", "SearchSourceConnectorUpdate", - "WorkspaceApiAccessUpdate", - # Workspace schemas - "WorkspaceBase", - "WorkspaceCreate", - "WorkspaceRead", - "WorkspaceUpdate", - "WorkspaceWithStats", "StripeWebhookResponse", "ThreadHistoryLoadResponse", "ThreadListItem", @@ -257,12 +250,19 @@ __all__ = [ # User schemas "UserCreate", "UserRead", - "UserWorkspaceAccess", "UserUpdate", + "UserWorkspaceAccess", "VerifyConnectionResponse", # Video Presentation schemas "VideoPresentationBase", "VideoPresentationCreate", "VideoPresentationRead", "VideoPresentationUpdate", + "WorkspaceApiAccessUpdate", + # Workspace schemas + "WorkspaceBase", + "WorkspaceCreate", + "WorkspaceRead", + "WorkspaceUpdate", + "WorkspaceWithStats", ] diff --git a/surfsense_backend/app/services/billable_calls.py b/surfsense_backend/app/services/billable_calls.py index 9529dd493..5c6f5dd76 100644 --- a/surfsense_backend/app/services/billable_calls.py +++ b/surfsense_backend/app/services/billable_calls.py @@ -492,9 +492,7 @@ async def _resolve_agent_billing_for_workspace( chat_model_id = workspace.chat_model_id if chat_model_id is None: - raise ValueError( - f"Workspace {workspace_id} has no chat_model_id configured" - ) + raise ValueError(f"Workspace {workspace_id} has no chat_model_id configured") owner_user_id: UUID = workspace.user_id diff --git a/surfsense_backend/app/services/chat_comments_service.py b/surfsense_backend/app/services/chat_comments_service.py index dfe25c267..2dcad48b9 100644 --- a/surfsense_backend/app/services/chat_comments_service.py +++ b/surfsense_backend/app/services/chat_comments_service.py @@ -17,8 +17,8 @@ from app.db import ( NewChatMessageRole, NewChatThread, Permission, - WorkspaceMembership, User, + WorkspaceMembership, has_permission, ) from app.notifications.service import NotificationService diff --git a/surfsense_backend/app/services/folder_service.py b/surfsense_backend/app/services/folder_service.py index fd31e41d3..c34d0132c 100644 --- a/surfsense_backend/app/services/folder_service.py +++ b/surfsense_backend/app/services/folder_service.py @@ -174,9 +174,7 @@ async def ensure_folder_hierarchy_with_depth_validation( if folder is None: await validate_folder_depth(session, parent_id, subtree_depth=0) - position = await generate_folder_position( - session, workspace_id, parent_id - ) + position = await generate_folder_position(session, workspace_id, parent_id) folder = Folder( name=name, workspace_id=workspace_id, diff --git a/surfsense_backend/app/services/linear/kb_sync_service.py b/surfsense_backend/app/services/linear/kb_sync_service.py index 49deb0113..b267428ec 100644 --- a/surfsense_backend/app/services/linear/kb_sync_service.py +++ b/surfsense_backend/app/services/linear/kb_sync_service.py @@ -213,9 +213,7 @@ class LinearKBSyncService: document.title = f"{issue_identifier}: {issue_title}" document.content = summary_content - document.content_hash = generate_content_hash( - issue_content, workspace_id - ) + document.content_hash = generate_content_hash(issue_content, workspace_id) document.embedding = summary_embedding from sqlalchemy.orm.attributes import flag_modified diff --git a/surfsense_backend/app/services/llm_service.py b/surfsense_backend/app/services/llm_service.py index 3991c3e89..6dbc6e6f7 100644 --- a/surfsense_backend/app/services/llm_service.py +++ b/surfsense_backend/app/services/llm_service.py @@ -469,9 +469,7 @@ async def get_vision_llm( if not candidates: logger.error("No vision-capable models available for Auto mode") return None - config_id = int( - choose_auto_model_candidate(candidates, workspace_id)["id"] - ) + config_id = int(choose_auto_model_candidate(candidates, workspace_id)["id"]) if config_id < 0: global_model = get_global_model(config_id) @@ -532,9 +530,7 @@ async def get_vision_llm( return SanitizedChatLiteLLM(**litellm_kwargs) except Exception as e: - logger.error( - f"Error getting vision LLM for workspace {workspace_id}: {e!s}" - ) + logger.error(f"Error getting vision LLM for workspace {workspace_id}: {e!s}") return None diff --git a/surfsense_backend/app/services/memory/service.py b/surfsense_backend/app/services/memory/service.py index 31169e55c..b25522c10 100644 --- a/surfsense_backend/app/services/memory/service.py +++ b/surfsense_backend/app/services/memory/service.py @@ -11,7 +11,7 @@ from uuid import UUID from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.db import Workspace, User +from app.db import User, Workspace from app.services.memory.document import parse_memory_document, render_memory_document from app.services.memory.rewrite import forced_rewrite from app.services.memory.schemas import MemoryLimits diff --git a/surfsense_backend/app/services/notion/tool_metadata_service.py b/surfsense_backend/app/services/notion/tool_metadata_service.py index efb09c7c2..e80e5889f 100644 --- a/surfsense_backend/app/services/notion/tool_metadata_service.py +++ b/surfsense_backend/app/services/notion/tool_metadata_service.py @@ -84,9 +84,7 @@ class NotionToolMetadataService: "error": "No Notion accounts connected", } - parent_pages = await self._get_parent_pages_by_account( - workspace_id, accounts - ) + parent_pages = await self._get_parent_pages_by_account(workspace_id, accounts) accounts_with_status = [] for acc in accounts: diff --git a/surfsense_backend/app/services/public_chat_service.py b/surfsense_backend/app/services/public_chat_service.py index 303c9abe8..88b9018de 100644 --- a/surfsense_backend/app/services/public_chat_service.py +++ b/surfsense_backend/app/services/public_chat_service.py @@ -31,10 +31,10 @@ from app.db import ( PodcastStatus, PublicChatSnapshot, Report, - WorkspaceMembership, User, VideoPresentation, VideoPresentationStatus, + WorkspaceMembership, ) from app.utils.rbac import check_permission diff --git a/surfsense_backend/app/services/web_crawl_credit_service.py b/surfsense_backend/app/services/web_crawl_credit_service.py index 8f2af5d47..368d8894d 100644 --- a/surfsense_backend/app/services/web_crawl_credit_service.py +++ b/surfsense_backend/app/services/web_crawl_credit_service.py @@ -184,9 +184,7 @@ class WebCrawlCreditService: return user.credit_micros_balance - async def charge_credits( - self, user_id: str | UUID, successes: int - ) -> int | None: + async def charge_credits(self, user_id: str | UUID, successes: int) -> int | None: """Debit the wallet for ``successes`` successful crawls. No-op when crawl billing is disabled or ``successes <= 0``. Returns the @@ -198,9 +196,7 @@ class WebCrawlCreditService: return None return await self._apply_debit(user_id, self.successes_to_micros(successes)) - async def charge_captcha( - self, user_id: str | UUID, attempts: int - ) -> int | None: + async def charge_captcha(self, user_id: str | UUID, attempts: int) -> int | None: """Debit the wallet for ``attempts`` captcha solves (Phase 3d). Per-attempt (not per-success): the solver charges for every attempt even diff --git a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py index 694bf6409..c936b0080 100644 --- a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py @@ -211,9 +211,7 @@ async def _delete_folder_documents( ) def delete_workspace_task(self, workspace_id: int): """Celery task to delete a workspace and heavy child rows in batches.""" - return run_async_celery_task( - lambda: _delete_workspace_background(workspace_id) - ) + return run_async_celery_task(lambda: _delete_workspace_background(workspace_id)) async def _delete_workspace_background(workspace_id: int) -> None: diff --git a/surfsense_backend/app/tasks/celery_tasks/video_presentation_tasks.py b/surfsense_backend/app/tasks/celery_tasks/video_presentation_tasks.py index d46bee689..4f934cc27 100644 --- a/surfsense_backend/app/tasks/celery_tasks/video_presentation_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/video_presentation_tasks.py @@ -127,8 +127,7 @@ async def _generate_video_presentation( ) except ValueError as resolve_err: logger.error( - "VideoPresentation %s: cannot resolve billing for " - "workspace=%s: %s", + "VideoPresentation %s: cannot resolve billing for workspace=%s: %s", video_pres.id, workspace_id, resolve_err, diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/llm_bundle.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/llm_bundle.py index a66a9e12a..5d1a4bc83 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/shared/llm_bundle.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/llm_bundle.py @@ -55,9 +55,7 @@ def _agent_config_from_resolved( ) -async def _load_workspace( - session: AsyncSession, workspace_id: int -) -> Workspace | None: +async def _load_workspace(session: AsyncSession, workspace_id: int) -> Workspace | None: result = await session.execute( select(Workspace).where(Workspace.id == workspace_id) ) diff --git a/surfsense_backend/app/users.py b/surfsense_backend/app/users.py index c3e58cea9..dbec93876 100644 --- a/surfsense_backend/app/users.py +++ b/surfsense_backend/app/users.py @@ -22,10 +22,10 @@ from app.auth.session_cookies import access_expires_at, write_session from app.config import config from app.db import ( Prompt, + User, Workspace, WorkspaceMembership, WorkspaceRole, - User, async_session_maker, get_async_session, get_default_roles_config, @@ -207,9 +207,7 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]): f"Created default workspace (ID: {default_workspace.id}) for user {user.id}" ) except Exception as e: - logger.error( - f"Failed to create default workspace for user {user.id}: {e}" - ) + logger.error(f"Failed to create default workspace for user {user.id}: {e}") async def on_after_forgot_password( self, user: User, token: str, request: Request | None = None @@ -326,7 +324,7 @@ def _token_meets_epoch(token: str) -> bool: return False issued_at = payload.get("iat") - return isinstance(issued_at, (int, float)) and int(issued_at) >= min_issued_at + return isinstance(issued_at, int | float) and int(issued_at) >= min_issued_at async def get_auth_context( diff --git a/surfsense_backend/app/utils/crawl/classifier.py b/surfsense_backend/app/utils/crawl/classifier.py index 8b3ba8e98..e58a55537 100644 --- a/surfsense_backend/app/utils/crawl/classifier.py +++ b/surfsense_backend/app/utils/crawl/classifier.py @@ -12,10 +12,10 @@ our enterprise targets). Regexes are compiled once at import (hot-path hygiene). """ import re -from enum import Enum +from enum import StrEnum -class BlockType(str, Enum): +class BlockType(StrEnum): """Coarse label for what a fetched page represents.""" OK = "ok" # usable content, no challenge detected diff --git a/surfsense_backend/app/utils/validators.py b/surfsense_backend/app/utils/validators.py index 187cba21c..6e71abf79 100644 --- a/surfsense_backend/app/utils/validators.py +++ b/surfsense_backend/app/utils/validators.py @@ -44,9 +44,7 @@ def validate_workspace_id(workspace_id: Any) -> int: if isinstance(workspace_id, str): # Check if it's a valid integer string if not workspace_id.strip(): - raise HTTPException( - status_code=400, detail="workspace_id cannot be empty" - ) + raise HTTPException(status_code=400, detail="workspace_id cannot be empty") # Check for valid integer format (no leading zeros, no decimal points) if not re.match(r"^[1-9]\d*$", workspace_id.strip()): diff --git a/surfsense_backend/scripts/e2e_google_maps_deep.py b/surfsense_backend/scripts/e2e_google_maps_deep.py new file mode 100644 index 000000000..efaf4f588 --- /dev/null +++ b/surfsense_backend/scripts/e2e_google_maps_deep.py @@ -0,0 +1,824 @@ +"""Deep functional verification for the Google Maps scraper (live network). + +Complements the fast smoke e2e (e2e_google_maps_scraper.py) with breadth: +diverse places (countries, scripts, categories), URL-kind coverage (name-only +URLs, CID), and review semantics (sort order, date cutoff, pagination +uniqueness, personal-data stripping, localization). + +Verification style: ground-truth invariants instead of screenshots — known +coordinates/websites/address keywords for world-famous places, and internal +consistency rules (newest sort is monotonically non-increasing, cutoff dates +hold, review IDs are unique across pages, etc.). + +Run from the backend directory: + .\\.venv\\Scripts\\python.exe scripts/e2e_google_maps_deep.py +""" + +import asyncio +import itertools +import sys +from datetime import datetime +from pathlib import Path + +from dotenv import load_dotenv + +if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + +_BACKEND_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_BACKEND_ROOT)) +for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): + if _candidate.exists(): + load_dotenv(_candidate) + break + +from app.proprietary.platforms.google_maps import ( # noqa: E402 + GoogleMapsReviewsInput, + GoogleMapsScrapeInput, + scrape_places, + scrape_reviews, +) + +_CHECKS: list[tuple[str, bool, str]] = [] + + +def _check(label: str, ok: bool, detail: str = "") -> bool: + _CHECKS.append((label, ok, detail)) + print(f" [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}") + return ok + + +def _hr(title: str) -> None: + print(f"\n{'=' * 70}\n{title}\n{'=' * 70}") + + +def _near(actual: float | None, expected: float, tol: float = 0.05) -> bool: + return actual is not None and abs(actual - expected) <= tol + + +def _iso(s: str | None) -> datetime | None: + if not s: + return None + try: + return datetime.fromisoformat(s.replace("Z", "+00:00")) + except ValueError: + return None + + +# Ground truth: world-famous places whose facts don't drift. Name-only URLs +# exercise the HTML -> fid resolution path for every one of them. +_PLACES = [ + { + "label": "Eiffel Tower (FR landmark)", + "url": "https://www.google.com/maps/place/Eiffel+Tower/", + "title_contains": "eiffel", + "lat": 48.8584, + "lng": 2.2945, + "address_contains": ["paris", "75007", "france"], + "website_contains": "toureiffel", + }, + { + "label": "Tokyo Tower (JP, non-Latin locale)", + "url": "https://www.google.com/maps/place/Tokyo+Tower/", + "title_contains": "tokyo tower", + "lat": 35.6586, + "lng": 139.7454, + "address_contains": ["tokyo", "japan", "minato"], + }, + { + "label": "Sydney Opera House (AU)", + "url": "https://www.google.com/maps/place/Sydney+Opera+House/", + "title_contains": "opera house", + "lat": -33.8568, + "lng": 151.2153, + "address_contains": ["sydney", "nsw", "australia"], + }, + { + "label": "The Plaza Hotel (US hotel category)", + "url": "https://www.google.com/maps/place/The+Plaza+Hotel+New+York/", + "title_contains": "plaza", + "lat": 40.7646, + "lng": -73.9744, + "address_contains": ["new york", "ny"], + "category_contains": "hotel", + }, +] + +_KIMS_CID_URL = "https://maps.google.com/?cid=7838756667406262025" # Kim's Island +_KIMS_PLACE_ID = "ChIJJQz5EZzKw4kRCZ95UajbyGw" +_LOUVRE_URL = "https://www.google.com/maps/place/Louvre+Museum/" + + +async def scrape_one(url: str, **kwargs) -> dict | None: + items = await scrape_places( + GoogleMapsScrapeInput(startUrls=[{"url": url}], **kwargs) + ) + return items[0] if items else None + + +async def step_diverse_places() -> None: + _hr("A — diverse places via name-only URLs (HTML -> fid path)") + for spec in _PLACES: + it = await scrape_one(spec["url"]) + if it is None: + _check(spec["label"], False, "no item returned") + continue + title = (it.get("title") or "").lower() + loc = it.get("location") or {} + addr = (it.get("address") or "").lower() + problems = [] + if spec["title_contains"] not in title: + problems.append(f"title={it.get('title')!r}") + if not _near(loc.get("lat"), spec["lat"]) or not _near( + loc.get("lng"), spec["lng"] + ): + problems.append(f"loc={loc}") + if not any(k in addr for k in spec["address_contains"]): + problems.append(f"address={it.get('address')!r}") + if not it.get("placeId"): + problems.append("no placeId") + if not it.get("categories"): + problems.append("no categories") + if spec.get("website_contains") and spec["website_contains"] not in ( + it.get("website") or "" + ): + problems.append(f"website={it.get('website')!r}") + if ( + spec.get("category_contains") + and spec["category_contains"] + not in ( + (it.get("categoryName") or "") + " ".join(it.get("categories") or []) + ).lower() + ): + problems.append(f"categories={it.get('categories')}") + _check( + spec["label"], + not problems, + "; ".join(problems) + or f"{it.get('title')!r} @ ({loc.get('lat'):.4f},{loc.get('lng'):.4f}) " + f"cat={it.get('categoryName')!r} score={it.get('totalScore')}", + ) + + +async def step_cid_url() -> None: + _hr("B — CID URL dispatch") + it = await scrape_one(_KIMS_CID_URL) + ok = it is not None and it.get("title") == "Kim's Island" + _check( + "?cid=... resolves to the right place", + ok, + f"title={it.get('title')!r}" if it else "no item", + ) + if it: + _check( + "cid place has full detail (phone+hours)", + bool(it.get("phone")) and bool(it.get("openingHours")), + f"phone={it.get('phone')!r}, hours={len(it.get('openingHours') or [])} days", + ) + + +async def step_review_sorts() -> None: + _hr("C — review sort semantics (Kim's Island)") + newest = await scrape_reviews( + GoogleMapsReviewsInput(placeIds=[_KIMS_PLACE_ID], maxReviews=10) + ) + dates = [_iso(r.get("publishedAtDate")) for r in newest] + dated = [d for d in dates if d] + _check( + "newest: publishedAtDate non-increasing", + len(dated) >= 5 and all(a >= b for a, b in itertools.pairwise(dated)), + f"{len(newest)} reviews, first={newest[0].get('publishAt') if newest else None}", + ) + + lowest = await scrape_reviews( + GoogleMapsReviewsInput( + placeIds=[_KIMS_PLACE_ID], maxReviews=10, reviewsSort="lowestRanking" + ) + ) + highest = await scrape_reviews( + GoogleMapsReviewsInput( + placeIds=[_KIMS_PLACE_ID], maxReviews=10, reviewsSort="highestRanking" + ) + ) + lo = [r["stars"] for r in lowest if r.get("stars")] + hi = [r["stars"] for r in highest if r.get("stars")] + lo_avg = sum(lo) / len(lo) if lo else 0 + hi_avg = sum(hi) / len(hi) if hi else 0 + _check( + "lowestRanking avg < highestRanking avg", + bool(lo and hi) and lo_avg < hi_avg, + f"lowest avg={lo_avg:.2f} (first={lo[:3]}), highest avg={hi_avg:.2f} (first={hi[:3]})", + ) + _check( + "highestRanking page is all 5 stars", + bool(hi) and all(s == 5 for s in hi), + f"stars={hi}", + ) + + +async def step_start_date_cutoff() -> None: + _hr("D — reviewsStartDate cutoff") + baseline = await scrape_reviews( + GoogleMapsReviewsInput(placeIds=[_KIMS_PLACE_ID], maxReviews=10) + ) + dated = [(_iso(r.get("publishedAtDate")), r) for r in baseline] + dated = [(d, r) for d, r in dated if d] + if len(dated) < 6: + _check("cutoff test has enough dated reviews", False, f"only {len(dated)}") + return + # Cut between the 4th and 5th newest review -> expect exactly 4 back. + cutoff_dt = dated[4][0] + cutoff = ( + dated[3][0].strftime("%Y-%m-%dT%H:%M:%S.000Z") + if dated[3][0] == cutoff_dt + else cutoff_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") + ) + got = await scrape_reviews( + GoogleMapsReviewsInput( + placeIds=[_KIMS_PLACE_ID], maxReviews=100, reviewsStartDate=cutoff + ) + ) + got_dates = [_iso(r.get("publishedAtDate")) for r in got] + cutoff_parsed = _iso(cutoff) + _check( + "all returned reviews >= cutoff", + bool(got) and all(d is None or d >= cutoff_parsed for d in got_dates), + f"cutoff={cutoff}, returned={len(got)} (expected ~4)", + ) + _check( + "cutoff actually limits the result", + 0 < len(got) < len(baseline) + 1 and len(got) <= 6, + f"{len(got)} vs baseline {len(baseline)}", + ) + + +async def step_personal_data() -> None: + _hr("E — personalData=false stripping") + items = await scrape_reviews( + GoogleMapsReviewsInput( + placeIds=[_KIMS_PLACE_ID], maxReviews=3, personalData=False + ) + ) + if not items: + _check("reviews returned", False) + return + leaked = [ + k + for k in ("name", "reviewerId", "reviewerUrl", "reviewerPhotoUrl") + if any(r.get(k) for r in items) + ] + _check( + "reviewer identity fields are absent", + not leaked, + f"leaked={leaked}" if leaked else f"{len(items)} reviews, ids+stars kept", + ) + _check( + "non-personal fields survive", + all(r.get("reviewId") and r.get("stars") for r in items), + ) + + +async def step_localization() -> None: + _hr("F — localization (language=fr)") + items = await scrape_reviews( + GoogleMapsReviewsInput( + startUrls=[{"url": "https://www.google.com/maps/place/Eiffel+Tower/"}], + maxReviews=5, + language="fr", + ) + ) + if not items: + _check("french reviews returned", False) + return + rel = [r.get("publishAt") or "" for r in items] + french = [s for s in rel if "il y a" in s or "mois" in s or "semaine" in s] + _check( + "relative dates come back in French", + len(french) >= 3, + f"publishAt={rel}", + ) + _check( + "items stamped language=fr", + all(r.get("language") == "fr" for r in items), + ) + + +async def step_big_place_pagination() -> None: + _hr("G — big place, 30 reviews across >=3 pages (Louvre)") + items = await scrape_reviews( + GoogleMapsReviewsInput(startUrls=[{"url": _LOUVRE_URL}], maxReviews=30) + ) + ids = [r.get("reviewId") for r in items] + _check( + "30 reviews with unique IDs", + len(items) == 30 and len(set(ids)) == 30, + f"{len(items)} reviews, {len(set(ids))} unique", + ) + ok_fields = all( + r.get("name") and r.get("stars") is not None and r.get("publishedAtDate") + for r in items + ) + _check("every review has author/stars/date", ok_fields) + _check( + "place header stamped on all (Louvre)", + all("louvre" in (r.get("title") or "").lower() for r in items), + f"title={items[0].get('title')!r}" if items else "", + ) + + +async def step_search_discovery() -> None: + _hr("I — search discovery: paging, rank, dedupe (pizza in New York)") + items = await scrape_places( + GoogleMapsScrapeInput( + searchStringsArray=["pizza"], + locationQuery="New York, NY", + maxCrawledPlacesPerSearch=25, + ) + ) + fids = [i.get("fid") for i in items] + _check( + "25 places across >1 page, all unique fids", + len(items) == 25 and len(set(fids)) == 25, + f"{len(items)} items, {len(set(fids))} unique", + ) + _check( + "ranks are 1..25 in order", + [i.get("rank") for i in items] == list(range(1, 26)), + ) + ny = sum( + 1 + for i in items + if i.get("location") + and 40.4 < (i["location"]["lat"] or 0) < 41.1 + and -74.3 < (i["location"]["lng"] or 0) < -73.6 + ) + _check( + "locationQuery scopes results to NYC", + ny >= 23, + f"{ny}/25 within NYC bounds", + ) + with_core = sum( + 1 for i in items if i.get("title") and i.get("placeId") and i.get("address") + ) + _check("all items have title/placeId/address", with_core == 25, f"{with_core}/25") + + +async def step_search_filters() -> None: + _hr("J — search filters (stars, website, matching)") + starred = await scrape_places( + GoogleMapsScrapeInput( + searchStringsArray=["restaurant"], + locationQuery="Chicago, IL", + maxCrawledPlacesPerSearch=10, + placeMinimumStars="fourAndHalf", + ) + ) + scores = [i.get("totalScore") for i in starred] + _check( + "placeMinimumStars=fourAndHalf holds", + bool(scores) and all(s is not None and s >= 4.5 for s in scores), + f"scores={scores}", + ) + + no_web = await scrape_places( + GoogleMapsScrapeInput( + searchStringsArray=["restaurant"], + locationQuery="Chicago, IL", + maxCrawledPlacesPerSearch=5, + website="withoutWebsite", + ) + ) + _check( + "website=withoutWebsite holds", + bool(no_web) and all(not i.get("website") for i in no_web), + f"{len(no_web)} items, websites={[i.get('website') for i in no_web]}", + ) + + matching = await scrape_places( + GoogleMapsScrapeInput( + searchStringsArray=["pizza"], + locationQuery="Boston, MA", + maxCrawledPlacesPerSearch=5, + searchMatching="only_includes", + ) + ) + titles = [i.get("title") for i in matching] + _check( + "searchMatching=only_includes keeps 'pizza' in titles", + bool(titles) and all("pizza" in (t or "").lower() for t in titles), + f"titles={titles}", + ) + + +async def step_search_closed() -> None: + _hr("K — closed-place detection + skipClosedPlaces (Dean & DeLuca)") + q = "Dean DeLuca 560 Broadway New York" + found = await scrape_places( + GoogleMapsScrapeInput(searchStringsArray=[q], maxCrawledPlacesPerSearch=3) + ) + dean = next((i for i in found if "DeLuca" in (i.get("title") or "")), None) + _check( + "permanently closed place flagged", + dean is not None and dean.get("permanentlyClosed") is True, + f"title={dean.get('title') if dean else None}, closed={dean.get('permanentlyClosed') if dean else None}", + ) + skipped = await scrape_places( + GoogleMapsScrapeInput( + searchStringsArray=[q], + maxCrawledPlacesPerSearch=3, + skipClosedPlaces=True, + ) + ) + _check( + "skipClosedPlaces filters it out", + not any("DeLuca" in (i.get("title") or "") for i in skipped), + f"{len(skipped)} items after skip", + ) + + +async def step_search_url_and_geo() -> None: + _hr("L — /maps/search/ URL routing + customGeolocation") + via_url = await scrape_places( + GoogleMapsScrapeInput( + startUrls=[ + {"url": "https://www.google.com/maps/search/ramen+in+Osaka+Japan/"} + ], + maxCrawledPlacesPerSearch=5, + ) + ) + osaka = sum( + 1 + for i in via_url + if i.get("location") and 34.4 < (i["location"]["lat"] or 0) < 35.0 + ) + _check( + "/maps/search/ startUrl yields Osaka ramen places", + len(via_url) == 5 and osaka >= 4, + f"{len(via_url)} items, {osaka} in Osaka", + ) + + geo = await scrape_places( + GoogleMapsScrapeInput( + searchStringsArray=["museum"], + customGeolocation={ + "type": "Point", + "coordinates": [2.3522, 48.8566], # [lng, lat] Paris + "radiusKm": 10, + }, + maxCrawledPlacesPerSearch=5, + ) + ) + paris = sum( + 1 + for i in geo + if i.get("location") + and 48.5 < (i["location"]["lat"] or 0) < 49.2 + and 1.9 < (i["location"]["lng"] or 0) < 2.8 + ) + _check( + "customGeolocation Point scopes to Paris", + len(geo) >= 3 and paris >= 3, + f"{len(geo)} items, {paris} in Paris: {[i.get('title') for i in geo]}", + ) + + +async def step_search_pagination_stress() -> None: + _hr("M — search pagination stress (60 results / 3+ pages, exhaustion)") + items = await scrape_places( + GoogleMapsScrapeInput( + searchStringsArray=["restaurant"], + locationQuery="Manhattan, New York", + maxCrawledPlacesPerSearch=60, + ) + ) + fids = [i.get("fid") for i in items] + _check( + "60 places, all fids unique (offset paging + dedupe)", + len(items) == 60 and len(set(fids)) == 60, + f"{len(items)} items, {len(set(fids))} unique", + ) + _check( + "ranks strictly sequential 1..60", + [i.get("rank") for i in items] == list(range(1, 61)), + ) + manhattan = sum( + 1 + for i in items + if i.get("location") and 40.68 < (i["location"]["lat"] or 0) < 40.9 + ) + _check( + "results stay in Manhattan across pages", + manhattan >= 55, + f"{manhattan}/60 in bounds", + ) + + # A hyper-specific query has few results: paging must terminate on its + # own (no infinite loop, no error) well before the requested cap. + sparse = await scrape_places( + GoogleMapsScrapeInput( + searchStringsArray=["Kim's Island Staten Island"], + maxCrawledPlacesPerSearch=100, + ) + ) + _check( + "sparse query exhausts naturally below the cap", + 0 < len(sparse) < 25, + f"{len(sparse)} results", + ) + + +async def step_detail_extras() -> None: + _hr("N — detail-page extras (kgmid/cid/additionalInfo/links)") + # pin the exact place by fid — a name search returns a different Joe's + # branch run to run, which makes magnitude assertions flaky + items = await scrape_places( + GoogleMapsScrapeInput( + startUrls=[ + { + "url": "https://www.google.com/maps/place/Joe's+Pizza+Broadway/" + "data=!4m2!3m1!1s0x89c259ab3c1ef289:0x3b67a41175949f55" + } + ], + maxImages=5, + ) + ) + if not items: + _check("place returned", False) + return + it = items[0] + dist = it.get("reviewsDistribution") or {} + _check( + "reviewsCount + distribution (NID-session fields)", + (it.get("reviewsCount") or 0) > 20_000 + and (it.get("reviewsCount") == sum(dist.values())), + f"count={it.get('reviewsCount')}, dist={dist}", + ) + hist = it.get("popularTimesHistogram") or {} + _check( + "popular times histogram covers the week", + set(hist) == {"Su", "Mo", "Tu", "We", "Th", "Fr", "Sa"} + and all( + {"hour", "occupancyPercent"} <= set(slot) + for d in hist.values() + for slot in d + ), + f"days={sorted(hist)}", + ) + _check( + "image gallery fields (count/categories/urls capped at maxImages)", + (it.get("imagesCount") or 0) > 1000 + and "All" in (it.get("imageCategories") or []) + and len(it.get("imageUrls") or []) == 5, + f"count={it.get('imagesCount')}, cats={len(it.get('imageCategories') or [])}, " + f"urls={len(it.get('imageUrls') or [])}", + ) + tags = it.get("reviewsTags") or [] + _check( + "reviewsTags with counts", + len(tags) >= 5 and all(t.get("title") and t.get("count") for t in tags), + f"{tags[:2]}", + ) + _check( + "additionalInfo has full section set (not just Accessibility)", + len(it.get("additionalInfo") or {}) >= 8, + f"sections={list((it.get('additionalInfo') or {}).keys())}", + ) + + # scrapePlaceDetailPage on the SEARCH flow: search darrays lack the + # session-gated fields, so each hit must get enriched via a detail RPC. + enriched = await scrape_places( + GoogleMapsScrapeInput( + searchStringsArray=["ramen"], + locationQuery="Chicago, IL", + maxCrawledPlacesPerSearch=2, + scrapePlaceDetailPage=True, + ) + ) + _check( + "search flow + scrapePlaceDetailPage enriches every hit", + len(enriched) == 2 + and all( + (e.get("reviewsCount") or 0) > 0 and e.get("reviewsDistribution") + for e in enriched + ), + f"counts={[e.get('reviewsCount') for e in enriched]}", + ) + fid = it.get("fid") or "" + cid_ok = bool(fid) and it.get("cid") == str(int(fid.split(":")[1], 16)) + _check( + "kgmid + cid derived from fid", + bool(it.get("kgmid", "").startswith("/g/")) and cid_ok, + f"kgmid={it.get('kgmid')}, cid={it.get('cid')}", + ) + info = it.get("additionalInfo") or {} + _check( + "additionalInfo has sections with boolean options", + bool(info) + and all( + isinstance(v, list) and all(isinstance(e, dict) for e in v) + for v in info.values() + ), + f"sections={list(info.keys())}", + ) + + +async def step_hotel_fields() -> None: + _hr("O — hotel fields (The Plaza: stars, dates, similar, ads)") + items = await scrape_places( + GoogleMapsScrapeInput( + startUrls=[ + { + "url": "https://www.google.com/maps/place/The+Plaza/" + "data=!4m2!3m1!1s0x89c258f07d5da561:0x61f6aa300ba8339d" + } + ], + ) + ) + if not items: + _check("hotel returned", False) + return + it = items[0] + _check( + "hotelStars + check-in/out dates", + it.get("hotelStars") == "5 stars" + and (it.get("checkInDate") or "") < (it.get("checkOutDate") or ""), + f"stars={it.get('hotelStars')}, {it.get('checkInDate')}..{it.get('checkOutDate')}", + ) + similar = it.get("similarHotelsNearby") or [] + _check( + "similarHotelsNearby with fid/score", + len(similar) >= 3 and all(h.get("title") and h.get("fid") for h in similar), + f"{len(similar)} hotels, first={similar[0].get('title') if similar else None}", + ) + ads = it.get("hotelAds") or [] + _check( + "hotelAds with booking links", + bool(ads) and all(a.get("url", "").startswith("https://") for a in ads), + f"{len(ads)} ads", + ) + + +async def step_all_places_scan() -> None: + _hr("P — allPlacesNoSearchAction area scan (Times Square 400m)") + items = await scrape_places( + GoogleMapsScrapeInput( + allPlacesNoSearchAction="all_places_no_search_mouse", + customGeolocation={ + "type": "Point", + "coordinates": [-73.9855, 40.758], + "radiusKm": 0.4, + }, + maxCrawledPlacesPerSearch=25, + ) + ) + fids = [i.get("fid") for i in items] + _check( + "25 unique places without any search term", + len(items) == 25 and len(set(fids)) == 25, + f"{len(items)} items", + ) + cats = {i.get("categoryName") for i in items if i.get("categoryName")} + _check( + "multiple categories represented (sweep, not one query)", + len(cats) >= 5, + f"{len(cats)} categories", + ) + in_view = sum( + 1 for i in items if i.get("location") and 40.74 < i["location"]["lat"] < 40.78 + ) + _check("scan respects the viewport", in_view >= 23, f"{in_view}/25 in bounds") + + +async def step_short_url() -> None: + _hr("Q — short link (maps.app.goo.gl Firebase redirect -> place)") + # A real shared short link -> "Aux Merveilleux de Fred" (NYC). These are + # Firebase Dynamic Links (JS interstitial), so this exercises the browser- + # render redirect path in resolve_fid. fid is the ground-truth invariant. + it = await scrape_one("https://maps.app.goo.gl/8YUvDPbQPrasqC528") + _check( + "maps.app.goo.gl short link resolves to the right place", + it is not None + and it.get("fid") == "0x89c259957da502cd:0xed3eb58a4ca08a95" + and "merveilleux" in (it.get("title") or "").lower(), + f"title={it.get('title') if it else None!r}, fid={it.get('fid') if it else None}", + ) + + +async def step_filter_variants() -> None: + _hr("R — search filter variants (only_exact, categoryFilterWords)") + # only_exact: the parsed title must equal the query exactly (Seattle + # Starbucks are titled "Starbucks Coffee Company", not "Starbucks"). + exact = await scrape_places( + GoogleMapsScrapeInput( + searchStringsArray=["Starbucks Coffee Company"], + locationQuery="Seattle, WA", + maxCrawledPlacesPerSearch=8, + searchMatching="only_exact", + ) + ) + titles = [i.get("title") for i in exact] + _check( + "searchMatching=only_exact keeps only exact-title matches", + bool(titles) + and all((t or "").lower() == "starbucks coffee company" for t in titles), + f"{len(titles)} items, titles={titles[:3]}", + ) + # categoryFilterWords: drop places whose categories don't include a word. + coffee = await scrape_places( + GoogleMapsScrapeInput( + searchStringsArray=["food"], + locationQuery="Seattle, WA", + maxCrawledPlacesPerSearch=6, + categoryFilterWords=["coffee"], + ) + ) + _check( + "categoryFilterWords keeps only matching categories", + bool(coffee) + and all( + any("coffee" in c.lower() for c in (i.get("categories") or [])) + for i in coffee + ), + f"{len(coffee)} items, cats={[i.get('categories') for i in coffee][:2]}", + ) + + +async def step_reviews_origin() -> None: + _hr("S — reviewsOrigin=google filter") + # The public BOQ feed only carries Google-origin reviews (partner reviews + # aren't exposed anonymously), so the invariant is: nothing non-Google + # leaks through when origin is pinned to google. + items = await scrape_reviews( + GoogleMapsReviewsInput( + placeIds=[_KIMS_PLACE_ID], maxReviews=10, reviewsOrigin="google" + ) + ) + origins = {(r.get("reviewOrigin") or "Google") for r in items} + _check( + "reviewsOrigin=google -> only Google-origin reviews", + bool(items) and origins <= {"Google"}, + f"{len(items)} reviews, origins={origins}", + ) + + +async def step_inline_consistency() -> None: + _hr("H — inline reviews[] match the standalone reviews endpoint") + place = await scrape_one( + f"https://www.google.com/maps/place/?q=place_id:{_KIMS_PLACE_ID}", + maxReviews=5, + ) + standalone = await scrape_reviews( + GoogleMapsReviewsInput(placeIds=[_KIMS_PLACE_ID], maxReviews=5) + ) + if not place or not standalone: + _check("both sources returned data", False) + return + inline_ids = {r.get("reviewId") for r in (place.get("reviews") or [])} + standalone_ids = {r.get("reviewId") for r in standalone} + overlap = len(inline_ids & standalone_ids) + _check( + "inline and standalone reviews overlap (same feed)", + overlap >= 3, # feed ordering can shift slightly between calls + f"{overlap}/5 shared review IDs", + ) + + +async def main() -> int: + steps = [ + step_diverse_places(), + step_cid_url(), + step_review_sorts(), + step_start_date_cutoff(), + step_personal_data(), + step_localization(), + step_big_place_pagination(), + step_inline_consistency(), + step_search_discovery(), + step_search_filters(), + step_search_closed(), + step_search_url_and_geo(), + step_search_pagination_stress(), + step_detail_extras(), + step_hotel_fields(), + step_all_places_scan(), + step_short_url(), + step_filter_variants(), + step_reviews_origin(), + ] + for coro in steps: + try: + await coro + except Exception as e: # keep going; report the step as failed + _check(f"step crashed: {coro}", False, repr(e)) + + _hr("SUMMARY") + passed = sum(1 for _, ok, _ in _CHECKS if ok) + for label, ok, detail in _CHECKS: + if not ok: + print(f" FAILED: {label} — {detail}") + print(f" {passed}/{len(_CHECKS)} checks passed") + return 0 if passed == len(_CHECKS) else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/surfsense_backend/scripts/e2e_google_maps_scraper.py b/surfsense_backend/scripts/e2e_google_maps_scraper.py new file mode 100644 index 000000000..be452f3f1 --- /dev/null +++ b/surfsense_backend/scripts/e2e_google_maps_scraper.py @@ -0,0 +1,241 @@ +"""Manual functional e2e for the Google Maps scraper (app/proprietary/platforms/google_maps). + +Run from the backend directory: + cd surfsense_backend + uv run python scripts/e2e_google_maps_scraper.py + # or: .\\.venv\\Scripts\\python.exe scripts/e2e_google_maps_scraper.py + +NOT a pytest test (needs live network + optional proxy creds). It: + Step 1 — scrapes a known place URL and prints the core fields. + Step 2 — scrapes the same place by bare placeId (HTML -> fid -> RPC path). + Step 3 — dumps the raw place darray (jd[6] of /maps/preview/place) to + tests/unit/platforms/google_maps/fixtures/ for the offline parser test. + Step 4 — scrapes reviews via the Reviews endpoint (BOQ feed), checks fields. + Step 5 — paginates past one page (maxReviews=15) and checks the count. + Step 6 — place scrape with maxReviews>0 attaches inline reviews[]. + Step 7 — dumps a raw BOQ reviews page fixture for the offline parser test. + Step 8 — search discovery (searchStringsArray + locationQuery), checks items. + Step 9 — dumps a raw map-search response fixture for the offline parser test. +""" + +import asyncio +import json +import sys +from pathlib import Path + +from dotenv import load_dotenv + +# Windows consoles default to cp1252; reviews/ratings contain non-latin chars. +if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + +_BACKEND_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_BACKEND_ROOT)) +for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): + if _candidate.exists(): + load_dotenv(_candidate) + break + +from app.proprietary.platforms.google_maps import ( # noqa: E402 + GoogleMapsReviewsInput, + GoogleMapsScrapeInput, + scrape_places, + scrape_reviews, +) +from app.proprietary.platforms.google_maps.fetch import ( # noqa: E402 + build_search_url, + fetch_place_darray, + fetch_rpc_json, + iter_reviews_pages, +) +from app.proprietary.platforms.google_maps.url_resolver import extract_fid # noqa: E402 + +# A well-known, stable place (the restaurant used in the Apify output example). +_PLACE_URL = ( + "https://www.google.com/maps/place/Kim's+Island/" + "@40.5107736,-74.2482624,17z/data=!4m6!3m5!1s0x89c3ca9c11f90c25:" + "0x6cc8dba851799f09!8m2!3d40.5107736!4d-74.2482624!16s%2Fg%2F1tmgdcj8?hl=en" +) + +_FIXTURE_DIR = ( + _BACKEND_ROOT / "tests" / "unit" / "platforms" / "google_maps" / "fixtures" +) + + +def _hr(title: str) -> None: + print(f"\n{'=' * 70}\n{title}\n{'=' * 70}") + + +def _check(label: str, ok: bool, detail: str = "") -> bool: + print(f" [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}") + return ok + + +async def step1_place() -> bool: + _hr("STEP 1 — scrape a known place URL") + items = await scrape_places(GoogleMapsScrapeInput(startUrls=[{"url": _PLACE_URL}])) + if not items: + return _check("place scraped", False, "no items returned") + it = items[0] + print(json.dumps(it, indent=2, ensure_ascii=False)[:2500]) + ok = bool(it.get("title")) and it.get("placeId") is not None + return _check( + "place has title + placeId", + ok, + f"{it.get('title')!r} / {it.get('totalScore')}★ / {it.get('reviewsCount')} reviews", + ) + + +async def step2_place_id() -> bool: + _hr("STEP 2 — scrape by bare placeId (HTML -> fid -> RPC path)") + items = await scrape_places( + GoogleMapsScrapeInput(placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"]) + ) + if not items: + return _check("placeId scraped", False, "no items returned") + it = items[0] + return _check( + "placeId resolves to same place", + it.get("title") == "Kim's Island", + f"{it.get('title')!r}", + ) + + +async def step3_dump_fixture() -> bool: + _hr("STEP 3 — dump raw place darray fixture for offline test") + fid = extract_fid(_PLACE_URL) + if not fid: + return _check("extracted fid from URL", False) + darray = await fetch_place_darray(fid) + if not darray: + return _check("fetched place darray via RPC", False) + _FIXTURE_DIR.mkdir(parents=True, exist_ok=True) + (_FIXTURE_DIR / "place_darray.json").write_text( + json.dumps(darray), encoding="utf-8" + ) + return _check("dumped fixture", True, f"-> {_FIXTURE_DIR / 'place_darray.json'}") + + +async def step4_reviews() -> bool: + _hr("STEP 4 — reviews endpoint (one page, newest first)") + items = await scrape_reviews( + GoogleMapsReviewsInput(startUrls=[{"url": _PLACE_URL}], maxReviews=5) + ) + if not items: + return _check("reviews returned", False, "no items") + it = items[0] + print(json.dumps(it, indent=2, ensure_ascii=False)[:1800]) + ok = ( + bool(it.get("name")) + and it.get("stars") is not None + and bool(it.get("reviewId")) + and it.get("title") == "Kim's Island" # place header stamped on + and bool(it.get("publishedAtDate")) + ) + return _check( + "review has author/stars/id/place header", + ok and len(items) == 5, + f"{len(items)} reviews, first by {it.get('name')!r} ({it.get('stars')}★)", + ) + + +async def step5_pagination() -> bool: + _hr("STEP 5 — pagination past one page (maxReviews=15)") + items = await scrape_reviews( + GoogleMapsReviewsInput(placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"], maxReviews=15) + ) + dates = [i.get("publishedAtDate") for i in items[:3]] + return _check( + "got 15 reviews across >1 page", + len(items) == 15, + f"{len(items)} reviews; newest: {dates}", + ) + + +async def step6_inline_reviews() -> bool: + _hr("STEP 6 — place scrape with maxReviews=3 attaches inline reviews[]") + items = await scrape_places( + GoogleMapsScrapeInput(startUrls=[{"url": _PLACE_URL}], maxReviews=3) + ) + if not items: + return _check("place scraped", False, "no items") + reviews = items[0].get("reviews") or [] + return _check( + "place item carries 3 inline reviews", + len(reviews) == 3 and bool(reviews[0].get("name")), + f"{len(reviews)} reviews inline", + ) + + +async def step7_dump_reviews_fixture() -> bool: + _hr("STEP 7 — dump raw BOQ reviews page fixture for offline test") + fid = extract_fid(_PLACE_URL) + async for raw_page in iter_reviews_pages(fid, sort="newest", max_pages=1): + _FIXTURE_DIR.mkdir(parents=True, exist_ok=True) + (_FIXTURE_DIR / "boq_reviews_page.json").write_text( + json.dumps(raw_page), encoding="utf-8" + ) + return _check( + "dumped fixture", + len(raw_page) > 0, + f"{len(raw_page)} reviews -> {_FIXTURE_DIR / 'boq_reviews_page.json'}", + ) + return _check("fetched a reviews page", False) + + +async def step8_search() -> bool: + _hr("STEP 8 — search discovery (query + locationQuery)") + items = await scrape_places( + GoogleMapsScrapeInput( + searchStringsArray=["coffee shop"], + locationQuery="Seattle, WA", + maxCrawledPlacesPerSearch=5, + ) + ) + if not items: + return _check("search returned items", False, "no items") + it = items[0] + print(json.dumps(it, indent=2, ensure_ascii=False)[:1500]) + ok = ( + len(items) == 5 + and all(i.get("title") and i.get("placeId") and i.get("fid") for i in items) + and [i.get("rank") for i in items] == [1, 2, 3, 4, 5] + and it.get("searchString") == "coffee shop" + ) + return _check( + "5 ranked places with title/placeId/fid", + ok, + f"first: {it.get('title')!r} ({it.get('totalScore')}★, {it.get('city')})", + ) + + +async def step9_dump_search_fixture() -> bool: + _hr("STEP 9 — dump raw map-search response fixture for offline test") + url = build_search_url("pizza new york") + jd = await fetch_rpc_json(url) + if not isinstance(jd, list): + return _check("fetched search response", False) + _FIXTURE_DIR.mkdir(parents=True, exist_ok=True) + (_FIXTURE_DIR / "search_response.json").write_text(json.dumps(jd), encoding="utf-8") + return _check("dumped fixture", True, f"-> {_FIXTURE_DIR / 'search_response.json'}") + + +async def main() -> int: + results = [ + await step1_place(), + await step2_place_id(), + await step3_dump_fixture(), + await step4_reviews(), + await step5_pagination(), + await step6_inline_reviews(), + await step7_dump_reviews_fixture(), + await step8_search(), + await step9_dump_search_fixture(), + ] + _hr("SUMMARY") + print(f" {sum(results)}/{len(results)} steps passed") + return 0 if all(results) else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/surfsense_backend/scripts/e2e_youtube_scraper.py b/surfsense_backend/scripts/e2e_youtube_scraper.py index 4a4634674..ce409e051 100644 --- a/surfsense_backend/scripts/e2e_youtube_scraper.py +++ b/surfsense_backend/scripts/e2e_youtube_scraper.py @@ -1,4 +1,4 @@ -"""Manual functional e2e for the YouTube scraper (app/proprietary/scrapers/youtube). +"""Manual functional e2e for the YouTube scraper (app/proprietary/platforms/youtube). Run from the backend directory: cd surfsense_backend @@ -13,7 +13,7 @@ This is NOT a pytest test (it needs live network + optional proxy creds). It: Step 2 — runs a search query and prints the first few results. Step 3 — scrapes a small channel's latest videos. Step 4 — dumps trimmed raw ytInitialData / ytInitialPlayerResponse fixtures to - tests/unit/scrapers/youtube/fixtures/ for the offline parser test. + tests/unit/platforms/youtube/fixtures/ for the offline parser test. """ import asyncio @@ -58,7 +58,7 @@ _COLLAB_VIDEO_URL = "https://www.youtube.com/watch?v=AI2BwwLX_7s" # MrBeast localizes titles/descriptions into many languages (translation flow). _TRANSLATED_VIDEO_URL = "https://www.youtube.com/watch?v=iYlODtkyw_I" -_FIXTURE_DIR = _BACKEND_ROOT / "tests" / "unit" / "scrapers" / "youtube" / "fixtures" +_FIXTURE_DIR = _BACKEND_ROOT / "tests" / "unit" / "platforms" / "youtube" / "fixtures" def _hr(title: str) -> None: @@ -125,9 +125,7 @@ async def step4_dump_fixtures() -> bool: (_FIXTURE_DIR / "video_initial_data.json").write_text( json.dumps(initial), encoding="utf-8" ) - return _check( - "dumped fixtures", bool(player), f"-> {_FIXTURE_DIR}" - ) + return _check("dumped fixtures", bool(player), f"-> {_FIXTURE_DIR}") async def step5_comments() -> bool: @@ -137,11 +135,16 @@ async def step5_comments() -> bool: ) items = await scrape_comments(inp) for it in items[:6]: - print(f" - [{it.get('type')}] {it.get('author')} | {it.get('voteCount')} votes") + print( + f" - [{it.get('type')}] {it.get('author')} | {it.get('voteCount')} votes" + ) ok = bool(items) and all(it.get("cid") and it.get("videoId") for it in items) has_reply = any(it.get("type") == "reply" and it.get("replyToCid") for it in items) - return _check("comments scraped (cid+videoId, reply linkage)", ok and has_reply, - f"{len(items)} items") + return _check( + "comments scraped (cid+videoId, reply linkage)", + ok and has_reply, + f"{len(items)} items", + ) async def step6_location_collaborators() -> bool: @@ -165,13 +168,17 @@ async def step6_location_collaborators() -> bool: async def step7_translation() -> bool: _hr("STEP 7 — translatedTitle/translatedText (subtitlesLanguage=es)") items = await scrape_youtube( - YouTubeScrapeInput(startUrls=[{"url": _TRANSLATED_VIDEO_URL}], subtitlesLanguage="es") + YouTubeScrapeInput( + startUrls=[{"url": _TRANSLATED_VIDEO_URL}], subtitlesLanguage="es" + ) ) it = items[0] if items else {} print(f" title : {it.get('title')}") print(f" translatedTitle: {it.get('translatedTitle')}") # A localized video's translated title differs from the canonical English one. - ok = bool(it.get("translatedTitle")) and it.get("translatedTitle") != it.get("title") + ok = bool(it.get("translatedTitle")) and it.get("translatedTitle") != it.get( + "title" + ) return _check("translatedTitle differs from original", ok) diff --git a/surfsense_backend/tests/integration/chat/test_append_message_recovery.py b/surfsense_backend/tests/integration/chat/test_append_message_recovery.py index b07d60b8f..4c866f5a5 100644 --- a/surfsense_backend/tests/integration/chat/test_append_message_recovery.py +++ b/surfsense_backend/tests/integration/chat/test_append_message_recovery.py @@ -46,9 +46,9 @@ from app.db import ( NewChatMessage, NewChatMessageRole, NewChatThread, - Workspace, TokenUsage, User, + Workspace, ) from app.routes import new_chat_routes from app.services.token_tracking_service import TurnTokenAccumulator diff --git a/surfsense_backend/tests/integration/chat/test_message_id_sse.py b/surfsense_backend/tests/integration/chat/test_message_id_sse.py index bea2b71dd..d1631215e 100644 --- a/surfsense_backend/tests/integration/chat/test_message_id_sse.py +++ b/surfsense_backend/tests/integration/chat/test_message_id_sse.py @@ -48,8 +48,8 @@ from app.db import ( NewChatMessage, NewChatMessageRole, NewChatThread, - Workspace, User, + Workspace, ) from app.services.new_streaming_service import VercelStreamingService from app.tasks.chat import persistence as persistence_module diff --git a/surfsense_backend/tests/integration/chat/test_persistence.py b/surfsense_backend/tests/integration/chat/test_persistence.py index 8282ab287..b79946ade 100644 --- a/surfsense_backend/tests/integration/chat/test_persistence.py +++ b/surfsense_backend/tests/integration/chat/test_persistence.py @@ -38,9 +38,9 @@ from app.db import ( NewChatMessage, NewChatMessageRole, NewChatThread, - Workspace, TokenUsage, User, + Workspace, ) from app.services.token_tracking_service import TurnTokenAccumulator from app.tasks.chat import persistence as persistence_module diff --git a/surfsense_backend/tests/integration/chat/test_thread_visibility.py b/surfsense_backend/tests/integration/chat/test_thread_visibility.py index f843a0684..888f31439 100644 --- a/surfsense_backend/tests/integration/chat/test_thread_visibility.py +++ b/surfsense_backend/tests/integration/chat/test_thread_visibility.py @@ -19,10 +19,10 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.auth.context import AuthContext from app.db import ( ChatVisibility, + User, Workspace, WorkspaceMembership, WorkspaceRole, - User, ) from app.routes import new_chat_routes from app.schemas.new_chat import ( diff --git a/surfsense_backend/tests/integration/composio/test_oauth_callback.py b/surfsense_backend/tests/integration/composio/test_oauth_callback.py index 0c95f0c4a..5fdc3d51e 100644 --- a/surfsense_backend/tests/integration/composio/test_oauth_callback.py +++ b/surfsense_backend/tests/integration/composio/test_oauth_callback.py @@ -9,8 +9,8 @@ from app.config import config from app.db import ( SearchSourceConnector, SearchSourceConnectorType, - Workspace, User, + Workspace, ) from app.utils.oauth_security import OAuthStateManager @@ -57,8 +57,7 @@ async def test_callback_with_error_param_redirects_to_denied_page( assert response.status_code in {302, 303, 307} location = response.headers["location"] assert ( - f"/dashboard/{db_workspace.id}/connectors/callback?" - "error=composio_oauth_denied" + f"/dashboard/{db_workspace.id}/connectors/callback?error=composio_oauth_denied" ) in location connectors = await _drive_connectors( diff --git a/surfsense_backend/tests/integration/document_upload/test_document_upload.py b/surfsense_backend/tests/integration/document_upload/test_document_upload.py index 5ca01d148..2e8ff005b 100644 --- a/surfsense_backend/tests/integration/document_upload/test_document_upload.py +++ b/surfsense_backend/tests/integration/document_upload/test_document_upload.py @@ -320,9 +320,7 @@ class TestDocumentSearchability: doc_ids = resp.json()["document_ids"] cleanup_doc_ids.extend(doc_ids) - await poll_document_status( - client, headers, doc_ids, workspace_id=workspace_id - ) + await poll_document_status(client, headers, doc_ids, workspace_id=workspace_id) search_resp = await client.get( "/api/v1/documents/search", diff --git a/surfsense_backend/tests/integration/document_upload/test_etl_credits.py b/surfsense_backend/tests/integration/document_upload/test_etl_credits.py index ee8f8c4c8..ec31c6d6c 100644 --- a/surfsense_backend/tests/integration/document_upload/test_etl_credits.py +++ b/surfsense_backend/tests/integration/document_upload/test_etl_credits.py @@ -219,9 +219,7 @@ class TestDocumentProcessingNotification: doc_ids = resp.json()["document_ids"] cleanup_doc_ids.extend(doc_ids) - await poll_document_status( - client, headers, doc_ids, workspace_id=workspace_id - ) + await poll_document_status(client, headers, doc_ids, workspace_id=workspace_id) notifications = await get_notifications( client, diff --git a/surfsense_backend/tests/integration/google_unification/conftest.py b/surfsense_backend/tests/integration/google_unification/conftest.py index 8c9d4d9c1..861f171f0 100644 --- a/surfsense_backend/tests/integration/google_unification/conftest.py +++ b/surfsense_backend/tests/integration/google_unification/conftest.py @@ -18,8 +18,8 @@ from app.db import ( DocumentType, SearchSourceConnector, SearchSourceConnectorType, - Workspace, User, + Workspace, ) EMBEDDING_DIM = app_config.embedding_model_instance.dimension @@ -275,9 +275,7 @@ async def seed_connector( session.add(user) await session.flush() - space = Workspace( - name=f"{name_prefix} {uuid.uuid4().hex[:6]}", user_id=user.id - ) + space = Workspace(name=f"{name_prefix} {uuid.uuid4().hex[:6]}", user_id=user.id) session.add(space) await session.flush() space_id = space.id diff --git a/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py b/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py index 23829eebd..9b5c25504 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py @@ -119,9 +119,7 @@ async def test_reindex_updates_content(db_session, db_workspace, db_user, mocker @pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") -async def test_reindex_updates_content_hash( - db_session, db_workspace, db_user, mocker -): +async def test_reindex_updates_content_hash(db_session, db_workspace, db_user, mocker): """Content hash is recomputed after reindexing with new source markdown.""" adapter = UploadDocumentAdapter(db_session) await adapter.index( diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_index_editions.py b/surfsense_backend/tests/integration/indexing_pipeline/test_index_editions.py index d93a4c483..26e89435c 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_index_editions.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_index_editions.py @@ -55,9 +55,7 @@ async def test_edit_keeps_unchanged_rows_and_embeds_only_the_new_text( patched_embed_texts, ): service = IndexingPipelineService(session=db_session) - doc_v1 = make_connector_document( - workspace_id=db_workspace.id, source_markdown=_V1 - ) + doc_v1 = make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1) document = await _index(service, doc_v1) ids_v1 = {c.content: c.id for c in await _load_chunks(db_session, document.id)} @@ -100,9 +98,7 @@ async def test_head_insert_shifts_positions_without_new_rows_for_old_text( service = IndexingPipelineService(session=db_session) document = await _index( service, - make_connector_document( - workspace_id=db_workspace.id, source_markdown=_V1 - ), + make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1), ) ids_v1 = {c.content: c.id for c in await _load_chunks(db_session, document.id)} @@ -136,9 +132,7 @@ async def test_removed_paragraph_is_deleted_and_order_compacts( service = IndexingPipelineService(session=db_session) document = await _index( service, - make_connector_document( - workspace_id=db_workspace.id, source_markdown=_V1 - ), + make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1), ) ids_v1 = {c.content: c.id for c in await _load_chunks(db_session, document.id)} @@ -171,9 +165,7 @@ async def test_kill_switch_falls_back_to_full_replace( service = IndexingPipelineService(session=db_session) document = await _index( service, - make_connector_document( - workspace_id=db_workspace.id, source_markdown=_V1 - ), + make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1), ) ids_v1 = {c.id for c in await _load_chunks(db_session, document.id)} diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py index 1a120843e..76ee82209 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py @@ -14,8 +14,8 @@ from app.db import ( DocumentType, DocumentVersion, Folder, - Workspace, User, + Workspace, ) pytestmark = pytest.mark.integration diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_migrate_legacy_docs.py b/surfsense_backend/tests/integration/indexing_pipeline/test_migrate_legacy_docs.py index be67a887b..1580a1736 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_migrate_legacy_docs.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_migrate_legacy_docs.py @@ -59,9 +59,7 @@ async def test_legacy_composio_gmail_doc_migrated_in_db( assert reloaded.document_type == DocumentType.GOOGLE_GMAIL_CONNECTOR -async def test_no_legacy_doc_is_noop( - db_session, db_workspace, make_connector_document -): +async def test_no_legacy_doc_is_noop(db_session, db_workspace, make_connector_document): """When no legacy document exists, migrate_legacy_docs does nothing.""" connector_doc = make_connector_document( document_type=DocumentType.GOOGLE_CALENDAR_CONNECTOR, diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py b/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py index 3f8914045..70d1d7a24 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py @@ -225,9 +225,7 @@ async def test_updated_at_advances_when_title_only_changes( db_session, db_workspace, make_connector_document ): """updated_at advances even when only the title changes.""" - original = make_connector_document( - workspace_id=db_workspace.id, title="Old Title" - ) + original = make_connector_document(workspace_id=db_workspace.id, title="Old Title") service = IndexingPipelineService(session=db_session) first = await service.prepare_for_indexing([original]) @@ -238,9 +236,7 @@ async def test_updated_at_advances_when_title_only_changes( ) updated_at_v1 = result.scalars().first().updated_at - renamed = make_connector_document( - workspace_id=db_workspace.id, title="New Title" - ) + renamed = make_connector_document(workspace_id=db_workspace.id, title="New Title") await service.prepare_for_indexing([renamed]) result = await db_session.execute( diff --git a/surfsense_backend/tests/integration/notifications/test_base_handler.py b/surfsense_backend/tests/integration/notifications/test_base_handler.py index df73edf49..56092d19a 100644 --- a/surfsense_backend/tests/integration/notifications/test_base_handler.py +++ b/surfsense_backend/tests/integration/notifications/test_base_handler.py @@ -10,7 +10,7 @@ import pytest from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession -from app.db import Workspace, User +from app.db import User, Workspace from app.notifications.persistence import Notification from app.notifications.service import NotificationService diff --git a/surfsense_backend/tests/integration/notifications/test_comment_reply_handler.py b/surfsense_backend/tests/integration/notifications/test_comment_reply_handler.py index 004075975..50d85ad5f 100644 --- a/surfsense_backend/tests/integration/notifications/test_comment_reply_handler.py +++ b/surfsense_backend/tests/integration/notifications/test_comment_reply_handler.py @@ -5,7 +5,7 @@ from __future__ import annotations import pytest from sqlalchemy.ext.asyncio import AsyncSession -from app.db import Workspace, User +from app.db import User, Workspace from app.notifications.service import NotificationService pytestmark = pytest.mark.integration @@ -47,9 +47,7 @@ async def test_comment_reply_truncates_long_preview( db_session: AsyncSession, db_user: User, db_workspace: Workspace ): """A long comment preview is truncated in the reply message.""" - notification = await _notify( - db_session, db_user, db_workspace, preview="y" * 150 - ) + notification = await _notify(db_session, db_user, db_workspace, preview="y" * 150) assert notification.message == "y" * 100 + "..." diff --git a/surfsense_backend/tests/integration/notifications/test_connector_indexing_handler.py b/surfsense_backend/tests/integration/notifications/test_connector_indexing_handler.py index 596c02943..b3629ca70 100644 --- a/surfsense_backend/tests/integration/notifications/test_connector_indexing_handler.py +++ b/surfsense_backend/tests/integration/notifications/test_connector_indexing_handler.py @@ -10,7 +10,7 @@ from __future__ import annotations import pytest from sqlalchemy.ext.asyncio import AsyncSession -from app.db import Workspace, User +from app.db import User, Workspace from app.notifications.service import NotificationService pytestmark = pytest.mark.integration diff --git a/surfsense_backend/tests/integration/notifications/test_document_processing_handler.py b/surfsense_backend/tests/integration/notifications/test_document_processing_handler.py index c5a9a9bd5..806c6d331 100644 --- a/surfsense_backend/tests/integration/notifications/test_document_processing_handler.py +++ b/surfsense_backend/tests/integration/notifications/test_document_processing_handler.py @@ -5,7 +5,7 @@ from __future__ import annotations import pytest from sqlalchemy.ext.asyncio import AsyncSession -from app.db import Workspace, User +from app.db import User, Workspace from app.notifications.service import NotificationService pytestmark = pytest.mark.integration diff --git a/surfsense_backend/tests/integration/notifications/test_insufficient_credits_handler.py b/surfsense_backend/tests/integration/notifications/test_insufficient_credits_handler.py index 1cc934a08..2c1ec32a8 100644 --- a/surfsense_backend/tests/integration/notifications/test_insufficient_credits_handler.py +++ b/surfsense_backend/tests/integration/notifications/test_insufficient_credits_handler.py @@ -5,7 +5,7 @@ from __future__ import annotations import pytest from sqlalchemy.ext.asyncio import AsyncSession -from app.db import Workspace, User +from app.db import User, Workspace from app.notifications.service import NotificationService pytestmark = pytest.mark.integration diff --git a/surfsense_backend/tests/integration/notifications/test_mention_handler.py b/surfsense_backend/tests/integration/notifications/test_mention_handler.py index b2995bcca..59bb5c1ad 100644 --- a/surfsense_backend/tests/integration/notifications/test_mention_handler.py +++ b/surfsense_backend/tests/integration/notifications/test_mention_handler.py @@ -5,7 +5,7 @@ from __future__ import annotations import pytest from sqlalchemy.ext.asyncio import AsyncSession -from app.db import Workspace, User +from app.db import User, Workspace from app.notifications.service import NotificationService pytestmark = pytest.mark.integration @@ -47,9 +47,7 @@ async def test_new_mention_truncates_long_preview( db_session: AsyncSession, db_user: User, db_workspace: Workspace ): """A long comment preview is truncated in the mention message.""" - notification = await _notify( - db_session, db_user, db_workspace, preview="x" * 150 - ) + notification = await _notify(db_session, db_user, db_workspace, preview="x" * 150) assert notification.message == "x" * 100 + "..." diff --git a/surfsense_backend/tests/integration/podcasts/conftest.py b/surfsense_backend/tests/integration/podcasts/conftest.py index a3e7a2c09..5f29d9d3c 100644 --- a/surfsense_backend/tests/integration/podcasts/conftest.py +++ b/surfsense_backend/tests/integration/podcasts/conftest.py @@ -26,7 +26,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.app import app, limiter from app.auth.context import AuthContext from app.config import config as app_config -from app.db import Workspace, User, get_async_session +from app.db import User, Workspace, get_async_session from app.podcasts.persistence import Podcast, PodcastStatus from app.podcasts.schemas import ( DurationTarget, diff --git a/surfsense_backend/tests/integration/podcasts/test_cancel.py b/surfsense_backend/tests/integration/podcasts/test_cancel.py index c8307d89e..33daeef28 100644 --- a/surfsense_backend/tests/integration/podcasts/test_cancel.py +++ b/surfsense_backend/tests/integration/podcasts/test_cancel.py @@ -38,9 +38,7 @@ async def test_cancel_from_a_terminal_state_conflicts( assert resp.status_code == 409 -async def test_cancel_of_a_regeneration_is_rejected( - client, db_workspace, make_podcast -): +async def test_cancel_of_a_regeneration_is_rejected(client, db_workspace, make_podcast): # Cancelling here would destroy a playable episode; reverting the # regeneration is the way back. podcast = await make_podcast( diff --git a/surfsense_backend/tests/integration/retriever/conftest.py b/surfsense_backend/tests/integration/retriever/conftest.py index a8dbc090f..096b5c0dd 100644 --- a/surfsense_backend/tests/integration/retriever/conftest.py +++ b/surfsense_backend/tests/integration/retriever/conftest.py @@ -9,7 +9,7 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession from app.config import config as app_config -from app.db import Chunk, Document, DocumentType, Workspace, User +from app.db import Chunk, Document, DocumentType, User, Workspace EMBEDDING_DIM = app_config.embedding_model_instance.dimension DUMMY_EMBEDDING = [0.1] * EMBEDDING_DIM diff --git a/surfsense_backend/tests/integration/test_connector_index_authz.py b/surfsense_backend/tests/integration/test_connector_index_authz.py index 32a2b3e50..61aba762d 100644 --- a/surfsense_backend/tests/integration/test_connector_index_authz.py +++ b/surfsense_backend/tests/integration/test_connector_index_authz.py @@ -27,8 +27,8 @@ from app.auth.context import AuthContext from app.db import ( SearchSourceConnector, SearchSourceConnectorType, - Workspace, User, + Workspace, ) from app.routes.search_source_connectors_routes import index_connector_content from app.routes.workspaces_routes import create_default_roles_and_membership diff --git a/surfsense_backend/tests/integration/test_document_versioning.py b/surfsense_backend/tests/integration/test_document_versioning.py index b98f164fd..64da21b1a 100644 --- a/surfsense_backend/tests/integration/test_document_versioning.py +++ b/surfsense_backend/tests/integration/test_document_versioning.py @@ -7,7 +7,7 @@ import pytest_asyncio from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession -from app.db import Document, DocumentType, DocumentVersion, Workspace, User +from app.db import Document, DocumentType, DocumentVersion, User, Workspace pytestmark = pytest.mark.integration diff --git a/surfsense_backend/tests/integration/test_obsidian_plugin_routes.py b/surfsense_backend/tests/integration/test_obsidian_plugin_routes.py index ae9870ac7..025bf2075 100644 --- a/surfsense_backend/tests/integration/test_obsidian_plugin_routes.py +++ b/surfsense_backend/tests/integration/test_obsidian_plugin_routes.py @@ -32,8 +32,8 @@ from app.auth.context import AuthContext from app.db import ( SearchSourceConnector, SearchSourceConnectorType, - Workspace, User, + Workspace, ) from app.routes.obsidian_plugin_routes import ( obsidian_connect, diff --git a/surfsense_backend/tests/integration/test_pat_fail_closed_authz.py b/surfsense_backend/tests/integration/test_pat_fail_closed_authz.py index b8cca33bf..a398029d0 100644 --- a/surfsense_backend/tests/integration/test_pat_fail_closed_authz.py +++ b/surfsense_backend/tests/integration/test_pat_fail_closed_authz.py @@ -7,7 +7,7 @@ from fastapi import HTTPException from sqlalchemy.ext.asyncio import AsyncSession from app.auth.context import AuthContext -from app.db import PersonalAccessToken, Workspace, User +from app.db import PersonalAccessToken, User, Workspace from app.users import allow_any_principal, require_session_context from app.utils.rbac import check_workspace_access diff --git a/surfsense_backend/tests/integration/test_zero_authz_context.py b/surfsense_backend/tests/integration/test_zero_authz_context.py index 12c42098d..a2d042a9d 100644 --- a/surfsense_backend/tests/integration/test_zero_authz_context.py +++ b/surfsense_backend/tests/integration/test_zero_authz_context.py @@ -7,7 +7,7 @@ from fastapi import HTTPException from sqlalchemy.ext.asyncio import AsyncSession from app.auth.context import AuthContext -from app.db import PersonalAccessToken, Workspace, User +from app.db import PersonalAccessToken, User, Workspace from app.routes.workspaces_routes import create_default_roles_and_membership from app.utils.rbac import check_workspace_access, get_allowed_read_space_ids diff --git a/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py b/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py index 15ef61418..895eae571 100644 --- a/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py +++ b/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py @@ -158,9 +158,7 @@ async def test_build_dependencies_falls_back_to_workspace( ) workspace = SimpleNamespace(chat_model_id=-7) - result = await build_dependencies( - session=_FakeSession(workspace), workspace_id=42 - ) + result = await build_dependencies(session=_FakeSession(workspace), workspace_id=42) assert captured == {"config_id": -7} assert result.llm.name == "llm" diff --git a/surfsense_backend/tests/unit/capabilities/__init__.py b/surfsense_backend/tests/unit/capabilities/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/web/__init__.py b/surfsense_backend/tests/unit/capabilities/web/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/web/crawl/__init__.py b/surfsense_backend/tests/unit/capabilities/web/crawl/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/youtube/__init__.py b/surfsense_backend/tests/unit/capabilities/youtube/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/youtube/comments/__init__.py b/surfsense_backend/tests/unit/capabilities/youtube/comments/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/youtube/scrape/__init__.py b/surfsense_backend/tests/unit/capabilities/youtube/scrape/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/indexing_pipeline/test_create_placeholder_documents.py b/surfsense_backend/tests/unit/indexing_pipeline/test_create_placeholder_documents.py index 44603dcf8..374f5156f 100644 --- a/surfsense_backend/tests/unit/indexing_pipeline/test_create_placeholder_documents.py +++ b/surfsense_backend/tests/unit/indexing_pipeline/test_create_placeholder_documents.py @@ -36,9 +36,7 @@ def _make_placeholder(**overrides) -> PlaceholderInfo: def _uid_hash(p: PlaceholderInfo) -> str: - return compute_identifier_hash( - p.document_type.value, p.unique_id, p.workspace_id - ) + return compute_identifier_hash(p.document_type.value, p.unique_id, p.workspace_id) def _session_with_existing_hashes(existing: set[str] | None = None): diff --git a/surfsense_backend/tests/unit/platforms/google_maps/__init__.py b/surfsense_backend/tests/unit/platforms/google_maps/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/platforms/google_maps/test_parsers.py b/surfsense_backend/tests/unit/platforms/google_maps/test_parsers.py new file mode 100644 index 000000000..a0643c8b6 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/google_maps/test_parsers.py @@ -0,0 +1,175 @@ +"""Offline parser tests against a captured real place ``darray`` fixture. + +The fixture is the ``jd[6]`` array from a live ``/maps/preview/place`` RPC +response for "Kim's Island" (the restaurant used in the Apify output example), +regenerated by ``scripts/e2e_google_maps_scraper.py``. These tests pin the +array-index paths so a Google structure shift is caught offline. +""" + +import json +from pathlib import Path + +import pytest + +from app.proprietary.platforms.google_maps.fetch import strip_xssi +from app.proprietary.platforms.google_maps.parsers import ( + brace_match_json, + dig, + parse_place, +) + +_FIXTURE = Path(__file__).parent / "fixtures" / "place_darray.json" + + +@pytest.fixture +def darray() -> list: + return json.loads(_FIXTURE.read_text(encoding="utf-8")) + + +def test_parse_place_core_fields(darray): + place = parse_place(darray) + assert place["title"] == "Kim's Island" + assert place["placeId"] == "ChIJJQz5EZzKw4kRCZ95UajbyGw" + assert place["fid"] == "0x89c3ca9c11f90c25:0x6cc8dba851799f09" + assert place["categoryName"] == "Chinese restaurant" + assert "Chinese restaurant" in place["categories"] + assert place["totalScore"] == 4.5 + assert place["phone"] == "(718) 356-5168" + assert place["website"] == "http://kimsislandsi.com/" + assert place["plusCode"].startswith("GQ62+8M") + + +def test_parse_place_address_components(darray): + place = parse_place(darray) + assert place["neighborhood"] == "Tottenville" + assert place["street"] == "175 Main St" + assert place["city"] == "Staten Island" + assert place["postalCode"] == "10307" + assert place["state"] == "New York" + assert place["countryCode"] == "US" + + +def test_parse_place_location_and_hours(darray): + place = parse_place(darray) + assert place["location"]["lat"] == pytest.approx(40.5107736) + assert place["location"]["lng"] == pytest.approx(-74.2482624) + hours = place["openingHours"] + assert hours and all("day" in h and "hours" in h for h in hours) + + +def test_parse_place_detail_extras(darray): + place = parse_place(darray) + assert place["kgmid"] == "/g/1tmgdcj8" + # cid = decimal of the fid's second hex half + assert place["cid"] == str(int("0x6cc8dba851799f09", 16)) + info = place["additionalInfo"] + assert "Accessibility" in info + assert {"Wheelchair accessible entrance": True} in info["Accessibility"] + # every option is a single {name: bool} pair + for entries in info.values(): + for entry in entries: + assert len(entry) == 1 + assert all(isinstance(v, bool) for v in entry.values()) + + +def test_parse_place_session_gated_fields(darray): + # These fields only appear when the RPC is sent with an NID session + # cookie + the full-page pb selector; the fixture pins that payload. + place = parse_place(darray) + dist = place["reviewsDistribution"] + assert set(dist) == {"oneStar", "twoStar", "threeStar", "fourStar", "fiveStar"} + assert place["reviewsCount"] == sum(dist.values()) + assert place["imagesCount"] > 0 + assert "All" in place["imageCategories"] + assert all(u.startswith("https://") for u in place["imageUrls"]) + tags = place["reviewsTags"] + assert tags and all( + isinstance(t["title"], str) and isinstance(t["count"], int) for t in tags + ) + assert len(place["additionalInfo"]) >= 5 # full sections, not just Accessibility + + +def test_parse_place_hotel_fields(): + # Fixture: live darray for The Plaza (NYC), captured via the detail RPC. + fixture = Path(__file__).parent / "fixtures" / "hotel_darray.json" + place = parse_place(json.loads(fixture.read_text(encoding="utf-8"))) + assert place["title"] == "The Plaza" + assert place["hotelStars"] == "5 stars" + assert place["checkInDate"] < place["checkOutDate"] # ISO dates compare + assert place["hotelDescription"] + similar = place["similarHotelsNearby"] + assert len(similar) >= 3 + assert all(h["title"] and h["fid"] for h in similar) + assert any("hotel" in (h.get("description") or "").lower() for h in similar) + ads = place["hotelAds"] + assert ads and all(a["url"].startswith("https://") for a in ads) + assert any(a.get("price", "").startswith("$") for a in ads) + + +def test_hotel_fields_absent_for_non_hotels(darray): + # Kim's Island (restaurant) must not grow hotel fields. + place = parse_place(darray) + for key in ("hotelStars", "checkInDate", "similarHotelsNearby", "hotelAds"): + assert key not in place + + +def test_popular_times_shapes(): + from app.proprietary.platforms.google_maps.parsers import _popular_times + + darray: list = [None] * 100 + darray[84] = [ + [[7, [[6, 0, "", "None", "6 AM", "No wait", "6a"], [7, 35]]]], + None, + None, + None, + None, + None, + "A little busy", + [22, 76], + ] + out = _popular_times(darray) + assert out["popularTimesHistogram"] == { + "Su": [ + {"hour": 6, "occupancyPercent": 0}, + {"hour": 7, "occupancyPercent": 35}, + ] + } + assert out["popularTimesLiveText"] == "A little busy" + assert out["popularTimesLivePercent"] == 76 + assert _popular_times([None] * 100) == {} + + +def test_parse_place_reservation_links(): + fixture = Path(__file__).parent / "fixtures" / "search_response.json" + from app.proprietary.platforms.google_maps.fetch import _search_darrays + + jd = json.loads(fixture.read_text(encoding="utf-8")) + place = parse_place(_search_darrays(jd)[0]) + # the first search hit in the fixture carries a reservation provider + links = place["tableReservationLinks"] + assert all(set(link) == {"url", "source"} for link in links) + assert place["reserveTableUrl"] == links[0]["url"] + + +def test_parse_place_omits_missing_fields(darray): + # parse_place drops keys whose path missed, rather than emitting nulls. + place = parse_place(darray) + assert all(v is not None for v in place.values()) + + +def test_dig_tolerates_ragged_paths(): + assert dig([1, [2, 3]], 1, 0) == 2 + assert dig([1, [2, 3]], 5) is None + assert dig([1, None], 1, 0) is None + assert dig("not a list", 0) is None + + +def test_strip_xssi_and_brace_match_handle_html_wrapper(): + # Mirrors the real proxy response: text/plain body wrapped in

. + raw = ')]}\'\n[1,[2,3],"x"]

' + body = strip_xssi(raw) + blob = brace_match_json(body, body.index("[")) + assert json.loads(blob) == [1, [2, 3], "x"] + + wrapped = '

)]}\'\n[["a"]]' + assert strip_xssi(wrapped).startswith("[[") diff --git a/surfsense_backend/tests/unit/platforms/google_maps/test_reviews.py b/surfsense_backend/tests/unit/platforms/google_maps/test_reviews.py new file mode 100644 index 000000000..abe6567c8 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/google_maps/test_reviews.py @@ -0,0 +1,116 @@ +"""Offline tests for the Google Maps reviews parsing + flow helpers. + +``fixtures/boq_reviews_page.json`` is a real ``GetLocalBoqProxy`` page (the +raw review list, ``jd[1][10][2]``) captured by scripts/e2e_google_maps_scraper.py +step 7. Regenerate it with that script if Google shifts the structure. +""" + +import json +from datetime import datetime +from pathlib import Path + +import pytest + +from app.proprietary.platforms.google_maps.fetch import build_reviews_url +from app.proprietary.platforms.google_maps.parsers import ( + parse_review, + parse_reviews_page, + strip_personal_data, +) +from app.proprietary.platforms.google_maps.reviews import _before_cutoff, _keep + +_FIXTURE = Path(__file__).parent / "fixtures" / "boq_reviews_page.json" + + +@pytest.fixture +def reviews_raw() -> list: + return json.loads(_FIXTURE.read_text(encoding="utf-8")) + + +def test_parse_reviews_page_full_page(reviews_raw): + parsed = parse_reviews_page(reviews_raw) + assert len(parsed) == len(reviews_raw) == 10 + for review in parsed: + assert review["name"] + assert review["reviewId"] + assert 1 <= review["stars"] <= 5 + assert review["publishedAtDate"].endswith("Z") + + +def test_parse_review_fields(reviews_raw): + review = parse_review(reviews_raw[0]) + assert review["name"] == "Greg" + assert review["stars"] == 5 + assert review["reviewerId"] == "108156147079988470963" + assert review["reviewerUrl"].startswith("https://www.google.com/maps/contrib/") + assert review["reviewerNumberOfReviews"] == 63 + assert review["isLocalGuide"] is True + assert review["reviewOrigin"] == "Google" + assert review["originalLanguage"] == "en" + assert review["publishAt"] == "2 months ago" + # 1776469524140 ms epoch -> UTC ISO + assert review["publishedAtDate"] == "2026-04-17T23:45:24.140Z" + assert "spiced" in review["text"] + # Guided answers split into context vs per-aspect ratings. + assert review["reviewContext"]["Order type"] == "Take out" + assert review["reviewDetailedRating"]["Food"] == 5 + + +def test_parse_review_owner_response_and_images(reviews_raw): + with_reply = [ + r + for r in (parse_review(x) for x in reviews_raw) + if r and r.get("responseFromOwnerText") + ] + assert with_reply, "fixture should contain at least one owner reply" + assert with_reply[0]["responseFromOwnerText"] + + with_images = [ + r + for r in (parse_review(x) for x in reviews_raw) + if r and r.get("reviewImageUrls") + ] + assert with_images, "fixture should contain at least one review with photos" + assert with_images[0]["reviewImageUrls"][0].startswith("https://") + + +def test_parse_review_rejects_garbage(): + assert parse_review(None) is None + assert parse_review([]) is None + assert parse_review([None, 5]) is None # no author block + + +def test_strip_personal_data(reviews_raw): + review = parse_review(reviews_raw[0]) + strip_personal_data(review) + for key in ("name", "reviewerId", "reviewerUrl", "reviewerPhotoUrl"): + assert key not in review + assert review["reviewId"] # non-personal fields stay + assert review["stars"] == 5 + + +def test_before_cutoff(): + cutoff = datetime(2026, 3, 1) + assert _before_cutoff({"publishedAtDate": "2026-02-01T00:00:00.000Z"}, cutoff) + assert not _before_cutoff({"publishedAtDate": "2026-04-01T00:00:00.000Z"}, cutoff) + assert not _before_cutoff({}, cutoff) # undated reviews are kept + + +def test_keep_filters(): + review = {"text": "Great NOODLES here", "reviewOrigin": "Google"} + assert _keep(review, filter_string="", origin="all") + assert _keep(review, filter_string="noodles", origin="google") + assert not _keep(review, filter_string="pizza", origin="all") + assert not _keep({"reviewOrigin": "TripAdvisor"}, filter_string="", origin="google") + + +def test_build_reviews_url_shape(): + fid = "0x89c3ca9c11f90c25:0x6cc8dba851799f09" + first = build_reviews_url(fid, sort_code=2) + assert "GetLocalBoqProxy" in first + assert "hl=en" in first + assert fid in json.dumps(first) or "0x89c3ca9c11f90c25" in first + # First page requests a page size; later pages carry the token instead. + paged = build_reviews_url(fid, sort_code=2, page_token="TOKEN123") + assert "TOKEN123" in paged + assert paged != first diff --git a/surfsense_backend/tests/unit/platforms/google_maps/test_search.py b/surfsense_backend/tests/unit/platforms/google_maps/test_search.py new file mode 100644 index 000000000..6aeb9e8a4 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/google_maps/test_search.py @@ -0,0 +1,150 @@ +"""Offline tests for the map-search flow: response extraction, URL building, +and the Apify result filters. The fixture is a real ``search?tbm=map`` response +captured by scripts/e2e_google_maps_scraper.py (step 9, query "pizza new york"). +""" + +import json +from pathlib import Path + +import pytest + +from app.proprietary.platforms.google_maps.fetch import ( + _search_darrays, + build_search_url, +) +from app.proprietary.platforms.google_maps.parsers import parse_place +from app.proprietary.platforms.google_maps.schemas import GoogleMapsScrapeInput +from app.proprietary.platforms.google_maps.scraper import ( + _custom_point, + _location_text, + _passes_filters, +) + +_FIXTURE = Path(__file__).parent / "fixtures" / "search_response.json" + + +@pytest.fixture(scope="module") +def search_jd(): + return json.loads(_FIXTURE.read_text(encoding="utf-8")) + + +def test_search_darrays_extracts_full_page(search_jd): + darrays = _search_darrays(search_jd) + assert len(darrays) == 20 + parsed = [parse_place(d) for d in darrays] + for fields in parsed: + assert fields["title"] + assert fields["fid"].startswith("0x") + assert fields["placeId"].startswith("ChIJ") + assert "location" in fields + fids = [f["fid"] for f in parsed] + assert len(set(fids)) == 20 # no dupes within a page + + +def test_search_result_has_detail_fields(search_jd): + fields = parse_place(_search_darrays(search_jd)[0]) + # Search entries carry the full darray, so detail fields come through. + assert fields["categories"] + assert fields["address"] + assert fields["totalScore"] > 0 + assert fields["city"] + + +def test_search_darrays_rejects_garbage(): + assert _search_darrays(None) == [] + assert _search_darrays([]) == [] + assert _search_darrays([[None, None]]) == [] + + +def test_build_search_url_shape(): + url = build_search_url("pizza new york", offset=20, language="de") + assert url.startswith("https://www.google.com/search?tbm=map") + assert "hl=de" in url + assert "q=pizza%20new%20york" in url + assert "!8i20" in url # offset + assert "!7i20" in url # page size + # whole-earth viewport when no coordinates given + assert "!1d25000000" in url + + geo_url = build_search_url("museum", lat=48.85, lng=2.35, radius_m=5000) + assert "!2d2.35" in geo_url and "!3d48.85" in geo_url + assert "!1d10000" in geo_url # radius -> diameter + + +def test_location_text_prefers_location_query(): + assert ( + _location_text(GoogleMapsScrapeInput(locationQuery="Berlin, Germany")) + == "Berlin, Germany" + ) + assert ( + _location_text( + GoogleMapsScrapeInput(city="Austin", state="TX", countryCode="US") + ) + == "Austin, TX, US" + ) + assert _location_text(GoogleMapsScrapeInput()) is None + + +def test_custom_point(): + lat, lng, radius = _custom_point( + GoogleMapsScrapeInput( + customGeolocation={ + "type": "Point", + "coordinates": [2.35, 48.85], + "radiusKm": 5, + } + ) + ) + assert (lat, lng, radius) == (48.85, 2.35, 5000) + assert _custom_point(GoogleMapsScrapeInput()) == (None, None, None) + assert _custom_point( + GoogleMapsScrapeInput(customGeolocation={"type": "Polygon"}) + ) == (None, None, None) + + +def test_passes_filters(): + fields = { + "title": "Joe's Pizza", + "categories": ["Pizza restaurant"], + "totalScore": 4.4, + "website": "https://joes.example", + } + default = GoogleMapsScrapeInput() + assert _passes_filters(fields, "pizza", default) + + assert not _passes_filters( + fields, "pizza", GoogleMapsScrapeInput(searchMatching="only_exact") + ) + assert _passes_filters( + fields, "pizza", GoogleMapsScrapeInput(searchMatching="only_includes") + ) + assert not _passes_filters( + fields, "burger", GoogleMapsScrapeInput(searchMatching="only_includes") + ) + + assert _passes_filters( + fields, "pizza", GoogleMapsScrapeInput(categoryFilterWords=["pizza"]) + ) + assert not _passes_filters( + fields, "pizza", GoogleMapsScrapeInput(categoryFilterWords=["barber"]) + ) + + assert not _passes_filters( + fields, "pizza", GoogleMapsScrapeInput(placeMinimumStars="fourAndHalf") + ) + assert _passes_filters( + fields, "pizza", GoogleMapsScrapeInput(placeMinimumStars="four") + ) + + assert _passes_filters( + fields, "pizza", GoogleMapsScrapeInput(website="withWebsite") + ) + assert not _passes_filters( + fields, "pizza", GoogleMapsScrapeInput(website="withoutWebsite") + ) + + closed = {**fields, "permanentlyClosed": True} + assert not _passes_filters( + closed, "pizza", GoogleMapsScrapeInput(skipClosedPlaces=True) + ) + assert _passes_filters(closed, "pizza", default) diff --git a/surfsense_backend/tests/unit/platforms/google_maps/test_skeleton.py b/surfsense_backend/tests/unit/platforms/google_maps/test_skeleton.py new file mode 100644 index 000000000..4c21cc7fb --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/google_maps/test_skeleton.py @@ -0,0 +1,97 @@ +"""Offline checks for the Google Maps scraper skeleton. + +Covers the pure parts: URL classification and Apify-spec schema defaults/ +serialization. The live flows (place / reviews / search) are covered by the +e2e scripts and the fixture-based parser tests. +""" + +import pytest + +from app.proprietary.platforms.google_maps import ( + GoogleMapsReviewsInput, + GoogleMapsScrapeInput, + PlaceItem, + ReviewItem, +) +from app.proprietary.platforms.google_maps.url_resolver import extract_fid, resolve_url + + +@pytest.mark.parametrize( + ("url", "kind", "value"), + [ + ( + "https://www.google.com/maps/place/Kim's+Island/@40.51,-74.24,17z/data=!4m6", + "place", + "Kim's Island", + ), + ( + "https://www.google.com/maps/search/restaurants/@52.5190603,13.388574,13z/", + "search", + "restaurants", + ), + ("https://www.google.com/maps/reviews/data=!4m8!14m7", "reviews", None), + ( + "https://www.google.com/maps?cid=7838756667406262025", + "cid", + "7838756667406262025", + ), + ("https://goo.gl/maps/abc123", "shortlink", None), + ("https://maps.app.goo.gl/xyz", "shortlink", None), + ], +) +def test_resolve_url(url, kind, value): + resolved = resolve_url(url) + assert resolved is not None + assert resolved.kind == kind + if value is not None: + assert resolved.value == value + + +def test_resolve_url_rejects_non_maps(): + assert resolve_url("https://example.com/maps/place/foo") is None + assert resolve_url("https://www.google.com/search?q=pizza") is None + + +def test_extract_fid(): + url = ( + "https://www.google.com/maps/place/Kim's+Island/@40.51,-74.24,17z/" + "data=!4m6!3m5!1s0x89c3ca9c11f90c25:0x6cc8dba851799f09!8m2" + ) + assert extract_fid(url) == "0x89c3ca9c11f90c25:0x6cc8dba851799f09" + assert extract_fid("https://www.google.com/maps/place/Foo") is None + assert resolve_url(url).fid == "0x89c3ca9c11f90c25:0x6cc8dba851799f09" + + +def test_scrape_input_defaults_match_apify_spec(): + inp = GoogleMapsScrapeInput() + assert inp.language == "en" + assert inp.maxCrawledPlacesPerSearch is None # empty = all places + assert inp.searchMatching == "all" + assert inp.website == "allPlaces" + assert inp.reviewsSort == "newest" + assert inp.reviewsOrigin == "all" + assert inp.scrapeReviewsPersonalData is True + assert inp.maxCompetitorsToAnalyze == 30 + assert inp.scrapeSocialMediaProfiles.facebooks is False + # Unknown fields are accepted (extra="allow") so parity is additive. + GoogleMapsScrapeInput(someFutureField=1) + + +def test_reviews_input_defaults_match_apify_spec(): + inp = GoogleMapsReviewsInput() + assert inp.maxReviews == 10_000_000 + assert inp.reviewsSort == "newest" + assert inp.personalData is True + + +def test_output_items_serialize_full_shape(): + place = PlaceItem(title="Kim's Island", placeId="ChIJx").to_output() + assert place["title"] == "Kim's Island" + assert place["permanentlyClosed"] is False + assert place["categories"] == [] + assert "reviewsDistribution" in place # unsourced fields still emitted + + review = ReviewItem(reviewId="abc", stars=5).to_output() + assert review["stars"] == 5 + assert review["reviewImageUrls"] == [] + assert "responseFromOwnerText" in review diff --git a/surfsense_backend/tests/unit/platforms/youtube/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/youtube/test_fetch_resilience.py index d69da5cb2..9f20271be 100644 --- a/surfsense_backend/tests/unit/platforms/youtube/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/youtube/test_fetch_resilience.py @@ -93,7 +93,9 @@ async def test_post_rotates_on_connection_error_then_succeeds(): async def test_post_gives_up_after_max_rotations(): # Every IP is blocked -> rotate up to the cap, then return None. - holder = _FakeHolder([_FakeSession(429) for _ in range(innertube._MAX_ROTATIONS + 1)]) + holder = _FakeHolder( + [_FakeSession(429) for _ in range(innertube._MAX_ROTATIONS + 1)] + ) token = _current_session.set(holder) try: result = await post_innertube(INNERTUBE_SEARCH_URL, _payload()) @@ -134,7 +136,9 @@ async def test_fetch_html_falls_back_to_stealthy_when_all_blocked(monkeypatch): return "stealthy" monkeypatch.setattr(innertube, "_fetch_html_stealthy", _fake_stealthy) - holder = _FakeHolder([_FakeSession(429) for _ in range(innertube._MAX_ROTATIONS + 1)]) + holder = _FakeHolder( + [_FakeSession(429) for _ in range(innertube._MAX_ROTATIONS + 1)] + ) token = _current_session.set(holder) try: html = await fetch_html("https://www.youtube.com/watch?v=zzz") diff --git a/surfsense_backend/tests/unit/platforms/youtube/test_parsers.py b/surfsense_backend/tests/unit/platforms/youtube/test_parsers.py index 47a457260..55f591c63 100644 --- a/surfsense_backend/tests/unit/platforms/youtube/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/youtube/test_parsers.py @@ -105,12 +105,19 @@ def _video_html() -> str: "keywords": ["#tech"], "thumbnail": { "thumbnails": [ - {"url": "https://i.ytimg.com/vi/abc123/hq.jpg", "width": 480, "height": 360} + { + "url": "https://i.ytimg.com/vi/abc123/hq.jpg", + "width": 480, + "height": 360, + } ] }, }, "microformat": { - "playerMicroformatRenderer": {"publishDate": "2024-08-27", "isFamilySafe": True} + "playerMicroformatRenderer": { + "publishDate": "2024-08-27", + "isFamilySafe": True, + } }, "adPlacements": [{"adPlacementRenderer": {}}], } @@ -200,7 +207,9 @@ def test_parse_video_page_members_only_and_paid(): player = { "videoDetails": {"videoId": "m", "title": "t"}, "playabilityStatus": {"errorScreen": {"x": {"offerId": "sponsors_only_video"}}}, - "paidContentOverlay": {"paidContentOverlayRenderer": {"text": "Includes paid promotion"}}, + "paidContentOverlay": { + "paidContentOverlayRenderer": {"text": "Includes paid promotion"} + }, } initial = { "badges": [ @@ -237,7 +246,11 @@ def test_parse_channel_metadata(): "externalId": "UCabc123", "description": "We scrape the web.", "vanityChannelUrl": "https://www.youtube.com/@Apify", - "avatar": {"thumbnails": [{"url": "https://a/avatar.jpg", "width": 88, "height": 88}]}, + "avatar": { + "thumbnails": [ + {"url": "https://a/avatar.jpg", "width": 88, "height": 88} + ] + }, } }, "header": { @@ -246,7 +259,11 @@ def test_parse_channel_metadata(): "imageBannerViewModel": { "image": { "sources": [ - {"url": "https://a/banner.jpg", "width": 1060, "height": 175} + { + "url": "https://a/banner.jpg", + "width": 1060, + "height": 175, + } ] } } @@ -290,7 +307,9 @@ def test_parse_description_links(): { "startIndex": 4, "length": 4, - "onTap": {"innertubeCommand": {"urlEndpoint": {"url": "/hashtag/tag"}}}, + "onTap": { + "innertubeCommand": {"urlEndpoint": {"url": "/hashtag/tag"}} + }, }, { "startIndex": 21, @@ -318,8 +337,16 @@ def test_parse_description_links(): def test_channel_about_tokens(): initial = { - "a": {"showEngagementPanelEndpoint": {"x": {"continuationCommand": {"token": "T1"}}}}, - "b": {"showEngagementPanelEndpoint": {"y": {"continuationCommand": {"token": "T2"}}}}, + "a": { + "showEngagementPanelEndpoint": { + "x": {"continuationCommand": {"token": "T1"}} + } + }, + "b": { + "showEngagementPanelEndpoint": { + "y": {"continuationCommand": {"token": "T2"}} + } + }, } assert channel_about_tokens(initial) == ["T1", "T2"] assert channel_about_tokens({}) == [] @@ -340,7 +367,11 @@ def test_parse_search_response(): "shortViewCountText": {"simpleText": "451K views"}, "thumbnail": { "thumbnails": [ - {"url": "https://i.ytimg.com/vi/x/hq.jpg", "width": 360, "height": 202} + { + "url": "https://i.ytimg.com/vi/x/hq.jpg", + "width": 360, + "height": 202, + } ] }, "ownerText": { @@ -395,7 +426,11 @@ def test_parse_channel_shorts(): "thumbnailViewModel": { "image": { "sources": [ - {"url": "https://i.ytimg.com/s.jpg", "width": 405, "height": 720} + { + "url": "https://i.ytimg.com/s.jpg", + "width": 405, + "height": 720, + } ] } }, @@ -422,13 +457,30 @@ def test_parse_playlist_video_ids(): data = { "contents": { "items": [ - {"lockupViewModel": {"contentId": "fNk_zzaMoSs", "contentType": "VIDEO"}}, - {"lockupViewModel": {"contentId": "k7RM-ot2NWY", "contentType": "VIDEO"}}, - {"lockupViewModel": {"contentId": "fNk_zzaMoSs", "contentType": "VIDEO"}}, # dup + { + "lockupViewModel": { + "contentId": "fNk_zzaMoSs", + "contentType": "VIDEO", + } + }, + { + "lockupViewModel": { + "contentId": "k7RM-ot2NWY", + "contentType": "VIDEO", + } + }, + { + "lockupViewModel": { + "contentId": "fNk_zzaMoSs", + "contentType": "VIDEO", + } + }, # dup ] }, # a playlist self-lockup (non-video, longer id) that must be ignored - "sidebar": {"lockupViewModel": {"contentId": "PLZHQObOWTQDPD3MizzM2xVFitgF8hE_ab"}}, + "sidebar": { + "lockupViewModel": {"contentId": "PLZHQObOWTQDPD3MizzM2xVFitgF8hE_ab"} + }, "continuationItemRenderer": { "continuationEndpoint": {"continuationCommand": {"token": "PAGE2"}} }, @@ -494,7 +546,11 @@ def _primary_info(super_title_link): "results": { "results": { "contents": [ - {"videoPrimaryInfoRenderer": {"superTitleLink": super_title_link}} + { + "videoPrimaryInfoRenderer": { + "superTitleLink": super_title_link + } + } ] } } @@ -536,7 +592,9 @@ def _owner(video_owner_renderer): "contents": [ { "videoSecondaryInfoRenderer": { - "owner": {"videoOwnerRenderer": video_owner_renderer} + "owner": { + "videoOwnerRenderer": video_owner_renderer + } } } ] @@ -556,14 +614,19 @@ def _collab_row(name, base_url): { "onTap": { "innertubeCommand": { - "browseEndpoint": {"browseId": "UCx", "canonicalBaseUrl": base_url} + "browseEndpoint": { + "browseId": "UCx", + "canonicalBaseUrl": base_url, + } } } } ], }, # A nested subscribe submenu — must NOT be picked up as a collaborator. - "subscribeMenu": {"listItemViewModel": {"title": {"content": "Unsubscribe"}}}, + "subscribeMenu": { + "listItemViewModel": {"title": {"content": "Unsubscribe"}} + }, } } @@ -583,15 +646,20 @@ def test_parse_collaborators_from_dialog(): "dialogViewModel": { "header": { "dialogHeaderViewModel": { - "headline": {"content": "Collaborators"} + "headline": { + "content": "Collaborators" + } } }, "customContent": { "listViewModel": { "listItems": [ - _collab_row("Alice", "/@alice"), _collab_row( - "Bob", "/channel/UCbob123" + "Alice", "/@alice" + ), + _collab_row( + "Bob", + "/channel/UCbob123", ), ] } @@ -610,7 +678,11 @@ def test_parse_collaborators_from_dialog(): collaborators = parse_collaborators(initial) assert collaborators == [ {"name": "Alice", "username": "alice", "url": "https://www.youtube.com/@alice"}, - {"name": "Bob", "username": None, "url": "https://www.youtube.com/channel/UCbob123"}, + { + "name": "Bob", + "username": None, + "url": "https://www.youtube.com/channel/UCbob123", + }, ] @@ -627,8 +699,23 @@ def test_parse_translation_from_next(): "results": { "results": { "contents": [ - {"videoPrimaryInfoRenderer": {"title": {"runs": [{"text": "Título "}, {"text": "traducido"}]}}}, - {"videoSecondaryInfoRenderer": {"attributedDescription": {"content": "Descripción traducida"}}}, + { + "videoPrimaryInfoRenderer": { + "title": { + "runs": [ + {"text": "Título "}, + {"text": "traducido"}, + ] + } + } + }, + { + "videoSecondaryInfoRenderer": { + "attributedDescription": { + "content": "Descripción traducida" + } + } + }, ] } } @@ -642,7 +729,9 @@ def test_parse_translation_from_next(): def test_parse_translation_description_runs_fallback(): next_data = { - "videoSecondaryInfoRenderer": {"description": {"runs": [{"text": "old "}, {"text": "style"}]}} + "videoSecondaryInfoRenderer": { + "description": {"runs": [{"text": "old "}, {"text": "style"}]} + } } title, description = parse_translation(next_data) assert title is None @@ -671,6 +760,7 @@ def _comment_cep(cid, *, level=0, hearted=False, owner=False, replies="5"): def _next_comments_response(): """A /next comments response: sort menu, two threads, a page token.""" + def _reply_loader(token): return { "continuationItemRenderer": { @@ -692,7 +782,13 @@ def _next_comments_response(): "frameworkUpdates": { "entityBatchUpdate": { "mutations": [ - {"payload": {"commentEntityPayload": _comment_cep("C1", hearted=True, owner=True)}}, + { + "payload": { + "commentEntityPayload": _comment_cep( + "C1", hearted=True, owner=True + ) + } + }, {"payload": {"commentEntityPayload": _comment_cep("C2")}}, ] } @@ -717,8 +813,14 @@ def _next_comments_response(): "header": { "sortFilterSubMenuRenderer": { "subMenuItems": [ - {"title": "Top", "serviceEndpoint": {"continuationCommand": {"token": "TOPTOK"}}}, - {"title": "Newest", "serviceEndpoint": {"continuationCommand": {"token": "NEWTOK"}}}, + { + "title": "Top", + "serviceEndpoint": {"continuationCommand": {"token": "TOPTOK"}}, + }, + { + "title": "Newest", + "serviceEndpoint": {"continuationCommand": {"token": "NEWTOK"}}, + }, ] } }, @@ -749,7 +851,13 @@ def test_comment_entity_reply_level_and_empty_reply_count(): "frameworkUpdates": { "entityBatchUpdate": { "mutations": [ - {"payload": {"commentEntityPayload": _comment_cep("R1", level=1, replies="")}} + { + "payload": { + "commentEntityPayload": _comment_cep( + "R1", level=1, replies="" + ) + } + } ] } } @@ -789,7 +897,9 @@ def test_comment_section_token_from_watch_page(): { "continuationItemRenderer": { "continuationEndpoint": { - "continuationCommand": {"token": "SECTIONTOK"} + "continuationCommand": { + "token": "SECTIONTOK" + } } } } @@ -815,9 +925,17 @@ def test_comment_section_token_from_watch_page(): ("https://youtu.be/dQw4w9WgXcQ", "video", "dQw4w9WgXcQ"), ("https://www.youtube.com/shorts/abc123", "video", "abc123"), ("https://www.youtube.com/@Apify", "channel", "Apify"), - ("https://www.youtube.com/channel/UC123456789abc/videos", "channel", "UC123456789abc"), + ( + "https://www.youtube.com/channel/UC123456789abc/videos", + "channel", + "UC123456789abc", + ), ("https://www.youtube.com/playlist?list=PL123", "playlist", "PL123"), - ("https://www.youtube.com/results?search_query=web+scraping", "search", "web scraping"), + ( + "https://www.youtube.com/results?search_query=web+scraping", + "search", + "web scraping", + ), ("https://www.youtube.com/hashtag/tech", "hashtag", "tech"), ], ) diff --git a/surfsense_backend/tests/unit/proprietary/web_crawler/test_captcha.py b/surfsense_backend/tests/unit/proprietary/web_crawler/test_captcha.py index 51a57bf87..de7228a76 100644 --- a/surfsense_backend/tests/unit/proprietary/web_crawler/test_captcha.py +++ b/surfsense_backend/tests/unit/proprietary/web_crawler/test_captcha.py @@ -15,16 +15,16 @@ pytestmark = pytest.mark.unit def _cfg(**overrides) -> CaptchaConfig: - base = dict( - enabled=True, - solving_site="capsolver", - api_key="key-123", - max_attempts_per_url=1, - timeout_s=30, - captcha_type_default="v2", - v3_min_score=0.7, - v3_action="verify", - ) + base = { + "enabled": True, + "solving_site": "capsolver", + "api_key": "key-123", + "max_attempts_per_url": 1, + "timeout_s": 30, + "captcha_type_default": "v2", + "v3_min_score": 0.7, + "v3_action": "verify", + } base.update(overrides) return CaptchaConfig(**base) @@ -40,7 +40,9 @@ class _FakeEl: class _FakePage: """Minimal sync-Playwright-ish page for detection/injection tests.""" - def __init__(self, widgets=None, iframes=None, scripts=None, url="https://t.test/p"): + def __init__( + self, widgets=None, iframes=None, scripts=None, url="https://t.test/p" + ): self._widgets = widgets or {} # selector -> _FakeEl self._iframes = iframes or [] # list[_FakeEl] self._scripts = scripts or [] # list[_FakeEl] @@ -116,7 +118,9 @@ class TestDetect: def test_v3_render_param_when_default_v3(self): page = _FakePage( - scripts=[_FakeEl({"src": "https://www.google.com/recaptcha/api.js?render=SK_V3"})] + scripts=[ + _FakeEl({"src": "https://www.google.com/recaptcha/api.js?render=SK_V3"}) + ] ) assert cap.detect_challenge(page, _cfg(captcha_type_default="v3")) == ( "v3", @@ -217,11 +221,11 @@ class TestPageAction: assert state == {"attempts": 1, "solved": False} def test_no_balance_latches_and_stops(self, monkeypatch): - class ErrNoBalance(Exception): + class NoBalanceError(Exception): pass def _boom(*_a, **_k): - raise ErrNoBalance("out of balance") + raise NoBalanceError("out of balance") monkeypatch.setattr(cap, "_harvest_token", _boom) state = {"attempts": 0, "solved": False} diff --git a/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py b/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py index 2c749b2cd..f33d8c007 100644 --- a/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py +++ b/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py @@ -10,8 +10,8 @@ import pytest from app.proprietary.web_crawler import ( CrawlOutcomeStatus, WebCrawlerConnector, + connector as connector_module, ) -from app.proprietary.web_crawler import connector as connector_module from app.utils.crawl import BlockType pytestmark = pytest.mark.unit @@ -230,9 +230,7 @@ async def test_captcha_attempts_surface_even_when_crawl_fails( monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty) monkeypatch.setattr(crawler, "_crawl_with_dynamic", _empty) - monkeypatch.setattr( - crawler, "_crawl_with_stealthy", _stealthy_attempt_then_empty - ) + monkeypatch.setattr(crawler, "_crawl_with_stealthy", _stealthy_attempt_then_empty) outcome = await crawler.crawl_url("https://example.com") @@ -341,9 +339,7 @@ async def test_static_4xx_is_classified( monkeypatch.setattr(connector_module, "get_proxy_url", lambda: None) block_state: dict = {"block_type": BlockType.UNKNOWN} - result = await crawler._crawl_with_async_fetcher( - "https://example.com", block_state - ) + result = await crawler._crawl_with_async_fetcher("https://example.com", block_state) assert result is None # 4xx => fall through to next tier assert block_state["block_type"] is BlockType.CLOUDFLARE @@ -354,7 +350,9 @@ def test_build_result_ok_on_real_content() -> None: crawler = WebCrawlerConnector() block_state: dict = {"block_type": BlockType.UNKNOWN} - html = "

" + ("Real content. " * 40) + "
" + html = ( + "
" + ("Real content. " * 40) + "
" + ) crawler._build_result( html, "https://example.com", diff --git a/surfsense_backend/tests/unit/tasks/chat/streaming/test_llm_bundle.py b/surfsense_backend/tests/unit/tasks/chat/streaming/test_llm_bundle.py index bc3f0f7e5..09a0351b7 100644 --- a/surfsense_backend/tests/unit/tasks/chat/streaming/test_llm_bundle.py +++ b/surfsense_backend/tests/unit/tasks/chat/streaming/test_llm_bundle.py @@ -32,9 +32,7 @@ def _patch_common_bundle_dependencies(monkeypatch: pytest.MonkeyPatch): _CapturedChatLiteLLM.calls = [] - async def _fake_workspace( - _session: Any, _workspace_id: int - ) -> SimpleNamespace: + async def _fake_workspace(_session: Any, _workspace_id: int) -> SimpleNamespace: return SimpleNamespace(id=42, user_id="user-1") monkeypatch.setattr(llm_bundle, "_load_workspace", _fake_workspace) diff --git a/surfsense_backend/tests/unit/test_pat_fail_closed_static.py b/surfsense_backend/tests/unit/test_pat_fail_closed_static.py index 0b4f3aa9c..17bdc4ca0 100644 --- a/surfsense_backend/tests/unit/test_pat_fail_closed_static.py +++ b/surfsense_backend/tests/unit/test_pat_fail_closed_static.py @@ -16,9 +16,7 @@ ALLOW_ANY_EXPECTED = { "routes/obsidian_plugin_routes.py": ( "_auth: AuthContext = Depends(allow_any_principal)" ), - "routes/workspaces_routes.py": ( - "auth: AuthContext = Depends(allow_any_principal)" - ), + "routes/workspaces_routes.py": ("auth: AuthContext = Depends(allow_any_principal)"), } CONNECTOR_LISTERS = [