mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
feat(native-connector): added google maps places & reviews scrapers
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
cc99bc4cae
commit
7185079bd6
114 changed files with 4688 additions and 425 deletions
|
|
@ -6,6 +6,8 @@ Revises: 167
|
|||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "168"
|
||||
|
|
@ -48,9 +50,7 @@ def upgrade() -> None:
|
|||
"""
|
||||
)
|
||||
|
||||
op.execute(
|
||||
"ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(64)"
|
||||
)
|
||||
op.execute("ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(64)")
|
||||
op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS is_revoked")
|
||||
|
||||
|
||||
|
|
@ -74,8 +74,6 @@ def downgrade() -> None:
|
|||
"""
|
||||
)
|
||||
op.execute("ALTER TABLE refresh_tokens ALTER COLUMN is_revoked DROP DEFAULT")
|
||||
op.execute(
|
||||
"ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(256)"
|
||||
)
|
||||
op.execute("ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(256)")
|
||||
op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS absolute_expiry")
|
||||
op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS revoked_at")
|
||||
|
|
|
|||
|
|
@ -373,7 +373,9 @@ def _neutralize_column_list_tables(conn) -> None:
|
|||
for table in PUBLICATION_COLUMN_LIST_TABLES:
|
||||
if _is_publication_member(conn, table):
|
||||
conn.execute(
|
||||
sa.text(f"ALTER PUBLICATION {_quote(PUBLICATION_NAME)} DROP TABLE {table}")
|
||||
sa.text(
|
||||
f"ALTER PUBLICATION {_quote(PUBLICATION_NAME)} DROP TABLE {table}"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -440,7 +442,9 @@ def _restore_downgrade_publication(conn) -> None:
|
|||
for table in PUBLICATION_COLUMN_LIST_TABLES:
|
||||
if _is_publication_member(conn, table):
|
||||
conn.execute(
|
||||
sa.text(f"ALTER PUBLICATION {_quote(PUBLICATION_NAME)} DROP TABLE {table}")
|
||||
sa.text(
|
||||
f"ALTER PUBLICATION {_quote(PUBLICATION_NAME)} DROP TABLE {table}"
|
||||
)
|
||||
)
|
||||
|
||||
entries: list[str] = []
|
||||
|
|
|
|||
|
|
@ -111,9 +111,7 @@ async def _ensure_folder_hierarchy(
|
|||
sibling_query = (
|
||||
select(Folder.position).order_by(Folder.position.desc()).limit(1)
|
||||
)
|
||||
sibling_query = sibling_query.where(
|
||||
Folder.workspace_id == workspace_id
|
||||
)
|
||||
sibling_query = sibling_query.where(Folder.workspace_id == workspace_id)
|
||||
if parent_id is None:
|
||||
sibling_query = sibling_query.where(Folder.parent_id.is_(None))
|
||||
else:
|
||||
|
|
@ -1241,9 +1239,7 @@ async def commit_staged_filesystem_state(
|
|||
|
||||
await session.commit()
|
||||
except Exception: # pragma: no cover - rollback safety net
|
||||
logger.exception(
|
||||
"kb_persistence: commit failed (workspace=%s)", workspace_id
|
||||
)
|
||||
logger.exception("kb_persistence: commit failed (workspace=%s)", workspace_id)
|
||||
# Outer commit raised: everything above rolled back, so drop the
|
||||
# deferred dispatches.
|
||||
deferred_dispatches.clear()
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from langgraph.runtime import Runtime
|
|||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db import ChatVisibility, Workspace, User, shielded_async_session
|
||||
from app.db import ChatVisibility, User, Workspace, shielded_async_session
|
||||
from app.services.memory import MEMORY_HARD_LIMIT, MEMORY_SOFT_LIMIT
|
||||
from app.utils.perf import get_perf_logger
|
||||
|
||||
|
|
|
|||
|
|
@ -98,9 +98,7 @@ async def create_multi_agent_chat_deep_agent(
|
|||
|
||||
_t0 = time.perf_counter()
|
||||
try:
|
||||
connector_types = await connector_service.get_available_connectors(
|
||||
workspace_id
|
||||
)
|
||||
connector_types = await connector_service.get_available_connectors(workspace_id)
|
||||
available_connectors = map_connectors_to_searchable_types(connector_types)
|
||||
|
||||
available_document_types = await connector_service.get_available_document_types(
|
||||
|
|
@ -153,9 +151,7 @@ async def create_multi_agent_chat_deep_agent(
|
|||
|
||||
_t0 = time.perf_counter()
|
||||
try:
|
||||
mcp_tools_by_agent = await load_mcp_tools_by_connector(
|
||||
db_session, workspace_id
|
||||
)
|
||||
mcp_tools_by_agent = await load_mcp_tools_by_connector(db_session, workspace_id)
|
||||
except Exception as e:
|
||||
# Degrade to builtins-only rather than aborting the turn: a transient
|
||||
# DB or MCP-server hiccup should not deny the user a response.
|
||||
|
|
|
|||
|
|
@ -116,9 +116,7 @@ def create_generate_image_tool(
|
|||
# later workspace changes. No workspace read needed.
|
||||
config_id = image_gen_model_id_override or IMAGE_GEN_AUTO_MODE_ID
|
||||
else:
|
||||
config_id = (
|
||||
workspace.image_gen_model_id or IMAGE_GEN_AUTO_MODE_ID
|
||||
)
|
||||
config_id = workspace.image_gen_model_id or IMAGE_GEN_AUTO_MODE_ID
|
||||
|
||||
# size/quality/style are intentionally omitted: valid values
|
||||
# differ per model, so we let each model use its own defaults.
|
||||
|
|
@ -191,10 +189,7 @@ def create_generate_image_tool(
|
|||
):
|
||||
err = f"Image generation model {config_id} not found"
|
||||
return _failed({"error": err}, error=err)
|
||||
if (
|
||||
conn.user_id is not None
|
||||
and conn.user_id != workspace.user_id
|
||||
):
|
||||
if conn.user_id is not None and conn.user_id != workspace.user_id:
|
||||
err = f"Image generation model {config_id} not found"
|
||||
return _failed({"error": err}, error=err)
|
||||
if not has_capability(db_model, "image_gen"):
|
||||
|
|
|
|||
|
|
@ -70,9 +70,7 @@ def create_create_calendar_event_tool(
|
|||
|
||||
try:
|
||||
metadata_service = GoogleCalendarToolMetadataService(db_session)
|
||||
context = await metadata_service.get_creation_context(
|
||||
workspace_id, user_id
|
||||
)
|
||||
context = await metadata_service.get_creation_context(workspace_id, user_id)
|
||||
|
||||
if "error" in context:
|
||||
logger.error(f"Failed to fetch creation context: {context['error']}")
|
||||
|
|
|
|||
|
|
@ -52,9 +52,7 @@ def create_create_confluence_page_tool(
|
|||
|
||||
try:
|
||||
metadata_service = ConfluenceToolMetadataService(db_session)
|
||||
context = await metadata_service.get_creation_context(
|
||||
workspace_id, user_id
|
||||
)
|
||||
context = await metadata_service.get_creation_context(workspace_id, user_id)
|
||||
|
||||
if "error" in context:
|
||||
return {"status": "error", "message": context["error"]}
|
||||
|
|
|
|||
|
|
@ -29,9 +29,7 @@ def create_list_discord_channels_tool(
|
|||
}
|
||||
|
||||
try:
|
||||
connector = await get_discord_connector(
|
||||
db_session, workspace_id, user_id
|
||||
)
|
||||
connector = await get_discord_connector(db_session, workspace_id, user_id)
|
||||
if not connector:
|
||||
return {"status": "error", "message": "No Discord connector found."}
|
||||
|
||||
|
|
|
|||
|
|
@ -39,9 +39,7 @@ def create_read_discord_messages_tool(
|
|||
limit = min(limit, 50)
|
||||
|
||||
try:
|
||||
connector = await get_discord_connector(
|
||||
db_session, workspace_id, user_id
|
||||
)
|
||||
connector = await get_discord_connector(db_session, workspace_id, user_id)
|
||||
if not connector:
|
||||
return {"status": "error", "message": "No Discord connector found."}
|
||||
|
||||
|
|
|
|||
|
|
@ -49,9 +49,7 @@ def create_send_discord_message_tool(
|
|||
}
|
||||
|
||||
try:
|
||||
connector = await get_discord_connector(
|
||||
db_session, workspace_id, user_id
|
||||
)
|
||||
connector = await get_discord_connector(db_session, workspace_id, user_id)
|
||||
if not connector:
|
||||
return {"status": "error", "message": "No Discord connector found."}
|
||||
|
||||
|
|
|
|||
|
|
@ -67,9 +67,7 @@ def create_create_gmail_draft_tool(
|
|||
|
||||
try:
|
||||
metadata_service = GmailToolMetadataService(db_session)
|
||||
context = await metadata_service.get_creation_context(
|
||||
workspace_id, user_id
|
||||
)
|
||||
context = await metadata_service.get_creation_context(workspace_id, user_id)
|
||||
|
||||
if "error" in context:
|
||||
logger.error(f"Failed to fetch creation context: {context['error']}")
|
||||
|
|
|
|||
|
|
@ -96,9 +96,7 @@ def create_send_gmail_email_tool(
|
|||
|
||||
try:
|
||||
metadata_service = GmailToolMetadataService(db_session)
|
||||
context = await metadata_service.get_creation_context(
|
||||
workspace_id, user_id
|
||||
)
|
||||
context = await metadata_service.get_creation_context(workspace_id, user_id)
|
||||
|
||||
if "error" in context:
|
||||
logger.error(f"Failed to fetch creation context: {context['error']}")
|
||||
|
|
|
|||
|
|
@ -81,9 +81,7 @@ def create_create_google_drive_file_tool(
|
|||
|
||||
try:
|
||||
metadata_service = GoogleDriveToolMetadataService(db_session)
|
||||
context = await metadata_service.get_creation_context(
|
||||
workspace_id, user_id
|
||||
)
|
||||
context = await metadata_service.get_creation_context(workspace_id, user_id)
|
||||
|
||||
if "error" in context:
|
||||
logger.error(f"Failed to fetch creation context: {context['error']}")
|
||||
|
|
|
|||
|
|
@ -80,9 +80,7 @@ def create_create_notion_page_tool(
|
|||
|
||||
try:
|
||||
metadata_service = NotionToolMetadataService(db_session)
|
||||
context = await metadata_service.get_creation_context(
|
||||
workspace_id, user_id
|
||||
)
|
||||
context = await metadata_service.get_creation_context(workspace_id, user_id)
|
||||
|
||||
if "error" in context:
|
||||
logger.error(f"Failed to fetch creation context: {context['error']}")
|
||||
|
|
|
|||
|
|
@ -43,9 +43,7 @@ class AutomationService:
|
|||
|
||||
async def create(self, payload: AutomationCreate) -> Automation:
|
||||
"""Create an automation and its initial triggers in one transaction."""
|
||||
await self._authorize(
|
||||
payload.workspace_id, Permission.AUTOMATIONS_CREATE.value
|
||||
)
|
||||
await self._authorize(payload.workspace_id, Permission.AUTOMATIONS_CREATE.value)
|
||||
|
||||
# Capture the model profile onto the definition so runs are insulated
|
||||
# from later chat/workspace model changes. Two sources:
|
||||
|
|
|
|||
|
|
@ -1075,9 +1075,7 @@ class Config:
|
|||
CAPTCHA_SOLVER_PROVIDER = os.getenv("CAPTCHA_SOLVER_PROVIDER", "capsolver")
|
||||
CAPTCHA_SOLVER_API_KEY = os.getenv("CAPTCHA_SOLVER_API_KEY")
|
||||
# Per-URL solve cap so one hostile page can't burn unbounded solver credit.
|
||||
CAPTCHA_MAX_ATTEMPTS_PER_URL = int(
|
||||
os.getenv("CAPTCHA_MAX_ATTEMPTS_PER_URL", "1")
|
||||
)
|
||||
CAPTCHA_MAX_ATTEMPTS_PER_URL = int(os.getenv("CAPTCHA_MAX_ATTEMPTS_PER_URL", "1"))
|
||||
# Abort a single solve after this many seconds (solves take 10-60s).
|
||||
CAPTCHA_SOLVE_TIMEOUT_S = int(os.getenv("CAPTCHA_SOLVE_TIMEOUT_S", "120"))
|
||||
# Default captcha type when detection is ambiguous: v2 | v3 | hcaptcha.
|
||||
|
|
@ -1114,9 +1112,7 @@ class Config:
|
|||
# Route DNS via Cloudflare DoH (anti DNS-leak behind proxies). Adds a DNS
|
||||
# round-trip => default FALSE to honor the "no speed regression" bar; flip on
|
||||
# when leak-safety outweighs the marginal latency.
|
||||
CRAWL_DNS_OVER_HTTPS = (
|
||||
os.getenv("CRAWL_DNS_OVER_HTTPS", "FALSE").upper() == "TRUE"
|
||||
)
|
||||
CRAWL_DNS_OVER_HTTPS = os.getenv("CRAWL_DNS_OVER_HTTPS", "FALSE").upper() == "TRUE"
|
||||
|
||||
# Litellm TTS Configuration
|
||||
TTS_SERVICE = os.getenv("TTS_SERVICE")
|
||||
|
|
|
|||
|
|
@ -823,9 +823,7 @@ class ExternalChatAccount(Base, TimestampMixin):
|
|||
)
|
||||
|
||||
owner = relationship("User", foreign_keys=[owner_user_id])
|
||||
owner_workspace = relationship(
|
||||
"Workspace", foreign_keys=[owner_workspace_id]
|
||||
)
|
||||
owner_workspace = relationship("Workspace", foreign_keys=[owner_workspace_id])
|
||||
bindings = relationship(
|
||||
"ExternalChatBinding",
|
||||
back_populates="account",
|
||||
|
|
@ -984,9 +982,7 @@ class ExternalChatBinding(Base, TimestampMixin):
|
|||
postgresql_where=text("state = 'pending'"),
|
||||
),
|
||||
Index("ix_external_chat_bindings_user_state", "user_id", "state"),
|
||||
Index(
|
||||
"ix_external_chat_bindings_workspace_state", "workspace_id", "state"
|
||||
),
|
||||
Index("ix_external_chat_bindings_workspace_state", "workspace_id", "state"),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1902,9 +1898,7 @@ class SearchSourceConnector(BaseModel, TimestampMixin):
|
|||
ForeignKey("workspaces.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
workspace = relationship(
|
||||
"Workspace", back_populates="search_source_connectors"
|
||||
)
|
||||
workspace = relationship("Workspace", back_populates="search_source_connectors")
|
||||
|
||||
user_id = Column(
|
||||
UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False
|
||||
|
|
|
|||
|
|
@ -99,9 +99,7 @@ def finalize(state: TranscriptState, config: RunnableConfig) -> dict[str, Any]:
|
|||
async def _require_llm(state: TranscriptState, tc: TranscriptConfig):
|
||||
llm = await get_agent_llm(state.db_session, tc.workspace_id)
|
||||
if llm is None:
|
||||
raise RuntimeError(
|
||||
f"no agent LLM configured for workspace {tc.workspace_id}"
|
||||
)
|
||||
raise RuntimeError(f"no agent LLM configured for workspace {tc.workspace_id}")
|
||||
return llm
|
||||
|
||||
|
||||
|
|
|
|||
341
surfsense_backend/app/proprietary/scrapers/google_maps/README.md
Normal file
341
surfsense_backend/app/proprietary/scrapers/google_maps/README.md
Normal file
|
|
@ -0,0 +1,341 @@
|
|||
# Google Maps Scraper
|
||||
|
||||
A platform-native Google Maps scraper intended as a **drop-in clone of the
|
||||
Apify "Google Maps Scraper" and "Google Maps Reviews Scraper" actors** — same
|
||||
input surface, same output item shape. Built on the same layout and
|
||||
progressive-implementation approach as the sibling `../youtube` scraper.
|
||||
|
||||
**Current status: search + place details + reviews live.** Search terms
|
||||
(`searchStringsArray` x `locationQuery`/geolocation) are discovered via the
|
||||
public `search?tbm=map` RPC with offset paging and the Apify result filters.
|
||||
Direct place URLs and bare `placeIds` are scraped via Google's public
|
||||
`/maps/preview/place` RPC; reviews (both the standalone Reviews endpoint and
|
||||
inline `reviews[]` on places when `maxReviews > 0`) come from the public
|
||||
`GetLocalBoqProxy` review feed with full token-based pagination. No login,
|
||||
proxy-only egress.
|
||||
|
||||
## Public vs. gated data
|
||||
|
||||
All core Maps data is **public — no Google account needed**: search results,
|
||||
place details (phone, website, hours, price, coordinates, plus code, address
|
||||
components), and reviews (text, ratings, reviewer profiles, owner responses)
|
||||
are served by Google's internal endpoints (`/maps/preview/place`,
|
||||
`GetLocalBoqProxy`, `search?tbm=map`) which return XSSI-guarded JSON to
|
||||
anonymous requests. Only a residential proxy is required (Google blocks
|
||||
datacenter IPs) — already wired via `app/utils/proxy`.
|
||||
|
||||
**Session-gated (but still login-free) fields:** Google trims the rich detail
|
||||
fields — `reviewsCount`, `reviewsDistribution`, popular times, image
|
||||
galleries, `reviewsTags`, and most `additionalInfo` sections — from responses
|
||||
that carry **no session cookie**. A plain GET to `/maps` mints an `NID` cookie
|
||||
that unlocks all of them (no login, no browser needed at runtime;
|
||||
`fetch.get_session_cookies` mints them into a small rotating pool with a
|
||||
30-minute TTL). The full-page `pb` selector these fields also require was
|
||||
captured once from a headless browser render of a place page and genericized
|
||||
into `_PLACE_DETAIL_PB`.
|
||||
|
||||
> Note: the older review RPCs other scrapers documented (`listugcposts`,
|
||||
> `listentitiesreviews`) now return empty pages / 404 to anonymous callers —
|
||||
> they appear to require a signed-in session. The `GetLocalBoqProxy` feed (used
|
||||
> by Google's own search local panel) is the one that still works publicly and
|
||||
> is what `fetch.iter_reviews_pages` uses.
|
||||
|
||||
If Google ever serves a sign-in/consent wall instead of data,
|
||||
`SignInRequiredError` is raised and the route returns **403 "Google sign in
|
||||
required"** rather than an empty result.
|
||||
|
||||
Not sourced from Maps at all (Apify enrichment add-ons that hit third-party
|
||||
data brokers / the business's own website — out of scope for public-Maps-only):
|
||||
`maximumLeadsEnrichmentRecords`, `leadsEnrichmentDepartments`,
|
||||
`verifyLeadsEnrichmentEmails`, `scrapeSocialMediaProfiles`, `scrapeContacts`,
|
||||
`enableCompetitorAnalysis`. These stay at their schema defaults (`None`/`[]`).
|
||||
|
||||
## Quick start
|
||||
|
||||
```python
|
||||
from app.proprietary.scrapers.google_maps import (
|
||||
GoogleMapsScrapeInput, scrape_places,
|
||||
GoogleMapsReviewsInput, scrape_reviews,
|
||||
)
|
||||
|
||||
# Places — search terms, direct URLs, and placeIds are all additive.
|
||||
# maxReviews > 0 also attaches inline reviews[] to each place item.
|
||||
places = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
searchStringsArray=["coffee shop"],
|
||||
locationQuery="Seattle, WA",
|
||||
maxCrawledPlacesPerSearch=20,
|
||||
startUrls=[{"url": "https://www.google.com/maps/place/..."}],
|
||||
placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"],
|
||||
maxReviews=10,
|
||||
)
|
||||
)
|
||||
|
||||
# Reviews — one flat item per review (review fields + place header fields)
|
||||
reviews = await scrape_reviews(
|
||||
GoogleMapsReviewsInput(
|
||||
placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"],
|
||||
maxReviews=100,
|
||||
reviewsSort="newest",
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
Both have a streaming twin — `iter_places()` / `iter_reviews()`. The HTTP
|
||||
surface lives in `app/routes/google_maps_routes.py` (`POST /google-maps/scrape`
|
||||
and `POST /google-maps/reviews`).
|
||||
|
||||
## Module map
|
||||
|
||||
| File | Responsibility |
|
||||
| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `__init__.py` | Public exports (entry points + schemas). |
|
||||
| `schemas.py` | Pydantic input/output models mirroring the Apify camelCase specs. `extra="allow"` on outputs keeps the contract open. |
|
||||
| `scraper.py` | Places orchestrator. `_place_flow` (live): fid → detail RPC → `PlaceItem` (+ inline reviews). `_search_flow` (live): tbm=map paging + filters. |
|
||||
| `reviews.py` | Reviews orchestrator (live). Pages the BOQ feed per place, applies sort/date-cutoff/origin/personal-data rules, one flat item per review. |
|
||||
| `fetch.py` | Proxy-only fetch seam: `fetch_html`, `fetch_rpc_json` (XSSI-strip + HTML-wrapper tolerant), `fetch_place_darray`, `iter_reviews_pages`, `iter_search_pages`. |
|
||||
| `parsers.py` | Pure array-path parsing: place `darray` → PlaceItem fields (paths from gosom), BOQ review arrays → ReviewFields dicts. |
|
||||
| `url_resolver.py` | Classify a URL into `place` / `search` / `reviews` / `cid` / `shortlink`; extract the feature ID (`0x..:0x..`). |
|
||||
|
||||
## How place scraping works
|
||||
|
||||
1. Resolve the **feature ID** (`0x<hex>:0x<hex>`): from the URL's `data=!1s…`
|
||||
blob when present, else fetch the place page once and regex it from HTML
|
||||
(covers `?q=place_id:ChIJ…` and `?cid=…`). Name-only URLs
|
||||
(`/maps/place/Eiffel+Tower/`) serve a JS shell with no fid in the HTML, so
|
||||
those fall back to a `search?tbm=map` lookup of the name
|
||||
(`fetch.fid_from_search`) and take the top result's fid. Short links
|
||||
(`maps.app.goo.gl` / `goo.gl`) are Firebase Dynamic Links — a plain GET only
|
||||
returns a JS interstitial, so they're browser-rendered so the JS follows the
|
||||
redirect to the real `/maps/place/…` URL, then the fid is read from that.
|
||||
2. GET `/maps/preview/place?…&pb=!1m13!1s{fid}…` through the proxy, with the
|
||||
NID session cookie. The `pb` is the same selector the Maps web app sends
|
||||
(browser-captured, genericized), so the response is the **full** payload
|
||||
including the session-gated fields. `)]}'`-guarded JSON; `jd[6]` is the
|
||||
place detail array ("darray") that all field paths index into.
|
||||
3. `parse_place()` maps darray positions → Apify-shaped fields (title,
|
||||
categories, address components, phone, website, rating, plus code, hours,
|
||||
coordinates, placeId…). Only keys whose path hit are set; the rest keep
|
||||
schema defaults.
|
||||
|
||||
Detail extras parsed from the same darray (no extra requests):
|
||||
|
||||
- `kgmid` (`darray[89]`, e.g. `/g/11bw4ws2mt`) and `cid` — the CID is just the
|
||||
decimal value of the fid's second hex half, so it's derived, not fetched.
|
||||
- `additionalInfo` — all about sections at `darray[100][1]` in Apify's shape
|
||||
(`{"Accessibility": [{"Wheelchair accessible entrance": true}, …]}`).
|
||||
- `reviewsCount` (`[4][8]`) + `reviewsDistribution` (`[175][3]`).
|
||||
- `popularTimesHistogram` / `popularTimesLiveText` / `popularTimesLivePercent`
|
||||
(`darray[84]`).
|
||||
- `imagesCount` (`[37][1]`), `imageCategories` (gallery tab names at
|
||||
`[171][0]`), `imageUrls` (hero photos + tab thumbnails; emitted only when
|
||||
`maxImages > 0`, capped at it). Full multi-thousand-photo galleries would
|
||||
need the signed-in photo-listing RPC, so `imageUrls` tops out around a
|
||||
dozen per place.
|
||||
- `reviewsTags` (`[153][0]` → `[{title, count}]`).
|
||||
- `tableReservationLinks` + `reserveTableUrl` (`darray[46]`), `orderBy` /
|
||||
`googleFoodUrl` (order-online providers at `darray[75]`), `menu`
|
||||
(`darray[38][0]`).
|
||||
|
||||
Hotel places (detected by the star string at `darray[35][6]`) additionally
|
||||
get: `hotelStars`, `hotelDescription`, `checkInDate`/`checkOutDate` (the
|
||||
dates Google quoted prices for, `[35][0..1]`), `similarHotelsNearby`
|
||||
(`[35][29][0]` — title/fid/score/count/location/description), and `hotelAds`
|
||||
(`[35][44]` — booking-partner title/url/price). Probed live on The Plaza NYC.
|
||||
|
||||
Search-result darrays are served **without** the session-gated fields (the
|
||||
NID cookie doesn't help `search?tbm=map` — verified live), so when
|
||||
`scrapePlaceDetailPage=true` or `maxImages > 0` the search flow makes one
|
||||
detail RPC per emitted place and merges the full payload over the search
|
||||
fields — same trade Apify makes (search-only is one request per ~20 places;
|
||||
detail adds one per place).
|
||||
|
||||
**`allPlacesNoSearchAction`** (area scan with no search term): Apify's
|
||||
implementation OCRs / mouse-overs rendered map pins — the public RPC has no
|
||||
"list everything" query (`*` and empty return nothing). Ours approximates the
|
||||
scan with a broad category sweep (`restaurant`, `store`, `hotel`, …17 terms)
|
||||
over the requested viewport, deduped by fid, until
|
||||
`maxCrawledPlacesPerSearch`. `searchString` carries the action value on scan
|
||||
items. Verified live: 25 unique places / 14 distinct categories from a 400 m
|
||||
Times Square viewport.
|
||||
|
||||
## How search discovery works
|
||||
|
||||
1. Build the query: `"{search term} in {location}"` when `locationQuery` (or
|
||||
the city/state/... geolocation fields) is set — Google localizes from the
|
||||
query text, so no geocoding round-trip is needed. A `customGeolocation`
|
||||
GeoJSON Point instead sets a real lat/lng/radius viewport.
|
||||
2. Page `search?tbm=map` (~20 results per page, `!8i` offset). Each result
|
||||
entry embeds a **full place darray** at `entry[14]` — same shape as the
|
||||
detail RPC — so `parse_place()` runs directly on it and no per-place detail
|
||||
request is needed. Single-match queries put the one place in slot 0 of the
|
||||
results list; multi-result pages start at slot 1 (both handled).
|
||||
3. Dedupe by fid (Google reshuffles results between pages), stamp
|
||||
`rank`/`searchString`/`searchPageUrl`, apply the Apify filters client-side
|
||||
(`searchMatching`, `categoryFilterWords`, `placeMinimumStars`, `website`,
|
||||
`skipClosedPlaces`), and stop at `maxCrawledPlacesPerSearch`, an empty
|
||||
page, or a page with no new fids. Verified live to 60 unique results
|
||||
(3+ pages) with strictly sequential ranks, and to terminate on its own for
|
||||
sparse queries that exhaust before the cap.
|
||||
|
||||
Closed places: `darray[88][0]` carries a status enum (`CLOSED` verified live
|
||||
on a permanently-closed place) that maps to `permanentlyClosed`.
|
||||
|
||||
## Performance
|
||||
|
||||
Every RPC is a ~2s proxy round-trip, so wall-clock time is dominated by how
|
||||
many requests run in series, not by parsing. Independent requests are
|
||||
overlapped (bounded, order preserved via `gather_bounded`):
|
||||
|
||||
- **Detail enrichment** (`scrapePlaceDetailPage` / `maxImages`) and inline
|
||||
reviews fire concurrently across a search page instead of one place at a
|
||||
time — the biggest lever (a 20-place enriched search went from ~50s to ~10s,
|
||||
`_DETAIL_CONCURRENCY=8`).
|
||||
- **Search pages** are prefetched in waves sized from the result cap
|
||||
(`_prefetch_for`) so a 60-result / 3-page search overlaps its pages.
|
||||
- **Bulk `placeIds`** are fetched in parallel.
|
||||
- **NID mint** coalesces concurrent cold callers onto one in-flight request
|
||||
(the pool lock isn't held across the network call), so a burst of parallel
|
||||
detail fetches doesn't serialize behind N sequential ~2s mints.
|
||||
|
||||
Reviews within a single place stay sequential — pagination is
|
||||
continuation-token based (each token embeds the previous page's last-review
|
||||
key plus a signature, so page N+1's token can't be forged or precomputed).
|
||||
The only lever there is page size: the feed is asked for the ~60/page ceiling
|
||||
(`_REVIEWS_PAGE_SIZE`) instead of the old 10/20, cutting the sequential
|
||||
round-trips ~3x (1000 reviews: ~113s → ~40s).
|
||||
|
||||
## How review scraping works
|
||||
|
||||
1. Resolve the feature ID exactly as above (reviews accept the same
|
||||
`startUrls` / `placeIds` inputs).
|
||||
2. Fetch the place detail once for the header fields (title, address,
|
||||
location, …) that Apify stamps onto every review item.
|
||||
3. Page `GetLocalBoqProxy` (~60 reviews per page — the feed's ceiling, asked
|
||||
via `_REVIEWS_PAGE_SIZE`; opaque continuation token in `node[6]`) until
|
||||
`maxReviews`, the `reviewsStartDate` cutoff (newest-first), or the feed is
|
||||
exhausted. Sort modes map to codes 1–4
|
||||
(mostRelevant/newest/highestRanking/lowestRanking).
|
||||
4. `parse_review()` maps each ~48-slot review array → ReviewFields (author,
|
||||
stars, text, ISO publish date, images, owner response, guided
|
||||
context/per-aspect ratings, origin). `personalData=false` strips reviewer
|
||||
name/id/URL/photo (reviewId stays), per Apify semantics.
|
||||
|
||||
## API spec
|
||||
|
||||
Mirrors the Apify "Google Maps Scraper" and "Google Maps Reviews Scraper"
|
||||
actors (camelCase, `extra="allow"`). Unknown inputs are accepted, unsourced
|
||||
outputs come back as `None`/`[]`/`{}` — parity grows without breaking
|
||||
consumers. `schemas.py` is the source of truth; the tables below list what the
|
||||
implementation actually **sources** vs. still **stubs**.
|
||||
|
||||
### Places — input (`GoogleMapsScrapeInput`)
|
||||
|
||||
| Field | Type / default | Status | Notes |
|
||||
|---|---|---|---|
|
||||
| `searchStringsArray` | `list[str]` `[]` | ✅ | search-term discovery; additive with the others |
|
||||
| `startUrls` | `list[{url}]` `[]` | ✅ | place / CID / short / search / reviews URLs |
|
||||
| `placeIds` | `list[str]` `[]` | ✅ | bare `ChIJ…` ids (fetched in parallel) |
|
||||
| `allPlacesNoSearchAction` | enum `""` | ✅* | area scan via broad-category sweep (approximation) |
|
||||
| `locationQuery` | `str` | ✅ | e.g. `"San Jose, CA"`, appended as `"{q} in {loc}"` |
|
||||
| `city`/`state`/`county`/`postalCode`/`countryCode` | `str` | ✅ | discrete location parts (alt. to `locationQuery`) |
|
||||
| `customGeolocation` | GeoJSON `Point`+`radiusKm` | ✅ | real lat/lng/radius viewport |
|
||||
| `maxCrawledPlacesPerSearch` | `int\|None` | ✅ | `None` = all; caps emitted places |
|
||||
| `language` | `str` `"en"` | ✅ | `hl=` on every RPC |
|
||||
| `searchMatching` | `all\|only_includes\|only_exact` | ✅ | title-match filter |
|
||||
| `categoryFilterWords` | `list[str]` `[]` | ✅ | category client-filter |
|
||||
| `placeMinimumStars` | enum `""` | ✅ | min `totalScore` filter |
|
||||
| `website` | `allPlaces\|withWebsite\|withoutWebsite` | ✅ | website-presence filter |
|
||||
| `skipClosedPlaces` | `bool` `false` | ✅ | drops permanently/temporarily closed |
|
||||
| `scrapePlaceDetailPage` | `bool` `false` | ✅ | adds detail-RPC extras (see below) |
|
||||
| `maxReviews` | `int` `0` | ✅ | `0` = none; attaches `reviews[]` |
|
||||
| `reviewsSort`/`reviewsStartDate`/`reviewsFilterString`/`reviewsOrigin`/`scrapeReviewsPersonalData` | — | ✅ | as in the Reviews scraper |
|
||||
| `maxImages` | `int` `0` | ✅ | caps `imageUrls` (URLs only, no author/date) |
|
||||
| `scrapeContacts`, `scrapeSocialMediaProfiles`, `*LeadsEnrichment*`, `enableCompetitorAnalysis`, `maxCompetitorsToAnalyze`, `scrapeTableReservationProvider`, `scrapeOrderOnline`, `includeWebResults`, `scrapeDirectories`, `maxQuestions`, `scrapeImageAuthors` | — | ⛔ stub | accepted; out of scope / login-walled (see TODO) |
|
||||
|
||||
`✅*` = approximation, not pin-complete.
|
||||
|
||||
### Places — output (`PlaceItem`), sourced fields
|
||||
|
||||
- **Identity:** `title`, `description`, `price`, `categoryName`, `categories`,
|
||||
`placeId`, `fid`, `cid`, `kgmid`
|
||||
- **Location:** `address`, `neighborhood`, `street`, `city`, `postalCode`,
|
||||
`state`, `countryCode`, `location{lat,lng}`, `plusCode`
|
||||
- **Contact:** `website`, `phone`, `phoneUnformatted`, `menu`
|
||||
- **Ratings/status:** `totalScore`, `reviewsCount`, `reviewsDistribution`,
|
||||
`permanentlyClosed`, `temporarilyClosed`
|
||||
- **Images:** `imageUrl`, `imagesCount`, `imageCategories`, `imageUrls`
|
||||
- **Detail-page (with `scrapePlaceDetailPage`):** `openingHours`,
|
||||
`additionalInfo`, `reviewsTags`, `popularTimesHistogram`,
|
||||
`popularTimesLiveText`, `popularTimesLivePercent`
|
||||
- **Hotels (hotel places):** `hotelStars`, `hotelDescription`, `checkInDate`,
|
||||
`checkOutDate`, `similarHotelsNearby`, `hotelAds`
|
||||
- **Reviews (with `maxReviews>0`):** `reviews[]` (see review fields below)
|
||||
- **Provenance/meta:** `searchString`, `rank`, `searchPageUrl`, `url`,
|
||||
`scrapedAt`
|
||||
- **Stubbed** (`[]`/`{}`/`None`): `peopleAlsoSearch`, `questionsAndAnswers`,
|
||||
`images` (author/date objects), `webResults`, `bookingLinks`,
|
||||
`tableReservationLinks`, `gasPrices`, `ownerUpdates`, leads/contact/social
|
||||
enrichment.
|
||||
|
||||
### Reviews — input (`GoogleMapsReviewsInput`) & output (`ReviewItem`)
|
||||
|
||||
Input: `startUrls`, `placeIds`, `maxReviews` (default `10000000` = "all"),
|
||||
`reviewsSort`, `reviewsStartDate`, `language`, `reviewsOrigin`, `personalData`.
|
||||
|
||||
Output is **one flat item per review** — the review fields merged with the
|
||||
parent place header. Sourced review fields: `reviewId`, `name`/`text`/
|
||||
`textTranslated`, `stars`, `publishAt` (relative) + `publishedAtDate` (ISO),
|
||||
`likesCount`, `reviewerId`/`reviewerUrl`/`reviewerPhotoUrl`/
|
||||
`reviewerNumberOfReviews`/`isLocalGuide`, `reviewOrigin`, `reviewImageUrls`,
|
||||
`reviewContext`, `reviewDetailedRating`, `responseFromOwnerText`
|
||||
(+ `responseFromOwnerDate` as relative text only — see TODO).
|
||||
|
||||
Notable input semantics (matching Apify):
|
||||
|
||||
- Places: `searchStringsArray`, `startUrls`, and `placeIds` are **additive**
|
||||
(unlike YouTube where startUrls override queries).
|
||||
- `maxCrawledPlacesPerSearch=None` (unset) means "all places".
|
||||
- Reviews: `maxReviews` defaults to `10000000` ("all"); `reviewsStartDate`
|
||||
forces newest-first and stops at the cutoff.
|
||||
- `personalData` / `scrapeReviewsPersonalData` default `true`; when off,
|
||||
reviewer id/name/URL/photo must be stripped (reviewId always stays).
|
||||
|
||||
## Testing
|
||||
|
||||
Offline unit tests (no network; parser tests pin real captured fixtures):
|
||||
|
||||
```bash
|
||||
cd surfsense_backend
|
||||
.venv/Scripts/python.exe -m pytest tests/unit/scrapers/google_maps/
|
||||
```
|
||||
|
||||
Live end-to-end (real network + proxy; also regenerates fixtures):
|
||||
|
||||
```bash
|
||||
.venv/Scripts/python.exe scripts/e2e_google_maps_scraper.py
|
||||
```
|
||||
|
||||
Deep live verification (diverse places worldwide + review semantics: sort
|
||||
order, date cutoff, pagination uniqueness, personal-data stripping, French
|
||||
localization, CID URLs, name-only URLs, short links, search-filter variants
|
||||
`only_exact` / `categoryFilterWords`, and `reviewsOrigin=google`):
|
||||
|
||||
```bash
|
||||
.venv/Scripts/python.exe scripts/e2e_google_maps_deep.py
|
||||
```
|
||||
|
||||
## Implementation TODO (progressive, like YouTube)
|
||||
|
||||
- Owner-response dates are only exposed as relative text ("11 months ago") in
|
||||
the BOQ feed; `responseFromOwnerDate` carries that string, not an ISO date.
|
||||
(The detail RPC's inline reviews don't carry reply timestamps either, and
|
||||
`listugcposts` still returns no reviews even with the NID cookie.)
|
||||
- Full image galleries (`images` with author/date, beyond the ~dozen
|
||||
`imageUrls`): needs the signed-in photo-listing RPC.
|
||||
- Q&A (`questionsAndAnswers`): `darray[126]` is empty everywhere we probed —
|
||||
Google appears to have retired public Q&A. `peopleAlsoSearch`: not in the
|
||||
detail darray; likely a separate RPC.
|
||||
- A pin-complete `allPlacesNoSearchAction` (the category sweep covers most
|
||||
pins but not businesses outside the swept categories); would need browser
|
||||
rendering + tile OCR like Apify's.
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
"""Platform-native Google Maps scraper (Apify Google Maps Scraper-compatible)."""
|
||||
|
||||
from .reviews import iter_reviews, scrape_reviews
|
||||
from .schemas import (
|
||||
GoogleMapsReviewsInput,
|
||||
GoogleMapsScrapeInput,
|
||||
PlaceItem,
|
||||
ReviewItem,
|
||||
)
|
||||
from .scraper import iter_places, scrape_places
|
||||
|
||||
__all__ = [
|
||||
"GoogleMapsReviewsInput",
|
||||
"GoogleMapsScrapeInput",
|
||||
"PlaceItem",
|
||||
"ReviewItem",
|
||||
"iter_places",
|
||||
"iter_reviews",
|
||||
"scrape_places",
|
||||
"scrape_reviews",
|
||||
]
|
||||
595
surfsense_backend/app/proprietary/scrapers/google_maps/fetch.py
Normal file
595
surfsense_backend/app/proprietary/scrapers/google_maps/fetch.py
Normal file
|
|
@ -0,0 +1,595 @@
|
|||
"""Proxy-aware fetch seam for the Google Maps scraper.
|
||||
|
||||
All network I/O flows through here and always egresses through the residential
|
||||
proxy (Google blocks datacenter IPs outright; a direct hit also risk-blocks the
|
||||
server IP). Mirrors the design of ``../youtube/innertube.py`` but trimmed to
|
||||
what Maps needs today: a GET that returns HTML and a GET that returns the
|
||||
XSSI-stripped JSON that Maps' internal ``/maps/preview/*`` RPC endpoints emit.
|
||||
|
||||
Google Maps place/search/review data is **public** — no Google account is
|
||||
required. :func:`looks_like_signin_wall` flags the rare responses where Google
|
||||
serves a consent/sign-in interstitial instead of data, so callers can surface a
|
||||
clear "sign-in required" error instead of silently returning nothing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from scrapling.fetchers import AsyncFetcher
|
||||
|
||||
from app.utils.proxy import get_proxy_url
|
||||
|
||||
from .parsers import brace_match_json
|
||||
from .url_resolver import ResolvedUrl, extract_fid
|
||||
|
||||
try: # browser tier is optional (needs patchright browsers installed)
|
||||
from scrapling.fetchers import StealthyFetcher
|
||||
except Exception: # pragma: no cover - import guard
|
||||
StealthyFetcher = None # type: ignore[assignment]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SignInRequiredError(RuntimeError):
|
||||
"""Raised when Google serves a sign-in/consent wall instead of public data.
|
||||
|
||||
Public Maps data does not need a Google account, so this is rare; when it
|
||||
happens the route surfaces it as a clear "Google sign in required" error
|
||||
rather than returning an empty result.
|
||||
"""
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
"""UTC timestamp in the millisecond ISO shape Apify stamps on items."""
|
||||
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
||||
|
||||
|
||||
async def gather_bounded(coro_factories, *, concurrency: int) -> list:
|
||||
"""Run zero-arg async callables with bounded concurrency, results in order.
|
||||
|
||||
Takes callables (``() -> awaitable``), not live coroutines, so nothing
|
||||
starts until a semaphore slot frees up — lets callers queue far more work
|
||||
than ``concurrency`` without opening every socket at once. Every RPC here
|
||||
is ~2s of proxy round-trip, so overlapping independent ones is the single
|
||||
biggest lever on wall-clock time.
|
||||
"""
|
||||
if not coro_factories:
|
||||
return []
|
||||
sem = asyncio.Semaphore(concurrency)
|
||||
|
||||
async def _run(factory):
|
||||
async with sem:
|
||||
return await factory()
|
||||
|
||||
return await asyncio.gather(*(_run(f) for f in coro_factories))
|
||||
|
||||
|
||||
# XSSI guard Google prepends to its RPC JSON responses.
|
||||
_XSSI_PREFIX = ")]}'"
|
||||
|
||||
# Consent cookie to dodge the EU consent interstitial that otherwise returns a
|
||||
# page with no APP_INITIALIZATION_STATE. Mirrors the YouTube fetcher's SOCS.
|
||||
CONSENT_COOKIES = {
|
||||
"SOCS": "CAISNQgDEitib3FfaWRlbnRpdHlmcm9udGVuZHVpc2VydmVyXzIwMjMwODI5LjA3X3AxGgJlbiADGgYIgOa_pgY",
|
||||
}
|
||||
|
||||
_HEADERS = {"Accept-Language": "en-US,en;q=0.9"}
|
||||
|
||||
|
||||
def looks_like_signin_wall(html_or_text: str | None) -> bool:
|
||||
"""Heuristic: did Google serve a sign-in / consent wall instead of data?
|
||||
|
||||
Public Maps data never needs login, so this should be rare — it mainly
|
||||
fires on consent redirects (``consent.google.com``) or an account-picker
|
||||
page. Callers turn a True here into a clear "sign-in required" error.
|
||||
"""
|
||||
if not html_or_text:
|
||||
return False
|
||||
text = html_or_text[:5000].lower()
|
||||
markers = (
|
||||
"consent.google.com",
|
||||
"accounts.google.com/servicelogin",
|
||||
"sign in to continue",
|
||||
"before you continue to google",
|
||||
)
|
||||
return any(m in text for m in markers)
|
||||
|
||||
|
||||
def strip_xssi(text: str) -> str:
|
||||
"""Return the JSON body after Google's ``)]}'`` anti-XSSI guard.
|
||||
|
||||
The guard is normally at position 0, but a proxy/HTML fetcher may wrap the
|
||||
text/plain body in ``<html><body><p>…`` — so we locate the guard rather than
|
||||
require it at the start, then return everything after its line.
|
||||
"""
|
||||
idx = text.find(_XSSI_PREFIX)
|
||||
if idx == -1:
|
||||
return text
|
||||
nl = text.find("\n", idx)
|
||||
return text[nl + 1 :] if nl != -1 else text[idx + len(_XSSI_PREFIX) :]
|
||||
|
||||
|
||||
async def resolve_fid(resolved: ResolvedUrl) -> str | None:
|
||||
"""Get a place's feature ID: from the URL if present, else from its HTML.
|
||||
|
||||
A ``/maps/place/...`` URL usually embeds the fid in its ``data=`` blob. A
|
||||
``?q=place_id:...`` URL or a shortlink does not, so we fetch the page once
|
||||
and pull the fid out of the returned HTML. Raises :class:`SignInRequiredError`
|
||||
if Google serves a consent/sign-in wall instead of the page.
|
||||
"""
|
||||
if resolved.fid:
|
||||
return resolved.fid
|
||||
# Short links (maps.app.goo.gl / goo.gl) are Firebase Dynamic Links: a plain
|
||||
# GET only returns a JS interstitial, so render it — the JS follows the
|
||||
# redirect to the real /maps/place/… URL, from which we read the fid.
|
||||
# ponytail: browser render is slow (~30s) but it's the only way these
|
||||
# links resolve; they're rare and cached downstream by fid.
|
||||
if resolved.kind == "shortlink":
|
||||
rendered = await _fetch_html_stealthy(resolved.url, dict(CONSENT_COOKIES))
|
||||
return extract_fid(rendered) if rendered else None
|
||||
html = await fetch_html(resolved.url)
|
||||
if html is not None:
|
||||
if looks_like_signin_wall(html):
|
||||
raise SignInRequiredError(
|
||||
f"Google returned a sign-in/consent wall for {resolved.url}"
|
||||
)
|
||||
fid = extract_fid(html)
|
||||
if fid:
|
||||
return fid
|
||||
# Name-only place URLs (e.g. /maps/place/Eiffel+Tower/) serve a JS-only
|
||||
# shell whose static HTML never contains the fid (even a headless render
|
||||
# won't navigate to the place). The lite ``tbm=map`` search endpoint DOES
|
||||
# embed fids in plain HTML, so look the name up there instead.
|
||||
if resolved.kind == "place" and resolved.value and resolved.value != resolved.url:
|
||||
return await fid_from_search(resolved.value)
|
||||
return None
|
||||
|
||||
|
||||
async def fid_from_search(query: str, *, language: str = "en") -> str | None:
|
||||
"""Resolve a free-text place name to its feature ID via ``tbm=map`` search.
|
||||
|
||||
This is the lite map-search page (the same endpoint the search-discovery
|
||||
flow will paginate later); its static HTML embeds every result's fid.
|
||||
``ponytail:`` takes the first fid in the page — that's the top-ranked
|
||||
result. Good enough for name lookups; the full search flow will parse the
|
||||
structured payload instead.
|
||||
"""
|
||||
url = f"https://www.google.com/search?tbm=map&hl={language}&gl=us&q={quote(query)}"
|
||||
html = await fetch_html(url)
|
||||
if html is None:
|
||||
return None
|
||||
if looks_like_signin_wall(html):
|
||||
raise SignInRequiredError(
|
||||
f"Google returned a sign-in/consent wall for map search {query!r}"
|
||||
)
|
||||
return extract_fid(html)
|
||||
|
||||
|
||||
async def fetch_html(url: str, *, cookies: dict[str, str] | None = None) -> str | None:
|
||||
"""GET a Google Maps page and return raw HTML (proxy, stealthy fallback)."""
|
||||
merged = {**CONSENT_COOKIES, **(cookies or {})}
|
||||
try:
|
||||
started = time.perf_counter()
|
||||
page = await AsyncFetcher.get(
|
||||
url,
|
||||
headers=_HEADERS,
|
||||
cookies=merged,
|
||||
proxy=get_proxy_url(),
|
||||
stealthy_headers=True,
|
||||
)
|
||||
fetch_ms = (time.perf_counter() - started) * 1000
|
||||
logger.info(
|
||||
"[google_maps][perf] source=html url=%s status=%s fetch_ms=%.1f",
|
||||
url,
|
||||
page.status,
|
||||
fetch_ms,
|
||||
)
|
||||
if page.status == 200:
|
||||
return page.html_content
|
||||
logger.warning("Maps HTML GET %s returned %s", url, page.status)
|
||||
except Exception as e:
|
||||
logger.warning("Maps HTML GET %s failed: %s", url, e)
|
||||
return await _fetch_html_stealthy(url, merged)
|
||||
|
||||
|
||||
async def _fetch_html_stealthy(url: str, cookies: dict[str, str]) -> str | None:
|
||||
"""Last-resort browser fetch for anti-bot walls (mirrors the YouTube tier)."""
|
||||
if StealthyFetcher is None:
|
||||
return None
|
||||
try:
|
||||
started = time.perf_counter()
|
||||
page = await asyncio.to_thread(
|
||||
StealthyFetcher.fetch,
|
||||
url,
|
||||
headless=True,
|
||||
network_idle=True,
|
||||
solve_cloudflare=True,
|
||||
proxy=get_proxy_url(),
|
||||
)
|
||||
fetch_ms = (time.perf_counter() - started) * 1000
|
||||
logger.info(
|
||||
"[google_maps][perf] source=html tier=stealthy url=%s status=%s fetch_ms=%.1f",
|
||||
url,
|
||||
page.status,
|
||||
fetch_ms,
|
||||
)
|
||||
if page.status == 200:
|
||||
return page.html_content
|
||||
logger.warning("Maps HTML GET %s tier=stealthy returned %s", url, page.status)
|
||||
except Exception as e:
|
||||
logger.warning("Maps HTML GET %s tier=stealthy failed: %s", url, e)
|
||||
return None
|
||||
|
||||
|
||||
# Place-detail RPC field selector (protobuf-over-URL). ``{fid}`` is the place's
|
||||
# feature ID (hex:hex). This is the selector the Maps web app itself sends when
|
||||
# rendering a place page (captured live via a headless browser, then
|
||||
# genericized: the free-text ``!2s`` name and per-place ``!15m2…!4s/g/…`` kgmid
|
||||
# hint dropped with counts adjusted, coords zeroed, and the ``!14m3!1s<EI>``
|
||||
# session token replaced with a filler — verified the token value is not
|
||||
# checked). Combined with an NID session cookie it returns the FULL payload:
|
||||
# reviewsCount, reviews distribution, popular times, image galleries, review
|
||||
# tags, and every about section. The returned ``jd[6]`` is the ``darray``
|
||||
# parsers read.
|
||||
_PLACE_DETAIL_PB = (
|
||||
"!1m13!1s{fid}"
|
||||
"!3m8!1m3!1d5000!2d0!3d0!3m2!1i1024!2i768!4f13.1"
|
||||
"!4m2!3d0!4d0"
|
||||
"!12m4!2m3!1i360!2i120!4i8"
|
||||
"!13m57!2m2!1i203!2i100!3m2!2i4!5b1"
|
||||
"!6m6!1m2!1i86!2i86!1m2!1i408!2i240"
|
||||
"!7m33!1m3!1e1!2b0!3e3!1m3!1e2!2b1!3e2!1m3!1e2!2b0!3e3"
|
||||
"!1m3!1e8!2b0!3e3!1m3!1e10!2b0!3e3!1m3!1e10!2b1!3e2"
|
||||
"!1m3!1e10!2b0!3e4!1m3!1e9!2b1!3e2!2b1!9b0"
|
||||
"!15m8!1m7!1m2!1m1!1e2!2m2!1i195!2i195!3i20"
|
||||
"!14m3!1s0ahUKEwixxxxxxxxxxxxxxxxxxxxxxxxx!7e81!15i10112"
|
||||
"!15m108!1m26!13m9!2b1!3b1!4b1!6i1!8b1!9b1!14b1!20b1!25b1"
|
||||
"!18m15!3b1!4b1!5b1!6b1!13b1!14b1!17b1!21b1!22b1!30b1!32b1!33m1!1b1!34b1!36e2"
|
||||
"!10m1!8e3!11m1!3e1!17b1!20m2!1e3!1e6!24b1!25b1!26b1!27b1!29b1"
|
||||
"!30m1!2b1!36b1!37b1!39m3!2m2!2i1!3i1!43b1!52b1!54m1!1b1!55b1!56m1!1b1"
|
||||
"!61m2!1m1!1e1!65m5!3m4!1m3!1m2!1i224!2i298"
|
||||
"!72m22!1m8!2b1!5b1!7b1!12m4!1b1!2b1!4m1!1e1!4b1"
|
||||
"!8m10!1m6!4m1!1e1!4m1!1e3!4m1!1e4"
|
||||
"!3sother_user_google_review_posts__and__hotel_and_vr_partner_review_posts"
|
||||
"!6m1!1e1!9b1!89b1!90m2!1m1!1e2!98m3!1b1!2b1!3b1!103b1!113b1"
|
||||
"!114m3!1b1!2m1!1b1!117b1!122m1!1b1!126b1!127b1!128m1!1b0"
|
||||
"!21m0!22m1!1e81!30m8!3b1!6m2!1b1!2b1!7m2!1e3!2b1!9b1"
|
||||
"!34m5!7b1!10b1!14b1!15m1!1b0!37i785"
|
||||
)
|
||||
|
||||
# NID session cookie pool. Google trims the rich detail fields (counts,
|
||||
# distribution, popular times, galleries, tags, most about sections) from
|
||||
# responses that carry no session cookie; a plain GET to /maps mints an NID
|
||||
# that unlocks them — no login, no browser. A small pool is rotated
|
||||
# round-robin (mirrors the YouTube fetcher's ``_RotatingSession``) so no
|
||||
# single session accumulates the whole request volume; each expires after a
|
||||
# TTL and is re-minted lazily.
|
||||
_NID_TTL_S = 30 * 60
|
||||
_NID_POOL_SIZE = 3
|
||||
_nid_pool: list[dict[str, Any]] = [
|
||||
{"value": None, "at": 0.0} for _ in range(_NID_POOL_SIZE)
|
||||
]
|
||||
_nid_rr = 0
|
||||
_nid_lock = asyncio.Lock()
|
||||
_nid_inflight: asyncio.Future | None = None
|
||||
|
||||
|
||||
async def _mint_nid() -> str | None:
|
||||
"""Mint a fresh NID session cookie with a plain GET to /maps."""
|
||||
try:
|
||||
page = await AsyncFetcher.get(
|
||||
"https://www.google.com/maps?hl=en",
|
||||
headers=_HEADERS,
|
||||
cookies=CONSENT_COOKIES,
|
||||
proxy=get_proxy_url(),
|
||||
stealthy_headers=True,
|
||||
)
|
||||
return (page.cookies or {}).get("NID")
|
||||
except Exception as e:
|
||||
logger.warning("NID mint failed: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
async def get_session_cookies() -> dict[str, str]:
|
||||
"""Consent cookies plus an NID session cookie from the rotating pool.
|
||||
|
||||
Concurrent cold callers coalesce onto a single in-flight mint (the lock is
|
||||
not held across the network call), so a burst of parallel detail fetches
|
||||
doesn't serialize behind N sequential ~2s mints on a cold pool.
|
||||
"""
|
||||
global _nid_rr, _nid_inflight
|
||||
async with _nid_lock:
|
||||
slot = _nid_pool[_nid_rr % _NID_POOL_SIZE]
|
||||
_nid_rr += 1
|
||||
if slot["value"] and time.time() - slot["at"] < _NID_TTL_S:
|
||||
return {**CONSENT_COOKIES, "NID": slot["value"]}
|
||||
if _nid_inflight is None or _nid_inflight.done():
|
||||
_nid_inflight = asyncio.ensure_future(_mint_nid())
|
||||
fut = _nid_inflight
|
||||
nid = await fut
|
||||
if nid:
|
||||
async with _nid_lock:
|
||||
slot["value"] = nid
|
||||
slot["at"] = time.time()
|
||||
return {**CONSENT_COOKIES, "NID": nid}
|
||||
return dict(CONSENT_COOKIES)
|
||||
|
||||
|
||||
async def fetch_place_darray(fid: str, *, language: str = "en") -> list | None:
|
||||
"""Fetch a place's detail ``darray`` via the ``/maps/preview/place`` RPC.
|
||||
|
||||
``fid`` is the feature ID (``0x..:0x..``). Returns ``jd[6]`` (the array all
|
||||
place-field paths index into), or ``None`` on failure. Sent with an NID
|
||||
session cookie so Google returns the full payload (see ``_PLACE_DETAIL_PB``).
|
||||
"""
|
||||
pb = _PLACE_DETAIL_PB.format(fid=fid)
|
||||
url = (
|
||||
"https://www.google.com/maps/preview/place"
|
||||
f"?authuser=0&hl={language}&gl=us&pb={quote(pb, safe='!')}"
|
||||
)
|
||||
jd = await fetch_rpc_json(url, cookies=await get_session_cookies())
|
||||
darray = jd[6] if isinstance(jd, list) and len(jd) > 6 else None
|
||||
return darray if isinstance(darray, list) else None
|
||||
|
||||
|
||||
# Map-search RPC (``search?tbm=map``) field selector. Viewport block first
|
||||
# (``!1d`` diameter-meters, ``!2d`` lng, ``!3d`` lat), then paging (``!7i``
|
||||
# per-page, ``!8i`` offset), then the same detail selector blocks the place RPC
|
||||
# uses so each result embeds a full place darray at ``entry[14]``.
|
||||
_SEARCH_PB = (
|
||||
"!4m12!1m3!1d{diameter}!2d{lng}!3d{lat}"
|
||||
"!2m3!1f0!2f0!3f0!3m2!1i1024!2i768!4f13.1"
|
||||
"!7i{per_page}!8i{offset}!10b1"
|
||||
"!12m6!1m1!18b1!2m1!20e3!6m1!114b1"
|
||||
"!17m1!3e1"
|
||||
"!20m57!2m2!1i203!2i100!3m2!2i4!5b1"
|
||||
"!6m6!1m2!1i86!2i86!1m2!1i408!2i240"
|
||||
"!7m33!1m3!1e1!2b0!3e3!1m3!1e2!2b1!3e2!1m3!1e2!2b0!3e3"
|
||||
"!1m3!1e8!2b0!3e3!1m3!1e10!2b0!3e3!1m3!1e10!2b1!3e2"
|
||||
"!1m3!1e10!2b0!3e4!1m3!1e9!2b1!3e2!2b1!9b0"
|
||||
"!15m8!1m7!1m2!1m1!1e2!2m2!1i195!2i195!3i20"
|
||||
)
|
||||
|
||||
# Whole-earth viewport: Google localizes from the query text itself (verified
|
||||
# live — "pizza new york" returns NYC results with this viewport), so callers
|
||||
# only need coordinates when the input provides an explicit geolocation.
|
||||
_EARTH = {"diameter": 25_000_000.0, "lat": 0.0, "lng": 0.0}
|
||||
|
||||
|
||||
def build_search_url(
|
||||
query: str,
|
||||
*,
|
||||
offset: int = 0,
|
||||
per_page: int = 20,
|
||||
language: str = "en",
|
||||
lat: float | None = None,
|
||||
lng: float | None = None,
|
||||
radius_m: float | None = None,
|
||||
) -> str:
|
||||
"""Build a ``search?tbm=map`` URL returning protobuf-over-JSON results."""
|
||||
viewport = dict(_EARTH)
|
||||
if lat is not None and lng is not None:
|
||||
viewport = {
|
||||
"diameter": (radius_m or 10_000) * 2,
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
}
|
||||
pb = _SEARCH_PB.format(per_page=per_page, offset=offset, **viewport)
|
||||
return (
|
||||
f"https://www.google.com/search?tbm=map&authuser=0&hl={language}&gl=us"
|
||||
f"&q={quote(query)}&pb={quote(pb, safe='!')}"
|
||||
)
|
||||
|
||||
|
||||
def _search_darrays(jd: Any) -> list[list]:
|
||||
"""Extract the place darrays from a map-search response.
|
||||
|
||||
Results live at ``jd[0][1]``; each result entry holds its place darray at
|
||||
``entry[14]`` (same shape as the place detail RPC's ``jd[6]``). Multi-result
|
||||
pages put metadata (no ``[14]``) in slot 0; single-match responses put the
|
||||
one place directly in slot 0 — scanning every entry handles both.
|
||||
"""
|
||||
entries = (
|
||||
jd[0][1]
|
||||
if isinstance(jd, list) and jd and isinstance(jd[0], list) and len(jd[0]) > 1
|
||||
else None
|
||||
)
|
||||
if not isinstance(entries, list):
|
||||
return []
|
||||
out = []
|
||||
for entry in entries:
|
||||
darray = entry[14] if isinstance(entry, list) and len(entry) > 14 else None
|
||||
if isinstance(darray, list):
|
||||
out.append(darray)
|
||||
return out
|
||||
|
||||
|
||||
async def iter_search_pages(
|
||||
query: str,
|
||||
*,
|
||||
language: str = "en",
|
||||
lat: float | None = None,
|
||||
lng: float | None = None,
|
||||
radius_m: float | None = None,
|
||||
max_pages: int = 25,
|
||||
prefetch: int = 1,
|
||||
):
|
||||
"""Yield lists of place darrays, one map-search page (~20 places) a time.
|
||||
|
||||
Paging is offset-based (``!8i``); Google reshuffles results between pages
|
||||
so callers must dedupe by fid. Stops on an empty page or ``max_pages``.
|
||||
|
||||
``prefetch`` pages are fetched concurrently per wave (still yielded in
|
||||
offset order): each page is a ~2s round-trip, so a caller that knows it
|
||||
needs several pages (large ``maxCrawledPlacesPerSearch``) gets them
|
||||
overlapped. ``prefetch=1`` keeps the old one-at-a-time behavior for callers
|
||||
that only need a page or two (no wasted fetches).
|
||||
"""
|
||||
per_page = 20
|
||||
page = 0
|
||||
while page < max_pages:
|
||||
wave = min(max(prefetch, 1), max_pages - page)
|
||||
urls = [
|
||||
build_search_url(
|
||||
query,
|
||||
offset=(page + i) * per_page,
|
||||
per_page=per_page,
|
||||
language=language,
|
||||
lat=lat,
|
||||
lng=lng,
|
||||
radius_m=radius_m,
|
||||
)
|
||||
for i in range(wave)
|
||||
]
|
||||
pages = await gather_bounded(
|
||||
[lambda u=u: fetch_rpc_json(u) for u in urls], concurrency=wave
|
||||
)
|
||||
for jd in pages:
|
||||
darrays = _search_darrays(jd)
|
||||
if not darrays:
|
||||
return
|
||||
yield darrays
|
||||
page += wave
|
||||
|
||||
|
||||
# Sort code the reviews RPC expects (slot 1 of the request payload).
|
||||
REVIEWS_SORT_CODES = {
|
||||
"mostRelevant": 1,
|
||||
"newest": 2,
|
||||
"highestRanking": 3,
|
||||
"lowestRanking": 4,
|
||||
}
|
||||
|
||||
# Reviews are cursor-paged (each page's continuation token embeds the previous
|
||||
# page's last-review key + a signature), so pages CANNOT be fetched in
|
||||
# parallel — the only lever on review throughput is page size. Google honors
|
||||
# this on both the first and continuation requests but caps the actual return
|
||||
# at ~60/page (asking higher just yields the max), so 100 grabs the ceiling and
|
||||
# cuts the sequential round-trips ~3x vs the old 10/20-per-page default.
|
||||
_REVIEWS_PAGE_SIZE = 100
|
||||
|
||||
|
||||
def build_reviews_url(
|
||||
fid: str, *, sort_code: int, page_token: str = "", language: str = "en"
|
||||
) -> str:
|
||||
"""Build a ``GetLocalBoqProxy`` reviews URL (public, no session token).
|
||||
|
||||
This is the review feed the Google search local panel uses. The older
|
||||
``listugcposts`` / ``listentitiesreviews`` RPCs now return empty pages for
|
||||
anonymous callers, so this proxy is the one that still works (approach from
|
||||
the maintained ``google-maps-review-scraper`` npm package).
|
||||
|
||||
``fid`` is the feature ID (``0x..:0x..``); ``page_token`` is the opaque
|
||||
continuation token from the previous page (empty for the first page).
|
||||
"""
|
||||
inner: list = [None] * 12
|
||||
inner[1] = sort_code
|
||||
inner[9] = _REVIEWS_PAGE_SIZE
|
||||
inner[11] = [fid]
|
||||
if page_token:
|
||||
inner.extend([None] * 8)
|
||||
inner[19] = page_token
|
||||
payload = [None, [None] * 9 + [inner]]
|
||||
return (
|
||||
"https://www.google.com/httpservice/web/PrivateLocalSearchUiDataService/"
|
||||
f"GetLocalBoqProxy?msc=gwsrpc&hl={language}"
|
||||
f"&reqpld={quote(json.dumps(payload))}"
|
||||
)
|
||||
|
||||
|
||||
def _reviews_node(jd: Any) -> list | None:
|
||||
"""The reviews node of a BOQ response: ``jd[1][10]`` = [.., reviews, .., token]."""
|
||||
node = (
|
||||
jd[1][10]
|
||||
if isinstance(jd, list)
|
||||
and len(jd) > 1
|
||||
and isinstance(jd[1], list)
|
||||
and len(jd[1]) > 10
|
||||
else None
|
||||
)
|
||||
return node if isinstance(node, list) else None
|
||||
|
||||
|
||||
async def iter_reviews_pages(
|
||||
fid: str,
|
||||
*,
|
||||
sort: str = "newest",
|
||||
language: str = "en",
|
||||
max_pages: int = 10_000,
|
||||
):
|
||||
"""Yield raw review arrays, one RPC page (~10 reviews) at a time.
|
||||
|
||||
Each yielded item is the page's review list (``node[2]``). Pagination
|
||||
follows ``node[6]`` (the continuation token); stops when it is missing or
|
||||
unchanged, or ``max_pages`` is reached. ``ponytail:`` sequential paging —
|
||||
the endpoint only hands out one continuation token at a time anyway.
|
||||
"""
|
||||
sort_code = REVIEWS_SORT_CODES.get(sort, 2)
|
||||
page_token = ""
|
||||
for _ in range(max_pages):
|
||||
url = build_reviews_url(
|
||||
fid, sort_code=sort_code, page_token=page_token, language=language
|
||||
)
|
||||
node = _reviews_node(await fetch_rpc_json(url))
|
||||
if not node or len(node) < 3 or not isinstance(node[2], list):
|
||||
return
|
||||
yield node[2]
|
||||
next_token = node[6] if len(node) > 6 else None
|
||||
if (
|
||||
not isinstance(next_token, str)
|
||||
or not next_token
|
||||
or next_token == page_token
|
||||
):
|
||||
return
|
||||
page_token = next_token
|
||||
|
||||
|
||||
async def fetch_rpc_json(
|
||||
url: str, *, cookies: dict[str, str] | None = None
|
||||
) -> Any | None:
|
||||
"""GET a Maps ``/maps/preview/*`` RPC URL and return parsed JSON.
|
||||
|
||||
These endpoints return an XSSI-guarded JSON array (not HTML). Returns the
|
||||
decoded structure, or ``None`` on failure / non-JSON (e.g. a sign-in wall).
|
||||
"""
|
||||
try:
|
||||
started = time.perf_counter()
|
||||
page = await AsyncFetcher.get(
|
||||
url,
|
||||
headers=_HEADERS,
|
||||
cookies=cookies or CONSENT_COOKIES,
|
||||
proxy=get_proxy_url(),
|
||||
stealthy_headers=True,
|
||||
)
|
||||
fetch_ms = (time.perf_counter() - started) * 1000
|
||||
logger.info(
|
||||
"[google_maps][perf] source=rpc url=%s status=%s fetch_ms=%.1f",
|
||||
url,
|
||||
page.status,
|
||||
fetch_ms,
|
||||
)
|
||||
if page.status != 200:
|
||||
logger.warning("Maps RPC GET %s returned %s", url, page.status)
|
||||
return None
|
||||
text = page.html_content
|
||||
if looks_like_signin_wall(text):
|
||||
logger.warning("Maps RPC GET %s hit a sign-in/consent wall", url)
|
||||
return None
|
||||
body = strip_xssi(text)
|
||||
# The proxy/HTML fetcher may wrap the JSON in <html><body><p>…</p>…,
|
||||
# so extract just the balanced top-level array/object.
|
||||
start = next((i for i, c in enumerate(body) if c in "[{"), -1)
|
||||
if start == -1:
|
||||
return None
|
||||
blob = brace_match_json(body, start)
|
||||
return json.loads(blob) if blob else None
|
||||
except Exception as e:
|
||||
logger.warning("Maps RPC GET %s failed: %s", url, e)
|
||||
return None
|
||||
|
|
@ -0,0 +1,526 @@
|
|||
"""Pure, I/O-free parsing of Google Maps' protobuf-over-JSON place data.
|
||||
|
||||
Google Maps returns a place's data as a deeply-nested JSON array (no field
|
||||
names, just positions) from the ``/maps/preview/place`` RPC (element ``jd[6]``,
|
||||
the ``darray``). Fields are read by fixed array paths.
|
||||
|
||||
The array paths are ported from the actively-maintained ``gosom/google-maps-
|
||||
scraper`` (Go), which tracks Google's periodic structure shifts and carries
|
||||
fallback paths (e.g. the Nov-2025 opening-hours move). Everything here is
|
||||
deterministic and offline-unit-testable against captured fixtures.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def dig(obj: Any, *path: int) -> Any:
|
||||
"""Safely walk a nested list by integer indices; ``None`` if any step misses.
|
||||
|
||||
The Python twin of gosom's ``getNthElementAndCast`` — Maps arrays are ragged
|
||||
and shift between updates, so every access must tolerate a short/absent path.
|
||||
"""
|
||||
cur = obj
|
||||
for idx in path:
|
||||
if not isinstance(cur, list) or idx >= len(cur) or cur[idx] is None:
|
||||
return None
|
||||
cur = cur[idx]
|
||||
return cur
|
||||
|
||||
|
||||
def _dig_str(obj: Any, *path: int) -> str | None:
|
||||
val = dig(obj, *path)
|
||||
return val if isinstance(val, str) else None
|
||||
|
||||
|
||||
def _dig_num(obj: Any, *path: int) -> float | None:
|
||||
val = dig(obj, *path)
|
||||
return val if isinstance(val, int | float) else None
|
||||
|
||||
|
||||
def brace_match_json(text: str, start: int) -> str | None:
|
||||
"""Return the balanced ``[...]`` / ``{...}`` blob beginning at ``text[start]``."""
|
||||
open_ch = text[start]
|
||||
close_ch = {"[": "]", "{": "}"}.get(open_ch)
|
||||
if close_ch is None:
|
||||
return None
|
||||
depth = 0
|
||||
i = start
|
||||
n = len(text)
|
||||
while i < n:
|
||||
ch = text[i]
|
||||
if ch == open_ch:
|
||||
depth += 1
|
||||
elif ch == close_ch:
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return text[start : i + 1]
|
||||
elif ch == '"':
|
||||
i += 1
|
||||
while i < n and text[i] != '"':
|
||||
if text[i] == "\\":
|
||||
i += 1
|
||||
i += 1
|
||||
i += 1
|
||||
return None
|
||||
|
||||
|
||||
def _opening_hours(darray: list) -> list[dict[str, str]]:
|
||||
"""Weekly hours. New layout (Nov 2025) at [203][0], old at [34][1]."""
|
||||
items = dig(darray, 203, 0) or dig(darray, 34, 1) or []
|
||||
hours: list[dict[str, str]] = []
|
||||
for item in items if isinstance(items, list) else []:
|
||||
day = _dig_str(item, 0)
|
||||
if not day:
|
||||
continue
|
||||
slots = dig(item, 3) # new format: [ [label, [[h,m],[h,m]]], ... ]
|
||||
times: list[str] = []
|
||||
if isinstance(slots, list) and slots:
|
||||
times = [_dig_str(s, 0) for s in slots if _dig_str(s, 0)]
|
||||
else: # old format: [1] is a flat list of strings
|
||||
old = dig(item, 1)
|
||||
times = (
|
||||
[t for t in old if isinstance(t, str)] if isinstance(old, list) else []
|
||||
)
|
||||
hours.append({"day": day, "hours": ", ".join(times) if times else "Closed"})
|
||||
return hours
|
||||
|
||||
|
||||
def _categories(darray: list) -> list[str]:
|
||||
cats = dig(darray, 13)
|
||||
return [c for c in cats if isinstance(c, str)] if isinstance(cats, list) else []
|
||||
|
||||
|
||||
def _website(darray: list) -> str | None:
|
||||
raw = _dig_str(darray, 7, 0)
|
||||
if raw and raw.startswith("/url?q="): # Google redirect wrapper
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
q = parse_qs(urlparse(raw).query).get("q", [None])[0]
|
||||
return q or raw
|
||||
return raw
|
||||
|
||||
|
||||
def _additional_info(darray: list) -> dict[str, Any] | None:
|
||||
"""About sections (``darray[100][1]``) in Apify's ``additionalInfo`` shape:
|
||||
``{"Accessibility": [{"Wheelchair accessible entrance": true}, ...], ...}``.
|
||||
|
||||
Option paths mirror gosom's About parser; enabled state is
|
||||
``option[2][1][0][0] == 1``.
|
||||
"""
|
||||
sections = dig(darray, 100, 1)
|
||||
if not isinstance(sections, list):
|
||||
return None
|
||||
info: dict[str, Any] = {}
|
||||
for section in sections:
|
||||
name = _dig_str(section, 1)
|
||||
options = dig(section, 2)
|
||||
if not name or not isinstance(options, list):
|
||||
continue
|
||||
entries = []
|
||||
for opt in options:
|
||||
opt_name = _dig_str(opt, 1)
|
||||
if opt_name:
|
||||
entries.append({opt_name: dig(opt, 2, 1, 0, 0) == 1})
|
||||
if entries:
|
||||
info[name] = entries
|
||||
return info or None
|
||||
|
||||
|
||||
def _link_sources(items: Any, link_path: tuple, source_path: tuple) -> list[dict]:
|
||||
"""Extract ``[{"url":…, "source":…}]`` pairs from a link-source array."""
|
||||
out = []
|
||||
for item in items if isinstance(items, list) else []:
|
||||
url = _dig_str(item, *link_path)
|
||||
source = _dig_str(item, *source_path)
|
||||
if url and source:
|
||||
out.append({"url": url, "source": source})
|
||||
return out
|
||||
|
||||
|
||||
def _order_links(darray: list) -> list[dict]:
|
||||
"""Order-online providers (paths from gosom, two known layouts)."""
|
||||
items = dig(darray, 75, 0, 1, 2) or dig(darray, 75, 0, 0, 2)
|
||||
return _link_sources(items, (1, 2, 0), (0, 0))
|
||||
|
||||
|
||||
# Day numbers Google uses in the popular-times block (gosom's mapping),
|
||||
# rendered as Apify's two-letter histogram keys.
|
||||
_DAY_KEYS = {1: "Mo", 2: "Tu", 3: "We", 4: "Th", 5: "Fr", 6: "Sa", 7: "Su"}
|
||||
|
||||
|
||||
def _popular_times(darray: list) -> dict[str, Any]:
|
||||
"""Popular-times block (``darray[84]``) in Apify's field shapes.
|
||||
|
||||
``[84][0]`` is a list of ``[day_num, [[hour, occupancy%, …], …]]`` days;
|
||||
``[84][6]`` is the live busyness text and ``[84][7][1]`` the live percent.
|
||||
"""
|
||||
out: dict[str, Any] = {}
|
||||
days = dig(darray, 84, 0)
|
||||
if isinstance(days, list):
|
||||
histogram: dict[str, list] = {}
|
||||
for day in days:
|
||||
key = _DAY_KEYS.get(dig(day, 0))
|
||||
hours = dig(day, 1)
|
||||
if not key or not isinstance(hours, list):
|
||||
continue
|
||||
histogram[key] = [
|
||||
{"hour": h[0], "occupancyPercent": h[1]}
|
||||
for h in hours
|
||||
if isinstance(h, list) and len(h) >= 2
|
||||
]
|
||||
if histogram:
|
||||
out["popularTimesHistogram"] = histogram
|
||||
live_text = _dig_str(darray, 84, 6)
|
||||
if live_text:
|
||||
out["popularTimesLiveText"] = live_text
|
||||
live_pct = _dig_num(darray, 84, 7, 1)
|
||||
if live_pct is not None:
|
||||
out["popularTimesLivePercent"] = int(live_pct)
|
||||
return out
|
||||
|
||||
|
||||
def _reviews_tags(darray: list) -> list[dict]:
|
||||
"""Review keyword tags (``darray[153][0]``): ``[{title, count}, …]``."""
|
||||
entries = dig(darray, 153, 0)
|
||||
out = []
|
||||
for e in entries if isinstance(entries, list) else []:
|
||||
title = _dig_str(e, 1)
|
||||
count = _dig_num(e, 3, 4)
|
||||
if title and count is not None:
|
||||
out.append({"title": title, "count": int(count)})
|
||||
return out
|
||||
|
||||
|
||||
def _image_fields(darray: list) -> dict[str, Any]:
|
||||
"""Image gallery fields: count, category tab names, and photo URLs.
|
||||
|
||||
``darray[37]`` holds ``[hero_photos, total_count]``; ``darray[171][0]``
|
||||
holds the gallery tabs (each with a thumbnail). ``ponytail:`` anonymous
|
||||
sessions only expose the few hero photos + one thumbnail per tab, not the
|
||||
full gallery — scraping thousands of photos would need the signed-in
|
||||
photo-listing RPC. ``imageUrls`` therefore tops out around a dozen.
|
||||
"""
|
||||
out: dict[str, Any] = {}
|
||||
count = _dig_num(darray, 37, 1)
|
||||
if count is not None:
|
||||
out["imagesCount"] = int(count)
|
||||
tabs = dig(darray, 171, 0)
|
||||
urls: list[str] = []
|
||||
if isinstance(tabs, list):
|
||||
categories = [t for tab in tabs if (t := _dig_str(tab, 2))]
|
||||
if categories:
|
||||
out["imageCategories"] = categories
|
||||
urls.extend(
|
||||
u for tab in tabs if (u := _dig_str(tab, 3, 0, 6, 0)) and u not in urls
|
||||
)
|
||||
heroes = dig(darray, 37, 0)
|
||||
for photo in heroes if isinstance(heroes, list) else []:
|
||||
u = _dig_str(photo, 6, 0)
|
||||
if u and u not in urls:
|
||||
urls.append(u)
|
||||
if urls:
|
||||
out["imageUrls"] = urls
|
||||
return out
|
||||
|
||||
|
||||
def _hotel_fields(darray: list) -> dict[str, Any]:
|
||||
"""Hotel block (``darray[35]`` + ``[64]``), only present for lodging.
|
||||
|
||||
Paths probed live on The Plaza (NYC): ``[35][6]`` star string,
|
||||
``[35][0]/[1]`` the check-in/out dates the price quotes are for,
|
||||
``[35][29][0]`` similar hotels, ``[35][44]`` booking-partner ads.
|
||||
"""
|
||||
stars = _dig_str(darray, 35, 6) or (
|
||||
f"{int(n)} stars" if (n := _dig_num(darray, 64, 0)) else None
|
||||
)
|
||||
if not stars:
|
||||
return {}
|
||||
out: dict[str, Any] = {"hotelStars": stars}
|
||||
description = _dig_str(darray, 32, 1, 1)
|
||||
if description:
|
||||
out["hotelDescription"] = description
|
||||
check_in = _dig_str(darray, 35, 0)
|
||||
check_out = _dig_str(darray, 35, 1)
|
||||
if check_in:
|
||||
out["checkInDate"] = check_in
|
||||
if check_out:
|
||||
out["checkOutDate"] = check_out
|
||||
|
||||
similar = []
|
||||
for entry in dig(darray, 35, 29, 0) or []:
|
||||
info = dig(entry, 0)
|
||||
title = _dig_str(info, 4)
|
||||
if not title:
|
||||
continue
|
||||
hotel: dict[str, Any] = {"title": title, "fid": _dig_str(info, 2)}
|
||||
if (score := _dig_num(info, 7)) is not None:
|
||||
hotel["totalScore"] = score
|
||||
if (count := _dig_num(info, 8)) is not None:
|
||||
hotel["reviewsCount"] = int(count)
|
||||
lat, lng = _dig_num(info, 3, 2), _dig_num(info, 3, 3)
|
||||
if lat is not None and lng is not None:
|
||||
hotel["location"] = {"lat": lat, "lng": lng}
|
||||
if desc := _dig_str(entry, 1):
|
||||
hotel["description"] = desc
|
||||
similar.append(hotel)
|
||||
if similar:
|
||||
out["similarHotelsNearby"] = similar
|
||||
|
||||
ads = []
|
||||
for ad in dig(darray, 35, 44) or []:
|
||||
title = _dig_str(ad, 0)
|
||||
url = _dig_str(ad, 5, 0)
|
||||
if not title or not url:
|
||||
continue
|
||||
item: dict[str, Any] = {"title": title, "url": url}
|
||||
if price := _dig_str(ad, 1):
|
||||
item["price"] = price
|
||||
ads.append(item)
|
||||
if ads:
|
||||
out["hotelAds"] = ads
|
||||
return out
|
||||
|
||||
|
||||
def _cid_from_fid(fid: str | None) -> str | None:
|
||||
"""CID (a.k.a. ludocid) = the decimal value of the fid's second hex half."""
|
||||
if not fid or ":" not in fid:
|
||||
return None
|
||||
try:
|
||||
return str(int(fid.split(":")[1], 16))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def parse_place(darray: list) -> dict[str, Any]:
|
||||
"""Map a place ``darray`` to PlaceItem-shaped fields (public Maps data only).
|
||||
|
||||
Only fields with a stable, known array path are populated; the rest stay at
|
||||
their schema defaults. Paths mirror gosom's ``EntryFromJSON``.
|
||||
"""
|
||||
lat = _dig_num(darray, 9, 2)
|
||||
lng = _dig_num(darray, 9, 3)
|
||||
dist = dig(darray, 175, 3) # per-star review counts [1★,2★,3★,4★,5★]
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"title": _dig_str(darray, 11),
|
||||
"categories": _categories(darray),
|
||||
"address": _dig_str(darray, 18),
|
||||
"website": _website(darray),
|
||||
"phone": _dig_str(darray, 178, 0, 0),
|
||||
"plusCode": _dig_str(darray, 183, 2, 2, 0),
|
||||
"totalScore": _dig_num(darray, 4, 7),
|
||||
"reviewsCount": (int(v) if (v := _dig_num(darray, 4, 8)) is not None else None),
|
||||
"price": _dig_str(darray, 4, 2),
|
||||
"description": _dig_str(darray, 32, 1, 1),
|
||||
"placeId": _dig_str(darray, 78),
|
||||
"fid": _dig_str(darray, 10),
|
||||
"kgmid": _dig_str(darray, 89),
|
||||
"menu": _dig_str(darray, 38, 0),
|
||||
"imageUrl": _dig_str(darray, 72, 0, 1, 6, 0),
|
||||
"additionalInfo": _additional_info(darray),
|
||||
}
|
||||
result["cid"] = _cid_from_fid(result["fid"])
|
||||
|
||||
reservations = _link_sources(dig(darray, 46), (0,), (1,))
|
||||
if reservations:
|
||||
result["tableReservationLinks"] = reservations
|
||||
result["reserveTableUrl"] = reservations[0]["url"]
|
||||
order_links = _order_links(darray)
|
||||
if order_links:
|
||||
result["orderBy"] = order_links # Apify's field name for these
|
||||
food = next(
|
||||
(o["url"] for o in order_links if "food.google.com" in o["url"]), None
|
||||
)
|
||||
if food:
|
||||
result["googleFoodUrl"] = food
|
||||
if result["categories"]:
|
||||
result["categoryName"] = result["categories"][0]
|
||||
if lat is not None and lng is not None:
|
||||
result["location"] = {"lat": lat, "lng": lng}
|
||||
if isinstance(dist, list) and len(dist) >= 5:
|
||||
result["reviewsDistribution"] = {
|
||||
"oneStar": int(dist[0] or 0),
|
||||
"twoStar": int(dist[1] or 0),
|
||||
"threeStar": int(dist[2] or 0),
|
||||
"fourStar": int(dist[3] or 0),
|
||||
"fiveStar": int(dist[4] or 0),
|
||||
}
|
||||
hours = _opening_hours(darray)
|
||||
if hours:
|
||||
result["openingHours"] = hours
|
||||
|
||||
result.update(_popular_times(darray))
|
||||
result.update(_image_fields(darray))
|
||||
result.update(_hotel_fields(darray))
|
||||
tags = _reviews_tags(darray)
|
||||
if tags:
|
||||
result["reviewsTags"] = tags
|
||||
|
||||
# Closed status: darray[88][0] carries a status enum for closed places
|
||||
# ('CLOSED' verified live on a permanently-closed place; open places have
|
||||
# None or an editorial snippet there, so only exact enums count).
|
||||
status = _dig_str(darray, 88, 0)
|
||||
if status in ("CLOSED", "CLOSED_PERMANENTLY"):
|
||||
result["permanentlyClosed"] = True
|
||||
elif status == "CLOSED_TEMPORARILY":
|
||||
result["temporarilyClosed"] = True
|
||||
|
||||
# Complete address components (borough/street/city/postal/state/country).
|
||||
result["neighborhood"] = _dig_str(darray, 183, 1, 0)
|
||||
result["street"] = _dig_str(darray, 183, 1, 1)
|
||||
result["city"] = _dig_str(darray, 183, 1, 3)
|
||||
result["postalCode"] = _dig_str(darray, 183, 1, 4)
|
||||
result["state"] = _dig_str(darray, 183, 1, 5)
|
||||
result["countryCode"] = _dig_str(darray, 183, 1, 6)
|
||||
|
||||
return {k: v for k, v in result.items() if v is not None}
|
||||
|
||||
|
||||
# --- reviews (GetLocalBoqProxy) ----------------------------------------------
|
||||
#
|
||||
# The BOQ proxy returns ``jd[1][10]`` = ``[.., reviews, .., nextToken]``. Each
|
||||
# review ``r`` is a flat ~48-slot array. Index map (verified against live
|
||||
# captures, matching the ``google-maps-review-scraper`` npm package):
|
||||
# r[1] stars r[2] [relativeDate, ?, publishedMs]
|
||||
# r[3] [name, photoUrl, contribUrl, nReviews, nPhotos, [?, localGuide]]
|
||||
# r[4] owner reply [?, relativeDate, text, ...]
|
||||
# r[5] reviewId r[12] reviewUrl
|
||||
# r[13]/r[14] images [[url, caption, ...], ...]
|
||||
# r[26] language r[27] text r[28] translated text
|
||||
# r[30] guided Q&A (context + per-aspect ratings) r[44] [origin, ...]
|
||||
|
||||
|
||||
_CONTRIB_ID_RE = re.compile(r"/contrib/(\d+)")
|
||||
|
||||
|
||||
def _ms_to_iso(ms: Any) -> str | None:
|
||||
"""Convert a millisecond epoch (Google sends it as a string) to ISO-8601."""
|
||||
try:
|
||||
ms_val = float(ms)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not ms_val:
|
||||
return None
|
||||
from datetime import UTC, datetime
|
||||
|
||||
try:
|
||||
dt = datetime.fromtimestamp(ms_val / 1000, tz=UTC)
|
||||
except (ValueError, OSError, OverflowError):
|
||||
return None
|
||||
return dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
||||
|
||||
|
||||
def _review_images(r: Any) -> list[str]:
|
||||
for idx in (13, 14):
|
||||
imgs = dig(r, idx)
|
||||
if isinstance(imgs, list):
|
||||
urls = [_dig_str(imgs, j, 0) for j in range(len(imgs))]
|
||||
return [u for u in urls if u]
|
||||
return []
|
||||
|
||||
|
||||
def _guided_answers(r: Any) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
"""Split the guided Q&A block into (reviewContext, reviewDetailedRating).
|
||||
|
||||
Entries with a numeric rating (``entry[11] = [n]``) are per-aspect ratings
|
||||
(Food/Service/Atmosphere); entries with a chosen answer are context
|
||||
(e.g. ``Order type: Take out``).
|
||||
"""
|
||||
context: dict[str, Any] = {}
|
||||
detailed: dict[str, Any] = {}
|
||||
block = dig(r, 30)
|
||||
if not isinstance(block, list):
|
||||
return context, detailed
|
||||
for entry in block:
|
||||
label = _dig_str(entry, 5) or _dig_str(entry, 1)
|
||||
if not label:
|
||||
continue
|
||||
rating = _dig_num(entry, 11, 0)
|
||||
if rating is not None:
|
||||
detailed[label] = rating
|
||||
continue
|
||||
answer = _dig_str(entry, 2, 0, 0, 1)
|
||||
if answer:
|
||||
context[label] = answer
|
||||
return context, detailed
|
||||
|
||||
|
||||
def parse_review(r: Any) -> dict[str, Any] | None:
|
||||
"""Map one BOQ review array to ReviewFields-shaped fields.
|
||||
|
||||
Returns ``None`` for malformed entries (no author block).
|
||||
"""
|
||||
if not isinstance(r, list):
|
||||
return None
|
||||
name = _dig_str(r, 3, 0)
|
||||
if not name:
|
||||
return None
|
||||
|
||||
reviewer_url = _dig_str(r, 3, 2)
|
||||
reviewer_id = None
|
||||
if reviewer_url:
|
||||
m = _CONTRIB_ID_RE.search(reviewer_url)
|
||||
reviewer_id = m.group(1) if m else None
|
||||
|
||||
n_reviews = _dig_num(r, 3, 3)
|
||||
local_guide = _dig_num(r, 3, 5, 1)
|
||||
text = _dig_str(r, 27)
|
||||
translated = _dig_str(r, 28)
|
||||
context, detailed = _guided_answers(r)
|
||||
|
||||
fields: dict[str, Any] = {
|
||||
"reviewId": _dig_str(r, 5),
|
||||
"reviewUrl": _dig_str(r, 12),
|
||||
"name": name,
|
||||
"reviewerId": reviewer_id,
|
||||
"reviewerUrl": reviewer_url,
|
||||
"reviewerPhotoUrl": _dig_str(r, 3, 1),
|
||||
"reviewerNumberOfReviews": int(n_reviews) if n_reviews is not None else None,
|
||||
"isLocalGuide": bool(local_guide) if local_guide is not None else None,
|
||||
"stars": _dig_num(r, 1),
|
||||
"text": text,
|
||||
"textTranslated": translated if translated and translated != text else None,
|
||||
"publishAt": _dig_str(r, 2, 0),
|
||||
"publishedAtDate": _ms_to_iso(dig(r, 2, 2)),
|
||||
"originalLanguage": _dig_str(r, 26),
|
||||
"reviewOrigin": _dig_str(r, 44, 0),
|
||||
"reviewImageUrls": _review_images(r),
|
||||
"reviewContext": context or None,
|
||||
"reviewDetailedRating": detailed or None,
|
||||
}
|
||||
|
||||
# Owner response (r[4]); only a relative date is exposed here.
|
||||
reply_text = _dig_str(r, 4, 2)
|
||||
if reply_text:
|
||||
fields["responseFromOwnerText"] = reply_text
|
||||
fields["responseFromOwnerDate"] = _dig_str(r, 4, 1)
|
||||
|
||||
return {k: v for k, v in fields.items() if v not in (None, [])}
|
||||
|
||||
|
||||
def parse_reviews_page(reviews: Any) -> list[dict[str, Any]]:
|
||||
"""Parse one BOQ page's raw review list into ReviewFields dicts."""
|
||||
if not isinstance(reviews, list):
|
||||
return []
|
||||
out = []
|
||||
for entry in reviews:
|
||||
parsed = parse_review(entry)
|
||||
if parsed:
|
||||
out.append(parsed)
|
||||
return out
|
||||
|
||||
|
||||
def strip_personal_data(review: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Drop reviewer identity fields (Apify ``personalData=false``).
|
||||
|
||||
``reviewId`` and the review content stay; name/id/url/photo are removed.
|
||||
"""
|
||||
for key in ("name", "reviewerId", "reviewerUrl", "reviewerPhotoUrl"):
|
||||
review.pop(key, None)
|
||||
return review
|
||||
|
|
@ -0,0 +1,210 @@
|
|||
"""Orchestrator for the Google Maps Reviews scraper (Apify-compatible).
|
||||
|
||||
Distinct from the places scraper: one flat output item per review (the review
|
||||
fields merged with the parent place fields), per the Apify "Google Maps Reviews
|
||||
Scraper" output schema.
|
||||
|
||||
Reviews come from Google's public ``GetLocalBoqProxy`` review feed (no login,
|
||||
~10 reviews per page, opaque continuation-token pagination). The place header
|
||||
fields stamped onto each item come from the same ``/maps/preview/place`` RPC the
|
||||
places scraper uses.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from .fetch import fetch_place_darray, iter_reviews_pages, now_iso, resolve_fid
|
||||
from .parsers import parse_place, parse_reviews_page, strip_personal_data
|
||||
from .schemas import GoogleMapsReviewsInput, ReviewItem
|
||||
from .url_resolver import ResolvedUrl, resolve_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Place header keys copied from a parsed place onto every ReviewItem.
|
||||
_PLACE_KEYS = (
|
||||
"title",
|
||||
"placeId",
|
||||
"address",
|
||||
"location",
|
||||
"categories",
|
||||
"categoryName",
|
||||
"totalScore",
|
||||
"reviewsCount",
|
||||
"price",
|
||||
"cid",
|
||||
"fid",
|
||||
"imageUrl",
|
||||
"neighborhood",
|
||||
"street",
|
||||
"city",
|
||||
"countryCode",
|
||||
"postalCode",
|
||||
"state",
|
||||
)
|
||||
|
||||
|
||||
def _parse_start_date(value: str | None) -> datetime | None:
|
||||
"""Parse ``reviewsStartDate`` (ISO date/datetime) into a UTC-naive cutoff."""
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00")).replace(tzinfo=None)
|
||||
except ValueError:
|
||||
logger.warning("[google_maps] bad reviewsStartDate: %r", value)
|
||||
return None
|
||||
|
||||
|
||||
def _before_cutoff(review: dict[str, Any], cutoff: datetime) -> bool:
|
||||
"""True if a review predates the cutoff (newest-first stop condition)."""
|
||||
iso = review.get("publishedAtDate")
|
||||
if not iso:
|
||||
return False
|
||||
try:
|
||||
when = datetime.fromisoformat(iso.replace("Z", "+00:00")).replace(tzinfo=None)
|
||||
except ValueError:
|
||||
return False
|
||||
return when < cutoff
|
||||
|
||||
|
||||
def _keep(review: dict[str, Any], *, filter_string: str, origin: str) -> bool:
|
||||
"""Apply Apify's reviewsFilterString / reviewsOrigin filters client-side."""
|
||||
if origin == "google" and (review.get("reviewOrigin") or "Google") != "Google":
|
||||
return False
|
||||
if filter_string:
|
||||
text = (review.get("text") or "") + " " + (review.get("textTranslated") or "")
|
||||
if filter_string.lower() not in text.lower():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
async def collect_place_reviews(
|
||||
fid: str,
|
||||
*,
|
||||
max_reviews: int,
|
||||
sort: str = "newest",
|
||||
language: str = "en",
|
||||
filter_string: str = "",
|
||||
origin: str = "all",
|
||||
personal_data: bool = True,
|
||||
start_date: str | None = None,
|
||||
) -> tuple[list[dict[str, Any]], int | None]:
|
||||
"""Page a place's reviews via the BOQ feed.
|
||||
|
||||
Returns ``(reviews, exact_total_or_None)``. The feed does not expose the
|
||||
place's total review count up front, so the total is only known when
|
||||
pagination exhausts before ``max_reviews`` is hit (and no filters dropped
|
||||
anything) — the common case for small/medium places.
|
||||
"""
|
||||
cutoff = _parse_start_date(start_date)
|
||||
reviews: list[dict[str, Any]] = []
|
||||
seen = 0
|
||||
exhausted = True
|
||||
filtered = False
|
||||
|
||||
async for raw_page in iter_reviews_pages(fid, sort=sort, language=language):
|
||||
stop = False
|
||||
for review in parse_reviews_page(raw_page):
|
||||
seen += 1
|
||||
if cutoff and _before_cutoff(review, cutoff):
|
||||
stop = True
|
||||
break
|
||||
if not _keep(review, filter_string=filter_string, origin=origin):
|
||||
filtered = True
|
||||
continue
|
||||
if not personal_data:
|
||||
strip_personal_data(review)
|
||||
reviews.append(review)
|
||||
if len(reviews) >= max_reviews:
|
||||
stop = True
|
||||
break
|
||||
if stop:
|
||||
exhausted = False
|
||||
break
|
||||
|
||||
total = seen if exhausted and not filtered and cutoff is None else None
|
||||
return reviews, total
|
||||
|
||||
|
||||
async def _reviews_for_place(
|
||||
resolved: ResolvedUrl,
|
||||
*,
|
||||
input_model: GoogleMapsReviewsInput,
|
||||
input_place_id: str | None,
|
||||
input_start_url: str | None,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Page one place's reviews and yield a flat ReviewItem per review."""
|
||||
fid = await resolve_fid(resolved)
|
||||
if not fid:
|
||||
logger.warning("[google_maps] reviews: no feature id for %s", resolved.url)
|
||||
return
|
||||
|
||||
darray = await fetch_place_darray(fid, language=input_model.language)
|
||||
place = parse_place(darray) if darray else {}
|
||||
place_header = {k: place[k] for k in _PLACE_KEYS if k in place}
|
||||
scraped_at = now_iso()
|
||||
|
||||
reviews, total = await collect_place_reviews(
|
||||
place.get("fid", fid),
|
||||
max_reviews=input_model.maxReviews,
|
||||
sort=input_model.reviewsSort,
|
||||
language=input_model.language,
|
||||
origin=input_model.reviewsOrigin,
|
||||
personal_data=input_model.personalData,
|
||||
start_date=input_model.reviewsStartDate,
|
||||
)
|
||||
if total is not None and "reviewsCount" not in place_header:
|
||||
place_header["reviewsCount"] = total
|
||||
|
||||
for review in reviews:
|
||||
item = ReviewItem(**{**place_header, **review})
|
||||
item.scrapedAt = scraped_at
|
||||
item.language = input_model.language
|
||||
item.inputPlaceId = input_place_id
|
||||
item.inputStartUrl = input_start_url
|
||||
item.url = place.get("url") or resolved.url
|
||||
yield item.to_output()
|
||||
|
||||
|
||||
async def iter_reviews(
|
||||
input_model: GoogleMapsReviewsInput,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Yield Apify-shaped review items for every startUrl and placeId."""
|
||||
for entry in input_model.startUrls:
|
||||
resolved = resolve_url(entry.url)
|
||||
if not resolved:
|
||||
logger.warning("Reviews: unrecognized Google Maps URL: %s", entry.url)
|
||||
continue
|
||||
async for item in _reviews_for_place(
|
||||
resolved,
|
||||
input_model=input_model,
|
||||
input_place_id=None,
|
||||
input_start_url=entry.url,
|
||||
):
|
||||
yield item
|
||||
|
||||
for place_id in input_model.placeIds:
|
||||
url = f"https://www.google.com/maps/place/?q=place_id:{place_id}"
|
||||
resolved = ResolvedUrl("place", place_id, url)
|
||||
async for item in _reviews_for_place(
|
||||
resolved,
|
||||
input_model=input_model,
|
||||
input_place_id=place_id,
|
||||
input_start_url=None,
|
||||
):
|
||||
yield item
|
||||
|
||||
|
||||
async def scrape_reviews(
|
||||
input_model: GoogleMapsReviewsInput, *, limit: int | None = None
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Collect :func:`iter_reviews` into a list, honoring an optional guard."""
|
||||
results: list[dict[str, Any]] = []
|
||||
async for item in iter_reviews(input_model):
|
||||
results.append(item)
|
||||
if limit is not None and len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
|
|
@ -0,0 +1,360 @@
|
|||
# ruff: noqa: N815 - field names intentionally mirror the Apify camelCase spec
|
||||
"""Apify-compatible input/output models for the Google Maps scraper.
|
||||
|
||||
The models mirror the public Apify "Google Maps Scraper" and "Google Maps
|
||||
Reviews Scraper" actor specs so the endpoints can be drop-ins. The skeleton
|
||||
accepts the full input surface; output fields the implementation does not
|
||||
source yet are emitted as ``None``/``[]``/``{}`` so parity is additive.
|
||||
|
||||
Outputs use ``extra="allow"`` on purpose: it lets us grow the output shape
|
||||
without breaking existing consumers, exactly like the YouTube scraper models.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
SearchMatching = Literal["all", "only_includes", "only_exact"]
|
||||
PlaceMinimumStars = Literal[
|
||||
"", "two", "twoAndHalf", "three", "threeAndHalf", "four", "fourAndHalf"
|
||||
]
|
||||
WebsiteFilter = Literal["allPlaces", "withWebsite", "withoutWebsite"]
|
||||
ReviewsSort = Literal["newest", "mostRelevant", "highestRanking", "lowestRanking"]
|
||||
ReviewsOrigin = Literal["all", "google"]
|
||||
AllPlacesAction = Literal["", "all_places_no_search_ocr", "all_places_no_search_mouse"]
|
||||
|
||||
|
||||
class StartUrl(BaseModel):
|
||||
"""A single direct URL entry (Apify passes ``{"url": ...}`` objects)."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
url: str
|
||||
|
||||
|
||||
class SocialMediaEnrichment(BaseModel):
|
||||
"""Per-platform toggles for the social media profile enrichment add-on."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
facebooks: bool = False
|
||||
instagrams: bool = False
|
||||
youtubes: bool = False
|
||||
tiktoks: bool = False
|
||||
twitters: bool = False
|
||||
|
||||
|
||||
class GoogleMapsScrapeInput(BaseModel):
|
||||
"""Full Apify "Google Maps Scraper" input surface.
|
||||
|
||||
Semantics follow Apify: ``maxCrawledPlacesPerSearch=None`` means "all
|
||||
places"; add-on toggles default off; ``maxReviews``/``maxImages``/
|
||||
``maxQuestions`` of ``0`` mean "fetch none of that type".
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
# Discovery
|
||||
searchStringsArray: list[str] = Field(default_factory=list)
|
||||
locationQuery: str | None = None
|
||||
startUrls: list[StartUrl] = Field(default_factory=list)
|
||||
placeIds: list[str] = Field(default_factory=list)
|
||||
allPlacesNoSearchAction: AllPlacesAction = ""
|
||||
|
||||
# Caps / language
|
||||
maxCrawledPlacesPerSearch: int | None = Field(default=None, ge=1)
|
||||
language: str = "en"
|
||||
|
||||
# Filters ($)
|
||||
categoryFilterWords: list[str] = Field(default_factory=list)
|
||||
searchMatching: SearchMatching = "all"
|
||||
placeMinimumStars: PlaceMinimumStars = ""
|
||||
website: WebsiteFilter = "allPlaces"
|
||||
skipClosedPlaces: bool = False
|
||||
|
||||
# Place detail page ($) and its dependents
|
||||
scrapePlaceDetailPage: bool = False
|
||||
scrapeTableReservationProvider: bool = False
|
||||
scrapeOrderOnline: bool = False
|
||||
includeWebResults: bool = False
|
||||
scrapeDirectories: bool = False
|
||||
maxQuestions: int = Field(default=0, ge=0)
|
||||
|
||||
# Enrichment add-ons ($)
|
||||
scrapeContacts: bool = False
|
||||
scrapeSocialMediaProfiles: SocialMediaEnrichment = Field(
|
||||
default_factory=SocialMediaEnrichment
|
||||
)
|
||||
maximumLeadsEnrichmentRecords: int = Field(default=0, ge=0)
|
||||
leadsEnrichmentDepartments: list[str] = Field(default_factory=list)
|
||||
verifyLeadsEnrichmentEmails: bool = False
|
||||
|
||||
# Reviews ($)
|
||||
maxReviews: int = Field(default=0, ge=0)
|
||||
reviewsStartDate: str | None = None
|
||||
reviewsSort: ReviewsSort = "newest"
|
||||
reviewsFilterString: str = ""
|
||||
reviewsOrigin: ReviewsOrigin = "all"
|
||||
scrapeReviewsPersonalData: bool = True
|
||||
|
||||
# Images ($)
|
||||
maxImages: int = Field(default=0, ge=0)
|
||||
scrapeImageAuthors: bool = False
|
||||
|
||||
# Competitor analysis add-on ($)
|
||||
enableCompetitorAnalysis: bool = False
|
||||
maxCompetitorsToAnalyze: int = Field(default=30, ge=0, le=100)
|
||||
|
||||
# Geolocation parameters (use either these or locationQuery, not both)
|
||||
countryCode: str | None = None
|
||||
city: str | None = None
|
||||
state: str | None = None
|
||||
county: str | None = None
|
||||
postalCode: str | None = None
|
||||
customGeolocation: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class Location(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
lat: float | None = None
|
||||
lng: float | None = None
|
||||
|
||||
|
||||
class ReviewsDistribution(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
oneStar: int | None = None
|
||||
twoStar: int | None = None
|
||||
threeStar: int | None = None
|
||||
fourStar: int | None = None
|
||||
fiveStar: int | None = None
|
||||
|
||||
|
||||
class OpeningHour(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
day: str | None = None
|
||||
hours: str | None = None
|
||||
|
||||
|
||||
class PeopleAlsoSearchItem(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
category: str | None = None
|
||||
title: str | None = None
|
||||
reviewsCount: int | None = None
|
||||
totalScore: float | None = None
|
||||
|
||||
|
||||
class ReviewsTag(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
title: str | None = None
|
||||
count: int | None = None
|
||||
|
||||
|
||||
class ImageItem(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
imageUrl: str | None = None
|
||||
authorName: str | None = None
|
||||
authorUrl: str | None = None
|
||||
uploadedAt: str | None = None
|
||||
|
||||
|
||||
class ReviewFields(BaseModel):
|
||||
"""Review-level fields shared by the nested place ``reviews[]`` items and
|
||||
the standalone Reviews Scraper output items."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
name: str | None = None
|
||||
text: str | None = None
|
||||
textTranslated: str | None = None
|
||||
publishAt: str | None = None
|
||||
publishedAtDate: str | None = None
|
||||
likesCount: int | None = None
|
||||
reviewId: str | None = None
|
||||
reviewUrl: str | None = None
|
||||
reviewerId: str | None = None
|
||||
reviewerUrl: str | None = None
|
||||
reviewerPhotoUrl: str | None = None
|
||||
reviewerNumberOfReviews: int | None = None
|
||||
isLocalGuide: bool | None = None
|
||||
reviewOrigin: str | None = None
|
||||
stars: float | None = None
|
||||
rating: str | None = None
|
||||
responseFromOwnerDate: str | None = None
|
||||
responseFromOwnerText: str | None = None
|
||||
reviewImageUrls: list[str] = Field(default_factory=list)
|
||||
reviewContext: dict[str, Any] = Field(default_factory=dict)
|
||||
reviewDetailedRating: dict[str, Any] = Field(default_factory=dict)
|
||||
visitedIn: str | None = None
|
||||
originalLanguage: str | None = None
|
||||
translatedLanguage: str | None = None
|
||||
|
||||
|
||||
class PlaceItem(BaseModel):
|
||||
"""Apify "Google Maps Scraper" output item (one per place).
|
||||
|
||||
Mirrors the actor's example JSON. Unsourced fields default to
|
||||
``None``/``[]``/``{}``; ``extra="allow"`` keeps the contract open.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
# Provenance
|
||||
searchString: str | None = None
|
||||
rank: int | None = None
|
||||
searchPageUrl: str | None = None
|
||||
searchPageLoadedUrl: str | None = None
|
||||
isAdvertisement: bool = False
|
||||
|
||||
# Identity
|
||||
title: str | None = None
|
||||
subTitle: str | None = None
|
||||
description: str | None = None
|
||||
price: str | None = None
|
||||
categoryName: str | None = None
|
||||
categories: list[str] = Field(default_factory=list)
|
||||
placeId: str | None = None
|
||||
fid: str | None = None
|
||||
cid: str | None = None
|
||||
kgmid: str | None = None
|
||||
|
||||
# Address / location
|
||||
address: str | None = None
|
||||
neighborhood: str | None = None
|
||||
street: str | None = None
|
||||
city: str | None = None
|
||||
postalCode: str | None = None
|
||||
state: str | None = None
|
||||
countryCode: str | None = None
|
||||
location: Location | None = None
|
||||
plusCode: str | None = None
|
||||
locatedIn: str | None = None
|
||||
parentPlaceUrl: str | None = None
|
||||
|
||||
# Contact
|
||||
website: str | None = None
|
||||
phone: str | None = None
|
||||
phoneUnformatted: str | None = None
|
||||
menu: str | None = None
|
||||
servicesLink: str | None = None
|
||||
claimThisBusiness: bool | None = None
|
||||
|
||||
# Ratings / status
|
||||
totalScore: float | None = None
|
||||
reviewsCount: int | None = None
|
||||
reviewsDistribution: ReviewsDistribution | None = None
|
||||
permanentlyClosed: bool = False
|
||||
temporarilyClosed: bool = False
|
||||
|
||||
# Images
|
||||
imageUrl: str | None = None
|
||||
imagesCount: int | None = None
|
||||
imageCategories: list[str] = Field(default_factory=list)
|
||||
images: list[ImageItem] = Field(default_factory=list)
|
||||
imageUrls: list[str] = Field(default_factory=list)
|
||||
|
||||
# Detail-page fields (populated only when scrapePlaceDetailPage)
|
||||
openingHours: list[OpeningHour] = Field(default_factory=list)
|
||||
peopleAlsoSearch: list[PeopleAlsoSearchItem] = Field(default_factory=list)
|
||||
placesTags: list[Any] = Field(default_factory=list)
|
||||
reviewsTags: list[ReviewsTag] = Field(default_factory=list)
|
||||
additionalInfo: dict[str, Any] | None = None
|
||||
questionsAndAnswers: list[Any] = Field(default_factory=list)
|
||||
updatesFromCustomers: Any | None = None
|
||||
ownerUpdates: list[Any] = Field(default_factory=list)
|
||||
webResults: list[Any] = Field(default_factory=list)
|
||||
tableReservationLinks: list[Any] = Field(default_factory=list)
|
||||
bookingLinks: list[Any] = Field(default_factory=list)
|
||||
reserveTableUrl: str | None = None
|
||||
googleFoodUrl: str | None = None
|
||||
gasPrices: list[Any] = Field(default_factory=list)
|
||||
restaurantData: dict[str, Any] = Field(default_factory=dict)
|
||||
userPlaceNote: str | None = None
|
||||
|
||||
# Hotel fields
|
||||
hotelStars: str | None = None
|
||||
hotelDescription: str | None = None
|
||||
checkInDate: str | None = None
|
||||
checkOutDate: str | None = None
|
||||
similarHotelsNearby: Any | None = None
|
||||
hotelReviewSummary: Any | None = None
|
||||
hotelAds: list[Any] = Field(default_factory=list)
|
||||
|
||||
# Reviews (populated only when maxReviews > 0)
|
||||
reviews: list[ReviewFields] = Field(default_factory=list)
|
||||
|
||||
# Meta
|
||||
url: str | None = None
|
||||
scrapedAt: str | None = None
|
||||
|
||||
def to_output(self) -> dict[str, Any]:
|
||||
"""Serialize to the flat dict shape Apify emits (keeps extras)."""
|
||||
return self.model_dump(exclude_none=False)
|
||||
|
||||
|
||||
class GoogleMapsReviewsInput(BaseModel):
|
||||
"""Apify "Google Maps Reviews Scraper" input surface."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
startUrls: list[StartUrl] = Field(default_factory=list)
|
||||
placeIds: list[str] = Field(default_factory=list)
|
||||
maxReviews: int = Field(default=10_000_000, ge=1)
|
||||
reviewsSort: ReviewsSort = "newest"
|
||||
reviewsStartDate: str | None = None
|
||||
language: str = "en"
|
||||
reviewsOrigin: ReviewsOrigin = "all"
|
||||
personalData: bool = True
|
||||
|
||||
|
||||
class ReviewItem(ReviewFields):
|
||||
"""Apify "Google Maps Reviews Scraper" output item.
|
||||
|
||||
One flat item per review: the review fields (inherited) merged with the
|
||||
place fields, per the actor's output schema.
|
||||
"""
|
||||
|
||||
# Place
|
||||
title: str | None = None
|
||||
placeId: str | None = None
|
||||
address: str | None = None
|
||||
location: Location | None = None
|
||||
categories: list[str] = Field(default_factory=list)
|
||||
isAdvertisement: bool = False
|
||||
categoryName: str | None = None
|
||||
totalScore: float | None = None
|
||||
permanentlyClosed: bool = False
|
||||
temporarilyClosed: bool = False
|
||||
reviewsCount: int | None = None
|
||||
url: str | None = None
|
||||
price: str | None = None
|
||||
cid: str | None = None
|
||||
fid: str | None = None
|
||||
imageUrl: str | None = None
|
||||
hotelStars: str | None = None
|
||||
kgmid: str | None = None
|
||||
neighborhood: str | None = None
|
||||
street: str | None = None
|
||||
city: str | None = None
|
||||
countryCode: str | None = None
|
||||
postalCode: str | None = None
|
||||
state: str | None = None
|
||||
|
||||
# Provenance / meta
|
||||
scrapedAt: str | None = None
|
||||
searchPageUrl: str | None = None
|
||||
searchString: str | None = None
|
||||
inputPlaceId: str | None = None
|
||||
inputStartUrl: str | None = None
|
||||
language: str | None = None
|
||||
rank: int | None = None
|
||||
|
||||
def to_output(self) -> dict[str, Any]:
|
||||
return self.model_dump(exclude_none=False)
|
||||
|
|
@ -0,0 +1,463 @@
|
|||
"""Orchestrator for the Google Maps places scraper (Apify-compatible).
|
||||
|
||||
Skeleton mirroring the YouTube scraper layout: the core is the async generator
|
||||
:func:`iter_places` (unbounded), :func:`scrape_places` is a thin collector with
|
||||
a caller-supplied ``limit`` guard. Discovery inputs dispatch to per-kind flows
|
||||
(search / place URL / place ID) which are currently no-ops — each will be
|
||||
implemented progressively, exactly like the YouTube flows were.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from .fetch import (
|
||||
SignInRequiredError,
|
||||
fetch_place_darray,
|
||||
gather_bounded,
|
||||
iter_search_pages,
|
||||
now_iso,
|
||||
resolve_fid,
|
||||
)
|
||||
from .parsers import parse_place
|
||||
from .schemas import GoogleMapsScrapeInput, PlaceItem
|
||||
from .url_resolver import ResolvedUrl, resolve_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Re-exported so callers/routes can keep importing it from the orchestrator.
|
||||
__all__ = ["SignInRequiredError", "iter_places", "scrape_places"]
|
||||
|
||||
# Max concurrent per-place detail/review fetches. Each is a ~2s proxy
|
||||
# round-trip, so overlapping them is what turns a 20-place enriched search from
|
||||
# ~50s into a handful of seconds. ``ponytail:`` a fixed ceiling — high enough to
|
||||
# hide latency, low enough not to trip Google/proxy rate limits; make it
|
||||
# configurable if a faster proxy pool ever wants more.
|
||||
_DETAIL_CONCURRENCY = 8
|
||||
|
||||
_SEARCH_PAGE_SIZE = 20
|
||||
|
||||
|
||||
def _prefetch_for(cap: int | None) -> int:
|
||||
"""How many search pages to fetch per wave, from the result cap.
|
||||
|
||||
One page holds ~20 results, so a cap of 60 wants ~3 pages overlapped; an
|
||||
uncapped scan overlaps a small fixed wave. Capped at 5 to bound wasted
|
||||
fetches when dedupe or an early empty page cuts things short.
|
||||
"""
|
||||
if cap is None:
|
||||
return 5
|
||||
return max(1, min(5, (cap + _SEARCH_PAGE_SIZE - 1) // _SEARCH_PAGE_SIZE))
|
||||
|
||||
|
||||
# Apify's placeMinimumStars options -> numeric cutoff.
|
||||
_MIN_STARS = {
|
||||
"two": 2.0,
|
||||
"twoAndHalf": 2.5,
|
||||
"three": 3.0,
|
||||
"threeAndHalf": 3.5,
|
||||
"four": 4.0,
|
||||
"fourAndHalf": 4.5,
|
||||
}
|
||||
|
||||
|
||||
def _location_text(input_model: GoogleMapsScrapeInput) -> str | None:
|
||||
"""The location to scope searches to (Apify appends it to the query)."""
|
||||
if input_model.locationQuery:
|
||||
return input_model.locationQuery
|
||||
parts = [
|
||||
input_model.city,
|
||||
input_model.county,
|
||||
input_model.state,
|
||||
input_model.postalCode,
|
||||
input_model.countryCode,
|
||||
]
|
||||
joined = ", ".join(p for p in parts if p)
|
||||
return joined or None
|
||||
|
||||
|
||||
def _custom_point(
|
||||
input_model: GoogleMapsScrapeInput,
|
||||
) -> tuple[float | None, float | None, float | None]:
|
||||
"""(lat, lng, radius_m) from a GeoJSON-ish customGeolocation Point."""
|
||||
geo = input_model.customGeolocation
|
||||
if not isinstance(geo, dict) or geo.get("type") != "Point":
|
||||
return None, None, None
|
||||
coords = geo.get("coordinates")
|
||||
if not isinstance(coords, list | tuple) or len(coords) < 2:
|
||||
return None, None, None
|
||||
lng, lat = coords[0], coords[1]
|
||||
radius_km = geo.get("radiusKm") or geo.get("radius") or 10
|
||||
return float(lat), float(lng), float(radius_km) * 1000
|
||||
|
||||
|
||||
def _passes_filters(
|
||||
fields: dict[str, Any], query: str, input_model: GoogleMapsScrapeInput
|
||||
) -> bool:
|
||||
"""Apply Apify's search result filters to parsed place fields."""
|
||||
title = (fields.get("title") or "").lower()
|
||||
q = query.lower()
|
||||
if input_model.searchMatching == "only_exact" and title != q:
|
||||
return False
|
||||
if input_model.searchMatching == "only_includes" and q not in title:
|
||||
return False
|
||||
|
||||
if input_model.categoryFilterWords:
|
||||
cats = " ".join(fields.get("categories") or []).lower()
|
||||
if not any(w.lower() in cats for w in input_model.categoryFilterWords):
|
||||
return False
|
||||
|
||||
min_stars = _MIN_STARS.get(input_model.placeMinimumStars)
|
||||
if min_stars is not None and (fields.get("totalScore") or 0) < min_stars:
|
||||
return False
|
||||
|
||||
if input_model.website == "withWebsite" and not fields.get("website"):
|
||||
return False
|
||||
if input_model.website == "withoutWebsite" and fields.get("website"):
|
||||
return False
|
||||
|
||||
return not (
|
||||
input_model.skipClosedPlaces
|
||||
and (fields.get("permanentlyClosed") or fields.get("temporarilyClosed"))
|
||||
)
|
||||
|
||||
|
||||
def _apply_image_cap(
|
||||
fields: dict[str, Any], input_model: GoogleMapsScrapeInput
|
||||
) -> None:
|
||||
"""Apify semantics: gallery ``imageUrls`` only when ``maxImages > 0``."""
|
||||
if not input_model.maxImages:
|
||||
fields.pop("imageUrls", None)
|
||||
elif "imageUrls" in fields:
|
||||
fields["imageUrls"] = fields["imageUrls"][: input_model.maxImages]
|
||||
|
||||
|
||||
async def _enrich_from_detail(
|
||||
fields: dict[str, Any], input_model: GoogleMapsScrapeInput
|
||||
) -> dict[str, Any]:
|
||||
"""Merge the full detail-RPC payload over search-result fields.
|
||||
|
||||
Search darrays are served without the session-gated extras (reviewsCount,
|
||||
distribution, popular times, galleries, tags, full about sections); the
|
||||
detail RPC with an NID cookie has them all. Detail values win on overlap.
|
||||
"""
|
||||
fid = fields.get("fid")
|
||||
if not fid:
|
||||
return fields
|
||||
darray = await fetch_place_darray(fid, language=input_model.language)
|
||||
if not darray:
|
||||
return fields
|
||||
return {**fields, **parse_place(darray)}
|
||||
|
||||
|
||||
async def _build_items(
|
||||
candidates: list[tuple[dict[str, Any], str, int]],
|
||||
input_model: GoogleMapsScrapeInput,
|
||||
*,
|
||||
search_string: str,
|
||||
search_page_url: str | None,
|
||||
enrich: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Turn dedup'd/filtered place fields into output items, in parallel.
|
||||
|
||||
Each candidate is ``(fields, fid, rank)``. Detail enrichment and inline
|
||||
reviews are per-place round-trips; running the whole batch concurrently
|
||||
(bounded, order preserved) is the main speedup over the old one-at-a-time
|
||||
loop. Pure-CPU when neither enrichment nor reviews are requested.
|
||||
"""
|
||||
|
||||
async def _build(fields: dict[str, Any], fid: str, rank: int) -> dict[str, Any]:
|
||||
if enrich:
|
||||
fields = await _enrich_from_detail(fields, input_model)
|
||||
_apply_image_cap(fields, input_model)
|
||||
item = PlaceItem(**fields)
|
||||
item.searchString = search_string
|
||||
if search_page_url:
|
||||
item.searchPageUrl = search_page_url
|
||||
item.rank = rank
|
||||
item.url = (
|
||||
f"https://www.google.com/maps/place/?q=place_id:{fields.get('placeId')}"
|
||||
)
|
||||
item.scrapedAt = now_iso()
|
||||
if input_model.maxReviews:
|
||||
await _attach_reviews(item, fid, input_model)
|
||||
return item.to_output()
|
||||
|
||||
return await gather_bounded(
|
||||
[lambda c=c: _build(*c) for c in candidates],
|
||||
concurrency=_DETAIL_CONCURRENCY,
|
||||
)
|
||||
|
||||
|
||||
async def _search_flow(
|
||||
query: str, *, input_model: GoogleMapsScrapeInput
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Search-term discovery via the ``search?tbm=map`` RPC.
|
||||
|
||||
Pages offset-based results (~20/page), dedupes by fid (Google reshuffles
|
||||
between pages), applies the Apify filters, and emits full place items —
|
||||
each search hit already carries a place darray, so no per-place request is
|
||||
needed for the core fields. When ``scrapePlaceDetailPage`` or ``maxImages``
|
||||
is set, one detail RPC per place adds the session-gated extras; those are
|
||||
fetched concurrently across the page rather than one at a time.
|
||||
"""
|
||||
location = _location_text(input_model)
|
||||
search_query = f"{query} in {location}" if location else query
|
||||
lat, lng, radius_m = _custom_point(input_model)
|
||||
cap = input_model.maxCrawledPlacesPerSearch
|
||||
enrich = bool(input_model.scrapePlaceDetailPage or input_model.maxImages)
|
||||
search_page_url = (
|
||||
f"https://www.google.com/maps/search/{quote(search_query, safe='')}"
|
||||
)
|
||||
|
||||
seen: set[str] = set()
|
||||
emitted = 0
|
||||
async for darrays in iter_search_pages(
|
||||
search_query,
|
||||
language=input_model.language,
|
||||
lat=lat,
|
||||
lng=lng,
|
||||
radius_m=radius_m,
|
||||
prefetch=_prefetch_for(cap),
|
||||
):
|
||||
candidates: list[tuple[dict[str, Any], str, int]] = []
|
||||
new_on_page = 0
|
||||
for darray in darrays:
|
||||
fields = parse_place(darray)
|
||||
fid = fields.get("fid")
|
||||
if not fid or fid in seen:
|
||||
continue
|
||||
seen.add(fid)
|
||||
new_on_page += 1
|
||||
if _passes_filters(fields, query, input_model):
|
||||
candidates.append((fields, fid, len(seen)))
|
||||
if cap is not None:
|
||||
candidates = candidates[: max(0, cap - emitted)]
|
||||
for out in await _build_items(
|
||||
candidates,
|
||||
input_model,
|
||||
search_string=query,
|
||||
search_page_url=search_page_url,
|
||||
enrich=enrich,
|
||||
):
|
||||
yield out
|
||||
emitted += 1
|
||||
if cap is not None and emitted >= cap:
|
||||
return
|
||||
if not new_on_page: # page was all repeats -> feed is cycling, stop
|
||||
return
|
||||
|
||||
|
||||
# Broad category sweep for allPlacesNoSearchAction. Apify's implementation
|
||||
# OCRs / mouse-overs the rendered map pins (hence the enum names); the public
|
||||
# search RPC has no "list everything" query (verified: '*' and '' return
|
||||
# nothing). ponytail: approximate the scan with broad category searches over
|
||||
# the same viewport, deduped by fid — covers the vast majority of pins; a true
|
||||
# pin-complete scan would need browser rendering + tile OCR.
|
||||
_ALL_PLACES_SWEEP = [
|
||||
"restaurant",
|
||||
"cafe",
|
||||
"bar",
|
||||
"store",
|
||||
"shopping",
|
||||
"hotel",
|
||||
"tourist attraction",
|
||||
"park",
|
||||
"gym",
|
||||
"salon",
|
||||
"bank",
|
||||
"gas station",
|
||||
"pharmacy",
|
||||
"doctor",
|
||||
"school",
|
||||
"church",
|
||||
"services",
|
||||
]
|
||||
|
||||
|
||||
async def _all_places_flow(
|
||||
input_model: GoogleMapsScrapeInput,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Area scan without a search term: sweep broad categories, dedupe by fid.
|
||||
|
||||
Emits the same items the search flow would; ``searchString`` carries the
|
||||
action value so callers can tell scan hits from query hits.
|
||||
"""
|
||||
location = _location_text(input_model)
|
||||
lat, lng, radius_m = _custom_point(input_model)
|
||||
cap = input_model.maxCrawledPlacesPerSearch
|
||||
enrich = bool(input_model.scrapePlaceDetailPage or input_model.maxImages)
|
||||
seen: set[str] = set()
|
||||
emitted = 0
|
||||
for term in _ALL_PLACES_SWEEP:
|
||||
search_query = f"{term} in {location}" if location else term
|
||||
async for darrays in iter_search_pages(
|
||||
search_query,
|
||||
language=input_model.language,
|
||||
lat=lat,
|
||||
lng=lng,
|
||||
radius_m=radius_m,
|
||||
prefetch=_prefetch_for(cap),
|
||||
):
|
||||
candidates: list[tuple[dict[str, Any], str, int]] = []
|
||||
new_on_page = 0
|
||||
for darray in darrays:
|
||||
fields = parse_place(darray)
|
||||
fid = fields.get("fid")
|
||||
if not fid or fid in seen:
|
||||
continue
|
||||
seen.add(fid)
|
||||
new_on_page += 1
|
||||
if _passes_filters(fields, "", input_model):
|
||||
candidates.append((fields, fid, len(seen)))
|
||||
if cap is not None:
|
||||
candidates = candidates[: max(0, cap - emitted)]
|
||||
for out in await _build_items(
|
||||
candidates,
|
||||
input_model,
|
||||
search_string=input_model.allPlacesNoSearchAction,
|
||||
search_page_url=None,
|
||||
enrich=enrich,
|
||||
):
|
||||
yield out
|
||||
emitted += 1
|
||||
if cap is not None and emitted >= cap:
|
||||
return
|
||||
if not new_on_page:
|
||||
break # this term is exhausted; move to the next one
|
||||
|
||||
|
||||
async def _place_flow(
|
||||
resolved: ResolvedUrl, *, input_model: GoogleMapsScrapeInput
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Single place from a direct Maps place URL / place ID.
|
||||
|
||||
Resolves the feature ID, fetches the place detail via the ``/maps/preview/
|
||||
place`` RPC (full payload — see ``_PLACE_DETAIL_PB``), and maps the fields
|
||||
into a ``PlaceItem``. When ``maxReviews > 0`` the place's reviews are
|
||||
attached inline (Apify puts them on ``reviews[]``).
|
||||
"""
|
||||
fid = await resolve_fid(resolved)
|
||||
if not fid:
|
||||
logger.warning("[google_maps] could not resolve feature id: %s", resolved.url)
|
||||
return
|
||||
darray = await fetch_place_darray(fid, language=input_model.language)
|
||||
if not darray:
|
||||
logger.warning(
|
||||
"[google_maps] no place data (structure may have shifted): %s", resolved.url
|
||||
)
|
||||
return
|
||||
fields = parse_place(darray)
|
||||
_apply_image_cap(fields, input_model)
|
||||
item = PlaceItem(**fields)
|
||||
item.url = resolved.url
|
||||
item.searchString = f"Direct URL: {resolved.url}"
|
||||
item.scrapedAt = now_iso()
|
||||
if input_model.maxReviews and fields.get("fid"):
|
||||
await _attach_reviews(item, fields["fid"], input_model)
|
||||
yield item.to_output()
|
||||
|
||||
|
||||
async def _attach_reviews(
|
||||
item: PlaceItem, fid: str, input_model: GoogleMapsScrapeInput
|
||||
) -> None:
|
||||
"""Populate ``item.reviews`` (and total count when knowable) from the feed."""
|
||||
from .reviews import collect_place_reviews
|
||||
|
||||
reviews, total = await collect_place_reviews(
|
||||
fid,
|
||||
max_reviews=input_model.maxReviews,
|
||||
sort=input_model.reviewsSort,
|
||||
language=input_model.language,
|
||||
filter_string=input_model.reviewsFilterString,
|
||||
origin=input_model.reviewsOrigin,
|
||||
personal_data=input_model.scrapeReviewsPersonalData,
|
||||
start_date=input_model.reviewsStartDate,
|
||||
)
|
||||
item.reviews = reviews
|
||||
if total is not None and item.reviewsCount is None:
|
||||
item.reviewsCount = total
|
||||
|
||||
|
||||
async def _place_id_flow(
|
||||
place_id: str, *, input_model: GoogleMapsScrapeInput
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Single place from a bare place ID (format ``ChIJ...``).
|
||||
|
||||
Maps resolves ``?q=place_id:<id>`` to the place page, so we build that URL
|
||||
and reuse the place flow.
|
||||
"""
|
||||
url = f"https://www.google.com/maps/place/?q=place_id:{place_id}"
|
||||
resolved = ResolvedUrl("place", place_id, url)
|
||||
async for item in _place_flow(resolved, input_model=input_model):
|
||||
yield item
|
||||
|
||||
|
||||
async def _dispatch(
|
||||
resolved: ResolvedUrl, input_model: GoogleMapsScrapeInput
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
if resolved.kind == "search":
|
||||
async for item in _search_flow(resolved.value, input_model=input_model):
|
||||
yield item
|
||||
else: # place / cid / shortlink / reviews all resolve to a place page
|
||||
async for item in _place_flow(resolved, input_model=input_model):
|
||||
yield item
|
||||
|
||||
|
||||
async def iter_places(
|
||||
input_model: GoogleMapsScrapeInput,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Yield Apify-shaped place items from all discovery inputs.
|
||||
|
||||
Apify runs searches, startUrls, and placeIds side by side (they are
|
||||
additive, unlike the YouTube scraper where startUrls override queries).
|
||||
"""
|
||||
for entry in input_model.startUrls:
|
||||
resolved = resolve_url(entry.url)
|
||||
if not resolved:
|
||||
logger.warning("Unrecognized Google Maps URL: %s", entry.url)
|
||||
continue
|
||||
async for item in _dispatch(resolved, input_model):
|
||||
yield item
|
||||
|
||||
# placeIds are independent single-place detail fetches (~2s each); run the
|
||||
# batch concurrently, bounded, results in input order — a bulk list of IDs
|
||||
# is the common case and was previously fully sequential.
|
||||
if input_model.placeIds:
|
||||
|
||||
async def _collect(pid: str) -> list[dict[str, Any]]:
|
||||
return [it async for it in _place_id_flow(pid, input_model=input_model)]
|
||||
|
||||
for items in await gather_bounded(
|
||||
[lambda p=p: _collect(p) for p in input_model.placeIds],
|
||||
concurrency=_DETAIL_CONCURRENCY,
|
||||
):
|
||||
for item in items:
|
||||
yield item
|
||||
|
||||
for query in input_model.searchStringsArray:
|
||||
async for item in _search_flow(query, input_model=input_model):
|
||||
yield item
|
||||
|
||||
if input_model.allPlacesNoSearchAction:
|
||||
async for item in _all_places_flow(input_model):
|
||||
yield item
|
||||
|
||||
|
||||
async def scrape_places(
|
||||
input_model: GoogleMapsScrapeInput, *, limit: int | None = None
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Collect :func:`iter_places` into a list, honoring an optional ``limit``.
|
||||
|
||||
``limit`` is a request-time policy guard (used by the route), NOT a ceiling
|
||||
in the streaming core.
|
||||
"""
|
||||
results: list[dict[str, Any]] = []
|
||||
async for item in iter_places(input_model):
|
||||
results.append(item)
|
||||
if limit is not None and len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
"""Classify a Google Maps URL and extract its identifier.
|
||||
|
||||
Covers the ``startUrls`` shapes the Apify specs accept: place pages
|
||||
(``/maps/place``), search pages (``/maps/search``), review pages
|
||||
(``/maps/reviews``), CID links (``google.com/maps?cid=***``), and short links
|
||||
(``goo.gl/maps`` / ``maps.app.goo.gl``, which need a network redirect to
|
||||
resolve — classified here, resolved later in the fetch layer).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
ResolvedKind = Literal["place", "search", "reviews", "cid", "shortlink"]
|
||||
|
||||
_MAPS_HOSTS = ("google.", "www.google.")
|
||||
_SHORTLINK_HOSTS = ("goo.gl", "maps.app.goo.gl")
|
||||
|
||||
# Feature ID (a.k.a. "fid" / data_id): two hex halves, e.g.
|
||||
# 0x89c3ca9c11f90c25:0x6cc8dba851799f09. It appears in the URL ``data=`` blob as
|
||||
# ``!1s<hex:hex>`` and is the key that links a place to its detail/review RPCs.
|
||||
_FID_RE = re.compile(r"(0x[0-9a-f]+:0x[0-9a-f]+)")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedUrl:
|
||||
kind: ResolvedKind
|
||||
value: str # place slug, search query, cid, or the short URL itself
|
||||
url: str
|
||||
fid: str | None = None # feature id (hex:hex) when present in the URL
|
||||
|
||||
|
||||
def extract_fid(url: str) -> str | None:
|
||||
"""Pull the feature ID (``0x..:0x..``) out of a Google Maps URL, if present."""
|
||||
match = _FID_RE.search(url)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def resolve_url(url: str) -> ResolvedUrl | None:
|
||||
"""Classify a Google Maps URL into a scrape job, or ``None`` if unrecognized."""
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname or ""
|
||||
path = parsed.path or ""
|
||||
|
||||
if hostname in _SHORTLINK_HOSTS or hostname.endswith(".goo.gl"):
|
||||
return ResolvedUrl("shortlink", url, url)
|
||||
|
||||
if "google." not in hostname:
|
||||
return None
|
||||
|
||||
# google.com/maps?cid=123... (uncommon but supported by both actors)
|
||||
cid = parse_qs(parsed.query).get("cid", [None])[0]
|
||||
if cid:
|
||||
return ResolvedUrl("cid", cid, url)
|
||||
|
||||
for prefix, kind in (
|
||||
("/maps/place/", "place"),
|
||||
("/maps/search/", "search"),
|
||||
("/maps/reviews/", "reviews"),
|
||||
):
|
||||
if path.startswith(prefix):
|
||||
# Maps slugs encode spaces as "+" (e.g. /maps/place/Kim's+Island).
|
||||
value = unquote(path[len(prefix) :].split("/")[0].replace("+", " "))
|
||||
return ResolvedUrl(kind, value, url, extract_fid(url)) # type: ignore[arg-type]
|
||||
|
||||
# Bare /maps/search or /maps/reviews carrying data in the query/data blob.
|
||||
for suffix, kind in (("/maps/search", "search"), ("/maps/reviews", "reviews")):
|
||||
if path.rstrip("/") == suffix:
|
||||
return ResolvedUrl(kind, url, url) # type: ignore[arg-type]
|
||||
|
||||
return None
|
||||
|
|
@ -27,8 +27,8 @@ from .parsers import (
|
|||
parse_count,
|
||||
parse_video_page,
|
||||
)
|
||||
from .scraper import _post, _published_date, fan_out
|
||||
from .schemas import CommentItem, YouTubeCommentsInput
|
||||
from .scraper import _post, _published_date, fan_out
|
||||
from .url_resolver import resolve_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -39,10 +39,14 @@ _SORT_LABELS = {"TOP_COMMENTS": "Top", "NEWEST_FIRST": "Newest"}
|
|||
|
||||
|
||||
async def _post_next(token: str) -> dict[str, Any] | None:
|
||||
return await _post(INNERTUBE_NEXT_URL, build_innertube_payload(continuation_token=token))
|
||||
return await _post(
|
||||
INNERTUBE_NEXT_URL, build_innertube_payload(continuation_token=token)
|
||||
)
|
||||
|
||||
|
||||
def _finalize(partial: dict[str, Any], base: dict[str, Any], reply_to: str | None) -> dict:
|
||||
def _finalize(
|
||||
partial: dict[str, Any], base: dict[str, Any], reply_to: str | None
|
||||
) -> dict:
|
||||
return CommentItem(**{**base, **partial, "replyToCid": reply_to}).to_output()
|
||||
|
||||
|
||||
|
|
@ -132,7 +136,7 @@ async def _comments_for_video(
|
|||
for cid, tok in pending
|
||||
)
|
||||
)
|
||||
replies_by_cid = {cid: r for (cid, _), r in zip(pending, fetched)}
|
||||
replies_by_cid = {cid: r for (cid, _), r in zip(pending, fetched, strict=False)}
|
||||
|
||||
for entity in entities:
|
||||
if count >= limit:
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from contextvars import ContextVar
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -78,10 +78,8 @@ class _RotatingSession:
|
|||
|
||||
async def close(self) -> None:
|
||||
if self._cm is not None:
|
||||
try:
|
||||
with suppress(Exception): # best-effort teardown
|
||||
await self._cm.__aexit__(None, None, None)
|
||||
except Exception: # best-effort teardown
|
||||
pass
|
||||
self._cm = self.session = None
|
||||
|
||||
async def rotate(self) -> Any | None:
|
||||
|
|
@ -124,6 +122,7 @@ async def proxy_session():
|
|||
finally:
|
||||
await holder.close()
|
||||
|
||||
|
||||
# Consent cookies to dodge the EU consent interstitial that otherwise returns a
|
||||
# page with no ``ytInitialData``. Mirrors app/routes/youtube_routes.py.
|
||||
CONSENT_COOKIES = {
|
||||
|
|
@ -224,7 +223,9 @@ async def post_innertube(
|
|||
if page.status == 200:
|
||||
return page.json()
|
||||
logger.warning("InnerTube POST %s returned %s", base_url, page.status)
|
||||
if not (holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS):
|
||||
if not (
|
||||
holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS
|
||||
):
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning("InnerTube POST %s failed: %s", base_url, e)
|
||||
|
|
@ -270,7 +271,9 @@ async def fetch_html(url: str, *, cookies: dict[str, str] | None = None) -> str
|
|||
if page.status == 200:
|
||||
return page.html_content
|
||||
logger.warning("HTML GET %s returned %s", url, page.status)
|
||||
if not (holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS):
|
||||
if not (
|
||||
holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS
|
||||
):
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning("HTML GET %s failed: %s", url, e)
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ def parse_count(value: Any) -> int | None:
|
|||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
if isinstance(value, int | float):
|
||||
return int(value)
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
|
|
@ -267,7 +267,10 @@ def _comments_turned_off(initial: dict) -> bool | None:
|
|||
def _is_members_only(player: dict, initial: dict) -> bool:
|
||||
"""True for members-only videos (a members badge or a members-only offer)."""
|
||||
for badge in find_all(initial, "metadataBadgeRenderer"):
|
||||
if isinstance(badge, dict) and badge.get("style") == "BADGE_STYLE_TYPE_MEMBERS_ONLY":
|
||||
if (
|
||||
isinstance(badge, dict)
|
||||
and badge.get("style") == "BADGE_STYLE_TYPE_MEMBERS_ONLY"
|
||||
):
|
||||
return True
|
||||
status = player.get("playabilityStatus") or {}
|
||||
return find_first(status, "offerId") == "sponsors_only_video"
|
||||
|
|
@ -421,7 +424,9 @@ def parse_translation(next_data: dict) -> tuple[str | None, str | None]:
|
|||
language. Videos without a localization return their original text.
|
||||
"""
|
||||
vpir = find_first(next_data, "videoPrimaryInfoRenderer") or {}
|
||||
title = "".join(r.get("text", "") for r in (dig(vpir, "title", "runs") or [])) or None
|
||||
title = (
|
||||
"".join(r.get("text", "") for r in (dig(vpir, "title", "runs") or [])) or None
|
||||
)
|
||||
|
||||
vsir = find_first(next_data, "videoSecondaryInfoRenderer") or {}
|
||||
description = dig(vsir, "attributedDescription", "content")
|
||||
|
|
@ -440,7 +445,9 @@ def parse_collaborators(initial: dict) -> list[dict[str, str | None]] | None:
|
|||
``attributedTitle`` whose tap opens a dialog listing each credited channel
|
||||
(``listItemViewModel``). Returns ``None`` for ordinary single-owner videos.
|
||||
"""
|
||||
owner = dig(find_first(initial, "videoSecondaryInfoRenderer"), "owner", "videoOwnerRenderer")
|
||||
owner = dig(
|
||||
find_first(initial, "videoSecondaryInfoRenderer"), "owner", "videoOwnerRenderer"
|
||||
)
|
||||
attributed = owner.get("attributedTitle") if isinstance(owner, dict) else None
|
||||
if not isinstance(attributed, dict):
|
||||
return None
|
||||
|
|
@ -460,8 +467,12 @@ def parse_collaborators(initial: dict) -> list[dict[str, str | None]] | None:
|
|||
collaborators.append(
|
||||
{
|
||||
"name": name,
|
||||
"username": base[2:] if isinstance(base, str) and base.startswith("/@") else None,
|
||||
"url": f"https://www.youtube.com{base}" if isinstance(base, str) and base else None,
|
||||
"username": base[2:]
|
||||
if isinstance(base, str) and base.startswith("/@")
|
||||
else None,
|
||||
"url": f"https://www.youtube.com{base}"
|
||||
if isinstance(base, str) and base
|
||||
else None,
|
||||
}
|
||||
)
|
||||
return collaborators or None
|
||||
|
|
@ -537,9 +548,10 @@ def parse_video_page(html: str) -> dict[str, Any] | None:
|
|||
|
||||
# Verified badge on the channel owner.
|
||||
badges = find_all(initial, "metadataBadgeRenderer")
|
||||
result["isChannelVerified"] = any(
|
||||
isinstance(b, dict) and b.get("tooltip") == "Verified" for b in badges
|
||||
) or None
|
||||
result["isChannelVerified"] = (
|
||||
any(isinstance(b, dict) and b.get("tooltip") == "Verified" for b in badges)
|
||||
or None
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
|
@ -632,7 +644,9 @@ def parse_channel_videos(data: dict) -> tuple[list[dict[str, Any]], str | None]:
|
|||
"publishedTimeText": dig(parts, 1, "text", "content"),
|
||||
"duration": _lockup_duration(lockup),
|
||||
"thumbnailUrl": _best_thumbnail(
|
||||
dig(lockup, "contentImage", "thumbnailViewModel", "image", "sources")
|
||||
dig(
|
||||
lockup, "contentImage", "thumbnailViewModel", "image", "sources"
|
||||
)
|
||||
),
|
||||
}
|
||||
)
|
||||
|
|
@ -673,9 +687,7 @@ def parse_channel_shorts(data: dict) -> tuple[list[dict[str, Any]], str | None]:
|
|||
|
||||
|
||||
def _lockup_duration(lockup: dict) -> str | None:
|
||||
overlays = dig(
|
||||
lockup, "contentImage", "thumbnailViewModel", "overlays"
|
||||
) or []
|
||||
overlays = dig(lockup, "contentImage", "thumbnailViewModel", "overlays") or []
|
||||
for overlay in overlays:
|
||||
text = dig(
|
||||
overlay,
|
||||
|
|
|
|||
|
|
@ -100,9 +100,7 @@ async def fan_out(
|
|||
if holder is not None:
|
||||
await holder.close()
|
||||
|
||||
tasks = [
|
||||
asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))
|
||||
]
|
||||
tasks = [asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))]
|
||||
try:
|
||||
for _ in range(len(jobs)):
|
||||
for item in await results.get():
|
||||
|
|
@ -421,7 +419,7 @@ async def _playlist_flow(
|
|||
# is the bottleneck, so fan them out (each carries its playlist position in
|
||||
# ``order``; fan_out emits as they finish, not in playlist order).
|
||||
# ponytail: nested fan_out — when many playlist URLs run at once this can
|
||||
# stack pools (outer × inner) of proxy sessions. Fine for the common
|
||||
# stack pools (outer x inner) of proxy sessions. Fine for the common
|
||||
# single/few-playlist case; cap inner concurrency if bulk-playlist runs trip it.
|
||||
jobs = [
|
||||
_video_flow(
|
||||
|
|
|
|||
|
|
@ -56,7 +56,9 @@ def _select_transcript(transcript_list, language: str, prefer_generated: bool):
|
|||
return transcript_list.find_transcript(codes)
|
||||
|
||||
|
||||
def _fetch_subtitles_sync(video_id: str, language: str, fmt: str, prefer_generated: bool):
|
||||
def _fetch_subtitles_sync(
|
||||
video_id: str, language: str, fmt: str, prefer_generated: bool
|
||||
):
|
||||
api = _build_client()
|
||||
transcript_list = api.list(video_id)
|
||||
transcript = _select_transcript(transcript_list, language, prefer_generated)
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ balance can get the IP **temporarily banned**, so ``ErrNoBalance`` /
|
|||
won't change without a restart). ``reset_solver_latch()`` clears it for tests.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
|
|
@ -107,8 +108,10 @@ def detect_challenge(page: Any, cfg: CaptchaConfig) -> tuple[str, str] | None:
|
|||
m = _RENDER_SITEKEY.search(src)
|
||||
if m and m.group(1) != "explicit":
|
||||
return "v3", m.group(1)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("%s detection error (treated as no challenge): %s", _CAPTCHA_LOG, exc)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"%s detection error (treated as no challenge): %s", _CAPTCHA_LOG, exc
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -131,7 +134,7 @@ def proxy_url_to_captchatools(proxy_url: str | None) -> str | None:
|
|||
if p.username and p.password:
|
||||
return f"{p.hostname}:{p.port}:{p.username}:{p.password}"
|
||||
return f"{p.hostname}:{p.port}"
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -220,11 +223,9 @@ _INJECT_JS = r"""
|
|||
def _inject_and_submit(page: Any, challenge_type: str, token: str) -> None:
|
||||
try:
|
||||
page.evaluate(_INJECT_JS, {"token": token, "ctype": challenge_type})
|
||||
try:
|
||||
with contextlib.suppress(Exception):
|
||||
page.wait_for_load_state("networkidle", timeout=15000)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
except Exception as exc: # noqa: BLE001
|
||||
except Exception as exc:
|
||||
logger.warning("%s injection error: %s", _CAPTCHA_LOG, exc)
|
||||
|
||||
|
||||
|
|
@ -264,7 +265,7 @@ def build_captcha_page_action(
|
|||
page_url = getattr(page, "url", "") or ""
|
||||
try:
|
||||
user_agent = page.evaluate("() => navigator.userAgent")
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception:
|
||||
user_agent = None
|
||||
|
||||
# This counts as an attempt the moment we call the (paid) solver.
|
||||
|
|
@ -296,7 +297,7 @@ def build_captcha_page_action(
|
|||
except FuturesTimeout:
|
||||
logger.warning("%s solve timed out after %ss", _CAPTCHA_LOG, cfg.timeout_s)
|
||||
return page
|
||||
except Exception as exc: # noqa: BLE001
|
||||
except Exception as exc:
|
||||
if _is_unrecoverable(exc):
|
||||
_latch_solver(repr(exc))
|
||||
else:
|
||||
|
|
@ -310,7 +311,9 @@ def build_captcha_page_action(
|
|||
|
||||
_inject_and_submit(page, challenge_type, token)
|
||||
state["solved"] = True
|
||||
logger.info("%s solved type=%s site=%s", _CAPTCHA_LOG, challenge_type, sitekey[:12])
|
||||
logger.info(
|
||||
"%s solved type=%s site=%s", _CAPTCHA_LOG, challenge_type, sitekey[:12]
|
||||
)
|
||||
return page
|
||||
|
||||
return page_action
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ import logging
|
|||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
import trafilatura
|
||||
|
|
@ -48,7 +48,7 @@ logger = logging.getLogger(__name__)
|
|||
_PERF = "[webcrawler][perf]"
|
||||
|
||||
|
||||
class CrawlOutcomeStatus(str, Enum):
|
||||
class CrawlOutcomeStatus(StrEnum):
|
||||
"""Deterministic per-URL crawl result, single-sourcing the billable signal."""
|
||||
|
||||
SUCCESS = "success" # a tier returned usable extracted content
|
||||
|
|
@ -202,9 +202,7 @@ class WebCrawlerConnector:
|
|||
logger.info(f"[webcrawler] Using Scrapling StealthyFetcher for: {url}")
|
||||
result = await self._run_tier_with_proxy_retry(
|
||||
"scrapling-stealthy",
|
||||
lambda: self._crawl_with_stealthy(
|
||||
url, captcha_state, block_state
|
||||
),
|
||||
lambda: self._crawl_with_stealthy(url, captcha_state, block_state),
|
||||
)
|
||||
if result:
|
||||
self._log_tier_outcome(
|
||||
|
|
@ -395,9 +393,7 @@ class WebCrawlerConnector:
|
|||
Runs the sync fetch in a worker thread so it works on any event loop,
|
||||
including Windows ``SelectorEventLoop`` which cannot spawn subprocesses.
|
||||
"""
|
||||
return await asyncio.to_thread(
|
||||
self._crawl_with_dynamic_sync, url, block_state
|
||||
)
|
||||
return await asyncio.to_thread(self._crawl_with_dynamic_sync, url, block_state)
|
||||
|
||||
def _crawl_with_dynamic_sync(
|
||||
self, url: str, block_state: dict[str, Any] | None = None
|
||||
|
|
|
|||
|
|
@ -21,5 +21,5 @@ creds). Run it from the backend directory:
|
|||
|
||||
See ``README.md`` for the runbook and ``plans/backend/03f-undetectability-testing.md``
|
||||
for the design. This package is dev/operator tooling, untouched by the product
|
||||
rename, and only *measures* what 03a–03e built.
|
||||
rename, and only *measures* what 03a-03e built.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -14,11 +14,12 @@ change their DOM and the harness is manual.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
|
@ -30,7 +31,7 @@ RESULTS_DIR = _PKG_DIR / "results"
|
|||
SCREENSHOTS_DIR = RESULTS_DIR / "screenshots"
|
||||
|
||||
|
||||
class CheckStatus(str, Enum):
|
||||
class CheckStatus(StrEnum):
|
||||
"""Outcome of a single scorecard row."""
|
||||
|
||||
PASS = "PASS" # met the aspirational bar
|
||||
|
|
@ -76,7 +77,7 @@ class RunMeta:
|
|||
*, suites: str, proxy: str | None, headed: bool, scrapling_version: str
|
||||
) -> RunMeta:
|
||||
return RunMeta(
|
||||
timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
timestamp=datetime.now(UTC).isoformat(timespec="seconds"),
|
||||
suites=suites,
|
||||
proxy=mask_proxy(proxy),
|
||||
headed=headed,
|
||||
|
|
@ -120,15 +121,13 @@ def make_page_action(
|
|||
|
||||
def _action(page: Any) -> Any:
|
||||
if pre_wait_ms > 0:
|
||||
try:
|
||||
with contextlib.suppress(Exception):
|
||||
page.wait_for_timeout(pre_wait_ms)
|
||||
except Exception:
|
||||
pass
|
||||
if evaluate_js is not None:
|
||||
try:
|
||||
cell.value = page.evaluate(evaluate_js)
|
||||
cell.captured = True
|
||||
except Exception as exc: # noqa: BLE001 - tolerant by design
|
||||
except Exception as exc:
|
||||
cell.error = f"{type(exc).__name__}: {exc}"
|
||||
if screenshot_path is not None:
|
||||
try:
|
||||
|
|
@ -137,7 +136,7 @@ def make_page_action(
|
|||
# create it for us, so a missing dir silently drops every screenshot.
|
||||
Path(screenshot_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
page.screenshot(path=screenshot_path, full_page=True)
|
||||
except Exception as exc: # noqa: BLE001 - tolerant, but surface the cause
|
||||
except Exception as exc:
|
||||
if cell.error is None:
|
||||
cell.error = f"screenshot: {type(exc).__name__}: {exc}"
|
||||
return page
|
||||
|
|
@ -168,7 +167,7 @@ def ensure_dirs() -> None:
|
|||
|
||||
def scrapling_version() -> str:
|
||||
try:
|
||||
import scrapling # noqa: PLC0415
|
||||
import scrapling
|
||||
|
||||
return getattr(scrapling, "__version__", "unknown")
|
||||
except Exception:
|
||||
|
|
@ -178,9 +177,7 @@ def scrapling_version() -> str:
|
|||
# --- scorecard I/O -----------------------------------------------------------
|
||||
|
||||
|
||||
def write_scorecard(
|
||||
results: list[CheckResult], meta: RunMeta
|
||||
) -> tuple[Path, Path]:
|
||||
def write_scorecard(results: list[CheckResult], meta: RunMeta) -> tuple[Path, Path]:
|
||||
"""Write the JSON snapshot (committed, baseline) + a readable markdown report.
|
||||
|
||||
The JSON filename is timestamped so prior runs are kept as the baseline trail;
|
||||
|
|
@ -250,9 +247,7 @@ def diff_against_baseline(
|
|||
lines.append(f"+ NEW [{r.suite}] {r.name}: {r.status.value}")
|
||||
continue
|
||||
if was["status"] != r.status.value:
|
||||
lines.append(
|
||||
f"~ {r.name}: status {was['status']} -> {r.status.value}"
|
||||
)
|
||||
lines.append(f"~ {r.name}: status {was['status']} -> {r.status.value}")
|
||||
old_n, new_n = was.get("numeric"), r.numeric
|
||||
if old_n is not None and new_n is not None and abs(old_n - new_n) > 1e-9:
|
||||
lines.append(f"~ {r.name}: numeric {old_n} -> {new_n}")
|
||||
|
|
@ -281,8 +276,10 @@ def render_console(results: list[CheckResult]) -> None:
|
|||
by_suite.setdefault(r.suite, []).append(r)
|
||||
|
||||
for suite in sorted(by_suite):
|
||||
label = "Suite S - stealth/anti-bot" if suite == "S" else (
|
||||
"Suite E - extraction" if suite == "E" else f"Suite {suite}"
|
||||
label = (
|
||||
"Suite S - stealth/anti-bot"
|
||||
if suite == "S"
|
||||
else ("Suite E - extraction" if suite == "E" else f"Suite {suite}")
|
||||
)
|
||||
print(f"\n=== {label} ===")
|
||||
for r in by_suite[suite]:
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ async def _run_case(
|
|||
bar = f"SUCCESS + contains {len(case.must_contain)} marker(s)"
|
||||
try:
|
||||
outcome = await connector.crawl_url(case.url)
|
||||
except Exception as exc: # noqa: BLE001 - never crash the run
|
||||
except Exception as exc:
|
||||
return CheckResult(
|
||||
suite="E",
|
||||
name=case.name,
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ parser later is a one-function change (the spec list is the extension point).
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -83,10 +84,8 @@ def _full_text(page: Any, limit: int = 200_000) -> str:
|
|||
def _parse_sannysoft(page: Any, _cell: EvalCell):
|
||||
"""Count failed result cells (JS sets class='failed' on red rows)."""
|
||||
html = ""
|
||||
try:
|
||||
with contextlib.suppress(Exception):
|
||||
html = page.html_content or ""
|
||||
except Exception:
|
||||
pass
|
||||
fails = len(re.findall(r'class="[^"]*\bfailed\b[^"]*"', html))
|
||||
if not html:
|
||||
return (CheckStatus.ERROR, None, "no html returned")
|
||||
|
|
@ -184,8 +183,7 @@ def _parse_incolumitas(page: Any, _cell: EvalCell):
|
|||
if not fails and dc is None:
|
||||
return (CheckStatus.INFO, None, f"unparsed: {txt[:120]!r}")
|
||||
detail = (
|
||||
f"fpscanner FAIL={fails or 'none'} datacenter={datacenter} "
|
||||
f"behavioral={bscore}"
|
||||
f"fpscanner FAIL={fails or 'none'} datacenter={datacenter} behavioral={bscore}"
|
||||
)
|
||||
return (
|
||||
CheckStatus.PASS if not fails else CheckStatus.FAIL,
|
||||
|
|
@ -375,7 +373,7 @@ def _run_browser_site(
|
|||
|
||||
try:
|
||||
page = StealthyFetcher.fetch(site.url, **kwargs)
|
||||
except Exception as exc: # noqa: BLE001 - never crash the run
|
||||
except Exception as exc:
|
||||
return CheckResult(
|
||||
suite="S",
|
||||
name=site.name,
|
||||
|
|
@ -388,7 +386,7 @@ def _run_browser_site(
|
|||
|
||||
try:
|
||||
status, numeric, detail = site.parse(page, cell)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
except Exception as exc:
|
||||
status, numeric, detail = (
|
||||
CheckStatus.ERROR,
|
||||
None,
|
||||
|
|
@ -429,7 +427,7 @@ async def _run_tls(proxy: str | None) -> CheckResult:
|
|||
bar="informational (diff vs real Chrome)",
|
||||
detail=f"ja3={ja3} ja4={ja4} peet={peet}",
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
except Exception as exc:
|
||||
return CheckResult(
|
||||
suite="S",
|
||||
name="tls_fingerprint",
|
||||
|
|
@ -460,7 +458,7 @@ async def _run_proxy_leak(proxy: str | None) -> CheckResult:
|
|||
bar="not your real/datacenter IP",
|
||||
detail=f"exit_ip={ip} ({note})",
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
except Exception as exc:
|
||||
return CheckResult(
|
||||
suite="S",
|
||||
name="exit_ip",
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from .google_drive_add_connector_route import (
|
|||
from .google_gmail_add_connector_route import (
|
||||
router as google_gmail_add_connector_router,
|
||||
)
|
||||
from .google_maps_routes import router as google_maps_router
|
||||
from .image_generation_routes import router as image_generation_router
|
||||
from .incentive_tasks_routes import router as incentive_tasks_router
|
||||
from .jira_add_connector_route import router as jira_add_connector_router
|
||||
|
|
@ -61,12 +62,12 @@ from .rbac_routes import router as rbac_router
|
|||
from .reports_routes import router as reports_router
|
||||
from .sandbox_routes import router as sandbox_router
|
||||
from .search_source_connectors_routes import router as search_source_connectors_router
|
||||
from .workspaces_routes import router as workspaces_router
|
||||
from .slack_add_connector_route import router as slack_add_connector_router
|
||||
from .stripe_routes import router as stripe_router
|
||||
from .team_memory_routes import router as team_memory_router
|
||||
from .teams_add_connector_route import router as teams_add_connector_router
|
||||
from .video_presentations_routes import router as video_presentations_router
|
||||
from .workspaces_routes import router as workspaces_router
|
||||
from .youtube_routes import router as youtube_router
|
||||
|
||||
router = APIRouter()
|
||||
|
|
@ -133,6 +134,7 @@ router.include_router(public_chat_router) # Public chat sharing and cloning
|
|||
router.include_router(incentive_tasks_router) # Incentive tasks for earning free pages
|
||||
router.include_router(stripe_router) # Stripe checkout for additional page packs
|
||||
router.include_router(youtube_router) # YouTube playlist resolution
|
||||
router.include_router(google_maps_router) # Google Maps places + reviews scraper
|
||||
router.include_router(prompts_router)
|
||||
router.include_router(memory_router) # User personal memory (memory.md style)
|
||||
router.include_router(team_memory_router) # Workspace team memory
|
||||
|
|
|
|||
|
|
@ -172,9 +172,7 @@ async def get_editor_content(
|
|||
return _build_response(markdown_content)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/workspaces/{workspace_id}/documents/{document_id}/download-markdown"
|
||||
)
|
||||
@router.get("/workspaces/{workspace_id}/documents/{document_id}/download-markdown")
|
||||
async def download_document_markdown(
|
||||
workspace_id: int,
|
||||
document_id: int,
|
||||
|
|
|
|||
|
|
@ -517,9 +517,7 @@ async def bulk_move_documents(
|
|||
if not target:
|
||||
raise HTTPException(status_code=404, detail="Target folder not found")
|
||||
mismatched = [
|
||||
doc.id
|
||||
for doc in documents
|
||||
if doc.workspace_id != target.workspace_id
|
||||
doc.id for doc in documents if doc.workspace_id != target.workspace_id
|
||||
]
|
||||
if mismatched:
|
||||
raise HTTPException(
|
||||
|
|
|
|||
|
|
@ -886,9 +886,7 @@ async def list_connections(
|
|||
baileys_account_ids.add(int(account.id))
|
||||
route_type = "account"
|
||||
connection_id = account.id
|
||||
workspace_id = (
|
||||
account.owner_workspace_id or binding.workspace_id
|
||||
)
|
||||
workspace_id = account.owner_workspace_id or binding.workspace_id
|
||||
display_name = "WhatsApp Bridge"
|
||||
|
||||
connections.append(
|
||||
|
|
@ -907,7 +905,6 @@ async def list_connections(
|
|||
else binding.external_username
|
||||
),
|
||||
"workspace_name": workspace_name,
|
||||
"workspace_id": workspace_id,
|
||||
"health_status": account.health_status.value,
|
||||
"suspended_reason": binding.suspended_reason,
|
||||
}
|
||||
|
|
@ -940,7 +937,6 @@ async def list_connections(
|
|||
"display_name": "WhatsApp Bridge",
|
||||
"external_username": None,
|
||||
"workspace_name": account_state.get("display_phone_number"),
|
||||
"workspace_id": account_state.get("phone_number_id"),
|
||||
"health_status": account.health_status.value,
|
||||
"suspended_reason": account.suspended_reason,
|
||||
}
|
||||
|
|
|
|||
64
surfsense_backend/app/routes/google_maps_routes.py
Normal file
64
surfsense_backend/app/routes/google_maps_routes.py
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
"""Google Maps scraper routes (Apify actor-compatible)."""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from app.auth.context import AuthContext
|
||||
from app.proprietary.scrapers.google_maps import (
|
||||
GoogleMapsReviewsInput,
|
||||
GoogleMapsScrapeInput,
|
||||
scrape_places,
|
||||
scrape_reviews,
|
||||
)
|
||||
from app.proprietary.scrapers.google_maps.scraper import SignInRequiredError
|
||||
from app.users import require_session_context
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@router.post("/google-maps/scrape")
|
||||
async def scrape_places_route(
|
||||
payload: GoogleMapsScrapeInput,
|
||||
_auth: AuthContext = Depends(require_session_context),
|
||||
) -> list[dict]:
|
||||
"""Scrape Google Maps places (search terms / URLs / place IDs).
|
||||
|
||||
Apify "Google Maps Scraper"-compatible input/output. Runs inline and is
|
||||
bounded only by the request's own ``maxCrawledPlacesPerSearch``.
|
||||
"""
|
||||
try:
|
||||
return await scrape_places(payload)
|
||||
except SignInRequiredError as e:
|
||||
raise HTTPException(
|
||||
status_code=403, detail=f"Google sign in required: {e!s}"
|
||||
) from e
|
||||
except Exception as e:
|
||||
logger.error("Google Maps scrape failed: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=502, detail=f"Google Maps scrape failed: {e!s}"
|
||||
) from e
|
||||
|
||||
|
||||
@router.post("/google-maps/reviews")
|
||||
async def scrape_reviews_route(
|
||||
payload: GoogleMapsReviewsInput,
|
||||
_auth: AuthContext = Depends(require_session_context),
|
||||
) -> list[dict]:
|
||||
"""Scrape Google Maps reviews for the given place URLs / place IDs.
|
||||
|
||||
Apify "Google Maps Reviews Scraper"-compatible. Runs inline and is bounded
|
||||
only by the request's own ``maxReviews``.
|
||||
"""
|
||||
try:
|
||||
return await scrape_reviews(payload)
|
||||
except SignInRequiredError as e:
|
||||
raise HTTPException(
|
||||
status_code=403, detail=f"Google sign in required: {e!s}"
|
||||
) from e
|
||||
except Exception as e:
|
||||
logger.error("Google Maps reviews scrape failed: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=502, detail=f"Google Maps reviews scrape failed: {e!s}"
|
||||
) from e
|
||||
|
|
@ -277,9 +277,7 @@ async def get_logs_summary(
|
|||
# Get logs from the time window
|
||||
result = await session.execute(
|
||||
select(Log)
|
||||
.filter(
|
||||
and_(Log.workspace_id == workspace_id, Log.created_at >= since)
|
||||
)
|
||||
.filter(and_(Log.workspace_id == workspace_id, Log.created_at >= since))
|
||||
.order_by(desc(Log.created_at))
|
||||
)
|
||||
logs = result.scalars().all()
|
||||
|
|
|
|||
|
|
@ -206,9 +206,7 @@ async def _resolve_role_model_id(
|
|||
return 0
|
||||
|
||||
|
||||
async def _clear_invalid_roles(
|
||||
session: AsyncSession, workspace_id: int
|
||||
) -> Workspace:
|
||||
async def _clear_invalid_roles(session: AsyncSession, workspace_id: int) -> Workspace:
|
||||
workspace = await _get_workspace(session, workspace_id)
|
||||
workspace.chat_model_id = await _resolve_role_model_id(
|
||||
session,
|
||||
|
|
@ -752,9 +750,7 @@ async def test_connection_model(
|
|||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/workspaces/{workspace_id}/model-roles", response_model=ModelRolesRead
|
||||
)
|
||||
@router.get("/workspaces/{workspace_id}/model-roles", response_model=ModelRolesRead)
|
||||
async def get_model_roles(
|
||||
workspace_id: int,
|
||||
session: AsyncSession = Depends(get_async_session),
|
||||
|
|
@ -777,9 +773,7 @@ async def get_model_roles(
|
|||
)
|
||||
|
||||
|
||||
@router.put(
|
||||
"/workspaces/{workspace_id}/model-roles", response_model=ModelRolesRead
|
||||
)
|
||||
@router.put("/workspaces/{workspace_id}/model-roles", response_model=ModelRolesRead)
|
||||
async def update_model_roles(
|
||||
workspace_id: int,
|
||||
data: ModelRolesUpdate,
|
||||
|
|
|
|||
|
|
@ -45,9 +45,9 @@ from app.db import (
|
|||
NewChatMessageRole,
|
||||
NewChatThread,
|
||||
Permission,
|
||||
Workspace,
|
||||
TokenUsage,
|
||||
User,
|
||||
Workspace,
|
||||
get_async_session,
|
||||
shielded_async_session,
|
||||
)
|
||||
|
|
@ -560,9 +560,7 @@ async def check_thread_access(
|
|||
# For legacy threads (created before visibility feature),
|
||||
# only the workspace owner can access
|
||||
if is_legacy:
|
||||
workspace_query = select(Workspace).filter(
|
||||
Workspace.id == thread.workspace_id
|
||||
)
|
||||
workspace_query = select(Workspace).filter(Workspace.id == thread.workspace_id)
|
||||
workspace_result = await session.execute(workspace_query)
|
||||
workspace = workspace_result.scalar_one_or_none()
|
||||
is_workspace_owner = workspace and workspace.user_id == user.id
|
||||
|
|
@ -624,9 +622,7 @@ async def list_threads(
|
|||
)
|
||||
|
||||
# Check if user is the workspace owner (for legacy thread visibility)
|
||||
workspace_query = select(Workspace).filter(
|
||||
Workspace.id == workspace_id
|
||||
)
|
||||
workspace_query = select(Workspace).filter(Workspace.id == workspace_id)
|
||||
workspace_result = await session.execute(workspace_query)
|
||||
workspace = workspace_result.scalar_one_or_none()
|
||||
is_workspace_owner = workspace and workspace.user_id == user.id
|
||||
|
|
@ -731,9 +727,7 @@ async def search_threads(
|
|||
)
|
||||
|
||||
# Check if user is the workspace owner (for legacy thread visibility)
|
||||
workspace_query = select(Workspace).filter(
|
||||
Workspace.id == workspace_id
|
||||
)
|
||||
workspace_query = select(Workspace).filter(Workspace.id == workspace_id)
|
||||
workspace_result = await session.execute(workspace_query)
|
||||
workspace = workspace_result.scalar_one_or_none()
|
||||
is_workspace_owner = workspace and workspace.user_id == user.id
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ from app.db import (
|
|||
DocumentType,
|
||||
SearchSourceConnector,
|
||||
SearchSourceConnectorType,
|
||||
Workspace,
|
||||
User,
|
||||
Workspace,
|
||||
get_async_session,
|
||||
)
|
||||
from app.notifications.service import NotificationService
|
||||
|
|
|
|||
|
|
@ -21,11 +21,11 @@ from sqlalchemy.orm import selectinload
|
|||
from app.auth.context import AuthContext
|
||||
from app.db import (
|
||||
Permission,
|
||||
User,
|
||||
Workspace,
|
||||
WorkspaceInvite,
|
||||
WorkspaceMembership,
|
||||
WorkspaceRole,
|
||||
User,
|
||||
get_async_session,
|
||||
)
|
||||
from app.schemas import (
|
||||
|
|
@ -256,9 +256,7 @@ async def list_roles(
|
|||
)
|
||||
|
||||
result = await session.execute(
|
||||
select(WorkspaceRole).filter(
|
||||
WorkspaceRole.workspace_id == workspace_id
|
||||
)
|
||||
select(WorkspaceRole).filter(WorkspaceRole.workspace_id == workspace_id)
|
||||
)
|
||||
return result.scalars().all()
|
||||
|
||||
|
|
@ -982,9 +980,7 @@ async def get_invite_info(
|
|||
# Check if invite is still valid
|
||||
if not invite.is_active:
|
||||
return InviteInfoResponse(
|
||||
workspace_name=invite.workspace.name
|
||||
if invite.workspace
|
||||
else "",
|
||||
workspace_name=invite.workspace.name if invite.workspace else "",
|
||||
role_name=invite.role.name if invite.role else None,
|
||||
is_valid=False,
|
||||
message="This invite is no longer active",
|
||||
|
|
@ -992,9 +988,7 @@ async def get_invite_info(
|
|||
|
||||
if invite.expires_at and invite.expires_at < datetime.now(UTC):
|
||||
return InviteInfoResponse(
|
||||
workspace_name=invite.workspace.name
|
||||
if invite.workspace
|
||||
else "",
|
||||
workspace_name=invite.workspace.name if invite.workspace else "",
|
||||
role_name=invite.role.name if invite.role else None,
|
||||
is_valid=False,
|
||||
message="This invite has expired",
|
||||
|
|
@ -1002,9 +996,7 @@ async def get_invite_info(
|
|||
|
||||
if invite.max_uses and invite.uses_count >= invite.max_uses:
|
||||
return InviteInfoResponse(
|
||||
workspace_name=invite.workspace.name
|
||||
if invite.workspace
|
||||
else "",
|
||||
workspace_name=invite.workspace.name if invite.workspace else "",
|
||||
role_name=invite.role.name if invite.role else None,
|
||||
is_valid=False,
|
||||
message="This invite has reached its maximum uses",
|
||||
|
|
|
|||
|
|
@ -489,8 +489,7 @@ async def update_search_source_connector(
|
|||
if key == "connector_type" and value != db_connector.connector_type:
|
||||
check_result = await session.execute(
|
||||
select(SearchSourceConnector).filter(
|
||||
SearchSourceConnector.workspace_id
|
||||
== db_connector.workspace_id,
|
||||
SearchSourceConnector.workspace_id == db_connector.workspace_id,
|
||||
SearchSourceConnector.connector_type == value,
|
||||
SearchSourceConnector.id != connector_id,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -19,9 +19,9 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||
from app.auth.context import AuthContext
|
||||
from app.db import (
|
||||
Permission,
|
||||
VideoPresentation,
|
||||
Workspace,
|
||||
WorkspaceMembership,
|
||||
VideoPresentation,
|
||||
get_async_session,
|
||||
)
|
||||
from app.schemas import VideoPresentationRead
|
||||
|
|
|
|||
|
|
@ -279,9 +279,7 @@ async def update_workspace(
|
|||
) from e
|
||||
|
||||
|
||||
@router.put(
|
||||
"/workspaces/{workspace_id}/api-access", response_model=WorkspaceRead
|
||||
)
|
||||
@router.put("/workspaces/{workspace_id}/api-access", response_model=WorkspaceRead)
|
||||
async def update_workspace_api_access(
|
||||
workspace_id: int,
|
||||
body: WorkspaceApiAccessUpdate,
|
||||
|
|
|
|||
|
|
@ -103,14 +103,6 @@ from .search_source_connector import (
|
|||
SearchSourceConnectorRead,
|
||||
SearchSourceConnectorUpdate,
|
||||
)
|
||||
from .workspace import (
|
||||
WorkspaceApiAccessUpdate,
|
||||
WorkspaceBase,
|
||||
WorkspaceCreate,
|
||||
WorkspaceRead,
|
||||
WorkspaceUpdate,
|
||||
WorkspaceWithStats,
|
||||
)
|
||||
from .stripe import (
|
||||
CreateCreditCheckoutSessionRequest,
|
||||
CreateCreditCheckoutSessionResponse,
|
||||
|
|
@ -128,6 +120,14 @@ from .video_presentations import (
|
|||
VideoPresentationRead,
|
||||
VideoPresentationUpdate,
|
||||
)
|
||||
from .workspace import (
|
||||
WorkspaceApiAccessUpdate,
|
||||
WorkspaceBase,
|
||||
WorkspaceCreate,
|
||||
WorkspaceRead,
|
||||
WorkspaceUpdate,
|
||||
WorkspaceWithStats,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Folder schemas
|
||||
|
|
@ -242,13 +242,6 @@ __all__ = [
|
|||
"SearchSourceConnectorCreate",
|
||||
"SearchSourceConnectorRead",
|
||||
"SearchSourceConnectorUpdate",
|
||||
"WorkspaceApiAccessUpdate",
|
||||
# Workspace schemas
|
||||
"WorkspaceBase",
|
||||
"WorkspaceCreate",
|
||||
"WorkspaceRead",
|
||||
"WorkspaceUpdate",
|
||||
"WorkspaceWithStats",
|
||||
"StripeWebhookResponse",
|
||||
"ThreadHistoryLoadResponse",
|
||||
"ThreadListItem",
|
||||
|
|
@ -257,12 +250,19 @@ __all__ = [
|
|||
# User schemas
|
||||
"UserCreate",
|
||||
"UserRead",
|
||||
"UserWorkspaceAccess",
|
||||
"UserUpdate",
|
||||
"UserWorkspaceAccess",
|
||||
"VerifyConnectionResponse",
|
||||
# Video Presentation schemas
|
||||
"VideoPresentationBase",
|
||||
"VideoPresentationCreate",
|
||||
"VideoPresentationRead",
|
||||
"VideoPresentationUpdate",
|
||||
"WorkspaceApiAccessUpdate",
|
||||
# Workspace schemas
|
||||
"WorkspaceBase",
|
||||
"WorkspaceCreate",
|
||||
"WorkspaceRead",
|
||||
"WorkspaceUpdate",
|
||||
"WorkspaceWithStats",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -492,9 +492,7 @@ async def _resolve_agent_billing_for_workspace(
|
|||
|
||||
chat_model_id = workspace.chat_model_id
|
||||
if chat_model_id is None:
|
||||
raise ValueError(
|
||||
f"Workspace {workspace_id} has no chat_model_id configured"
|
||||
)
|
||||
raise ValueError(f"Workspace {workspace_id} has no chat_model_id configured")
|
||||
|
||||
owner_user_id: UUID = workspace.user_id
|
||||
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ from app.db import (
|
|||
NewChatMessageRole,
|
||||
NewChatThread,
|
||||
Permission,
|
||||
WorkspaceMembership,
|
||||
User,
|
||||
WorkspaceMembership,
|
||||
has_permission,
|
||||
)
|
||||
from app.notifications.service import NotificationService
|
||||
|
|
|
|||
|
|
@ -174,9 +174,7 @@ async def ensure_folder_hierarchy_with_depth_validation(
|
|||
|
||||
if folder is None:
|
||||
await validate_folder_depth(session, parent_id, subtree_depth=0)
|
||||
position = await generate_folder_position(
|
||||
session, workspace_id, parent_id
|
||||
)
|
||||
position = await generate_folder_position(session, workspace_id, parent_id)
|
||||
folder = Folder(
|
||||
name=name,
|
||||
workspace_id=workspace_id,
|
||||
|
|
|
|||
|
|
@ -213,9 +213,7 @@ class LinearKBSyncService:
|
|||
|
||||
document.title = f"{issue_identifier}: {issue_title}"
|
||||
document.content = summary_content
|
||||
document.content_hash = generate_content_hash(
|
||||
issue_content, workspace_id
|
||||
)
|
||||
document.content_hash = generate_content_hash(issue_content, workspace_id)
|
||||
document.embedding = summary_embedding
|
||||
from sqlalchemy.orm.attributes import flag_modified
|
||||
|
||||
|
|
|
|||
|
|
@ -469,9 +469,7 @@ async def get_vision_llm(
|
|||
if not candidates:
|
||||
logger.error("No vision-capable models available for Auto mode")
|
||||
return None
|
||||
config_id = int(
|
||||
choose_auto_model_candidate(candidates, workspace_id)["id"]
|
||||
)
|
||||
config_id = int(choose_auto_model_candidate(candidates, workspace_id)["id"])
|
||||
|
||||
if config_id < 0:
|
||||
global_model = get_global_model(config_id)
|
||||
|
|
@ -532,9 +530,7 @@ async def get_vision_llm(
|
|||
return SanitizedChatLiteLLM(**litellm_kwargs)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error getting vision LLM for workspace {workspace_id}: {e!s}"
|
||||
)
|
||||
logger.error(f"Error getting vision LLM for workspace {workspace_id}: {e!s}")
|
||||
return None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from uuid import UUID
|
|||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db import Workspace, User
|
||||
from app.db import User, Workspace
|
||||
from app.services.memory.document import parse_memory_document, render_memory_document
|
||||
from app.services.memory.rewrite import forced_rewrite
|
||||
from app.services.memory.schemas import MemoryLimits
|
||||
|
|
|
|||
|
|
@ -84,9 +84,7 @@ class NotionToolMetadataService:
|
|||
"error": "No Notion accounts connected",
|
||||
}
|
||||
|
||||
parent_pages = await self._get_parent_pages_by_account(
|
||||
workspace_id, accounts
|
||||
)
|
||||
parent_pages = await self._get_parent_pages_by_account(workspace_id, accounts)
|
||||
|
||||
accounts_with_status = []
|
||||
for acc in accounts:
|
||||
|
|
|
|||
|
|
@ -31,10 +31,10 @@ from app.db import (
|
|||
PodcastStatus,
|
||||
PublicChatSnapshot,
|
||||
Report,
|
||||
WorkspaceMembership,
|
||||
User,
|
||||
VideoPresentation,
|
||||
VideoPresentationStatus,
|
||||
WorkspaceMembership,
|
||||
)
|
||||
from app.utils.rbac import check_permission
|
||||
|
||||
|
|
|
|||
|
|
@ -184,9 +184,7 @@ class WebCrawlCreditService:
|
|||
|
||||
return user.credit_micros_balance
|
||||
|
||||
async def charge_credits(
|
||||
self, user_id: str | UUID, successes: int
|
||||
) -> int | None:
|
||||
async def charge_credits(self, user_id: str | UUID, successes: int) -> int | None:
|
||||
"""Debit the wallet for ``successes`` successful crawls.
|
||||
|
||||
No-op when crawl billing is disabled or ``successes <= 0``. Returns the
|
||||
|
|
@ -198,9 +196,7 @@ class WebCrawlCreditService:
|
|||
return None
|
||||
return await self._apply_debit(user_id, self.successes_to_micros(successes))
|
||||
|
||||
async def charge_captcha(
|
||||
self, user_id: str | UUID, attempts: int
|
||||
) -> int | None:
|
||||
async def charge_captcha(self, user_id: str | UUID, attempts: int) -> int | None:
|
||||
"""Debit the wallet for ``attempts`` captcha solves (Phase 3d).
|
||||
|
||||
Per-attempt (not per-success): the solver charges for every attempt even
|
||||
|
|
|
|||
|
|
@ -212,9 +212,7 @@ async def _delete_folder_documents(
|
|||
)
|
||||
def delete_workspace_task(self, workspace_id: int):
|
||||
"""Celery task to delete a workspace and heavy child rows in batches."""
|
||||
return run_async_celery_task(
|
||||
lambda: _delete_workspace_background(workspace_id)
|
||||
)
|
||||
return run_async_celery_task(lambda: _delete_workspace_background(workspace_id))
|
||||
|
||||
|
||||
async def _delete_workspace_background(workspace_id: int) -> None:
|
||||
|
|
|
|||
|
|
@ -127,8 +127,7 @@ async def _generate_video_presentation(
|
|||
)
|
||||
except ValueError as resolve_err:
|
||||
logger.error(
|
||||
"VideoPresentation %s: cannot resolve billing for "
|
||||
"workspace=%s: %s",
|
||||
"VideoPresentation %s: cannot resolve billing for workspace=%s: %s",
|
||||
video_pres.id,
|
||||
workspace_id,
|
||||
resolve_err,
|
||||
|
|
|
|||
|
|
@ -55,9 +55,7 @@ def _agent_config_from_resolved(
|
|||
)
|
||||
|
||||
|
||||
async def _load_workspace(
|
||||
session: AsyncSession, workspace_id: int
|
||||
) -> Workspace | None:
|
||||
async def _load_workspace(session: AsyncSession, workspace_id: int) -> Workspace | None:
|
||||
result = await session.execute(
|
||||
select(Workspace).where(Workspace.id == workspace_id)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,12 +13,12 @@ from datetime import datetime
|
|||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import config
|
||||
from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType
|
||||
from app.proprietary.web_crawler import (
|
||||
CrawlOutcomeStatus,
|
||||
WebCrawlerConnector,
|
||||
)
|
||||
from app.config import config
|
||||
from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType
|
||||
from app.services.etl_credit_service import InsufficientCreditsError
|
||||
from app.services.task_logging_service import TaskLoggingService
|
||||
from app.services.token_tracking_service import record_token_usage
|
||||
|
|
|
|||
|
|
@ -22,10 +22,10 @@ from app.auth.session_cookies import access_expires_at, write_session
|
|||
from app.config import config
|
||||
from app.db import (
|
||||
Prompt,
|
||||
User,
|
||||
Workspace,
|
||||
WorkspaceMembership,
|
||||
WorkspaceRole,
|
||||
User,
|
||||
async_session_maker,
|
||||
get_async_session,
|
||||
get_default_roles_config,
|
||||
|
|
@ -207,9 +207,7 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
|
|||
f"Created default workspace (ID: {default_workspace.id}) for user {user.id}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to create default workspace for user {user.id}: {e}"
|
||||
)
|
||||
logger.error(f"Failed to create default workspace for user {user.id}: {e}")
|
||||
|
||||
async def on_after_forgot_password(
|
||||
self, user: User, token: str, request: Request | None = None
|
||||
|
|
@ -326,7 +324,7 @@ def _token_meets_epoch(token: str) -> bool:
|
|||
return False
|
||||
|
||||
issued_at = payload.get("iat")
|
||||
return isinstance(issued_at, (int, float)) and int(issued_at) >= min_issued_at
|
||||
return isinstance(issued_at, int | float) and int(issued_at) >= min_issued_at
|
||||
|
||||
|
||||
async def get_auth_context(
|
||||
|
|
|
|||
|
|
@ -12,10 +12,10 @@ our enterprise targets). Regexes are compiled once at import (hot-path hygiene).
|
|||
"""
|
||||
|
||||
import re
|
||||
from enum import Enum
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class BlockType(str, Enum):
|
||||
class BlockType(StrEnum):
|
||||
"""Coarse label for what a fetched page represents."""
|
||||
|
||||
OK = "ok" # usable content, no challenge detected
|
||||
|
|
|
|||
|
|
@ -44,9 +44,7 @@ def validate_workspace_id(workspace_id: Any) -> int:
|
|||
if isinstance(workspace_id, str):
|
||||
# Check if it's a valid integer string
|
||||
if not workspace_id.strip():
|
||||
raise HTTPException(
|
||||
status_code=400, detail="workspace_id cannot be empty"
|
||||
)
|
||||
raise HTTPException(status_code=400, detail="workspace_id cannot be empty")
|
||||
|
||||
# Check for valid integer format (no leading zeros, no decimal points)
|
||||
if not re.match(r"^[1-9]\d*$", workspace_id.strip()):
|
||||
|
|
|
|||
824
surfsense_backend/scripts/e2e_google_maps_deep.py
Normal file
824
surfsense_backend/scripts/e2e_google_maps_deep.py
Normal file
|
|
@ -0,0 +1,824 @@
|
|||
"""Deep functional verification for the Google Maps scraper (live network).
|
||||
|
||||
Complements the fast smoke e2e (e2e_google_maps_scraper.py) with breadth:
|
||||
diverse places (countries, scripts, categories), URL-kind coverage (name-only
|
||||
URLs, CID), and review semantics (sort order, date cutoff, pagination
|
||||
uniqueness, personal-data stripping, localization).
|
||||
|
||||
Verification style: ground-truth invariants instead of screenshots — known
|
||||
coordinates/websites/address keywords for world-famous places, and internal
|
||||
consistency rules (newest sort is monotonically non-increasing, cutoff dates
|
||||
hold, review IDs are unique across pages, etc.).
|
||||
|
||||
Run from the backend directory:
|
||||
.\\.venv\\Scripts\\python.exe scripts/e2e_google_maps_deep.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
|
||||
if _candidate.exists():
|
||||
load_dotenv(_candidate)
|
||||
break
|
||||
|
||||
from app.proprietary.scrapers.google_maps import ( # noqa: E402
|
||||
GoogleMapsReviewsInput,
|
||||
GoogleMapsScrapeInput,
|
||||
scrape_places,
|
||||
scrape_reviews,
|
||||
)
|
||||
|
||||
_CHECKS: list[tuple[str, bool, str]] = []
|
||||
|
||||
|
||||
def _check(label: str, ok: bool, detail: str = "") -> bool:
|
||||
_CHECKS.append((label, ok, detail))
|
||||
print(f" [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}")
|
||||
return ok
|
||||
|
||||
|
||||
def _hr(title: str) -> None:
|
||||
print(f"\n{'=' * 70}\n{title}\n{'=' * 70}")
|
||||
|
||||
|
||||
def _near(actual: float | None, expected: float, tol: float = 0.05) -> bool:
|
||||
return actual is not None and abs(actual - expected) <= tol
|
||||
|
||||
|
||||
def _iso(s: str | None) -> datetime | None:
|
||||
if not s:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(s.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
# Ground truth: world-famous places whose facts don't drift. Name-only URLs
|
||||
# exercise the HTML -> fid resolution path for every one of them.
|
||||
_PLACES = [
|
||||
{
|
||||
"label": "Eiffel Tower (FR landmark)",
|
||||
"url": "https://www.google.com/maps/place/Eiffel+Tower/",
|
||||
"title_contains": "eiffel",
|
||||
"lat": 48.8584,
|
||||
"lng": 2.2945,
|
||||
"address_contains": ["paris", "75007", "france"],
|
||||
"website_contains": "toureiffel",
|
||||
},
|
||||
{
|
||||
"label": "Tokyo Tower (JP, non-Latin locale)",
|
||||
"url": "https://www.google.com/maps/place/Tokyo+Tower/",
|
||||
"title_contains": "tokyo tower",
|
||||
"lat": 35.6586,
|
||||
"lng": 139.7454,
|
||||
"address_contains": ["tokyo", "japan", "minato"],
|
||||
},
|
||||
{
|
||||
"label": "Sydney Opera House (AU)",
|
||||
"url": "https://www.google.com/maps/place/Sydney+Opera+House/",
|
||||
"title_contains": "opera house",
|
||||
"lat": -33.8568,
|
||||
"lng": 151.2153,
|
||||
"address_contains": ["sydney", "nsw", "australia"],
|
||||
},
|
||||
{
|
||||
"label": "The Plaza Hotel (US hotel category)",
|
||||
"url": "https://www.google.com/maps/place/The+Plaza+Hotel+New+York/",
|
||||
"title_contains": "plaza",
|
||||
"lat": 40.7646,
|
||||
"lng": -73.9744,
|
||||
"address_contains": ["new york", "ny"],
|
||||
"category_contains": "hotel",
|
||||
},
|
||||
]
|
||||
|
||||
_KIMS_CID_URL = "https://maps.google.com/?cid=7838756667406262025" # Kim's Island
|
||||
_KIMS_PLACE_ID = "ChIJJQz5EZzKw4kRCZ95UajbyGw"
|
||||
_LOUVRE_URL = "https://www.google.com/maps/place/Louvre+Museum/"
|
||||
|
||||
|
||||
async def scrape_one(url: str, **kwargs) -> dict | None:
|
||||
items = await scrape_places(
|
||||
GoogleMapsScrapeInput(startUrls=[{"url": url}], **kwargs)
|
||||
)
|
||||
return items[0] if items else None
|
||||
|
||||
|
||||
async def step_diverse_places() -> None:
|
||||
_hr("A — diverse places via name-only URLs (HTML -> fid path)")
|
||||
for spec in _PLACES:
|
||||
it = await scrape_one(spec["url"])
|
||||
if it is None:
|
||||
_check(spec["label"], False, "no item returned")
|
||||
continue
|
||||
title = (it.get("title") or "").lower()
|
||||
loc = it.get("location") or {}
|
||||
addr = (it.get("address") or "").lower()
|
||||
problems = []
|
||||
if spec["title_contains"] not in title:
|
||||
problems.append(f"title={it.get('title')!r}")
|
||||
if not _near(loc.get("lat"), spec["lat"]) or not _near(
|
||||
loc.get("lng"), spec["lng"]
|
||||
):
|
||||
problems.append(f"loc={loc}")
|
||||
if not any(k in addr for k in spec["address_contains"]):
|
||||
problems.append(f"address={it.get('address')!r}")
|
||||
if not it.get("placeId"):
|
||||
problems.append("no placeId")
|
||||
if not it.get("categories"):
|
||||
problems.append("no categories")
|
||||
if spec.get("website_contains") and spec["website_contains"] not in (
|
||||
it.get("website") or ""
|
||||
):
|
||||
problems.append(f"website={it.get('website')!r}")
|
||||
if (
|
||||
spec.get("category_contains")
|
||||
and spec["category_contains"]
|
||||
not in (
|
||||
(it.get("categoryName") or "") + " ".join(it.get("categories") or [])
|
||||
).lower()
|
||||
):
|
||||
problems.append(f"categories={it.get('categories')}")
|
||||
_check(
|
||||
spec["label"],
|
||||
not problems,
|
||||
"; ".join(problems)
|
||||
or f"{it.get('title')!r} @ ({loc.get('lat'):.4f},{loc.get('lng'):.4f}) "
|
||||
f"cat={it.get('categoryName')!r} score={it.get('totalScore')}",
|
||||
)
|
||||
|
||||
|
||||
async def step_cid_url() -> None:
|
||||
_hr("B — CID URL dispatch")
|
||||
it = await scrape_one(_KIMS_CID_URL)
|
||||
ok = it is not None and it.get("title") == "Kim's Island"
|
||||
_check(
|
||||
"?cid=... resolves to the right place",
|
||||
ok,
|
||||
f"title={it.get('title')!r}" if it else "no item",
|
||||
)
|
||||
if it:
|
||||
_check(
|
||||
"cid place has full detail (phone+hours)",
|
||||
bool(it.get("phone")) and bool(it.get("openingHours")),
|
||||
f"phone={it.get('phone')!r}, hours={len(it.get('openingHours') or [])} days",
|
||||
)
|
||||
|
||||
|
||||
async def step_review_sorts() -> None:
|
||||
_hr("C — review sort semantics (Kim's Island)")
|
||||
newest = await scrape_reviews(
|
||||
GoogleMapsReviewsInput(placeIds=[_KIMS_PLACE_ID], maxReviews=10)
|
||||
)
|
||||
dates = [_iso(r.get("publishedAtDate")) for r in newest]
|
||||
dated = [d for d in dates if d]
|
||||
_check(
|
||||
"newest: publishedAtDate non-increasing",
|
||||
len(dated) >= 5 and all(a >= b for a, b in itertools.pairwise(dated)),
|
||||
f"{len(newest)} reviews, first={newest[0].get('publishAt') if newest else None}",
|
||||
)
|
||||
|
||||
lowest = await scrape_reviews(
|
||||
GoogleMapsReviewsInput(
|
||||
placeIds=[_KIMS_PLACE_ID], maxReviews=10, reviewsSort="lowestRanking"
|
||||
)
|
||||
)
|
||||
highest = await scrape_reviews(
|
||||
GoogleMapsReviewsInput(
|
||||
placeIds=[_KIMS_PLACE_ID], maxReviews=10, reviewsSort="highestRanking"
|
||||
)
|
||||
)
|
||||
lo = [r["stars"] for r in lowest if r.get("stars")]
|
||||
hi = [r["stars"] for r in highest if r.get("stars")]
|
||||
lo_avg = sum(lo) / len(lo) if lo else 0
|
||||
hi_avg = sum(hi) / len(hi) if hi else 0
|
||||
_check(
|
||||
"lowestRanking avg < highestRanking avg",
|
||||
bool(lo and hi) and lo_avg < hi_avg,
|
||||
f"lowest avg={lo_avg:.2f} (first={lo[:3]}), highest avg={hi_avg:.2f} (first={hi[:3]})",
|
||||
)
|
||||
_check(
|
||||
"highestRanking page is all 5 stars",
|
||||
bool(hi) and all(s == 5 for s in hi),
|
||||
f"stars={hi}",
|
||||
)
|
||||
|
||||
|
||||
async def step_start_date_cutoff() -> None:
|
||||
_hr("D — reviewsStartDate cutoff")
|
||||
baseline = await scrape_reviews(
|
||||
GoogleMapsReviewsInput(placeIds=[_KIMS_PLACE_ID], maxReviews=10)
|
||||
)
|
||||
dated = [(_iso(r.get("publishedAtDate")), r) for r in baseline]
|
||||
dated = [(d, r) for d, r in dated if d]
|
||||
if len(dated) < 6:
|
||||
_check("cutoff test has enough dated reviews", False, f"only {len(dated)}")
|
||||
return
|
||||
# Cut between the 4th and 5th newest review -> expect exactly 4 back.
|
||||
cutoff_dt = dated[4][0]
|
||||
cutoff = (
|
||||
dated[3][0].strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
if dated[3][0] == cutoff_dt
|
||||
else cutoff_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
)
|
||||
got = await scrape_reviews(
|
||||
GoogleMapsReviewsInput(
|
||||
placeIds=[_KIMS_PLACE_ID], maxReviews=100, reviewsStartDate=cutoff
|
||||
)
|
||||
)
|
||||
got_dates = [_iso(r.get("publishedAtDate")) for r in got]
|
||||
cutoff_parsed = _iso(cutoff)
|
||||
_check(
|
||||
"all returned reviews >= cutoff",
|
||||
bool(got) and all(d is None or d >= cutoff_parsed for d in got_dates),
|
||||
f"cutoff={cutoff}, returned={len(got)} (expected ~4)",
|
||||
)
|
||||
_check(
|
||||
"cutoff actually limits the result",
|
||||
0 < len(got) < len(baseline) + 1 and len(got) <= 6,
|
||||
f"{len(got)} vs baseline {len(baseline)}",
|
||||
)
|
||||
|
||||
|
||||
async def step_personal_data() -> None:
|
||||
_hr("E — personalData=false stripping")
|
||||
items = await scrape_reviews(
|
||||
GoogleMapsReviewsInput(
|
||||
placeIds=[_KIMS_PLACE_ID], maxReviews=3, personalData=False
|
||||
)
|
||||
)
|
||||
if not items:
|
||||
_check("reviews returned", False)
|
||||
return
|
||||
leaked = [
|
||||
k
|
||||
for k in ("name", "reviewerId", "reviewerUrl", "reviewerPhotoUrl")
|
||||
if any(r.get(k) for r in items)
|
||||
]
|
||||
_check(
|
||||
"reviewer identity fields are absent",
|
||||
not leaked,
|
||||
f"leaked={leaked}" if leaked else f"{len(items)} reviews, ids+stars kept",
|
||||
)
|
||||
_check(
|
||||
"non-personal fields survive",
|
||||
all(r.get("reviewId") and r.get("stars") for r in items),
|
||||
)
|
||||
|
||||
|
||||
async def step_localization() -> None:
|
||||
_hr("F — localization (language=fr)")
|
||||
items = await scrape_reviews(
|
||||
GoogleMapsReviewsInput(
|
||||
startUrls=[{"url": "https://www.google.com/maps/place/Eiffel+Tower/"}],
|
||||
maxReviews=5,
|
||||
language="fr",
|
||||
)
|
||||
)
|
||||
if not items:
|
||||
_check("french reviews returned", False)
|
||||
return
|
||||
rel = [r.get("publishAt") or "" for r in items]
|
||||
french = [s for s in rel if "il y a" in s or "mois" in s or "semaine" in s]
|
||||
_check(
|
||||
"relative dates come back in French",
|
||||
len(french) >= 3,
|
||||
f"publishAt={rel}",
|
||||
)
|
||||
_check(
|
||||
"items stamped language=fr",
|
||||
all(r.get("language") == "fr" for r in items),
|
||||
)
|
||||
|
||||
|
||||
async def step_big_place_pagination() -> None:
|
||||
_hr("G — big place, 30 reviews across >=3 pages (Louvre)")
|
||||
items = await scrape_reviews(
|
||||
GoogleMapsReviewsInput(startUrls=[{"url": _LOUVRE_URL}], maxReviews=30)
|
||||
)
|
||||
ids = [r.get("reviewId") for r in items]
|
||||
_check(
|
||||
"30 reviews with unique IDs",
|
||||
len(items) == 30 and len(set(ids)) == 30,
|
||||
f"{len(items)} reviews, {len(set(ids))} unique",
|
||||
)
|
||||
ok_fields = all(
|
||||
r.get("name") and r.get("stars") is not None and r.get("publishedAtDate")
|
||||
for r in items
|
||||
)
|
||||
_check("every review has author/stars/date", ok_fields)
|
||||
_check(
|
||||
"place header stamped on all (Louvre)",
|
||||
all("louvre" in (r.get("title") or "").lower() for r in items),
|
||||
f"title={items[0].get('title')!r}" if items else "",
|
||||
)
|
||||
|
||||
|
||||
async def step_search_discovery() -> None:
|
||||
_hr("I — search discovery: paging, rank, dedupe (pizza in New York)")
|
||||
items = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
searchStringsArray=["pizza"],
|
||||
locationQuery="New York, NY",
|
||||
maxCrawledPlacesPerSearch=25,
|
||||
)
|
||||
)
|
||||
fids = [i.get("fid") for i in items]
|
||||
_check(
|
||||
"25 places across >1 page, all unique fids",
|
||||
len(items) == 25 and len(set(fids)) == 25,
|
||||
f"{len(items)} items, {len(set(fids))} unique",
|
||||
)
|
||||
_check(
|
||||
"ranks are 1..25 in order",
|
||||
[i.get("rank") for i in items] == list(range(1, 26)),
|
||||
)
|
||||
ny = sum(
|
||||
1
|
||||
for i in items
|
||||
if i.get("location")
|
||||
and 40.4 < (i["location"]["lat"] or 0) < 41.1
|
||||
and -74.3 < (i["location"]["lng"] or 0) < -73.6
|
||||
)
|
||||
_check(
|
||||
"locationQuery scopes results to NYC",
|
||||
ny >= 23,
|
||||
f"{ny}/25 within NYC bounds",
|
||||
)
|
||||
with_core = sum(
|
||||
1 for i in items if i.get("title") and i.get("placeId") and i.get("address")
|
||||
)
|
||||
_check("all items have title/placeId/address", with_core == 25, f"{with_core}/25")
|
||||
|
||||
|
||||
async def step_search_filters() -> None:
|
||||
_hr("J — search filters (stars, website, matching)")
|
||||
starred = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
searchStringsArray=["restaurant"],
|
||||
locationQuery="Chicago, IL",
|
||||
maxCrawledPlacesPerSearch=10,
|
||||
placeMinimumStars="fourAndHalf",
|
||||
)
|
||||
)
|
||||
scores = [i.get("totalScore") for i in starred]
|
||||
_check(
|
||||
"placeMinimumStars=fourAndHalf holds",
|
||||
bool(scores) and all(s is not None and s >= 4.5 for s in scores),
|
||||
f"scores={scores}",
|
||||
)
|
||||
|
||||
no_web = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
searchStringsArray=["restaurant"],
|
||||
locationQuery="Chicago, IL",
|
||||
maxCrawledPlacesPerSearch=5,
|
||||
website="withoutWebsite",
|
||||
)
|
||||
)
|
||||
_check(
|
||||
"website=withoutWebsite holds",
|
||||
bool(no_web) and all(not i.get("website") for i in no_web),
|
||||
f"{len(no_web)} items, websites={[i.get('website') for i in no_web]}",
|
||||
)
|
||||
|
||||
matching = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
searchStringsArray=["pizza"],
|
||||
locationQuery="Boston, MA",
|
||||
maxCrawledPlacesPerSearch=5,
|
||||
searchMatching="only_includes",
|
||||
)
|
||||
)
|
||||
titles = [i.get("title") for i in matching]
|
||||
_check(
|
||||
"searchMatching=only_includes keeps 'pizza' in titles",
|
||||
bool(titles) and all("pizza" in (t or "").lower() for t in titles),
|
||||
f"titles={titles}",
|
||||
)
|
||||
|
||||
|
||||
async def step_search_closed() -> None:
|
||||
_hr("K — closed-place detection + skipClosedPlaces (Dean & DeLuca)")
|
||||
q = "Dean DeLuca 560 Broadway New York"
|
||||
found = await scrape_places(
|
||||
GoogleMapsScrapeInput(searchStringsArray=[q], maxCrawledPlacesPerSearch=3)
|
||||
)
|
||||
dean = next((i for i in found if "DeLuca" in (i.get("title") or "")), None)
|
||||
_check(
|
||||
"permanently closed place flagged",
|
||||
dean is not None and dean.get("permanentlyClosed") is True,
|
||||
f"title={dean.get('title') if dean else None}, closed={dean.get('permanentlyClosed') if dean else None}",
|
||||
)
|
||||
skipped = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
searchStringsArray=[q],
|
||||
maxCrawledPlacesPerSearch=3,
|
||||
skipClosedPlaces=True,
|
||||
)
|
||||
)
|
||||
_check(
|
||||
"skipClosedPlaces filters it out",
|
||||
not any("DeLuca" in (i.get("title") or "") for i in skipped),
|
||||
f"{len(skipped)} items after skip",
|
||||
)
|
||||
|
||||
|
||||
async def step_search_url_and_geo() -> None:
|
||||
_hr("L — /maps/search/ URL routing + customGeolocation")
|
||||
via_url = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
startUrls=[
|
||||
{"url": "https://www.google.com/maps/search/ramen+in+Osaka+Japan/"}
|
||||
],
|
||||
maxCrawledPlacesPerSearch=5,
|
||||
)
|
||||
)
|
||||
osaka = sum(
|
||||
1
|
||||
for i in via_url
|
||||
if i.get("location") and 34.4 < (i["location"]["lat"] or 0) < 35.0
|
||||
)
|
||||
_check(
|
||||
"/maps/search/ startUrl yields Osaka ramen places",
|
||||
len(via_url) == 5 and osaka >= 4,
|
||||
f"{len(via_url)} items, {osaka} in Osaka",
|
||||
)
|
||||
|
||||
geo = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
searchStringsArray=["museum"],
|
||||
customGeolocation={
|
||||
"type": "Point",
|
||||
"coordinates": [2.3522, 48.8566], # [lng, lat] Paris
|
||||
"radiusKm": 10,
|
||||
},
|
||||
maxCrawledPlacesPerSearch=5,
|
||||
)
|
||||
)
|
||||
paris = sum(
|
||||
1
|
||||
for i in geo
|
||||
if i.get("location")
|
||||
and 48.5 < (i["location"]["lat"] or 0) < 49.2
|
||||
and 1.9 < (i["location"]["lng"] or 0) < 2.8
|
||||
)
|
||||
_check(
|
||||
"customGeolocation Point scopes to Paris",
|
||||
len(geo) >= 3 and paris >= 3,
|
||||
f"{len(geo)} items, {paris} in Paris: {[i.get('title') for i in geo]}",
|
||||
)
|
||||
|
||||
|
||||
async def step_search_pagination_stress() -> None:
|
||||
_hr("M — search pagination stress (60 results / 3+ pages, exhaustion)")
|
||||
items = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
searchStringsArray=["restaurant"],
|
||||
locationQuery="Manhattan, New York",
|
||||
maxCrawledPlacesPerSearch=60,
|
||||
)
|
||||
)
|
||||
fids = [i.get("fid") for i in items]
|
||||
_check(
|
||||
"60 places, all fids unique (offset paging + dedupe)",
|
||||
len(items) == 60 and len(set(fids)) == 60,
|
||||
f"{len(items)} items, {len(set(fids))} unique",
|
||||
)
|
||||
_check(
|
||||
"ranks strictly sequential 1..60",
|
||||
[i.get("rank") for i in items] == list(range(1, 61)),
|
||||
)
|
||||
manhattan = sum(
|
||||
1
|
||||
for i in items
|
||||
if i.get("location") and 40.68 < (i["location"]["lat"] or 0) < 40.9
|
||||
)
|
||||
_check(
|
||||
"results stay in Manhattan across pages",
|
||||
manhattan >= 55,
|
||||
f"{manhattan}/60 in bounds",
|
||||
)
|
||||
|
||||
# A hyper-specific query has few results: paging must terminate on its
|
||||
# own (no infinite loop, no error) well before the requested cap.
|
||||
sparse = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
searchStringsArray=["Kim's Island Staten Island"],
|
||||
maxCrawledPlacesPerSearch=100,
|
||||
)
|
||||
)
|
||||
_check(
|
||||
"sparse query exhausts naturally below the cap",
|
||||
0 < len(sparse) < 25,
|
||||
f"{len(sparse)} results",
|
||||
)
|
||||
|
||||
|
||||
async def step_detail_extras() -> None:
|
||||
_hr("N — detail-page extras (kgmid/cid/additionalInfo/links)")
|
||||
# pin the exact place by fid — a name search returns a different Joe's
|
||||
# branch run to run, which makes magnitude assertions flaky
|
||||
items = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
startUrls=[
|
||||
{
|
||||
"url": "https://www.google.com/maps/place/Joe's+Pizza+Broadway/"
|
||||
"data=!4m2!3m1!1s0x89c259ab3c1ef289:0x3b67a41175949f55"
|
||||
}
|
||||
],
|
||||
maxImages=5,
|
||||
)
|
||||
)
|
||||
if not items:
|
||||
_check("place returned", False)
|
||||
return
|
||||
it = items[0]
|
||||
dist = it.get("reviewsDistribution") or {}
|
||||
_check(
|
||||
"reviewsCount + distribution (NID-session fields)",
|
||||
(it.get("reviewsCount") or 0) > 20_000
|
||||
and (it.get("reviewsCount") == sum(dist.values())),
|
||||
f"count={it.get('reviewsCount')}, dist={dist}",
|
||||
)
|
||||
hist = it.get("popularTimesHistogram") or {}
|
||||
_check(
|
||||
"popular times histogram covers the week",
|
||||
set(hist) == {"Su", "Mo", "Tu", "We", "Th", "Fr", "Sa"}
|
||||
and all(
|
||||
{"hour", "occupancyPercent"} <= set(slot)
|
||||
for d in hist.values()
|
||||
for slot in d
|
||||
),
|
||||
f"days={sorted(hist)}",
|
||||
)
|
||||
_check(
|
||||
"image gallery fields (count/categories/urls capped at maxImages)",
|
||||
(it.get("imagesCount") or 0) > 1000
|
||||
and "All" in (it.get("imageCategories") or [])
|
||||
and len(it.get("imageUrls") or []) == 5,
|
||||
f"count={it.get('imagesCount')}, cats={len(it.get('imageCategories') or [])}, "
|
||||
f"urls={len(it.get('imageUrls') or [])}",
|
||||
)
|
||||
tags = it.get("reviewsTags") or []
|
||||
_check(
|
||||
"reviewsTags with counts",
|
||||
len(tags) >= 5 and all(t.get("title") and t.get("count") for t in tags),
|
||||
f"{tags[:2]}",
|
||||
)
|
||||
_check(
|
||||
"additionalInfo has full section set (not just Accessibility)",
|
||||
len(it.get("additionalInfo") or {}) >= 8,
|
||||
f"sections={list((it.get('additionalInfo') or {}).keys())}",
|
||||
)
|
||||
|
||||
# scrapePlaceDetailPage on the SEARCH flow: search darrays lack the
|
||||
# session-gated fields, so each hit must get enriched via a detail RPC.
|
||||
enriched = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
searchStringsArray=["ramen"],
|
||||
locationQuery="Chicago, IL",
|
||||
maxCrawledPlacesPerSearch=2,
|
||||
scrapePlaceDetailPage=True,
|
||||
)
|
||||
)
|
||||
_check(
|
||||
"search flow + scrapePlaceDetailPage enriches every hit",
|
||||
len(enriched) == 2
|
||||
and all(
|
||||
(e.get("reviewsCount") or 0) > 0 and e.get("reviewsDistribution")
|
||||
for e in enriched
|
||||
),
|
||||
f"counts={[e.get('reviewsCount') for e in enriched]}",
|
||||
)
|
||||
fid = it.get("fid") or ""
|
||||
cid_ok = bool(fid) and it.get("cid") == str(int(fid.split(":")[1], 16))
|
||||
_check(
|
||||
"kgmid + cid derived from fid",
|
||||
bool(it.get("kgmid", "").startswith("/g/")) and cid_ok,
|
||||
f"kgmid={it.get('kgmid')}, cid={it.get('cid')}",
|
||||
)
|
||||
info = it.get("additionalInfo") or {}
|
||||
_check(
|
||||
"additionalInfo has sections with boolean options",
|
||||
bool(info)
|
||||
and all(
|
||||
isinstance(v, list) and all(isinstance(e, dict) for e in v)
|
||||
for v in info.values()
|
||||
),
|
||||
f"sections={list(info.keys())}",
|
||||
)
|
||||
|
||||
|
||||
async def step_hotel_fields() -> None:
|
||||
_hr("O — hotel fields (The Plaza: stars, dates, similar, ads)")
|
||||
items = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
startUrls=[
|
||||
{
|
||||
"url": "https://www.google.com/maps/place/The+Plaza/"
|
||||
"data=!4m2!3m1!1s0x89c258f07d5da561:0x61f6aa300ba8339d"
|
||||
}
|
||||
],
|
||||
)
|
||||
)
|
||||
if not items:
|
||||
_check("hotel returned", False)
|
||||
return
|
||||
it = items[0]
|
||||
_check(
|
||||
"hotelStars + check-in/out dates",
|
||||
it.get("hotelStars") == "5 stars"
|
||||
and (it.get("checkInDate") or "") < (it.get("checkOutDate") or ""),
|
||||
f"stars={it.get('hotelStars')}, {it.get('checkInDate')}..{it.get('checkOutDate')}",
|
||||
)
|
||||
similar = it.get("similarHotelsNearby") or []
|
||||
_check(
|
||||
"similarHotelsNearby with fid/score",
|
||||
len(similar) >= 3 and all(h.get("title") and h.get("fid") for h in similar),
|
||||
f"{len(similar)} hotels, first={similar[0].get('title') if similar else None}",
|
||||
)
|
||||
ads = it.get("hotelAds") or []
|
||||
_check(
|
||||
"hotelAds with booking links",
|
||||
bool(ads) and all(a.get("url", "").startswith("https://") for a in ads),
|
||||
f"{len(ads)} ads",
|
||||
)
|
||||
|
||||
|
||||
async def step_all_places_scan() -> None:
|
||||
_hr("P — allPlacesNoSearchAction area scan (Times Square 400m)")
|
||||
items = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
allPlacesNoSearchAction="all_places_no_search_mouse",
|
||||
customGeolocation={
|
||||
"type": "Point",
|
||||
"coordinates": [-73.9855, 40.758],
|
||||
"radiusKm": 0.4,
|
||||
},
|
||||
maxCrawledPlacesPerSearch=25,
|
||||
)
|
||||
)
|
||||
fids = [i.get("fid") for i in items]
|
||||
_check(
|
||||
"25 unique places without any search term",
|
||||
len(items) == 25 and len(set(fids)) == 25,
|
||||
f"{len(items)} items",
|
||||
)
|
||||
cats = {i.get("categoryName") for i in items if i.get("categoryName")}
|
||||
_check(
|
||||
"multiple categories represented (sweep, not one query)",
|
||||
len(cats) >= 5,
|
||||
f"{len(cats)} categories",
|
||||
)
|
||||
in_view = sum(
|
||||
1 for i in items if i.get("location") and 40.74 < i["location"]["lat"] < 40.78
|
||||
)
|
||||
_check("scan respects the viewport", in_view >= 23, f"{in_view}/25 in bounds")
|
||||
|
||||
|
||||
async def step_short_url() -> None:
|
||||
_hr("Q — short link (maps.app.goo.gl Firebase redirect -> place)")
|
||||
# A real shared short link -> "Aux Merveilleux de Fred" (NYC). These are
|
||||
# Firebase Dynamic Links (JS interstitial), so this exercises the browser-
|
||||
# render redirect path in resolve_fid. fid is the ground-truth invariant.
|
||||
it = await scrape_one("https://maps.app.goo.gl/8YUvDPbQPrasqC528")
|
||||
_check(
|
||||
"maps.app.goo.gl short link resolves to the right place",
|
||||
it is not None
|
||||
and it.get("fid") == "0x89c259957da502cd:0xed3eb58a4ca08a95"
|
||||
and "merveilleux" in (it.get("title") or "").lower(),
|
||||
f"title={it.get('title') if it else None!r}, fid={it.get('fid') if it else None}",
|
||||
)
|
||||
|
||||
|
||||
async def step_filter_variants() -> None:
|
||||
_hr("R — search filter variants (only_exact, categoryFilterWords)")
|
||||
# only_exact: the parsed title must equal the query exactly (Seattle
|
||||
# Starbucks are titled "Starbucks Coffee Company", not "Starbucks").
|
||||
exact = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
searchStringsArray=["Starbucks Coffee Company"],
|
||||
locationQuery="Seattle, WA",
|
||||
maxCrawledPlacesPerSearch=8,
|
||||
searchMatching="only_exact",
|
||||
)
|
||||
)
|
||||
titles = [i.get("title") for i in exact]
|
||||
_check(
|
||||
"searchMatching=only_exact keeps only exact-title matches",
|
||||
bool(titles)
|
||||
and all((t or "").lower() == "starbucks coffee company" for t in titles),
|
||||
f"{len(titles)} items, titles={titles[:3]}",
|
||||
)
|
||||
# categoryFilterWords: drop places whose categories don't include a word.
|
||||
coffee = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
searchStringsArray=["food"],
|
||||
locationQuery="Seattle, WA",
|
||||
maxCrawledPlacesPerSearch=6,
|
||||
categoryFilterWords=["coffee"],
|
||||
)
|
||||
)
|
||||
_check(
|
||||
"categoryFilterWords keeps only matching categories",
|
||||
bool(coffee)
|
||||
and all(
|
||||
any("coffee" in c.lower() for c in (i.get("categories") or []))
|
||||
for i in coffee
|
||||
),
|
||||
f"{len(coffee)} items, cats={[i.get('categories') for i in coffee][:2]}",
|
||||
)
|
||||
|
||||
|
||||
async def step_reviews_origin() -> None:
|
||||
_hr("S — reviewsOrigin=google filter")
|
||||
# The public BOQ feed only carries Google-origin reviews (partner reviews
|
||||
# aren't exposed anonymously), so the invariant is: nothing non-Google
|
||||
# leaks through when origin is pinned to google.
|
||||
items = await scrape_reviews(
|
||||
GoogleMapsReviewsInput(
|
||||
placeIds=[_KIMS_PLACE_ID], maxReviews=10, reviewsOrigin="google"
|
||||
)
|
||||
)
|
||||
origins = {(r.get("reviewOrigin") or "Google") for r in items}
|
||||
_check(
|
||||
"reviewsOrigin=google -> only Google-origin reviews",
|
||||
bool(items) and origins <= {"Google"},
|
||||
f"{len(items)} reviews, origins={origins}",
|
||||
)
|
||||
|
||||
|
||||
async def step_inline_consistency() -> None:
|
||||
_hr("H — inline reviews[] match the standalone reviews endpoint")
|
||||
place = await scrape_one(
|
||||
f"https://www.google.com/maps/place/?q=place_id:{_KIMS_PLACE_ID}",
|
||||
maxReviews=5,
|
||||
)
|
||||
standalone = await scrape_reviews(
|
||||
GoogleMapsReviewsInput(placeIds=[_KIMS_PLACE_ID], maxReviews=5)
|
||||
)
|
||||
if not place or not standalone:
|
||||
_check("both sources returned data", False)
|
||||
return
|
||||
inline_ids = {r.get("reviewId") for r in (place.get("reviews") or [])}
|
||||
standalone_ids = {r.get("reviewId") for r in standalone}
|
||||
overlap = len(inline_ids & standalone_ids)
|
||||
_check(
|
||||
"inline and standalone reviews overlap (same feed)",
|
||||
overlap >= 3, # feed ordering can shift slightly between calls
|
||||
f"{overlap}/5 shared review IDs",
|
||||
)
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
steps = [
|
||||
step_diverse_places(),
|
||||
step_cid_url(),
|
||||
step_review_sorts(),
|
||||
step_start_date_cutoff(),
|
||||
step_personal_data(),
|
||||
step_localization(),
|
||||
step_big_place_pagination(),
|
||||
step_inline_consistency(),
|
||||
step_search_discovery(),
|
||||
step_search_filters(),
|
||||
step_search_closed(),
|
||||
step_search_url_and_geo(),
|
||||
step_search_pagination_stress(),
|
||||
step_detail_extras(),
|
||||
step_hotel_fields(),
|
||||
step_all_places_scan(),
|
||||
step_short_url(),
|
||||
step_filter_variants(),
|
||||
step_reviews_origin(),
|
||||
]
|
||||
for coro in steps:
|
||||
try:
|
||||
await coro
|
||||
except Exception as e: # keep going; report the step as failed
|
||||
_check(f"step crashed: {coro}", False, repr(e))
|
||||
|
||||
_hr("SUMMARY")
|
||||
passed = sum(1 for _, ok, _ in _CHECKS if ok)
|
||||
for label, ok, detail in _CHECKS:
|
||||
if not ok:
|
||||
print(f" FAILED: {label} — {detail}")
|
||||
print(f" {passed}/{len(_CHECKS)} checks passed")
|
||||
return 0 if passed == len(_CHECKS) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
241
surfsense_backend/scripts/e2e_google_maps_scraper.py
Normal file
241
surfsense_backend/scripts/e2e_google_maps_scraper.py
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
"""Manual functional e2e for the Google Maps scraper (app/proprietary/scrapers/google_maps).
|
||||
|
||||
Run from the backend directory:
|
||||
cd surfsense_backend
|
||||
uv run python scripts/e2e_google_maps_scraper.py
|
||||
# or: .\\.venv\\Scripts\\python.exe scripts/e2e_google_maps_scraper.py
|
||||
|
||||
NOT a pytest test (needs live network + optional proxy creds). It:
|
||||
Step 1 — scrapes a known place URL and prints the core fields.
|
||||
Step 2 — scrapes the same place by bare placeId (HTML -> fid -> RPC path).
|
||||
Step 3 — dumps the raw place darray (jd[6] of /maps/preview/place) to
|
||||
tests/unit/scrapers/google_maps/fixtures/ for the offline parser test.
|
||||
Step 4 — scrapes reviews via the Reviews endpoint (BOQ feed), checks fields.
|
||||
Step 5 — paginates past one page (maxReviews=15) and checks the count.
|
||||
Step 6 — place scrape with maxReviews>0 attaches inline reviews[].
|
||||
Step 7 — dumps a raw BOQ reviews page fixture for the offline parser test.
|
||||
Step 8 — search discovery (searchStringsArray + locationQuery), checks items.
|
||||
Step 9 — dumps a raw map-search response fixture for the offline parser test.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Windows consoles default to cp1252; reviews/ratings contain non-latin chars.
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
|
||||
if _candidate.exists():
|
||||
load_dotenv(_candidate)
|
||||
break
|
||||
|
||||
from app.proprietary.scrapers.google_maps import ( # noqa: E402
|
||||
GoogleMapsReviewsInput,
|
||||
GoogleMapsScrapeInput,
|
||||
scrape_places,
|
||||
scrape_reviews,
|
||||
)
|
||||
from app.proprietary.scrapers.google_maps.fetch import ( # noqa: E402
|
||||
build_search_url,
|
||||
fetch_place_darray,
|
||||
fetch_rpc_json,
|
||||
iter_reviews_pages,
|
||||
)
|
||||
from app.proprietary.scrapers.google_maps.url_resolver import extract_fid # noqa: E402
|
||||
|
||||
# A well-known, stable place (the restaurant used in the Apify output example).
|
||||
_PLACE_URL = (
|
||||
"https://www.google.com/maps/place/Kim's+Island/"
|
||||
"@40.5107736,-74.2482624,17z/data=!4m6!3m5!1s0x89c3ca9c11f90c25:"
|
||||
"0x6cc8dba851799f09!8m2!3d40.5107736!4d-74.2482624!16s%2Fg%2F1tmgdcj8?hl=en"
|
||||
)
|
||||
|
||||
_FIXTURE_DIR = (
|
||||
_BACKEND_ROOT / "tests" / "unit" / "scrapers" / "google_maps" / "fixtures"
|
||||
)
|
||||
|
||||
|
||||
def _hr(title: str) -> None:
|
||||
print(f"\n{'=' * 70}\n{title}\n{'=' * 70}")
|
||||
|
||||
|
||||
def _check(label: str, ok: bool, detail: str = "") -> bool:
|
||||
print(f" [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}")
|
||||
return ok
|
||||
|
||||
|
||||
async def step1_place() -> bool:
|
||||
_hr("STEP 1 — scrape a known place URL")
|
||||
items = await scrape_places(GoogleMapsScrapeInput(startUrls=[{"url": _PLACE_URL}]))
|
||||
if not items:
|
||||
return _check("place scraped", False, "no items returned")
|
||||
it = items[0]
|
||||
print(json.dumps(it, indent=2, ensure_ascii=False)[:2500])
|
||||
ok = bool(it.get("title")) and it.get("placeId") is not None
|
||||
return _check(
|
||||
"place has title + placeId",
|
||||
ok,
|
||||
f"{it.get('title')!r} / {it.get('totalScore')}★ / {it.get('reviewsCount')} reviews",
|
||||
)
|
||||
|
||||
|
||||
async def step2_place_id() -> bool:
|
||||
_hr("STEP 2 — scrape by bare placeId (HTML -> fid -> RPC path)")
|
||||
items = await scrape_places(
|
||||
GoogleMapsScrapeInput(placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"])
|
||||
)
|
||||
if not items:
|
||||
return _check("placeId scraped", False, "no items returned")
|
||||
it = items[0]
|
||||
return _check(
|
||||
"placeId resolves to same place",
|
||||
it.get("title") == "Kim's Island",
|
||||
f"{it.get('title')!r}",
|
||||
)
|
||||
|
||||
|
||||
async def step3_dump_fixture() -> bool:
|
||||
_hr("STEP 3 — dump raw place darray fixture for offline test")
|
||||
fid = extract_fid(_PLACE_URL)
|
||||
if not fid:
|
||||
return _check("extracted fid from URL", False)
|
||||
darray = await fetch_place_darray(fid)
|
||||
if not darray:
|
||||
return _check("fetched place darray via RPC", False)
|
||||
_FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(_FIXTURE_DIR / "place_darray.json").write_text(
|
||||
json.dumps(darray), encoding="utf-8"
|
||||
)
|
||||
return _check("dumped fixture", True, f"-> {_FIXTURE_DIR / 'place_darray.json'}")
|
||||
|
||||
|
||||
async def step4_reviews() -> bool:
|
||||
_hr("STEP 4 — reviews endpoint (one page, newest first)")
|
||||
items = await scrape_reviews(
|
||||
GoogleMapsReviewsInput(startUrls=[{"url": _PLACE_URL}], maxReviews=5)
|
||||
)
|
||||
if not items:
|
||||
return _check("reviews returned", False, "no items")
|
||||
it = items[0]
|
||||
print(json.dumps(it, indent=2, ensure_ascii=False)[:1800])
|
||||
ok = (
|
||||
bool(it.get("name"))
|
||||
and it.get("stars") is not None
|
||||
and bool(it.get("reviewId"))
|
||||
and it.get("title") == "Kim's Island" # place header stamped on
|
||||
and bool(it.get("publishedAtDate"))
|
||||
)
|
||||
return _check(
|
||||
"review has author/stars/id/place header",
|
||||
ok and len(items) == 5,
|
||||
f"{len(items)} reviews, first by {it.get('name')!r} ({it.get('stars')}★)",
|
||||
)
|
||||
|
||||
|
||||
async def step5_pagination() -> bool:
|
||||
_hr("STEP 5 — pagination past one page (maxReviews=15)")
|
||||
items = await scrape_reviews(
|
||||
GoogleMapsReviewsInput(placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"], maxReviews=15)
|
||||
)
|
||||
dates = [i.get("publishedAtDate") for i in items[:3]]
|
||||
return _check(
|
||||
"got 15 reviews across >1 page",
|
||||
len(items) == 15,
|
||||
f"{len(items)} reviews; newest: {dates}",
|
||||
)
|
||||
|
||||
|
||||
async def step6_inline_reviews() -> bool:
|
||||
_hr("STEP 6 — place scrape with maxReviews=3 attaches inline reviews[]")
|
||||
items = await scrape_places(
|
||||
GoogleMapsScrapeInput(startUrls=[{"url": _PLACE_URL}], maxReviews=3)
|
||||
)
|
||||
if not items:
|
||||
return _check("place scraped", False, "no items")
|
||||
reviews = items[0].get("reviews") or []
|
||||
return _check(
|
||||
"place item carries 3 inline reviews",
|
||||
len(reviews) == 3 and bool(reviews[0].get("name")),
|
||||
f"{len(reviews)} reviews inline",
|
||||
)
|
||||
|
||||
|
||||
async def step7_dump_reviews_fixture() -> bool:
|
||||
_hr("STEP 7 — dump raw BOQ reviews page fixture for offline test")
|
||||
fid = extract_fid(_PLACE_URL)
|
||||
async for raw_page in iter_reviews_pages(fid, sort="newest", max_pages=1):
|
||||
_FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(_FIXTURE_DIR / "boq_reviews_page.json").write_text(
|
||||
json.dumps(raw_page), encoding="utf-8"
|
||||
)
|
||||
return _check(
|
||||
"dumped fixture",
|
||||
len(raw_page) > 0,
|
||||
f"{len(raw_page)} reviews -> {_FIXTURE_DIR / 'boq_reviews_page.json'}",
|
||||
)
|
||||
return _check("fetched a reviews page", False)
|
||||
|
||||
|
||||
async def step8_search() -> bool:
|
||||
_hr("STEP 8 — search discovery (query + locationQuery)")
|
||||
items = await scrape_places(
|
||||
GoogleMapsScrapeInput(
|
||||
searchStringsArray=["coffee shop"],
|
||||
locationQuery="Seattle, WA",
|
||||
maxCrawledPlacesPerSearch=5,
|
||||
)
|
||||
)
|
||||
if not items:
|
||||
return _check("search returned items", False, "no items")
|
||||
it = items[0]
|
||||
print(json.dumps(it, indent=2, ensure_ascii=False)[:1500])
|
||||
ok = (
|
||||
len(items) == 5
|
||||
and all(i.get("title") and i.get("placeId") and i.get("fid") for i in items)
|
||||
and [i.get("rank") for i in items] == [1, 2, 3, 4, 5]
|
||||
and it.get("searchString") == "coffee shop"
|
||||
)
|
||||
return _check(
|
||||
"5 ranked places with title/placeId/fid",
|
||||
ok,
|
||||
f"first: {it.get('title')!r} ({it.get('totalScore')}★, {it.get('city')})",
|
||||
)
|
||||
|
||||
|
||||
async def step9_dump_search_fixture() -> bool:
|
||||
_hr("STEP 9 — dump raw map-search response fixture for offline test")
|
||||
url = build_search_url("pizza new york")
|
||||
jd = await fetch_rpc_json(url)
|
||||
if not isinstance(jd, list):
|
||||
return _check("fetched search response", False)
|
||||
_FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(_FIXTURE_DIR / "search_response.json").write_text(json.dumps(jd), encoding="utf-8")
|
||||
return _check("dumped fixture", True, f"-> {_FIXTURE_DIR / 'search_response.json'}")
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
results = [
|
||||
await step1_place(),
|
||||
await step2_place_id(),
|
||||
await step3_dump_fixture(),
|
||||
await step4_reviews(),
|
||||
await step5_pagination(),
|
||||
await step6_inline_reviews(),
|
||||
await step7_dump_reviews_fixture(),
|
||||
await step8_search(),
|
||||
await step9_dump_search_fixture(),
|
||||
]
|
||||
_hr("SUMMARY")
|
||||
print(f" {sum(results)}/{len(results)} steps passed")
|
||||
return 0 if all(results) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
|
|
@ -93,7 +93,7 @@ async def stage1_crawl_and_proxy() -> bool:
|
|||
try:
|
||||
direct = await AsyncFetcher.get(_IP_ECHO, impersonate="chrome", timeout=30)
|
||||
direct_ip = direct.json().get("ip")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
except Exception as exc:
|
||||
print(f" [INFO] direct IP fetch failed: {exc}")
|
||||
if proxy_url:
|
||||
try:
|
||||
|
|
@ -101,7 +101,7 @@ async def stage1_crawl_and_proxy() -> bool:
|
|||
_IP_ECHO, impersonate="chrome", proxy=proxy_url, timeout=45
|
||||
)
|
||||
proxied_ip = via.json().get("ip")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
except Exception as exc:
|
||||
print(f" [INFO] proxied IP fetch failed: {exc}")
|
||||
print(f" egress IP (direct) : {direct_ip}")
|
||||
print(f" egress IP (via proxy) : {proxied_ip}")
|
||||
|
|
@ -327,7 +327,7 @@ async def main() -> int:
|
|||
):
|
||||
try:
|
||||
results[name] = await coro()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
except Exception as exc:
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
|
|
|
|||
|
|
@ -125,9 +125,7 @@ async def step4_dump_fixtures() -> bool:
|
|||
(_FIXTURE_DIR / "video_initial_data.json").write_text(
|
||||
json.dumps(initial), encoding="utf-8"
|
||||
)
|
||||
return _check(
|
||||
"dumped fixtures", bool(player), f"-> {_FIXTURE_DIR}"
|
||||
)
|
||||
return _check("dumped fixtures", bool(player), f"-> {_FIXTURE_DIR}")
|
||||
|
||||
|
||||
async def step5_comments() -> bool:
|
||||
|
|
@ -137,11 +135,16 @@ async def step5_comments() -> bool:
|
|||
)
|
||||
items = await scrape_comments(inp)
|
||||
for it in items[:6]:
|
||||
print(f" - [{it.get('type')}] {it.get('author')} | {it.get('voteCount')} votes")
|
||||
print(
|
||||
f" - [{it.get('type')}] {it.get('author')} | {it.get('voteCount')} votes"
|
||||
)
|
||||
ok = bool(items) and all(it.get("cid") and it.get("videoId") for it in items)
|
||||
has_reply = any(it.get("type") == "reply" and it.get("replyToCid") for it in items)
|
||||
return _check("comments scraped (cid+videoId, reply linkage)", ok and has_reply,
|
||||
f"{len(items)} items")
|
||||
return _check(
|
||||
"comments scraped (cid+videoId, reply linkage)",
|
||||
ok and has_reply,
|
||||
f"{len(items)} items",
|
||||
)
|
||||
|
||||
|
||||
async def step6_location_collaborators() -> bool:
|
||||
|
|
@ -165,13 +168,17 @@ async def step6_location_collaborators() -> bool:
|
|||
async def step7_translation() -> bool:
|
||||
_hr("STEP 7 — translatedTitle/translatedText (subtitlesLanguage=es)")
|
||||
items = await scrape_youtube(
|
||||
YouTubeScrapeInput(startUrls=[{"url": _TRANSLATED_VIDEO_URL}], subtitlesLanguage="es")
|
||||
YouTubeScrapeInput(
|
||||
startUrls=[{"url": _TRANSLATED_VIDEO_URL}], subtitlesLanguage="es"
|
||||
)
|
||||
)
|
||||
it = items[0] if items else {}
|
||||
print(f" title : {it.get('title')}")
|
||||
print(f" translatedTitle: {it.get('translatedTitle')}")
|
||||
# A localized video's translated title differs from the canonical English one.
|
||||
ok = bool(it.get("translatedTitle")) and it.get("translatedTitle") != it.get("title")
|
||||
ok = bool(it.get("translatedTitle")) and it.get("translatedTitle") != it.get(
|
||||
"title"
|
||||
)
|
||||
return _check("translatedTitle differs from original", ok)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -46,9 +46,9 @@ from app.db import (
|
|||
NewChatMessage,
|
||||
NewChatMessageRole,
|
||||
NewChatThread,
|
||||
Workspace,
|
||||
TokenUsage,
|
||||
User,
|
||||
Workspace,
|
||||
)
|
||||
from app.routes import new_chat_routes
|
||||
from app.services.token_tracking_service import TurnTokenAccumulator
|
||||
|
|
|
|||
|
|
@ -48,8 +48,8 @@ from app.db import (
|
|||
NewChatMessage,
|
||||
NewChatMessageRole,
|
||||
NewChatThread,
|
||||
Workspace,
|
||||
User,
|
||||
Workspace,
|
||||
)
|
||||
from app.services.new_streaming_service import VercelStreamingService
|
||||
from app.tasks.chat import persistence as persistence_module
|
||||
|
|
|
|||
|
|
@ -38,9 +38,9 @@ from app.db import (
|
|||
NewChatMessage,
|
||||
NewChatMessageRole,
|
||||
NewChatThread,
|
||||
Workspace,
|
||||
TokenUsage,
|
||||
User,
|
||||
Workspace,
|
||||
)
|
||||
from app.services.token_tracking_service import TurnTokenAccumulator
|
||||
from app.tasks.chat import persistence as persistence_module
|
||||
|
|
|
|||
|
|
@ -19,10 +19,10 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||
from app.auth.context import AuthContext
|
||||
from app.db import (
|
||||
ChatVisibility,
|
||||
User,
|
||||
Workspace,
|
||||
WorkspaceMembership,
|
||||
WorkspaceRole,
|
||||
User,
|
||||
)
|
||||
from app.routes import new_chat_routes
|
||||
from app.schemas.new_chat import (
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ from app.config import config
|
|||
from app.db import (
|
||||
SearchSourceConnector,
|
||||
SearchSourceConnectorType,
|
||||
Workspace,
|
||||
User,
|
||||
Workspace,
|
||||
)
|
||||
from app.utils.oauth_security import OAuthStateManager
|
||||
|
||||
|
|
@ -57,8 +57,7 @@ async def test_callback_with_error_param_redirects_to_denied_page(
|
|||
assert response.status_code in {302, 303, 307}
|
||||
location = response.headers["location"]
|
||||
assert (
|
||||
f"/dashboard/{db_workspace.id}/connectors/callback?"
|
||||
"error=composio_oauth_denied"
|
||||
f"/dashboard/{db_workspace.id}/connectors/callback?error=composio_oauth_denied"
|
||||
) in location
|
||||
|
||||
connectors = await _drive_connectors(
|
||||
|
|
|
|||
|
|
@ -320,9 +320,7 @@ class TestDocumentSearchability:
|
|||
doc_ids = resp.json()["document_ids"]
|
||||
cleanup_doc_ids.extend(doc_ids)
|
||||
|
||||
await poll_document_status(
|
||||
client, headers, doc_ids, workspace_id=workspace_id
|
||||
)
|
||||
await poll_document_status(client, headers, doc_ids, workspace_id=workspace_id)
|
||||
|
||||
search_resp = await client.get(
|
||||
"/api/v1/documents/search",
|
||||
|
|
|
|||
|
|
@ -219,9 +219,7 @@ class TestDocumentProcessingNotification:
|
|||
doc_ids = resp.json()["document_ids"]
|
||||
cleanup_doc_ids.extend(doc_ids)
|
||||
|
||||
await poll_document_status(
|
||||
client, headers, doc_ids, workspace_id=workspace_id
|
||||
)
|
||||
await poll_document_status(client, headers, doc_ids, workspace_id=workspace_id)
|
||||
|
||||
notifications = await get_notifications(
|
||||
client,
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ from app.db import (
|
|||
DocumentType,
|
||||
SearchSourceConnector,
|
||||
SearchSourceConnectorType,
|
||||
Workspace,
|
||||
User,
|
||||
Workspace,
|
||||
)
|
||||
|
||||
EMBEDDING_DIM = app_config.embedding_model_instance.dimension
|
||||
|
|
@ -275,9 +275,7 @@ async def seed_connector(
|
|||
session.add(user)
|
||||
await session.flush()
|
||||
|
||||
space = Workspace(
|
||||
name=f"{name_prefix} {uuid.uuid4().hex[:6]}", user_id=user.id
|
||||
)
|
||||
space = Workspace(name=f"{name_prefix} {uuid.uuid4().hex[:6]}", user_id=user.id)
|
||||
session.add(space)
|
||||
await session.flush()
|
||||
space_id = space.id
|
||||
|
|
|
|||
|
|
@ -119,9 +119,7 @@ async def test_reindex_updates_content(db_session, db_workspace, db_user, mocker
|
|||
|
||||
|
||||
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
|
||||
async def test_reindex_updates_content_hash(
|
||||
db_session, db_workspace, db_user, mocker
|
||||
):
|
||||
async def test_reindex_updates_content_hash(db_session, db_workspace, db_user, mocker):
|
||||
"""Content hash is recomputed after reindexing with new source markdown."""
|
||||
adapter = UploadDocumentAdapter(db_session)
|
||||
await adapter.index(
|
||||
|
|
|
|||
|
|
@ -55,9 +55,7 @@ async def test_edit_keeps_unchanged_rows_and_embeds_only_the_new_text(
|
|||
patched_embed_texts,
|
||||
):
|
||||
service = IndexingPipelineService(session=db_session)
|
||||
doc_v1 = make_connector_document(
|
||||
workspace_id=db_workspace.id, source_markdown=_V1
|
||||
)
|
||||
doc_v1 = make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1)
|
||||
document = await _index(service, doc_v1)
|
||||
|
||||
ids_v1 = {c.content: c.id for c in await _load_chunks(db_session, document.id)}
|
||||
|
|
@ -100,9 +98,7 @@ async def test_head_insert_shifts_positions_without_new_rows_for_old_text(
|
|||
service = IndexingPipelineService(session=db_session)
|
||||
document = await _index(
|
||||
service,
|
||||
make_connector_document(
|
||||
workspace_id=db_workspace.id, source_markdown=_V1
|
||||
),
|
||||
make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1),
|
||||
)
|
||||
ids_v1 = {c.content: c.id for c in await _load_chunks(db_session, document.id)}
|
||||
|
||||
|
|
@ -136,9 +132,7 @@ async def test_removed_paragraph_is_deleted_and_order_compacts(
|
|||
service = IndexingPipelineService(session=db_session)
|
||||
document = await _index(
|
||||
service,
|
||||
make_connector_document(
|
||||
workspace_id=db_workspace.id, source_markdown=_V1
|
||||
),
|
||||
make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1),
|
||||
)
|
||||
ids_v1 = {c.content: c.id for c in await _load_chunks(db_session, document.id)}
|
||||
|
||||
|
|
@ -171,9 +165,7 @@ async def test_kill_switch_falls_back_to_full_replace(
|
|||
service = IndexingPipelineService(session=db_session)
|
||||
document = await _index(
|
||||
service,
|
||||
make_connector_document(
|
||||
workspace_id=db_workspace.id, source_markdown=_V1
|
||||
),
|
||||
make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1),
|
||||
)
|
||||
ids_v1 = {c.id for c in await _load_chunks(db_session, document.id)}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ from app.db import (
|
|||
DocumentType,
|
||||
DocumentVersion,
|
||||
Folder,
|
||||
Workspace,
|
||||
User,
|
||||
Workspace,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
|
|
|||
|
|
@ -59,9 +59,7 @@ async def test_legacy_composio_gmail_doc_migrated_in_db(
|
|||
assert reloaded.document_type == DocumentType.GOOGLE_GMAIL_CONNECTOR
|
||||
|
||||
|
||||
async def test_no_legacy_doc_is_noop(
|
||||
db_session, db_workspace, make_connector_document
|
||||
):
|
||||
async def test_no_legacy_doc_is_noop(db_session, db_workspace, make_connector_document):
|
||||
"""When no legacy document exists, migrate_legacy_docs does nothing."""
|
||||
connector_doc = make_connector_document(
|
||||
document_type=DocumentType.GOOGLE_CALENDAR_CONNECTOR,
|
||||
|
|
|
|||
|
|
@ -225,9 +225,7 @@ async def test_updated_at_advances_when_title_only_changes(
|
|||
db_session, db_workspace, make_connector_document
|
||||
):
|
||||
"""updated_at advances even when only the title changes."""
|
||||
original = make_connector_document(
|
||||
workspace_id=db_workspace.id, title="Old Title"
|
||||
)
|
||||
original = make_connector_document(workspace_id=db_workspace.id, title="Old Title")
|
||||
service = IndexingPipelineService(session=db_session)
|
||||
|
||||
first = await service.prepare_for_indexing([original])
|
||||
|
|
@ -238,9 +236,7 @@ async def test_updated_at_advances_when_title_only_changes(
|
|||
)
|
||||
updated_at_v1 = result.scalars().first().updated_at
|
||||
|
||||
renamed = make_connector_document(
|
||||
workspace_id=db_workspace.id, title="New Title"
|
||||
)
|
||||
renamed = make_connector_document(workspace_id=db_workspace.id, title="New Title")
|
||||
await service.prepare_for_indexing([renamed])
|
||||
|
||||
result = await db_session.execute(
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import pytest
|
|||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db import Workspace, User
|
||||
from app.db import User, Workspace
|
||||
from app.notifications.persistence import Notification
|
||||
from app.notifications.service import NotificationService
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from __future__ import annotations
|
|||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db import Workspace, User
|
||||
from app.db import User, Workspace
|
||||
from app.notifications.service import NotificationService
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
|
@ -47,9 +47,7 @@ async def test_comment_reply_truncates_long_preview(
|
|||
db_session: AsyncSession, db_user: User, db_workspace: Workspace
|
||||
):
|
||||
"""A long comment preview is truncated in the reply message."""
|
||||
notification = await _notify(
|
||||
db_session, db_user, db_workspace, preview="y" * 150
|
||||
)
|
||||
notification = await _notify(db_session, db_user, db_workspace, preview="y" * 150)
|
||||
|
||||
assert notification.message == "y" * 100 + "..."
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from __future__ import annotations
|
|||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db import Workspace, User
|
||||
from app.db import User, Workspace
|
||||
from app.notifications.service import NotificationService
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from __future__ import annotations
|
|||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db import Workspace, User
|
||||
from app.db import User, Workspace
|
||||
from app.notifications.service import NotificationService
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from __future__ import annotations
|
|||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db import Workspace, User
|
||||
from app.db import User, Workspace
|
||||
from app.notifications.service import NotificationService
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from __future__ import annotations
|
|||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db import Workspace, User
|
||||
from app.db import User, Workspace
|
||||
from app.notifications.service import NotificationService
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
|
@ -47,9 +47,7 @@ async def test_new_mention_truncates_long_preview(
|
|||
db_session: AsyncSession, db_user: User, db_workspace: Workspace
|
||||
):
|
||||
"""A long comment preview is truncated in the mention message."""
|
||||
notification = await _notify(
|
||||
db_session, db_user, db_workspace, preview="x" * 150
|
||||
)
|
||||
notification = await _notify(db_session, db_user, db_workspace, preview="x" * 150)
|
||||
|
||||
assert notification.message == "x" * 100 + "..."
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||
from app.app import app, limiter
|
||||
from app.auth.context import AuthContext
|
||||
from app.config import config as app_config
|
||||
from app.db import Workspace, User, get_async_session
|
||||
from app.db import User, Workspace, get_async_session
|
||||
from app.podcasts.persistence import Podcast, PodcastStatus
|
||||
from app.podcasts.schemas import (
|
||||
DurationTarget,
|
||||
|
|
|
|||
|
|
@ -38,9 +38,7 @@ async def test_cancel_from_a_terminal_state_conflicts(
|
|||
assert resp.status_code == 409
|
||||
|
||||
|
||||
async def test_cancel_of_a_regeneration_is_rejected(
|
||||
client, db_workspace, make_podcast
|
||||
):
|
||||
async def test_cancel_of_a_regeneration_is_rejected(client, db_workspace, make_podcast):
|
||||
# Cancelling here would destroy a playable episode; reverting the
|
||||
# regeneration is the way back.
|
||||
podcast = await make_podcast(
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import pytest_asyncio
|
|||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import config as app_config
|
||||
from app.db import Chunk, Document, DocumentType, Workspace, User
|
||||
from app.db import Chunk, Document, DocumentType, User, Workspace
|
||||
|
||||
EMBEDDING_DIM = app_config.embedding_model_instance.dimension
|
||||
DUMMY_EMBEDDING = [0.1] * EMBEDDING_DIM
|
||||
|
|
|
|||
|
|
@ -27,8 +27,8 @@ from app.auth.context import AuthContext
|
|||
from app.db import (
|
||||
SearchSourceConnector,
|
||||
SearchSourceConnectorType,
|
||||
Workspace,
|
||||
User,
|
||||
Workspace,
|
||||
)
|
||||
from app.routes.search_source_connectors_routes import index_connector_content
|
||||
from app.routes.workspaces_routes import create_default_roles_and_membership
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import pytest_asyncio
|
|||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db import Document, DocumentType, DocumentVersion, Workspace, User
|
||||
from app.db import Document, DocumentType, DocumentVersion, User, Workspace
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
|
|
|||
|
|
@ -32,8 +32,8 @@ from app.auth.context import AuthContext
|
|||
from app.db import (
|
||||
SearchSourceConnector,
|
||||
SearchSourceConnectorType,
|
||||
Workspace,
|
||||
User,
|
||||
Workspace,
|
||||
)
|
||||
from app.routes.obsidian_plugin_routes import (
|
||||
obsidian_connect,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from fastapi import HTTPException
|
|||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth.context import AuthContext
|
||||
from app.db import PersonalAccessToken, Workspace, User
|
||||
from app.db import PersonalAccessToken, User, Workspace
|
||||
from app.users import allow_any_principal, require_session_context
|
||||
from app.utils.rbac import check_workspace_access
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from fastapi import HTTPException
|
|||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth.context import AuthContext
|
||||
from app.db import PersonalAccessToken, Workspace, User
|
||||
from app.db import PersonalAccessToken, User, Workspace
|
||||
from app.routes.workspaces_routes import create_default_roles_and_membership
|
||||
from app.utils.rbac import check_workspace_access, get_allowed_read_space_ids
|
||||
|
||||
|
|
|
|||
|
|
@ -158,9 +158,7 @@ async def test_build_dependencies_falls_back_to_workspace(
|
|||
)
|
||||
|
||||
workspace = SimpleNamespace(chat_model_id=-7)
|
||||
result = await build_dependencies(
|
||||
session=_FakeSession(workspace), workspace_id=42
|
||||
)
|
||||
result = await build_dependencies(session=_FakeSession(workspace), workspace_id=42)
|
||||
|
||||
assert captured == {"config_id": -7}
|
||||
assert result.llm.name == "llm"
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue