feat(native-connector): added google maps places & reviews scrapers

surfsense_backend/scripts/e2e_google_maps_scraper.py

@@ -0,0 +1,241 @@
+"""Manual functional e2e for the Google Maps scraper (app/proprietary/scrapers/google_maps).
+
+Run from the backend directory:
+    cd surfsense_backend
+    uv run python scripts/e2e_google_maps_scraper.py
+    # or: .\\.venv\\Scripts\\python.exe scripts/e2e_google_maps_scraper.py
+
+NOT a pytest test (needs live network + optional proxy creds). It:
+  Step 1 — scrapes a known place URL and prints the core fields.
+  Step 2 — scrapes the same place by bare placeId (HTML -> fid -> RPC path).
+  Step 3 — dumps the raw place darray (jd[6] of /maps/preview/place) to
+      tests/unit/scrapers/google_maps/fixtures/ for the offline parser test.
+  Step 4 — scrapes reviews via the Reviews endpoint (BOQ feed), checks fields.
+  Step 5 — paginates past one page (maxReviews=15) and checks the count.
+  Step 6 — place scrape with maxReviews>0 attaches inline reviews[].
+  Step 7 — dumps a raw BOQ reviews page fixture for the offline parser test.
+  Step 8 — search discovery (searchStringsArray + locationQuery), checks items.
+  Step 9 — dumps a raw map-search response fixture for the offline parser test.
+"""
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+# Windows consoles default to cp1252; reviews/ratings contain non-latin chars.
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8")
+
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_BACKEND_ROOT))
+for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
+    if _candidate.exists():
+        load_dotenv(_candidate)
+        break
+
+from app.proprietary.scrapers.google_maps import (  # noqa: E402
+    GoogleMapsReviewsInput,
+    GoogleMapsScrapeInput,
+    scrape_places,
+    scrape_reviews,
+)
+from app.proprietary.scrapers.google_maps.fetch import (  # noqa: E402
+    build_search_url,
+    fetch_place_darray,
+    fetch_rpc_json,
+    iter_reviews_pages,
+)
+from app.proprietary.scrapers.google_maps.url_resolver import extract_fid  # noqa: E402
+
+# A well-known, stable place (the restaurant used in the Apify output example).
+_PLACE_URL = (
+    "https://www.google.com/maps/place/Kim's+Island/"
+    "@40.5107736,-74.2482624,17z/data=!4m6!3m5!1s0x89c3ca9c11f90c25:"
+    "0x6cc8dba851799f09!8m2!3d40.5107736!4d-74.2482624!16s%2Fg%2F1tmgdcj8?hl=en"
+)
+
+_FIXTURE_DIR = (
+    _BACKEND_ROOT / "tests" / "unit" / "scrapers" / "google_maps" / "fixtures"
+)
+
+
+def _hr(title: str) -> None:
+    print(f"\n{'=' * 70}\n{title}\n{'=' * 70}")
+
+
+def _check(label: str, ok: bool, detail: str = "") -> bool:
+    print(f"  [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}")
+    return ok
+
+
+async def step1_place() -> bool:
+    _hr("STEP 1 — scrape a known place URL")
+    items = await scrape_places(GoogleMapsScrapeInput(startUrls=[{"url": _PLACE_URL}]))
+    if not items:
+        return _check("place scraped", False, "no items returned")
+    it = items[0]
+    print(json.dumps(it, indent=2, ensure_ascii=False)[:2500])
+    ok = bool(it.get("title")) and it.get("placeId") is not None
+    return _check(
+        "place has title + placeId",
+        ok,
+        f"{it.get('title')!r} / {it.get('totalScore')}★ / {it.get('reviewsCount')} reviews",
+    )
+
+
+async def step2_place_id() -> bool:
+    _hr("STEP 2 — scrape by bare placeId (HTML -> fid -> RPC path)")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"])
+    )
+    if not items:
+        return _check("placeId scraped", False, "no items returned")
+    it = items[0]
+    return _check(
+        "placeId resolves to same place",
+        it.get("title") == "Kim's Island",
+        f"{it.get('title')!r}",
+    )
+
+
+async def step3_dump_fixture() -> bool:
+    _hr("STEP 3 — dump raw place darray fixture for offline test")
+    fid = extract_fid(_PLACE_URL)
+    if not fid:
+        return _check("extracted fid from URL", False)
+    darray = await fetch_place_darray(fid)
+    if not darray:
+        return _check("fetched place darray via RPC", False)
+    _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+    (_FIXTURE_DIR / "place_darray.json").write_text(
+        json.dumps(darray), encoding="utf-8"
+    )
+    return _check("dumped fixture", True, f"-> {_FIXTURE_DIR / 'place_darray.json'}")
+
+
+async def step4_reviews() -> bool:
+    _hr("STEP 4 — reviews endpoint (one page, newest first)")
+    items = await scrape_reviews(
+        GoogleMapsReviewsInput(startUrls=[{"url": _PLACE_URL}], maxReviews=5)
+    )
+    if not items:
+        return _check("reviews returned", False, "no items")
+    it = items[0]
+    print(json.dumps(it, indent=2, ensure_ascii=False)[:1800])
+    ok = (
+        bool(it.get("name"))
+        and it.get("stars") is not None
+        and bool(it.get("reviewId"))
+        and it.get("title") == "Kim's Island"  # place header stamped on
+        and bool(it.get("publishedAtDate"))
+    )
+    return _check(
+        "review has author/stars/id/place header",
+        ok and len(items) == 5,
+        f"{len(items)} reviews, first by {it.get('name')!r} ({it.get('stars')}★)",
+    )
+
+
+async def step5_pagination() -> bool:
+    _hr("STEP 5 — pagination past one page (maxReviews=15)")
+    items = await scrape_reviews(
+        GoogleMapsReviewsInput(placeIds=["ChIJJQz5EZzKw4kRCZ95UajbyGw"], maxReviews=15)
+    )
+    dates = [i.get("publishedAtDate") for i in items[:3]]
+    return _check(
+        "got 15 reviews across >1 page",
+        len(items) == 15,
+        f"{len(items)} reviews; newest: {dates}",
+    )
+
+
+async def step6_inline_reviews() -> bool:
+    _hr("STEP 6 — place scrape with maxReviews=3 attaches inline reviews[]")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(startUrls=[{"url": _PLACE_URL}], maxReviews=3)
+    )
+    if not items:
+        return _check("place scraped", False, "no items")
+    reviews = items[0].get("reviews") or []
+    return _check(
+        "place item carries 3 inline reviews",
+        len(reviews) == 3 and bool(reviews[0].get("name")),
+        f"{len(reviews)} reviews inline",
+    )
+
+
+async def step7_dump_reviews_fixture() -> bool:
+    _hr("STEP 7 — dump raw BOQ reviews page fixture for offline test")
+    fid = extract_fid(_PLACE_URL)
+    async for raw_page in iter_reviews_pages(fid, sort="newest", max_pages=1):
+        _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+        (_FIXTURE_DIR / "boq_reviews_page.json").write_text(
+            json.dumps(raw_page), encoding="utf-8"
+        )
+        return _check(
+            "dumped fixture",
+            len(raw_page) > 0,
+            f"{len(raw_page)} reviews -> {_FIXTURE_DIR / 'boq_reviews_page.json'}",
+        )
+    return _check("fetched a reviews page", False)
+
+
+async def step8_search() -> bool:
+    _hr("STEP 8 — search discovery (query + locationQuery)")
+    items = await scrape_places(
+        GoogleMapsScrapeInput(
+            searchStringsArray=["coffee shop"],
+            locationQuery="Seattle, WA",
+            maxCrawledPlacesPerSearch=5,
+        )
+    )
+    if not items:
+        return _check("search returned items", False, "no items")
+    it = items[0]
+    print(json.dumps(it, indent=2, ensure_ascii=False)[:1500])
+    ok = (
+        len(items) == 5
+        and all(i.get("title") and i.get("placeId") and i.get("fid") for i in items)
+        and [i.get("rank") for i in items] == [1, 2, 3, 4, 5]
+        and it.get("searchString") == "coffee shop"
+    )
+    return _check(
+        "5 ranked places with title/placeId/fid",
+        ok,
+        f"first: {it.get('title')!r} ({it.get('totalScore')}★, {it.get('city')})",
+    )
+
+
+async def step9_dump_search_fixture() -> bool:
+    _hr("STEP 9 — dump raw map-search response fixture for offline test")
+    url = build_search_url("pizza new york")
+    jd = await fetch_rpc_json(url)
+    if not isinstance(jd, list):
+        return _check("fetched search response", False)
+    _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+    (_FIXTURE_DIR / "search_response.json").write_text(json.dumps(jd), encoding="utf-8")
+    return _check("dumped fixture", True, f"-> {_FIXTURE_DIR / 'search_response.json'}")
+
+
+async def main() -> int:
+    results = [
+        await step1_place(),
+        await step2_place_id(),
+        await step3_dump_fixture(),
+        await step4_reviews(),
+        await step5_pagination(),
+        await step6_inline_reviews(),
+        await step7_dump_reviews_fixture(),
+        await step8_search(),
+        await step9_dump_search_fixture(),
+    ]
+    _hr("SUMMARY")
+    print(f"  {sum(results)}/{len(results)} steps passed")
+    return 0 if all(results) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))