feat(native-connector): added google maps places & reviews scrapers

surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py

@@ -158,9 +158,7 @@ async def test_build_dependencies_falls_back_to_workspace(
     )
 
     workspace = SimpleNamespace(chat_model_id=-7)
-    result = await build_dependencies(
-        session=_FakeSession(workspace), workspace_id=42
-    )
+    result = await build_dependencies(session=_FakeSession(workspace), workspace_id=42)
 
     assert captured == {"config_id": -7}
     assert result.llm.name == "llm"

surfsense_backend/tests/unit/connector_indexers/test_web_crawl_billing.py

@@ -142,7 +142,9 @@ async def test_charges_owner_for_successful_crawls(indexer_env):
     session, user = _make_session(_OWNER_USER, balance_micros=100_000)
     indexer_env["crawler"].crawl_url.side_effect = [_outcome(True), _outcome(True)]
 
-    total, warning = await _run(indexer_env, session, ["https://a.com", "https://b.com"])
+    total, warning = await _run(
+        indexer_env, session, ["https://a.com", "https://b.com"]
+    )
 
     assert total == 2
     assert warning is None

surfsense_backend/tests/unit/indexing_pipeline/test_create_placeholder_documents.py

@@ -36,9 +36,7 @@ def _make_placeholder(**overrides) -> PlaceholderInfo:
 
 
 def _uid_hash(p: PlaceholderInfo) -> str:
-    return compute_identifier_hash(
-        p.document_type.value, p.unique_id, p.workspace_id
-    )
+    return compute_identifier_hash(p.document_type.value, p.unique_id, p.workspace_id)
 
 
 def _session_with_existing_hashes(existing: set[str] | None = None):

surfsense_backend/tests/unit/proprietary/web_crawler/test_captcha.py

@@ -15,16 +15,16 @@ pytestmark = pytest.mark.unit
 
 
 def _cfg(**overrides) -> CaptchaConfig:
-    base = dict(
-        enabled=True,
-        solving_site="capsolver",
-        api_key="key-123",
-        max_attempts_per_url=1,
-        timeout_s=30,
-        captcha_type_default="v2",
-        v3_min_score=0.7,
-        v3_action="verify",
-    )
+    base = {
+        "enabled": True,
+        "solving_site": "capsolver",
+        "api_key": "key-123",
+        "max_attempts_per_url": 1,
+        "timeout_s": 30,
+        "captcha_type_default": "v2",
+        "v3_min_score": 0.7,
+        "v3_action": "verify",
+    }
     base.update(overrides)
     return CaptchaConfig(**base)
 
@@ -40,7 +40,9 @@ class _FakeEl:
 class _FakePage:
     """Minimal sync-Playwright-ish page for detection/injection tests."""
 
-    def __init__(self, widgets=None, iframes=None, scripts=None, url="https://t.test/p"):
+    def __init__(
+        self, widgets=None, iframes=None, scripts=None, url="https://t.test/p"
+    ):
         self._widgets = widgets or {}  # selector -> _FakeEl
         self._iframes = iframes or []  # list[_FakeEl]
         self._scripts = scripts or []  # list[_FakeEl]
@@ -116,7 +118,9 @@ class TestDetect:
 
     def test_v3_render_param_when_default_v3(self):
         page = _FakePage(
-            scripts=[_FakeEl({"src": "https://www.google.com/recaptcha/api.js?render=SK_V3"})]
+            scripts=[
+                _FakeEl({"src": "https://www.google.com/recaptcha/api.js?render=SK_V3"})
+            ]
         )
         assert cap.detect_challenge(page, _cfg(captcha_type_default="v3")) == (
             "v3",
@@ -217,11 +221,11 @@ class TestPageAction:
         assert state == {"attempts": 1, "solved": False}
 
     def test_no_balance_latches_and_stops(self, monkeypatch):
-        class ErrNoBalance(Exception):
+        class NoBalanceError(Exception):
             pass
 
         def _boom(*_a, **_k):
-            raise ErrNoBalance("out of balance")
+            raise NoBalanceError("out of balance")
 
         monkeypatch.setattr(cap, "_harvest_token", _boom)
         state = {"attempts": 0, "solved": False}

surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py

@@ -10,8 +10,8 @@ import pytest
 from app.proprietary.web_crawler import (
     CrawlOutcomeStatus,
     WebCrawlerConnector,
+    connector as connector_module,
 )
-from app.proprietary.web_crawler import connector as connector_module
 from app.utils.crawl import BlockType
 
 pytestmark = pytest.mark.unit
@@ -230,9 +230,7 @@ async def test_captcha_attempts_surface_even_when_crawl_fails(
 
     monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty)
     monkeypatch.setattr(crawler, "_crawl_with_dynamic", _empty)
-    monkeypatch.setattr(
-        crawler, "_crawl_with_stealthy", _stealthy_attempt_then_empty
-    )
+    monkeypatch.setattr(crawler, "_crawl_with_stealthy", _stealthy_attempt_then_empty)
 
     outcome = await crawler.crawl_url("https://example.com")
 
@@ -341,9 +339,7 @@ async def test_static_4xx_is_classified(
     monkeypatch.setattr(connector_module, "get_proxy_url", lambda: None)
 
     block_state: dict = {"block_type": BlockType.UNKNOWN}
-    result = await crawler._crawl_with_async_fetcher(
-        "https://example.com", block_state
-    )
+    result = await crawler._crawl_with_async_fetcher("https://example.com", block_state)
 
     assert result is None  # 4xx => fall through to next tier
     assert block_state["block_type"] is BlockType.CLOUDFLARE
@@ -354,7 +350,9 @@ def test_build_result_ok_on_real_content() -> None:
     crawler = WebCrawlerConnector()
     block_state: dict = {"block_type": BlockType.UNKNOWN}
 
-    html = "<html><body><article>" + ("Real content. " * 40) + "</article></body></html>"
+    html = (
+        "<html><body><article>" + ("Real content. " * 40) + "</article></body></html>"
+    )
     crawler._build_result(
         html,
         "https://example.com",

surfsense_backend/tests/unit/scrapers/google_maps/test_parsers.py

@@ -0,0 +1,175 @@
+"""Offline parser tests against a captured real place ``darray`` fixture.
+
+The fixture is the ``jd[6]`` array from a live ``/maps/preview/place`` RPC
+response for "Kim's Island" (the restaurant used in the Apify output example),
+regenerated by ``scripts/e2e_google_maps_scraper.py``. These tests pin the
+array-index paths so a Google structure shift is caught offline.
+"""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from app.proprietary.scrapers.google_maps.fetch import strip_xssi
+from app.proprietary.scrapers.google_maps.parsers import (
+    brace_match_json,
+    dig,
+    parse_place,
+)
+
+_FIXTURE = Path(__file__).parent / "fixtures" / "place_darray.json"
+
+
+@pytest.fixture
+def darray() -> list:
+    return json.loads(_FIXTURE.read_text(encoding="utf-8"))
+
+
+def test_parse_place_core_fields(darray):
+    place = parse_place(darray)
+    assert place["title"] == "Kim's Island"
+    assert place["placeId"] == "ChIJJQz5EZzKw4kRCZ95UajbyGw"
+    assert place["fid"] == "0x89c3ca9c11f90c25:0x6cc8dba851799f09"
+    assert place["categoryName"] == "Chinese restaurant"
+    assert "Chinese restaurant" in place["categories"]
+    assert place["totalScore"] == 4.5
+    assert place["phone"] == "(718) 356-5168"
+    assert place["website"] == "http://kimsislandsi.com/"
+    assert place["plusCode"].startswith("GQ62+8M")
+
+
+def test_parse_place_address_components(darray):
+    place = parse_place(darray)
+    assert place["neighborhood"] == "Tottenville"
+    assert place["street"] == "175 Main St"
+    assert place["city"] == "Staten Island"
+    assert place["postalCode"] == "10307"
+    assert place["state"] == "New York"
+    assert place["countryCode"] == "US"
+
+
+def test_parse_place_location_and_hours(darray):
+    place = parse_place(darray)
+    assert place["location"]["lat"] == pytest.approx(40.5107736)
+    assert place["location"]["lng"] == pytest.approx(-74.2482624)
+    hours = place["openingHours"]
+    assert hours and all("day" in h and "hours" in h for h in hours)
+
+
+def test_parse_place_detail_extras(darray):
+    place = parse_place(darray)
+    assert place["kgmid"] == "/g/1tmgdcj8"
+    # cid = decimal of the fid's second hex half
+    assert place["cid"] == str(int("0x6cc8dba851799f09", 16))
+    info = place["additionalInfo"]
+    assert "Accessibility" in info
+    assert {"Wheelchair accessible entrance": True} in info["Accessibility"]
+    # every option is a single {name: bool} pair
+    for entries in info.values():
+        for entry in entries:
+            assert len(entry) == 1
+            assert all(isinstance(v, bool) for v in entry.values())
+
+
+def test_parse_place_session_gated_fields(darray):
+    # These fields only appear when the RPC is sent with an NID session
+    # cookie + the full-page pb selector; the fixture pins that payload.
+    place = parse_place(darray)
+    dist = place["reviewsDistribution"]
+    assert set(dist) == {"oneStar", "twoStar", "threeStar", "fourStar", "fiveStar"}
+    assert place["reviewsCount"] == sum(dist.values())
+    assert place["imagesCount"] > 0
+    assert "All" in place["imageCategories"]
+    assert all(u.startswith("https://") for u in place["imageUrls"])
+    tags = place["reviewsTags"]
+    assert tags and all(
+        isinstance(t["title"], str) and isinstance(t["count"], int) for t in tags
+    )
+    assert len(place["additionalInfo"]) >= 5  # full sections, not just Accessibility
+
+
+def test_parse_place_hotel_fields():
+    # Fixture: live darray for The Plaza (NYC), captured via the detail RPC.
+    fixture = Path(__file__).parent / "fixtures" / "hotel_darray.json"
+    place = parse_place(json.loads(fixture.read_text(encoding="utf-8")))
+    assert place["title"] == "The Plaza"
+    assert place["hotelStars"] == "5 stars"
+    assert place["checkInDate"] < place["checkOutDate"]  # ISO dates compare
+    assert place["hotelDescription"]
+    similar = place["similarHotelsNearby"]
+    assert len(similar) >= 3
+    assert all(h["title"] and h["fid"] for h in similar)
+    assert any("hotel" in (h.get("description") or "").lower() for h in similar)
+    ads = place["hotelAds"]
+    assert ads and all(a["url"].startswith("https://") for a in ads)
+    assert any(a.get("price", "").startswith("$") for a in ads)
+
+
+def test_hotel_fields_absent_for_non_hotels(darray):
+    # Kim's Island (restaurant) must not grow hotel fields.
+    place = parse_place(darray)
+    for key in ("hotelStars", "checkInDate", "similarHotelsNearby", "hotelAds"):
+        assert key not in place
+
+
+def test_popular_times_shapes():
+    from app.proprietary.scrapers.google_maps.parsers import _popular_times
+
+    darray: list = [None] * 100
+    darray[84] = [
+        [[7, [[6, 0, "", "None", "6 AM", "No wait", "6a"], [7, 35]]]],
+        None,
+        None,
+        None,
+        None,
+        None,
+        "A little busy",
+        [22, 76],
+    ]
+    out = _popular_times(darray)
+    assert out["popularTimesHistogram"] == {
+        "Su": [
+            {"hour": 6, "occupancyPercent": 0},
+            {"hour": 7, "occupancyPercent": 35},
+        ]
+    }
+    assert out["popularTimesLiveText"] == "A little busy"
+    assert out["popularTimesLivePercent"] == 76
+    assert _popular_times([None] * 100) == {}
+
+
+def test_parse_place_reservation_links():
+    fixture = Path(__file__).parent / "fixtures" / "search_response.json"
+    from app.proprietary.scrapers.google_maps.fetch import _search_darrays
+
+    jd = json.loads(fixture.read_text(encoding="utf-8"))
+    place = parse_place(_search_darrays(jd)[0])
+    # the first search hit in the fixture carries a reservation provider
+    links = place["tableReservationLinks"]
+    assert all(set(link) == {"url", "source"} for link in links)
+    assert place["reserveTableUrl"] == links[0]["url"]
+
+
+def test_parse_place_omits_missing_fields(darray):
+    # parse_place drops keys whose path missed, rather than emitting nulls.
+    place = parse_place(darray)
+    assert all(v is not None for v in place.values())
+
+
+def test_dig_tolerates_ragged_paths():
+    assert dig([1, [2, 3]], 1, 0) == 2
+    assert dig([1, [2, 3]], 5) is None
+    assert dig([1, None], 1, 0) is None
+    assert dig("not a list", 0) is None
+
+
+def test_strip_xssi_and_brace_match_handle_html_wrapper():
+    # Mirrors the real proxy response: text/plain body wrapped in <p>…</p>.
+    raw = ')]}\'\n[1,[2,3],"x"]</p></body></html>'
+    body = strip_xssi(raw)
+    blob = brace_match_json(body, body.index("["))
+    assert json.loads(blob) == [1, [2, 3], "x"]
+
+    wrapped = '<html><body><p>)]}\'\n[["a"]]'
+    assert strip_xssi(wrapped).startswith("[[")

surfsense_backend/tests/unit/scrapers/google_maps/test_reviews.py

@@ -0,0 +1,116 @@
+"""Offline tests for the Google Maps reviews parsing + flow helpers.
+
+``fixtures/boq_reviews_page.json`` is a real ``GetLocalBoqProxy`` page (the
+raw review list, ``jd[1][10][2]``) captured by scripts/e2e_google_maps_scraper.py
+step 7. Regenerate it with that script if Google shifts the structure.
+"""
+
+import json
+from datetime import datetime
+from pathlib import Path
+
+import pytest
+
+from app.proprietary.scrapers.google_maps.fetch import build_reviews_url
+from app.proprietary.scrapers.google_maps.parsers import (
+    parse_review,
+    parse_reviews_page,
+    strip_personal_data,
+)
+from app.proprietary.scrapers.google_maps.reviews import _before_cutoff, _keep
+
+_FIXTURE = Path(__file__).parent / "fixtures" / "boq_reviews_page.json"
+
+
+@pytest.fixture
+def reviews_raw() -> list:
+    return json.loads(_FIXTURE.read_text(encoding="utf-8"))
+
+
+def test_parse_reviews_page_full_page(reviews_raw):
+    parsed = parse_reviews_page(reviews_raw)
+    assert len(parsed) == len(reviews_raw) == 10
+    for review in parsed:
+        assert review["name"]
+        assert review["reviewId"]
+        assert 1 <= review["stars"] <= 5
+        assert review["publishedAtDate"].endswith("Z")
+
+
+def test_parse_review_fields(reviews_raw):
+    review = parse_review(reviews_raw[0])
+    assert review["name"] == "Greg"
+    assert review["stars"] == 5
+    assert review["reviewerId"] == "108156147079988470963"
+    assert review["reviewerUrl"].startswith("https://www.google.com/maps/contrib/")
+    assert review["reviewerNumberOfReviews"] == 63
+    assert review["isLocalGuide"] is True
+    assert review["reviewOrigin"] == "Google"
+    assert review["originalLanguage"] == "en"
+    assert review["publishAt"] == "2 months ago"
+    # 1776469524140 ms epoch -> UTC ISO
+    assert review["publishedAtDate"] == "2026-04-17T23:45:24.140Z"
+    assert "spiced" in review["text"]
+    # Guided answers split into context vs per-aspect ratings.
+    assert review["reviewContext"]["Order type"] == "Take out"
+    assert review["reviewDetailedRating"]["Food"] == 5
+
+
+def test_parse_review_owner_response_and_images(reviews_raw):
+    with_reply = [
+        r
+        for r in (parse_review(x) for x in reviews_raw)
+        if r and r.get("responseFromOwnerText")
+    ]
+    assert with_reply, "fixture should contain at least one owner reply"
+    assert with_reply[0]["responseFromOwnerText"]
+
+    with_images = [
+        r
+        for r in (parse_review(x) for x in reviews_raw)
+        if r and r.get("reviewImageUrls")
+    ]
+    assert with_images, "fixture should contain at least one review with photos"
+    assert with_images[0]["reviewImageUrls"][0].startswith("https://")
+
+
+def test_parse_review_rejects_garbage():
+    assert parse_review(None) is None
+    assert parse_review([]) is None
+    assert parse_review([None, 5]) is None  # no author block
+
+
+def test_strip_personal_data(reviews_raw):
+    review = parse_review(reviews_raw[0])
+    strip_personal_data(review)
+    for key in ("name", "reviewerId", "reviewerUrl", "reviewerPhotoUrl"):
+        assert key not in review
+    assert review["reviewId"]  # non-personal fields stay
+    assert review["stars"] == 5
+
+
+def test_before_cutoff():
+    cutoff = datetime(2026, 3, 1)
+    assert _before_cutoff({"publishedAtDate": "2026-02-01T00:00:00.000Z"}, cutoff)
+    assert not _before_cutoff({"publishedAtDate": "2026-04-01T00:00:00.000Z"}, cutoff)
+    assert not _before_cutoff({}, cutoff)  # undated reviews are kept
+
+
+def test_keep_filters():
+    review = {"text": "Great NOODLES here", "reviewOrigin": "Google"}
+    assert _keep(review, filter_string="", origin="all")
+    assert _keep(review, filter_string="noodles", origin="google")
+    assert not _keep(review, filter_string="pizza", origin="all")
+    assert not _keep({"reviewOrigin": "TripAdvisor"}, filter_string="", origin="google")
+
+
+def test_build_reviews_url_shape():
+    fid = "0x89c3ca9c11f90c25:0x6cc8dba851799f09"
+    first = build_reviews_url(fid, sort_code=2)
+    assert "GetLocalBoqProxy" in first
+    assert "hl=en" in first
+    assert fid in json.dumps(first) or "0x89c3ca9c11f90c25" in first
+    # First page requests a page size; later pages carry the token instead.
+    paged = build_reviews_url(fid, sort_code=2, page_token="TOKEN123")
+    assert "TOKEN123" in paged
+    assert paged != first

surfsense_backend/tests/unit/scrapers/google_maps/test_search.py

@@ -0,0 +1,150 @@
+"""Offline tests for the map-search flow: response extraction, URL building,
+and the Apify result filters. The fixture is a real ``search?tbm=map`` response
+captured by scripts/e2e_google_maps_scraper.py (step 9, query "pizza new york").
+"""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from app.proprietary.scrapers.google_maps.fetch import (
+    _search_darrays,
+    build_search_url,
+)
+from app.proprietary.scrapers.google_maps.parsers import parse_place
+from app.proprietary.scrapers.google_maps.schemas import GoogleMapsScrapeInput
+from app.proprietary.scrapers.google_maps.scraper import (
+    _custom_point,
+    _location_text,
+    _passes_filters,
+)
+
+_FIXTURE = Path(__file__).parent / "fixtures" / "search_response.json"
+
+
+@pytest.fixture(scope="module")
+def search_jd():
+    return json.loads(_FIXTURE.read_text(encoding="utf-8"))
+
+
+def test_search_darrays_extracts_full_page(search_jd):
+    darrays = _search_darrays(search_jd)
+    assert len(darrays) == 20
+    parsed = [parse_place(d) for d in darrays]
+    for fields in parsed:
+        assert fields["title"]
+        assert fields["fid"].startswith("0x")
+        assert fields["placeId"].startswith("ChIJ")
+        assert "location" in fields
+    fids = [f["fid"] for f in parsed]
+    assert len(set(fids)) == 20  # no dupes within a page
+
+
+def test_search_result_has_detail_fields(search_jd):
+    fields = parse_place(_search_darrays(search_jd)[0])
+    # Search entries carry the full darray, so detail fields come through.
+    assert fields["categories"]
+    assert fields["address"]
+    assert fields["totalScore"] > 0
+    assert fields["city"]
+
+
+def test_search_darrays_rejects_garbage():
+    assert _search_darrays(None) == []
+    assert _search_darrays([]) == []
+    assert _search_darrays([[None, None]]) == []
+
+
+def test_build_search_url_shape():
+    url = build_search_url("pizza new york", offset=20, language="de")
+    assert url.startswith("https://www.google.com/search?tbm=map")
+    assert "hl=de" in url
+    assert "q=pizza%20new%20york" in url
+    assert "!8i20" in url  # offset
+    assert "!7i20" in url  # page size
+    # whole-earth viewport when no coordinates given
+    assert "!1d25000000" in url
+
+    geo_url = build_search_url("museum", lat=48.85, lng=2.35, radius_m=5000)
+    assert "!2d2.35" in geo_url and "!3d48.85" in geo_url
+    assert "!1d10000" in geo_url  # radius -> diameter
+
+
+def test_location_text_prefers_location_query():
+    assert (
+        _location_text(GoogleMapsScrapeInput(locationQuery="Berlin, Germany"))
+        == "Berlin, Germany"
+    )
+    assert (
+        _location_text(
+            GoogleMapsScrapeInput(city="Austin", state="TX", countryCode="US")
+        )
+        == "Austin, TX, US"
+    )
+    assert _location_text(GoogleMapsScrapeInput()) is None
+
+
+def test_custom_point():
+    lat, lng, radius = _custom_point(
+        GoogleMapsScrapeInput(
+            customGeolocation={
+                "type": "Point",
+                "coordinates": [2.35, 48.85],
+                "radiusKm": 5,
+            }
+        )
+    )
+    assert (lat, lng, radius) == (48.85, 2.35, 5000)
+    assert _custom_point(GoogleMapsScrapeInput()) == (None, None, None)
+    assert _custom_point(
+        GoogleMapsScrapeInput(customGeolocation={"type": "Polygon"})
+    ) == (None, None, None)
+
+
+def test_passes_filters():
+    fields = {
+        "title": "Joe's Pizza",
+        "categories": ["Pizza restaurant"],
+        "totalScore": 4.4,
+        "website": "https://joes.example",
+    }
+    default = GoogleMapsScrapeInput()
+    assert _passes_filters(fields, "pizza", default)
+
+    assert not _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(searchMatching="only_exact")
+    )
+    assert _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(searchMatching="only_includes")
+    )
+    assert not _passes_filters(
+        fields, "burger", GoogleMapsScrapeInput(searchMatching="only_includes")
+    )
+
+    assert _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(categoryFilterWords=["pizza"])
+    )
+    assert not _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(categoryFilterWords=["barber"])
+    )
+
+    assert not _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(placeMinimumStars="fourAndHalf")
+    )
+    assert _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(placeMinimumStars="four")
+    )
+
+    assert _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(website="withWebsite")
+    )
+    assert not _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(website="withoutWebsite")
+    )
+
+    closed = {**fields, "permanentlyClosed": True}
+    assert not _passes_filters(
+        closed, "pizza", GoogleMapsScrapeInput(skipClosedPlaces=True)
+    )
+    assert _passes_filters(closed, "pizza", default)

surfsense_backend/tests/unit/scrapers/google_maps/test_skeleton.py

@@ -0,0 +1,97 @@
+"""Offline checks for the Google Maps scraper skeleton.
+
+Covers the pure parts: URL classification and Apify-spec schema defaults/
+serialization. The live flows (place / reviews / search) are covered by the
+e2e scripts and the fixture-based parser tests.
+"""
+
+import pytest
+
+from app.proprietary.scrapers.google_maps import (
+    GoogleMapsReviewsInput,
+    GoogleMapsScrapeInput,
+    PlaceItem,
+    ReviewItem,
+)
+from app.proprietary.scrapers.google_maps.url_resolver import extract_fid, resolve_url
+
+
+@pytest.mark.parametrize(
+    ("url", "kind", "value"),
+    [
+        (
+            "https://www.google.com/maps/place/Kim's+Island/@40.51,-74.24,17z/data=!4m6",
+            "place",
+            "Kim's Island",
+        ),
+        (
+            "https://www.google.com/maps/search/restaurants/@52.5190603,13.388574,13z/",
+            "search",
+            "restaurants",
+        ),
+        ("https://www.google.com/maps/reviews/data=!4m8!14m7", "reviews", None),
+        (
+            "https://www.google.com/maps?cid=7838756667406262025",
+            "cid",
+            "7838756667406262025",
+        ),
+        ("https://goo.gl/maps/abc123", "shortlink", None),
+        ("https://maps.app.goo.gl/xyz", "shortlink", None),
+    ],
+)
+def test_resolve_url(url, kind, value):
+    resolved = resolve_url(url)
+    assert resolved is not None
+    assert resolved.kind == kind
+    if value is not None:
+        assert resolved.value == value
+
+
+def test_resolve_url_rejects_non_maps():
+    assert resolve_url("https://example.com/maps/place/foo") is None
+    assert resolve_url("https://www.google.com/search?q=pizza") is None
+
+
+def test_extract_fid():
+    url = (
+        "https://www.google.com/maps/place/Kim's+Island/@40.51,-74.24,17z/"
+        "data=!4m6!3m5!1s0x89c3ca9c11f90c25:0x6cc8dba851799f09!8m2"
+    )
+    assert extract_fid(url) == "0x89c3ca9c11f90c25:0x6cc8dba851799f09"
+    assert extract_fid("https://www.google.com/maps/place/Foo") is None
+    assert resolve_url(url).fid == "0x89c3ca9c11f90c25:0x6cc8dba851799f09"
+
+
+def test_scrape_input_defaults_match_apify_spec():
+    inp = GoogleMapsScrapeInput()
+    assert inp.language == "en"
+    assert inp.maxCrawledPlacesPerSearch is None  # empty = all places
+    assert inp.searchMatching == "all"
+    assert inp.website == "allPlaces"
+    assert inp.reviewsSort == "newest"
+    assert inp.reviewsOrigin == "all"
+    assert inp.scrapeReviewsPersonalData is True
+    assert inp.maxCompetitorsToAnalyze == 30
+    assert inp.scrapeSocialMediaProfiles.facebooks is False
+    # Unknown fields are accepted (extra="allow") so parity is additive.
+    GoogleMapsScrapeInput(someFutureField=1)
+
+
+def test_reviews_input_defaults_match_apify_spec():
+    inp = GoogleMapsReviewsInput()
+    assert inp.maxReviews == 10_000_000
+    assert inp.reviewsSort == "newest"
+    assert inp.personalData is True
+
+
+def test_output_items_serialize_full_shape():
+    place = PlaceItem(title="Kim's Island", placeId="ChIJx").to_output()
+    assert place["title"] == "Kim's Island"
+    assert place["permanentlyClosed"] is False
+    assert place["categories"] == []
+    assert "reviewsDistribution" in place  # unsourced fields still emitted
+
+    review = ReviewItem(reviewId="abc", stars=5).to_output()
+    assert review["stars"] == 5
+    assert review["reviewImageUrls"] == []
+    assert "responseFromOwnerText" in review

surfsense_backend/tests/unit/scrapers/youtube/test_fetch_resilience.py

@@ -9,8 +9,6 @@ from __future__ import annotations
 
 from collections.abc import AsyncIterator
 
-import pytest
-
 from app.proprietary.scrapers.youtube import innertube, scraper
 from app.proprietary.scrapers.youtube.innertube import (
     INNERTUBE_SEARCH_URL,
@@ -40,7 +38,7 @@ class _FakeSession:
         self.exc = exc
         self.calls = 0
 
-    async def post(self, url, json=None):  # noqa: A002 - mirror scrapling API
+    async def post(self, url, json=None):
         self.calls += 1
         if self.exc:
             raise ConnectionError("boom")
@@ -95,7 +93,9 @@ async def test_post_rotates_on_connection_error_then_succeeds():
 
 async def test_post_gives_up_after_max_rotations():
     # Every IP is blocked -> rotate up to the cap, then return None.
-    holder = _FakeHolder([_FakeSession(429) for _ in range(innertube._MAX_ROTATIONS + 1)])
+    holder = _FakeHolder(
+        [_FakeSession(429) for _ in range(innertube._MAX_ROTATIONS + 1)]
+    )
     token = _current_session.set(holder)
     try:
         result = await post_innertube(INNERTUBE_SEARCH_URL, _payload())
@@ -136,7 +136,9 @@ async def test_fetch_html_falls_back_to_stealthy_when_all_blocked(monkeypatch):
         return "<html>stealthy</html>"
 
     monkeypatch.setattr(innertube, "_fetch_html_stealthy", _fake_stealthy)
-    holder = _FakeHolder([_FakeSession(429) for _ in range(innertube._MAX_ROTATIONS + 1)])
+    holder = _FakeHolder(
+        [_FakeSession(429) for _ in range(innertube._MAX_ROTATIONS + 1)]
+    )
     token = _current_session.set(holder)
     try:
         html = await fetch_html("https://www.youtube.com/watch?v=zzz")

surfsense_backend/tests/unit/scrapers/youtube/test_parsers.py

@@ -21,17 +21,17 @@ from app.proprietary.scrapers.youtube.parsers import (
     dig,
     find_all,
     parse_channel_about,
-    parse_collaborators,
-    parse_comment_entities,
-    parse_description_links,
-    parse_location,
-    parse_translation,
     parse_channel_metadata,
     parse_channel_shorts,
-    parse_playlist_video_ids,
+    parse_collaborators,
+    parse_comment_entities,
     parse_count,
     parse_date,
+    parse_description_links,
+    parse_location,
+    parse_playlist_video_ids,
     parse_search_response,
+    parse_translation,
     parse_video_page,
     seconds_to_duration,
 )
@@ -105,12 +105,19 @@ def _video_html() -> str:
             "keywords": ["#tech"],
             "thumbnail": {
                 "thumbnails": [
-                    {"url": "https://i.ytimg.com/vi/abc123/hq.jpg", "width": 480, "height": 360}
+                    {
+                        "url": "https://i.ytimg.com/vi/abc123/hq.jpg",
+                        "width": 480,
+                        "height": 360,
+                    }
                 ]
             },
         },
         "microformat": {
-            "playerMicroformatRenderer": {"publishDate": "2024-08-27", "isFamilySafe": True}
+            "playerMicroformatRenderer": {
+                "publishDate": "2024-08-27",
+                "isFamilySafe": True,
+            }
         },
         "adPlacements": [{"adPlacementRenderer": {}}],
     }
@@ -200,7 +207,9 @@ def test_parse_video_page_members_only_and_paid():
     player = {
         "videoDetails": {"videoId": "m", "title": "t"},
         "playabilityStatus": {"errorScreen": {"x": {"offerId": "sponsors_only_video"}}},
-        "paidContentOverlay": {"paidContentOverlayRenderer": {"text": "Includes paid promotion"}},
+        "paidContentOverlay": {
+            "paidContentOverlayRenderer": {"text": "Includes paid promotion"}
+        },
     }
     initial = {
         "badges": [
@@ -237,7 +246,11 @@ def test_parse_channel_metadata():
                 "externalId": "UCabc123",
                 "description": "We scrape the web.",
                 "vanityChannelUrl": "https://www.youtube.com/@Apify",
-                "avatar": {"thumbnails": [{"url": "https://a/avatar.jpg", "width": 88, "height": 88}]},
+                "avatar": {
+                    "thumbnails": [
+                        {"url": "https://a/avatar.jpg", "width": 88, "height": 88}
+                    ]
+                },
             }
         },
         "header": {
@@ -246,7 +259,11 @@ def test_parse_channel_metadata():
                     "imageBannerViewModel": {
                         "image": {
                             "sources": [
-                                {"url": "https://a/banner.jpg", "width": 1060, "height": 175}
+                                {
+                                    "url": "https://a/banner.jpg",
+                                    "width": 1060,
+                                    "height": 175,
+                                }
                             ]
                         }
                     }
@@ -290,7 +307,9 @@ def test_parse_description_links():
                 {
                     "startIndex": 4,
                     "length": 4,
-                    "onTap": {"innertubeCommand": {"urlEndpoint": {"url": "/hashtag/tag"}}},
+                    "onTap": {
+                        "innertubeCommand": {"urlEndpoint": {"url": "/hashtag/tag"}}
+                    },
                 },
                 {
                     "startIndex": 21,
@@ -318,8 +337,16 @@ def test_parse_description_links():
 
 def test_channel_about_tokens():
     initial = {
-        "a": {"showEngagementPanelEndpoint": {"x": {"continuationCommand": {"token": "T1"}}}},
-        "b": {"showEngagementPanelEndpoint": {"y": {"continuationCommand": {"token": "T2"}}}},
+        "a": {
+            "showEngagementPanelEndpoint": {
+                "x": {"continuationCommand": {"token": "T1"}}
+            }
+        },
+        "b": {
+            "showEngagementPanelEndpoint": {
+                "y": {"continuationCommand": {"token": "T2"}}
+            }
+        },
     }
     assert channel_about_tokens(initial) == ["T1", "T2"]
     assert channel_about_tokens({}) == []
@@ -340,7 +367,11 @@ def test_parse_search_response():
                     "shortViewCountText": {"simpleText": "451K views"},
                     "thumbnail": {
                         "thumbnails": [
-                            {"url": "https://i.ytimg.com/vi/x/hq.jpg", "width": 360, "height": 202}
+                            {
+                                "url": "https://i.ytimg.com/vi/x/hq.jpg",
+                                "width": 360,
+                                "height": 202,
+                            }
                         ]
                     },
                     "ownerText": {
@@ -395,7 +426,11 @@ def test_parse_channel_shorts():
                     "thumbnailViewModel": {
                         "image": {
                             "sources": [
-                                {"url": "https://i.ytimg.com/s.jpg", "width": 405, "height": 720}
+                                {
+                                    "url": "https://i.ytimg.com/s.jpg",
+                                    "width": 405,
+                                    "height": 720,
+                                }
                             ]
                         }
                     },
@@ -422,13 +457,30 @@ def test_parse_playlist_video_ids():
     data = {
         "contents": {
             "items": [
-                {"lockupViewModel": {"contentId": "fNk_zzaMoSs", "contentType": "VIDEO"}},
-                {"lockupViewModel": {"contentId": "k7RM-ot2NWY", "contentType": "VIDEO"}},
-                {"lockupViewModel": {"contentId": "fNk_zzaMoSs", "contentType": "VIDEO"}},  # dup
+                {
+                    "lockupViewModel": {
+                        "contentId": "fNk_zzaMoSs",
+                        "contentType": "VIDEO",
+                    }
+                },
+                {
+                    "lockupViewModel": {
+                        "contentId": "k7RM-ot2NWY",
+                        "contentType": "VIDEO",
+                    }
+                },
+                {
+                    "lockupViewModel": {
+                        "contentId": "fNk_zzaMoSs",
+                        "contentType": "VIDEO",
+                    }
+                },  # dup
             ]
         },
         # a playlist self-lockup (non-video, longer id) that must be ignored
-        "sidebar": {"lockupViewModel": {"contentId": "PLZHQObOWTQDPD3MizzM2xVFitgF8hE_ab"}},
+        "sidebar": {
+            "lockupViewModel": {"contentId": "PLZHQObOWTQDPD3MizzM2xVFitgF8hE_ab"}
+        },
         "continuationItemRenderer": {
             "continuationEndpoint": {"continuationCommand": {"token": "PAGE2"}}
         },
@@ -494,7 +546,11 @@ def _primary_info(super_title_link):
                 "results": {
                     "results": {
                         "contents": [
-                            {"videoPrimaryInfoRenderer": {"superTitleLink": super_title_link}}
+                            {
+                                "videoPrimaryInfoRenderer": {
+                                    "superTitleLink": super_title_link
+                                }
+                            }
                         ]
                     }
                 }
@@ -536,7 +592,9 @@ def _owner(video_owner_renderer):
                         "contents": [
                             {
                                 "videoSecondaryInfoRenderer": {
-                                    "owner": {"videoOwnerRenderer": video_owner_renderer}
+                                    "owner": {
+                                        "videoOwnerRenderer": video_owner_renderer
+                                    }
                                 }
                             }
                         ]
@@ -556,14 +614,19 @@ def _collab_row(name, base_url):
                     {
                         "onTap": {
                             "innertubeCommand": {
-                                "browseEndpoint": {"browseId": "UCx", "canonicalBaseUrl": base_url}
+                                "browseEndpoint": {
+                                    "browseId": "UCx",
+                                    "canonicalBaseUrl": base_url,
+                                }
                             }
                         }
                     }
                 ],
             },
             # A nested subscribe submenu — must NOT be picked up as a collaborator.
-            "subscribeMenu": {"listItemViewModel": {"title": {"content": "Unsubscribe"}}},
+            "subscribeMenu": {
+                "listItemViewModel": {"title": {"content": "Unsubscribe"}}
+            },
         }
     }
 
@@ -583,15 +646,20 @@ def test_parse_collaborators_from_dialog():
                                             "dialogViewModel": {
                                                 "header": {
                                                     "dialogHeaderViewModel": {
-                                                        "headline": {"content": "Collaborators"}
+                                                        "headline": {
+                                                            "content": "Collaborators"
+                                                        }
                                                     }
                                                 },
                                                 "customContent": {
                                                     "listViewModel": {
                                                         "listItems": [
-                                                            _collab_row("Alice", "/@alice"),
                                                             _collab_row(
-                                                                "Bob", "/channel/UCbob123"
+                                                                "Alice", "/@alice"
+                                                            ),
+                                                            _collab_row(
+                                                                "Bob",
+                                                                "/channel/UCbob123",
                                                             ),
                                                         ]
                                                     }
@@ -610,7 +678,11 @@ def test_parse_collaborators_from_dialog():
     collaborators = parse_collaborators(initial)
     assert collaborators == [
         {"name": "Alice", "username": "alice", "url": "https://www.youtube.com/@alice"},
-        {"name": "Bob", "username": None, "url": "https://www.youtube.com/channel/UCbob123"},
+        {
+            "name": "Bob",
+            "username": None,
+            "url": "https://www.youtube.com/channel/UCbob123",
+        },
     ]
 
 
@@ -627,8 +699,23 @@ def test_parse_translation_from_next():
                 "results": {
                     "results": {
                         "contents": [
-                            {"videoPrimaryInfoRenderer": {"title": {"runs": [{"text": "Título "}, {"text": "traducido"}]}}},
-                            {"videoSecondaryInfoRenderer": {"attributedDescription": {"content": "Descripción traducida"}}},
+                            {
+                                "videoPrimaryInfoRenderer": {
+                                    "title": {
+                                        "runs": [
+                                            {"text": "Título "},
+                                            {"text": "traducido"},
+                                        ]
+                                    }
+                                }
+                            },
+                            {
+                                "videoSecondaryInfoRenderer": {
+                                    "attributedDescription": {
+                                        "content": "Descripción traducida"
+                                    }
+                                }
+                            },
                         ]
                     }
                 }
@@ -642,7 +729,9 @@ def test_parse_translation_from_next():
 
 def test_parse_translation_description_runs_fallback():
     next_data = {
-        "videoSecondaryInfoRenderer": {"description": {"runs": [{"text": "old "}, {"text": "style"}]}}
+        "videoSecondaryInfoRenderer": {
+            "description": {"runs": [{"text": "old "}, {"text": "style"}]}
+        }
     }
     title, description = parse_translation(next_data)
     assert title is None
@@ -671,6 +760,7 @@ def _comment_cep(cid, *, level=0, hearted=False, owner=False, replies="5"):
 
 def _next_comments_response():
     """A /next comments response: sort menu, two threads, a page token."""
+
     def _reply_loader(token):
         return {
             "continuationItemRenderer": {
@@ -692,7 +782,13 @@ def _next_comments_response():
         "frameworkUpdates": {
             "entityBatchUpdate": {
                 "mutations": [
-                    {"payload": {"commentEntityPayload": _comment_cep("C1", hearted=True, owner=True)}},
+                    {
+                        "payload": {
+                            "commentEntityPayload": _comment_cep(
+                                "C1", hearted=True, owner=True
+                            )
+                        }
+                    },
                     {"payload": {"commentEntityPayload": _comment_cep("C2")}},
                 ]
             }
@@ -717,8 +813,14 @@ def _next_comments_response():
         "header": {
             "sortFilterSubMenuRenderer": {
                 "subMenuItems": [
-                    {"title": "Top", "serviceEndpoint": {"continuationCommand": {"token": "TOPTOK"}}},
-                    {"title": "Newest", "serviceEndpoint": {"continuationCommand": {"token": "NEWTOK"}}},
+                    {
+                        "title": "Top",
+                        "serviceEndpoint": {"continuationCommand": {"token": "TOPTOK"}},
+                    },
+                    {
+                        "title": "Newest",
+                        "serviceEndpoint": {"continuationCommand": {"token": "NEWTOK"}},
+                    },
                 ]
             }
         },
@@ -749,7 +851,13 @@ def test_comment_entity_reply_level_and_empty_reply_count():
             "frameworkUpdates": {
                 "entityBatchUpdate": {
                     "mutations": [
-                        {"payload": {"commentEntityPayload": _comment_cep("R1", level=1, replies="")}}
+                        {
+                            "payload": {
+                                "commentEntityPayload": _comment_cep(
+                                    "R1", level=1, replies=""
+                                )
+                            }
+                        }
                     ]
                 }
             }
@@ -789,7 +897,9 @@ def test_comment_section_token_from_watch_page():
                                         {
                                             "continuationItemRenderer": {
                                                 "continuationEndpoint": {
-                                                    "continuationCommand": {"token": "SECTIONTOK"}
+                                                    "continuationCommand": {
+                                                        "token": "SECTIONTOK"
+                                                    }
                                                 }
                                             }
                                         }
@@ -815,9 +925,17 @@ def test_comment_section_token_from_watch_page():
         ("https://youtu.be/dQw4w9WgXcQ", "video", "dQw4w9WgXcQ"),
         ("https://www.youtube.com/shorts/abc123", "video", "abc123"),
         ("https://www.youtube.com/@Apify", "channel", "Apify"),
-        ("https://www.youtube.com/channel/UC123456789abc/videos", "channel", "UC123456789abc"),
+        (
+            "https://www.youtube.com/channel/UC123456789abc/videos",
+            "channel",
+            "UC123456789abc",
+        ),
         ("https://www.youtube.com/playlist?list=PL123", "playlist", "PL123"),
-        ("https://www.youtube.com/results?search_query=web+scraping", "search", "web scraping"),
+        (
+            "https://www.youtube.com/results?search_query=web+scraping",
+            "search",
+            "web scraping",
+        ),
         ("https://www.youtube.com/hashtag/tech", "hashtag", "tech"),
     ],
 )

surfsense_backend/tests/unit/tasks/chat/streaming/test_llm_bundle.py

@@ -32,9 +32,7 @@ def _patch_common_bundle_dependencies(monkeypatch: pytest.MonkeyPatch):
 
     _CapturedChatLiteLLM.calls = []
 
-    async def _fake_workspace(
-        _session: Any, _workspace_id: int
-    ) -> SimpleNamespace:
+    async def _fake_workspace(_session: Any, _workspace_id: int) -> SimpleNamespace:
         return SimpleNamespace(id=42, user_id="user-1")
 
     monkeypatch.setattr(llm_bundle, "_load_workspace", _fake_workspace)

surfsense_backend/tests/unit/test_pat_fail_closed_static.py

@@ -16,9 +16,7 @@ ALLOW_ANY_EXPECTED = {
     "routes/obsidian_plugin_routes.py": (
         "_auth: AuthContext = Depends(allow_any_principal)"
     ),
-    "routes/workspaces_routes.py": (
-        "auth: AuthContext = Depends(allow_any_principal)"
-    ),
+    "routes/workspaces_routes.py": ("auth: AuthContext = Depends(allow_any_principal)"),
 }
 
 CONNECTOR_LISTERS = [

surfsense_backend/tests/unit/utils/test_validators.py

@@ -11,10 +11,10 @@ from app.utils.validators import (
     validate_messages,
     validate_research_mode,
     validate_search_mode,
-    validate_workspace_id,
     validate_top_k,
     validate_url,
     validate_uuid,
+    validate_workspace_id,
 )
 
 pytestmark = pytest.mark.unit
@@ -334,6 +334,4 @@ def test_validate_connector_config_invalid():
 
     # WEBCRAWLER_CONNECTOR custom validation: malformed INITIAL_URLS rejected.
     with pytest.raises(ValueError):
-        validate_connector_config(
-            "WEBCRAWLER_CONNECTOR", {"INITIAL_URLS": "not-a-url"}
-        )
+        validate_connector_config("WEBCRAWLER_CONNECTOR", {"INITIAL_URLS": "not-a-url"})