refactor: implement cross-country proxy rotation for Reddit and TikTok scrapers, enhancing resilience against IP blocks and improving search query handling

surfsense_backend/tests/unit/platforms/reddit/test_fetch_resilience.py

@@ -266,3 +266,7 @@ async def test_fan_out_closes_all_sessions_on_early_stop(monkeypatch):
 async def test_fan_out_empty_jobs_is_noop():
     out = [x async for x in scraper.fan_out([])]
     assert out == []
+
+
+# Cross-country rotation lives in app.utils.proxy.rotation and is shared with the
+# TikTok sibling; its unit tests live in tests/unit/utils/proxy/test_rotation.py.

surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py

@@ -169,6 +169,37 @@ async def test_listing_dedupes_then_caps_per_target():
     assert [i["id"] for i in items] == ["1", "2"]
 
 
+async def test_search_query_resolves_to_listing_target():
+    # searchQueries must produce a search target (not be silently dropped): a
+    # query with results flows through the listing parse/dedupe/cap path.
+    async def fake_listing(url: str, _count: int) -> list[dict]:
+        assert "search?q=meal%20prep" in url  # query wired into the search URL
+        return [{"id": "1", "author": {"uniqueId": "a"}}]
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(searchQueries=["meal prep"], resultsPerPage=5),
+        fetch=_no_html,
+        fetch_listing=fake_listing,
+    )
+    assert [i["id"] for i in items] == ["1"]
+
+
+async def test_empty_search_query_degrades_to_error_item():
+    # A withheld anonymous search feed must surface one honest ErrorItem tagged
+    # with the query, never a silent empty.
+    async def fake_listing(_url: str, _count: int) -> list[dict]:
+        return []
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(searchQueries=["meal prep"], resultsPerPage=5),
+        fetch=_no_html,
+        fetch_listing=fake_listing,
+    )
+    assert len(items) == 1
+    assert items[0]["errorCode"] == "no_items"
+    assert items[0]["input"] == "meal prep"
+
+
 async def test_empty_listing_emits_error_item():
     # A trust-gated/empty feed (0 videos) must surface one honest ErrorItem,
     # tagged with errorCode, rather than vanishing silently.

surfsense_backend/tests/unit/utils/proxy/test_rotation.py

@@ -0,0 +1,53 @@
+"""Unit tests for the shared cross-country exit rotation helper.
+
+Offline: the provider is faked via monkeypatch, so no network/proxy is touched.
+"""
+
+from __future__ import annotations
+
+from app.utils.proxy import rotation
+
+
+class _Prov:
+    def __init__(self, location: str) -> None:
+        self._location = location
+
+    def get_location(self) -> str:
+        return self._location
+
+
+def test_lead_country_leads_and_dedupes(monkeypatch):
+    # The configured/default exit country leads and isn't duplicated in the walk.
+    monkeypatch.setattr(rotation, "get_active_provider", lambda: _Prov("gb"))
+    countries = rotation.rotation_countries()
+
+    assert countries[0] == "gb"
+    assert len(countries) == len(set(countries))  # de-duplicated
+    assert set(rotation.FALLBACK_COUNTRIES) <= set(countries)  # fallbacks kept
+
+
+def test_no_configured_country_uses_fallbacks_only(monkeypatch):
+    # A bare PROXY_URL (no country) leaves just the fallback pools, in order.
+    monkeypatch.setattr(rotation, "get_active_provider", lambda: _Prov(""))
+    assert rotation.rotation_countries() == rotation.FALLBACK_COUNTRIES
+
+
+def test_walk_covers_every_country_and_wraps(monkeypatch):
+    # A whole-pool block can't stall the walk: every country is reached, and the
+    # index cycles (wraps) rather than running off the end.
+    monkeypatch.setattr(rotation, "get_active_provider", lambda: _Prov("us"))
+    countries = rotation.rotation_countries()
+    tried = {rotation.country_for_rotation(n) for n in range(len(countries))}
+
+    assert tried == set(countries)
+    assert rotation.country_for_rotation(len(countries)) == countries[0]  # wraps
+
+
+def test_caller_budgets_cover_every_country():
+    # Each warm-on-block caller must budget enough rotations to try every pool at
+    # least once, else a wholly-blocked lead pool could fail a job prematurely.
+    from app.proprietary.platforms.reddit import fetch as reddit_fetch
+    from app.proprietary.platforms.tiktok.session import client as tiktok_client
+
+    assert reddit_fetch._MAX_ROTATIONS >= len(rotation.FALLBACK_COUNTRIES)
+    assert tiktok_client._MAX_ROTATIONS >= len(rotation.FALLBACK_COUNTRIES)