refactor: implement cross-country proxy rotation for Reddit and TikTok scrapers, enhancing resilience against IP blocks and improving search query handling

surfsense_backend/app/proprietary/platforms/reddit/fetch.py

@@ -37,7 +37,15 @@ from urllib.parse import urlencode
 
 from scrapling.fetchers import AsyncFetcher, FetcherSession
 
-from app.utils.proxy import get_proxy_url
+from app.utils.proxy import get_geo_proxy_url, get_proxy_url
+
+# Shared cross-country rotation walk (also used by the TikTok sibling). Kept under
+# the historical private names this module and its tests reference.
+from app.utils.proxy.rotation import (
+    FALLBACK_COUNTRIES as _FALLBACK_COUNTRIES,
+    country_for_rotation as _country_for_rotation,
+    rotation_countries as _rotation_countries,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -68,7 +76,16 @@ _current_session: ContextVar[_RotatingSession | None] = ContextVar(
 # different handling per status (spec section 3).
 _ROTATE_STATUS = 403
 _BACKOFF_STATUS = 429
-_MAX_ROTATIONS = 3
+# Rotating an IP is cheap (close + reopen one keep-alive connection through the
+# gateway + a 2-request warm ≈ a few seconds) and each rotation also walks to the
+# next country pool, so we spend rotations liberally: neither a dirty IP nor a
+# wholly-blocked country pool should fail a job. 8 ≥ len(_FALLBACK_COUNTRIES), so
+# a job tries every country at least once before giving up. Worst case (a genuine
+# global block) costs _MAX_ROTATIONS bounded warm attempts before
+# RedditAccessBlockedError.
+# ponytail: 8 caps that worst case at ~30s; raise if every pool gets dirty at
+# once, lower if a real global block is wasting time.
+_MAX_ROTATIONS = 8
 _MAX_BACKOFFS = 4
 _BACKOFF_BASE_S = 5.0
 
@@ -155,11 +172,13 @@ class _RotatingSession:
     """Owns one live ``FetcherSession`` (sticky IP) and can swap it for a fresh one.
 
     ``rotate()`` closes the current keep-alive connection and opens a new one, so
-    the rotating gateway hands out a different residential exit IP. Because the
-    ``loid`` cookie binds to the exit IP, ``rotate()`` also drops the warmed
-    state — the next fetch re-warms on the new IP. Used sequentially within a
-    single flow (never shared across concurrent tasks), so no locking is needed.
-    ``session`` is ``None`` only when no proxy is configured.
+    the rotating gateway hands out a different residential exit IP — walking to
+    the next country pool (see :func:`_country_for_rotation`) so a wholly-blocked
+    pool can't fail the flow. Because the ``loid`` cookie binds to the exit IP,
+    ``rotate()`` also drops the warmed state — the next fetch re-warms on the new
+    IP. Used sequentially within a single flow (never shared across concurrent
+    tasks), so no locking is needed. ``session`` is ``None`` only when no proxy
+    is configured.
     """
 
     def __init__(self) -> None:
@@ -167,11 +186,13 @@ class _RotatingSession:
         self.session: Any | None = None
         self.rotations = 0
         self.warmed = False
+        self.country = ""
         self._last_at = 0.0
 
     async def _open(self) -> None:
-        proxy = get_proxy_url()
         self.warmed = False
+        self.country = _country_for_rotation(self.rotations)
+        proxy = get_geo_proxy_url(self.country)
         if proxy is None:
             self._cm = self.session = None
             return
@@ -194,7 +215,11 @@ class _RotatingSession:
         await self.close()
         self.rotations += 1
         await self._open()
-        logger.info("[reddit] rotated proxy session (rotation #%d)", self.rotations)
+        logger.info(
+            "[reddit] rotated proxy session (rotation #%d, country=%s)",
+            self.rotations,
+            self.country,
+        )
         return self.session
 
     async def pace(self) -> None:

surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 from collections.abc import AsyncIterator
 from typing import Any
+from urllib.parse import quote
 
 from .extraction.timestamps import now_iso
 from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn
@@ -31,15 +32,18 @@ from .targets.types import TikTokTarget
 
 _PROFILE_URL = "https://www.tiktok.com/@{name}"
 _HASHTAG_URL = "https://www.tiktok.com/tag/{tag}"
+_SEARCH_URL = "https://www.tiktok.com/search?q={query}"
 _EXPLORE_URL = "https://www.tiktok.com/explore"
 
 
 def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
-    """Build the target list from the URL/profile/hashtag sources.
+    """Build the target list from every input source.
 
-    A raw ``tiktok.com/search?...`` URL passed explicitly in
-    ``startUrls``/``postURLs`` still resolves here and keeps its native listing
-    routing; there is no keyword-search shortcut.
+    ``searchQueries`` map to the same ``tiktok.com/search?q=`` targets that a raw
+    search URL in ``startUrls``/``postURLs`` resolves to, so both share the
+    listing flow's parse/dedupe/cap/empty-ErrorItem contract (the anonymous
+    search feed often withholds results, degrading to one honest ErrorItem rather
+    than a silent empty).
     """
     targets: list[TikTokTarget] = []
     for entry in input_model.startUrls:
@@ -55,6 +59,10 @@ def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
         targets.append(TikTokTarget("profile", name, _PROFILE_URL.format(name=name)))
     for tag in input_model.hashtags:
         targets.append(TikTokTarget("hashtag", tag, _HASHTAG_URL.format(tag=tag)))
+    for query in input_model.searchQueries:
+        resolved = resolve_target(_SEARCH_URL.format(query=quote(query)))
+        if resolved is not None:
+            targets.append(resolved)
     return targets
 
 

surfsense_backend/app/proprietary/platforms/tiktok/session/client.py

@@ -26,7 +26,12 @@ logger = logging.getLogger(__name__)
 # 403 => IP blocked; rotate and re-warm. 429 => rate limited; back off same IP.
 _ROTATE_STATUS = 403
 _BACKOFF_STATUS = 429
-_MAX_ROTATIONS = 3
+# Each rotation walks to the next country pool (see session/proxy.py). The bare
+# worldwide pool never mints ``ttwid`` (proven live 2026-07-17), so warming
+# relies on reaching a good country pool — budget enough rotations to try every
+# one at least once (>= len(rotation FALLBACK_COUNTRIES)). Rotating is cheap
+# (reopen one keep-alive connection + a 1-request warm), so spend liberally.
+_MAX_ROTATIONS = 8
 _MAX_BACKOFFS = 4
 _BACKOFF_BASE_S = 5.0
 

surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py

@@ -18,7 +18,8 @@ from typing import Any
 
 from scrapling.fetchers import FetcherSession
 
-from app.utils.proxy import get_proxy_url
+from app.utils.proxy import get_geo_proxy_url
+from app.utils.proxy.rotation import country_for_rotation
 
 logger = logging.getLogger(__name__)
 
@@ -37,9 +38,13 @@ _current_session: ContextVar[_RotatingSession | None] = ContextVar(
 class _RotatingSession:
     """Owns one live ``FetcherSession`` (sticky IP); ``rotate()`` swaps the IP.
 
-    Used sequentially within a single flow (never shared across concurrent
-    tasks), so no locking is needed. ``session`` is ``None`` only when no proxy
-    is configured.
+    Each open walks to the next country pool (see
+    :func:`app.utils.proxy.rotation.country_for_rotation`): TikTok withholds the
+    anonymous ``ttwid`` cookie from the provider's default worldwide pool but
+    mints it on country-pinned exits, so re-drawing within the same pool never
+    warms — spreading rotations across country pools does. Used sequentially
+    within a single flow (never shared across concurrent tasks), so no locking is
+    needed. ``session`` is ``None`` only when no proxy is configured.
     """
 
     def __init__(self) -> None:
@@ -47,11 +52,13 @@ class _RotatingSession:
         self.session: Any | None = None
         self.rotations = 0
         self.warmed = False
+        self.country = ""
         self._last_at = 0.0
 
     async def _open(self) -> None:
-        proxy = get_proxy_url()
         self.warmed = False
+        self.country = country_for_rotation(self.rotations)
+        proxy = get_geo_proxy_url(self.country)
         if proxy is None:
             self._cm = self.session = None
             return
@@ -74,7 +81,11 @@ class _RotatingSession:
         await self.close()
         self.rotations += 1
         await self._open()
-        logger.info("[tiktok] rotated proxy session (rotation #%d)", self.rotations)
+        logger.info(
+            "[tiktok] rotated proxy session (rotation #%d, country=%s)",
+            self.rotations,
+            self.country,
+        )
         return self.session
 
     async def pace(self) -> None:

surfsense_backend/app/utils/proxy/rotation.py

@@ -0,0 +1,34 @@
+"""Cross-country exit rotation for warm-session scrapers.
+
+Some targets (Reddit's ``loid``, TikTok's ``ttwid``) silently withhold their
+anonymous session cookie from the provider's *default worldwide* pool but hand
+it out freely on **country-pinned** exits (proven live: a bare-pool homepage
+hit returns 200 with an empty cookie jar, while a us/gb/de/nl-pinned hit mints
+the cookie every time). A warm-on-block flow that only re-draws from the same
+worldwide pool therefore burns every rotation on cookie-less IPs and fails.
+
+Walking a spread of country pools instead lets the flow escape a wholly-blocked
+pool. The provider's configured country leads (so an operator's choice is
+honoured first); the fallbacks are large, reliable residential pools. Non-geo
+providers (e.g. the custom single-URL provider) ignore the country and re-draw
+their one URL, so this is a harmless no-op there.
+"""
+
+from __future__ import annotations
+
+from app.utils.proxy.registry import get_active_provider
+
+# Walk order after the configured country. Ordered by pool size / reliability.
+FALLBACK_COUNTRIES = ("us", "gb", "de", "ca", "nl", "fr")
+
+
+def rotation_countries() -> tuple[str, ...]:
+    """Ordered, de-duplicated exit countries with the configured one leading."""
+    lead = get_active_provider().get_location()
+    return tuple(dict.fromkeys(c for c in (lead, *FALLBACK_COUNTRIES) if c))
+
+
+def country_for_rotation(n: int) -> str:
+    """Exit country for rotation index ``n`` (cycles the list, wrapping around)."""
+    countries = rotation_countries()
+    return countries[n % len(countries)]