fix(tiktok): warm browser listing + drop ctx-var-in-generator

surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py

@@ -16,7 +16,7 @@ from .flows import FetchFn, FetchListingFn
 from .flows.listing import iter_listing
 from .flows.video import iter_video
 from .schemas import TikTokScrapeInput
-from .session import fetch_html, fetch_item_list, proxy_session
+from .session import fetch_html, fetch_item_list
 from .targets import resolve_target
 from .targets.types import TikTokTarget
 
@@ -66,14 +66,18 @@ async def iter_tiktok(
     fetch: FetchFn = fetch_html,
     fetch_listing: FetchListingFn = fetch_item_list,
 ) -> AsyncIterator[dict[str, Any]]:
-    """Yield normalized items for every resolved target, in order."""
+    """Yield normalized items for every resolved target, in order.
+
+    The video flow's ``fetch_html`` opens its own warmed proxy session per call
+    when none is bound; the listing flow drives its own browser. Neither binds a
+    ContextVar across these ``yield``s, so the generator stays context-safe.
+    """
     cap = input_model.resultsPerPage
-    async with proxy_session():
-        for target in _resolve_targets(input_model):
-            async for item in _dispatch(
-                target, cap=cap, fetch=fetch, fetch_listing=fetch_listing
-            ):
-                yield item
+    for target in _resolve_targets(input_model):
+        async for item in _dispatch(
+            target, cap=cap, fetch=fetch, fetch_listing=fetch_listing
+        ):
+            yield item
 
 
 async def scrape_tiktok(

surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py

@@ -9,12 +9,17 @@ signs correctly for whatever version TikTok ships.
 
 The pure response-shape parsing lives in :func:`items_from_response`; this module
 is the untested browser-I/O glue (covered by the e2e smoke, not unit tests).
+
+Requires a residential proxy: TikTok throttles bare/datacenter IPs, returning
+empty ``item_list`` bodies (and 429s) after a few hits. Set
+``TIKTOK_LISTING_DEBUG=1`` to print captured XHR URLs while diagnosing.
 """
 
 from __future__ import annotations
 
 import asyncio
 import logging
+import os
 from typing import Any
 
 from scrapling.fetchers import StealthyFetcher
@@ -35,27 +40,68 @@ _ITEM_LIST_MARKERS = (
     "/api/challenge/item_list",
     "/api/search/",
 )
+_HOME_URL = "https://www.tiktok.com/"
+_MSTOKEN_COOKIE = "msToken"
 # Bounded scroll: a dead page can't loop forever, and a live one stops early
 # once enough items are captured.
 _SCROLL_MAX_ROUNDS = 20
-_SCROLL_SETTLE_MS = 1200
+_SCROLL_SETTLE_MS = 1500
+# Warm-up poll for the anonymous msToken cookie the item_list API requires.
+_WARM_POLLS = 8
+_WARM_POLL_MS = 500
 
 
-def _build_page_action(collected: list[dict[str, Any]], target_count: int):
-    """A sync ``page_action`` that captures item_list XHRs while scrolling."""
+def _has_mstoken(page: Any) -> bool:
+    try:
+        return any(c.get("name") == _MSTOKEN_COOKIE for c in page.context.cookies())
+    except Exception:
+        return False
+
+
+def _build_page_action(collected: list[dict[str, Any]], url: str, target_count: int):
+    """A sync ``page_action`` that warms the session then captures item_list XHRs.
+
+    A cold context returns an empty ``item_list`` body, so we first mint the
+    anonymous ``msToken`` (homepage hit), then navigate to the target with the
+    listener already attached so page-one fires into it; scrolling pages the rest.
+    """
+
+    debug = bool(os.getenv("TIKTOK_LISTING_DEBUG"))
 
     def _on_response(response: Any) -> None:
+        response_url = getattr(response, "url", "")
+        if debug and "/api/" in response_url:
+            print(f"    [xhr] {getattr(response, 'status', '?')} {response_url[:120]}")
         try:
-            if not any(marker in response.url for marker in _ITEM_LIST_MARKERS):
+            if not any(marker in response_url for marker in _ITEM_LIST_MARKERS):
                 return
             body = response.json()
-        except Exception:
+        except Exception as exc:
+            if debug:
+                print(f"    [xhr-parse-fail] {exc} :: {response_url[:120]}")
             return
-        collected.extend(items_from_response(body))
+        items = items_from_response(body)
+        if debug:
+            print(f"    [match] +{len(items)} items from {response_url[:100]}")
+        collected.extend(items)
+
+    def _warm(page: Any) -> None:
+        if _has_mstoken(page):
+            return
+        page.goto(_HOME_URL, wait_until="domcontentloaded")
+        for _ in range(_WARM_POLLS):
+            page.wait_for_timeout(_WARM_POLL_MS)
+            if _has_mstoken(page):
+                break
 
     def page_action(page: Any) -> Any:
         page.on("response", _on_response)
         try:
+            _warm(page)
+            # Navigate (back) to the target with the listener attached and a
+            # token in hand, so the page-one item_list fires into the capture.
+            page.goto(url, wait_until="domcontentloaded")
+            page.wait_for_timeout(_SCROLL_SETTLE_MS)
             last_height = 0
             for _ in range(_SCROLL_MAX_ROUNDS):
                 if len(collected) >= target_count:
@@ -81,7 +127,7 @@ def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]:
         headless=True,
         network_idle=False,
         proxy=get_proxy_url(),
-        page_action=_build_page_action(collected, target_count),
+        page_action=_build_page_action(collected, url, target_count),
         **kwargs,
     )
     return collected[:target_count]