diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index c7f2e9008..95bc10fb9 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -16,7 +16,7 @@ from .flows import FetchFn, FetchListingFn from .flows.listing import iter_listing from .flows.video import iter_video from .schemas import TikTokScrapeInput -from .session import fetch_html, fetch_item_list, proxy_session +from .session import fetch_html, fetch_item_list from .targets import resolve_target from .targets.types import TikTokTarget @@ -66,14 +66,18 @@ async def iter_tiktok( fetch: FetchFn = fetch_html, fetch_listing: FetchListingFn = fetch_item_list, ) -> AsyncIterator[dict[str, Any]]: - """Yield normalized items for every resolved target, in order.""" + """Yield normalized items for every resolved target, in order. + + The video flow's ``fetch_html`` opens its own warmed proxy session per call + when none is bound; the listing flow drives its own browser. Neither binds a + ContextVar across these ``yield``s, so the generator stays context-safe. + """ cap = input_model.resultsPerPage - async with proxy_session(): - for target in _resolve_targets(input_model): - async for item in _dispatch( - target, cap=cap, fetch=fetch, fetch_listing=fetch_listing - ): - yield item + for target in _resolve_targets(input_model): + async for item in _dispatch( + target, cap=cap, fetch=fetch, fetch_listing=fetch_listing + ): + yield item async def scrape_tiktok( diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index 0d19281a2..98b2e6e8b 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -9,12 +9,17 @@ signs correctly for whatever version TikTok ships. The pure response-shape parsing lives in :func:`items_from_response`; this module is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). + +Requires a residential proxy: TikTok throttles bare/datacenter IPs, returning +empty ``item_list`` bodies (and 429s) after a few hits. Set +``TIKTOK_LISTING_DEBUG=1`` to print captured XHR URLs while diagnosing. """ from __future__ import annotations import asyncio import logging +import os from typing import Any from scrapling.fetchers import StealthyFetcher @@ -35,27 +40,68 @@ _ITEM_LIST_MARKERS = ( "/api/challenge/item_list", "/api/search/", ) +_HOME_URL = "https://www.tiktok.com/" +_MSTOKEN_COOKIE = "msToken" # Bounded scroll: a dead page can't loop forever, and a live one stops early # once enough items are captured. _SCROLL_MAX_ROUNDS = 20 -_SCROLL_SETTLE_MS = 1200 +_SCROLL_SETTLE_MS = 1500 +# Warm-up poll for the anonymous msToken cookie the item_list API requires. +_WARM_POLLS = 8 +_WARM_POLL_MS = 500 -def _build_page_action(collected: list[dict[str, Any]], target_count: int): - """A sync ``page_action`` that captures item_list XHRs while scrolling.""" +def _has_mstoken(page: Any) -> bool: + try: + return any(c.get("name") == _MSTOKEN_COOKIE for c in page.context.cookies()) + except Exception: + return False + + +def _build_page_action(collected: list[dict[str, Any]], url: str, target_count: int): + """A sync ``page_action`` that warms the session then captures item_list XHRs. + + A cold context returns an empty ``item_list`` body, so we first mint the + anonymous ``msToken`` (homepage hit), then navigate to the target with the + listener already attached so page-one fires into it; scrolling pages the rest. + """ + + debug = bool(os.getenv("TIKTOK_LISTING_DEBUG")) def _on_response(response: Any) -> None: + response_url = getattr(response, "url", "") + if debug and "/api/" in response_url: + print(f" [xhr] {getattr(response, 'status', '?')} {response_url[:120]}") try: - if not any(marker in response.url for marker in _ITEM_LIST_MARKERS): + if not any(marker in response_url for marker in _ITEM_LIST_MARKERS): return body = response.json() - except Exception: + except Exception as exc: + if debug: + print(f" [xhr-parse-fail] {exc} :: {response_url[:120]}") return - collected.extend(items_from_response(body)) + items = items_from_response(body) + if debug: + print(f" [match] +{len(items)} items from {response_url[:100]}") + collected.extend(items) + + def _warm(page: Any) -> None: + if _has_mstoken(page): + return + page.goto(_HOME_URL, wait_until="domcontentloaded") + for _ in range(_WARM_POLLS): + page.wait_for_timeout(_WARM_POLL_MS) + if _has_mstoken(page): + break def page_action(page: Any) -> Any: page.on("response", _on_response) try: + _warm(page) + # Navigate (back) to the target with the listener attached and a + # token in hand, so the page-one item_list fires into the capture. + page.goto(url, wait_until="domcontentloaded") + page.wait_for_timeout(_SCROLL_SETTLE_MS) last_height = 0 for _ in range(_SCROLL_MAX_ROUNDS): if len(collected) >= target_count: @@ -81,7 +127,7 @@ def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]: headless=True, network_idle=False, proxy=get_proxy_url(), - page_action=_build_page_action(collected, target_count), + page_action=_build_page_action(collected, url, target_count), **kwargs, ) return collected[:target_count]