diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py new file mode 100644 index 000000000..659305055 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py @@ -0,0 +1,15 @@ +"""Cookie-warmed, rotate-on-block proxy session and page-fetch seam.""" + +from __future__ import annotations + +from .client import fetch_html +from .errors import TikTokAccessBlockedError +from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session + +__all__ = [ + "TikTokAccessBlockedError", + "bind_proxy_holder", + "fetch_html", + "open_proxy_holder", + "proxy_session", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py new file mode 100644 index 000000000..e5be1e419 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py @@ -0,0 +1,130 @@ +"""GET TikTok page HTML through a cookie-warmed, sticky-IP proxy session. + +The warm-up mints TikTok's anonymous device cookie (``ttwid``) on the first +homepage hit; the target page then server-renders its rehydration blob. Rotates +the residential IP and re-warms on 403, backs off on 429, and raises +:class:`TikTokAccessBlockedError` only when every rotated IP refuses access. +""" + +from __future__ import annotations + +import asyncio +import logging +import random +from contextlib import suppress +from typing import Any + +from scrapling.fetchers import AsyncFetcher + +from app.utils.proxy import get_proxy_url + +from .errors import TikTokAccessBlockedError +from .proxy import _REQUEST_TIMEOUT_S, _current_session, proxy_session + +logger = logging.getLogger(__name__) + +# 403 => IP blocked; rotate and re-warm. 429 => rate limited; back off same IP. +_ROTATE_STATUS = 403 +_BACKOFF_STATUS = 429 +_MAX_ROTATIONS = 3 +_MAX_BACKOFFS = 4 +_BACKOFF_BASE_S = 5.0 + +_HOME_URL = "https://www.tiktok.com/" +_TTWID_COOKIE = "ttwid" +_HEADERS = {"Accept-Language": "en-US,en;q=0.9"} + + +def _response_cookie_names(page: Any) -> set[str]: + cookies = getattr(page, "cookies", None) + return set(cookies.keys()) if isinstance(cookies, dict) else set() + + +def _page_html(page: Any) -> str | None: + for attr in ("text", "body"): + val = getattr(page, attr, None) + if isinstance(val, bytes): + val = val.decode("utf-8", "replace") + if isinstance(val, str) and val.strip(): + return val + return None + + +async def warm_session(session: Any) -> bool: + """Mint an anonymous ``ttwid`` cookie; ``True`` if the session can now fetch.""" + with suppress(Exception): + page = await session.get(_HOME_URL, headers=_HEADERS) + if _TTWID_COOKIE in _response_cookie_names(page): + return True + return False + + +async def _get_page(session: Any, url: str) -> Any: + if session is not None: + return await session.get(url, headers=_HEADERS) + return await AsyncFetcher.get( + url, + headers=_HEADERS, + proxy=get_proxy_url(), + stealthy_headers=True, + timeout=_REQUEST_TIMEOUT_S, + ) + + +async def fetch_html(url: str) -> str | None: + """Return page HTML, or ``None`` on 404 / non-block failure.""" + holder = _current_session.get() + if holder is None: + async with proxy_session(): + return await fetch_html(url) + + attempt = 0 + backoffs = 0 + while True: + session = holder.session + try: + if session is not None and not holder.warmed: + warmed_ok = await warm_session(session) + holder.warmed = True + if not warmed_ok: + if attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + raise TikTokAccessBlockedError( + f"could not warm session after {attempt} IP rotations: {url}" + ) + + await holder.pace() + page = await _get_page(session, url) + status = page.status + + if status == 200: + return _page_html(page) + if status == 404: + return None + if status == _BACKOFF_STATUS and backoffs < _MAX_BACKOFFS: + backoffs += 1 + delay = _BACKOFF_BASE_S * (2 ** (backoffs - 1)) + logger.warning("[tiktok] 429 on %s; backing off %.1fs", url, delay) + await asyncio.sleep(delay + random.uniform(0, 1)) + continue + if status == _ROTATE_STATUS and attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + if status == _ROTATE_STATUS: + raise TikTokAccessBlockedError( + f"TikTok refused {url} on {attempt} rotated IPs (403)" + ) + logger.warning("[tiktok] GET %s returned %s", url, status) + return None + except TikTokAccessBlockedError: + raise + except Exception as e: + logger.warning("[tiktok] GET %s failed: %s", url, e) + if attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + return None diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py new file mode 100644 index 000000000..4d9fc712d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py @@ -0,0 +1,11 @@ +"""Fetch-seam errors surfaced to the capability layer.""" + +from __future__ import annotations + + +class TikTokAccessBlockedError(RuntimeError): + """Raised when every rotated IP is refused anonymous access. + + Anonymous-only: we cannot log in, so a hard block is surfaced loudly rather + than returning empty data. The route maps it to a 403. + """ diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py new file mode 100644 index 000000000..dca6c776b --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py @@ -0,0 +1,113 @@ +"""Rotate-on-block sticky proxy session, bound per-flow via a ContextVar. + +Reusing one keep-alive connection pins a single residential exit IP so the +warmed cookie jar (``ttwid``/``msToken``, bound to that IP) stays valid across +the warm-up and every subsequent fetch. Ported from the Reddit sibling; the +TikTok-specific warm-up lives in :mod:`client`. +""" + +from __future__ import annotations + +import asyncio +import logging +import random +import time +from contextlib import asynccontextmanager, suppress +from contextvars import ContextVar +from typing import Any + +from scrapling.fetchers import FetcherSession + +from app.utils.proxy import get_proxy_url + +logger = logging.getLogger(__name__) + +# Pace each sticky IP so a fast exit can't burst past TikTok's per-IP threshold. +_MIN_INTERVAL_S = 0.5 +_PACE_JITTER_S = 0.25 +# A healthy fetch lands in ~1-2s; cap a dead IP at one bounded wait before it +# falls through to a rotation. +_REQUEST_TIMEOUT_S = 15.0 + +_current_session: ContextVar[_RotatingSession | None] = ContextVar( + "tiktok_proxy_session", default=None +) + + +class _RotatingSession: + """Owns one live ``FetcherSession`` (sticky IP); ``rotate()`` swaps the IP. + + Used sequentially within a single flow (never shared across concurrent + tasks), so no locking is needed. ``session`` is ``None`` only when no proxy + is configured. + """ + + def __init__(self) -> None: + self._cm: Any | None = None + self.session: Any | None = None + self.rotations = 0 + self.warmed = False + self._last_at = 0.0 + + async def _open(self) -> None: + proxy = get_proxy_url() + self.warmed = False + if proxy is None: + self._cm = self.session = None + return + self._cm = FetcherSession( + proxy=proxy, + stealthy_headers=True, + impersonate="chrome", + timeout=_REQUEST_TIMEOUT_S, + ) + self.session = await self._cm.__aenter__() + + async def close(self) -> None: + if self._cm is not None: + with suppress(Exception): + await self._cm.__aexit__(None, None, None) + self._cm = self.session = None + + async def rotate(self) -> Any | None: + """Drop the current IP and connect through a fresh one.""" + await self.close() + self.rotations += 1 + await self._open() + logger.info("[tiktok] rotated proxy session (rotation #%d)", self.rotations) + return self.session + + async def pace(self) -> None: + """Sleep to hold this sticky IP under TikTok's per-IP rate threshold.""" + wait = _MIN_INTERVAL_S - (time.monotonic() - self._last_at) + if wait > 0: + await asyncio.sleep(wait + random.uniform(0, _PACE_JITTER_S)) + self._last_at = time.monotonic() + + +async def open_proxy_holder() -> _RotatingSession: + """Open a warm rotate-on-block session holder (caller owns ``close()``).""" + holder = _RotatingSession() + await holder._open() + return holder + + +@asynccontextmanager +async def bind_proxy_holder(holder: _RotatingSession): + """Route this task's fetches through ``holder`` for the enclosed block.""" + token = _current_session.set(holder) + try: + yield holder + finally: + _current_session.reset(token) + + +@asynccontextmanager +async def proxy_session(): + """Open one reused, rotate-on-block proxy session for a continuation chain.""" + holder = await open_proxy_holder() + try: + async with bind_proxy_holder(holder): + yield holder + finally: + await holder.close() diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py new file mode 100644 index 000000000..484157964 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py @@ -0,0 +1,135 @@ +"""Fetch-seam resilience for the TikTok scraper (no network, fake sessions). + +Fake sessions drive the cookie warm-up + rotate-on-block + backoff branches +deterministically; a live first IP normally warms and returns 200s. +""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.session import ( + TikTokAccessBlockedError, + client, +) +from app.proprietary.platforms.tiktok.session.proxy import _current_session + +_HTML = "ok" + + +class _FakePage: + def __init__(self, status: int, *, cookies: dict | None = None, body: str = _HTML): + self.status = status + self.cookies = cookies or {} + self.body = body + + @property + def text(self) -> str: + return self.body + + +class _FakeSession: + """One 'IP': homepage warm mints ``ttwid`` per flag; page GETs return ``status``.""" + + def __init__(self, status: int = 200, *, warms: bool = True, body: str = _HTML): + self.status = status + self.warms = warms + self.body = body + self.page_calls = 0 + self.warm_calls = 0 + + async def get(self, url, headers=None, cookies=None): + if url.rstrip("/") == "https://www.tiktok.com": + self.warm_calls += 1 + return _FakePage(200, cookies={"ttwid": "x"} if self.warms else {}) + self.page_calls += 1 + return _FakePage(self.status, body=self.body) + + +class _FakeHolder: + def __init__(self, sessions: list[_FakeSession]) -> None: + self._sessions = sessions + self.session = sessions[0] + self.rotations = 0 + self.warmed = False + + async def rotate(self): + self.rotations += 1 + self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)] + self.warmed = False + return self.session + + async def pace(self) -> None: + return None + + async def close(self) -> None: + return None + + +def _no_sleep(monkeypatch) -> None: + async def _noop(_seconds): + return None + + monkeypatch.setattr(client.asyncio, "sleep", _noop) + + +async def test_warms_then_returns_html(): + holder = _FakeHolder([_FakeSession(200, warms=True)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@scout2015") + finally: + _current_session.reset(token) + assert result == _HTML + assert holder.rotations == 0 + assert holder.session.warm_calls == 1 + + +async def test_rotates_when_warm_fails_then_succeeds(): + holder = _FakeHolder([_FakeSession(200, warms=False), _FakeSession(200, warms=True)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@scout2015") + finally: + _current_session.reset(token) + assert result == _HTML + assert holder.rotations == 1 + + +async def test_404_returns_none_without_rotating(): + holder = _FakeHolder([_FakeSession(404), _FakeSession(200)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@missing") + finally: + _current_session.reset(token) + assert result is None + assert holder.rotations == 0 + + +async def test_rotates_and_rewarms_on_403(): + holder = _FakeHolder([_FakeSession(403), _FakeSession(200, warms=True)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@scout2015") + finally: + _current_session.reset(token) + assert result == _HTML + assert holder.rotations == 1 + assert holder.session.warm_calls == 1 + + +async def test_persistent_403_raises_blocked(monkeypatch): + _no_sleep(monkeypatch) + holder = _FakeHolder( + [_FakeSession(403) for _ in range(client._MAX_ROTATIONS + 1)] + ) + token = _current_session.set(holder) + try: + raised = False + try: + await client.fetch_html("https://www.tiktok.com/@scout2015") + except TikTokAccessBlockedError: + raised = True + finally: + _current_session.reset(token) + assert raised + assert holder.rotations == client._MAX_ROTATIONS