feat(tiktok): rotate-on-block fetch session

surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py

@@ -0,0 +1,15 @@
+"""Cookie-warmed, rotate-on-block proxy session and page-fetch seam."""
+
+from __future__ import annotations
+
+from .client import fetch_html
+from .errors import TikTokAccessBlockedError
+from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session
+
+__all__ = [
+    "TikTokAccessBlockedError",
+    "bind_proxy_holder",
+    "fetch_html",
+    "open_proxy_holder",
+    "proxy_session",
+]

surfsense_backend/app/proprietary/platforms/tiktok/session/client.py

@@ -0,0 +1,130 @@
+"""GET TikTok page HTML through a cookie-warmed, sticky-IP proxy session.
+
+The warm-up mints TikTok's anonymous device cookie (``ttwid``) on the first
+homepage hit; the target page then server-renders its rehydration blob. Rotates
+the residential IP and re-warms on 403, backs off on 429, and raises
+:class:`TikTokAccessBlockedError` only when every rotated IP refuses access.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import random
+from contextlib import suppress
+from typing import Any
+
+from scrapling.fetchers import AsyncFetcher
+
+from app.utils.proxy import get_proxy_url
+
+from .errors import TikTokAccessBlockedError
+from .proxy import _REQUEST_TIMEOUT_S, _current_session, proxy_session
+
+logger = logging.getLogger(__name__)
+
+# 403 => IP blocked; rotate and re-warm. 429 => rate limited; back off same IP.
+_ROTATE_STATUS = 403
+_BACKOFF_STATUS = 429
+_MAX_ROTATIONS = 3
+_MAX_BACKOFFS = 4
+_BACKOFF_BASE_S = 5.0
+
+_HOME_URL = "https://www.tiktok.com/"
+_TTWID_COOKIE = "ttwid"
+_HEADERS = {"Accept-Language": "en-US,en;q=0.9"}
+
+
+def _response_cookie_names(page: Any) -> set[str]:
+    cookies = getattr(page, "cookies", None)
+    return set(cookies.keys()) if isinstance(cookies, dict) else set()
+
+
+def _page_html(page: Any) -> str | None:
+    for attr in ("text", "body"):
+        val = getattr(page, attr, None)
+        if isinstance(val, bytes):
+            val = val.decode("utf-8", "replace")
+        if isinstance(val, str) and val.strip():
+            return val
+    return None
+
+
+async def warm_session(session: Any) -> bool:
+    """Mint an anonymous ``ttwid`` cookie; ``True`` if the session can now fetch."""
+    with suppress(Exception):
+        page = await session.get(_HOME_URL, headers=_HEADERS)
+        if _TTWID_COOKIE in _response_cookie_names(page):
+            return True
+    return False
+
+
+async def _get_page(session: Any, url: str) -> Any:
+    if session is not None:
+        return await session.get(url, headers=_HEADERS)
+    return await AsyncFetcher.get(
+        url,
+        headers=_HEADERS,
+        proxy=get_proxy_url(),
+        stealthy_headers=True,
+        timeout=_REQUEST_TIMEOUT_S,
+    )
+
+
+async def fetch_html(url: str) -> str | None:
+    """Return page HTML, or ``None`` on 404 / non-block failure."""
+    holder = _current_session.get()
+    if holder is None:
+        async with proxy_session():
+            return await fetch_html(url)
+
+    attempt = 0
+    backoffs = 0
+    while True:
+        session = holder.session
+        try:
+            if session is not None and not holder.warmed:
+                warmed_ok = await warm_session(session)
+                holder.warmed = True
+                if not warmed_ok:
+                    if attempt < _MAX_ROTATIONS:
+                        attempt += 1
+                        await holder.rotate()
+                        continue
+                    raise TikTokAccessBlockedError(
+                        f"could not warm session after {attempt} IP rotations: {url}"
+                    )
+
+            await holder.pace()
+            page = await _get_page(session, url)
+            status = page.status
+
+            if status == 200:
+                return _page_html(page)
+            if status == 404:
+                return None
+            if status == _BACKOFF_STATUS and backoffs < _MAX_BACKOFFS:
+                backoffs += 1
+                delay = _BACKOFF_BASE_S * (2 ** (backoffs - 1))
+                logger.warning("[tiktok] 429 on %s; backing off %.1fs", url, delay)
+                await asyncio.sleep(delay + random.uniform(0, 1))
+                continue
+            if status == _ROTATE_STATUS and attempt < _MAX_ROTATIONS:
+                attempt += 1
+                await holder.rotate()
+                continue
+            if status == _ROTATE_STATUS:
+                raise TikTokAccessBlockedError(
+                    f"TikTok refused {url} on {attempt} rotated IPs (403)"
+                )
+            logger.warning("[tiktok] GET %s returned %s", url, status)
+            return None
+        except TikTokAccessBlockedError:
+            raise
+        except Exception as e:
+            logger.warning("[tiktok] GET %s failed: %s", url, e)
+            if attempt < _MAX_ROTATIONS:
+                attempt += 1
+                await holder.rotate()
+                continue
+            return None

surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py

@@ -0,0 +1,11 @@
+"""Fetch-seam errors surfaced to the capability layer."""
+
+from __future__ import annotations
+
+
+class TikTokAccessBlockedError(RuntimeError):
+    """Raised when every rotated IP is refused anonymous access.
+
+    Anonymous-only: we cannot log in, so a hard block is surfaced loudly rather
+    than returning empty data. The route maps it to a 403.
+    """

surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py

@@ -0,0 +1,113 @@
+"""Rotate-on-block sticky proxy session, bound per-flow via a ContextVar.
+
+Reusing one keep-alive connection pins a single residential exit IP so the
+warmed cookie jar (``ttwid``/``msToken``, bound to that IP) stays valid across
+the warm-up and every subsequent fetch. Ported from the Reddit sibling; the
+TikTok-specific warm-up lives in :mod:`client`.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import random
+import time
+from contextlib import asynccontextmanager, suppress
+from contextvars import ContextVar
+from typing import Any
+
+from scrapling.fetchers import FetcherSession
+
+from app.utils.proxy import get_proxy_url
+
+logger = logging.getLogger(__name__)
+
+# Pace each sticky IP so a fast exit can't burst past TikTok's per-IP threshold.
+_MIN_INTERVAL_S = 0.5
+_PACE_JITTER_S = 0.25
+# A healthy fetch lands in ~1-2s; cap a dead IP at one bounded wait before it
+# falls through to a rotation.
+_REQUEST_TIMEOUT_S = 15.0
+
+_current_session: ContextVar[_RotatingSession | None] = ContextVar(
+    "tiktok_proxy_session", default=None
+)
+
+
+class _RotatingSession:
+    """Owns one live ``FetcherSession`` (sticky IP); ``rotate()`` swaps the IP.
+
+    Used sequentially within a single flow (never shared across concurrent
+    tasks), so no locking is needed. ``session`` is ``None`` only when no proxy
+    is configured.
+    """
+
+    def __init__(self) -> None:
+        self._cm: Any | None = None
+        self.session: Any | None = None
+        self.rotations = 0
+        self.warmed = False
+        self._last_at = 0.0
+
+    async def _open(self) -> None:
+        proxy = get_proxy_url()
+        self.warmed = False
+        if proxy is None:
+            self._cm = self.session = None
+            return
+        self._cm = FetcherSession(
+            proxy=proxy,
+            stealthy_headers=True,
+            impersonate="chrome",
+            timeout=_REQUEST_TIMEOUT_S,
+        )
+        self.session = await self._cm.__aenter__()
+
+    async def close(self) -> None:
+        if self._cm is not None:
+            with suppress(Exception):
+                await self._cm.__aexit__(None, None, None)
+        self._cm = self.session = None
+
+    async def rotate(self) -> Any | None:
+        """Drop the current IP and connect through a fresh one."""
+        await self.close()
+        self.rotations += 1
+        await self._open()
+        logger.info("[tiktok] rotated proxy session (rotation #%d)", self.rotations)
+        return self.session
+
+    async def pace(self) -> None:
+        """Sleep to hold this sticky IP under TikTok's per-IP rate threshold."""
+        wait = _MIN_INTERVAL_S - (time.monotonic() - self._last_at)
+        if wait > 0:
+            await asyncio.sleep(wait + random.uniform(0, _PACE_JITTER_S))
+        self._last_at = time.monotonic()
+
+
+async def open_proxy_holder() -> _RotatingSession:
+    """Open a warm rotate-on-block session holder (caller owns ``close()``)."""
+    holder = _RotatingSession()
+    await holder._open()
+    return holder
+
+
+@asynccontextmanager
+async def bind_proxy_holder(holder: _RotatingSession):
+    """Route this task's fetches through ``holder`` for the enclosed block."""
+    token = _current_session.set(holder)
+    try:
+        yield holder
+    finally:
+        _current_session.reset(token)
+
+
+@asynccontextmanager
+async def proxy_session():
+    """Open one reused, rotate-on-block proxy session for a continuation chain."""
+    holder = await open_proxy_holder()
+    try:
+        async with bind_proxy_holder(holder):
+            yield holder
+    finally:
+        await holder.close()

surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py

@@ -0,0 +1,135 @@
+"""Fetch-seam resilience for the TikTok scraper (no network, fake sessions).
+
+Fake sessions drive the cookie warm-up + rotate-on-block + backoff branches
+deterministically; a live first IP normally warms and returns 200s.
+"""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.session import (
+    TikTokAccessBlockedError,
+    client,
+)
+from app.proprietary.platforms.tiktok.session.proxy import _current_session
+
+_HTML = "<html><body>ok</body></html>"
+
+
+class _FakePage:
+    def __init__(self, status: int, *, cookies: dict | None = None, body: str = _HTML):
+        self.status = status
+        self.cookies = cookies or {}
+        self.body = body
+
+    @property
+    def text(self) -> str:
+        return self.body
+
+
+class _FakeSession:
+    """One 'IP': homepage warm mints ``ttwid`` per flag; page GETs return ``status``."""
+
+    def __init__(self, status: int = 200, *, warms: bool = True, body: str = _HTML):
+        self.status = status
+        self.warms = warms
+        self.body = body
+        self.page_calls = 0
+        self.warm_calls = 0
+
+    async def get(self, url, headers=None, cookies=None):
+        if url.rstrip("/") == "https://www.tiktok.com":
+            self.warm_calls += 1
+            return _FakePage(200, cookies={"ttwid": "x"} if self.warms else {})
+        self.page_calls += 1
+        return _FakePage(self.status, body=self.body)
+
+
+class _FakeHolder:
+    def __init__(self, sessions: list[_FakeSession]) -> None:
+        self._sessions = sessions
+        self.session = sessions[0]
+        self.rotations = 0
+        self.warmed = False
+
+    async def rotate(self):
+        self.rotations += 1
+        self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)]
+        self.warmed = False
+        return self.session
+
+    async def pace(self) -> None:
+        return None
+
+    async def close(self) -> None:
+        return None
+
+
+def _no_sleep(monkeypatch) -> None:
+    async def _noop(_seconds):
+        return None
+
+    monkeypatch.setattr(client.asyncio, "sleep", _noop)
+
+
+async def test_warms_then_returns_html():
+    holder = _FakeHolder([_FakeSession(200, warms=True)])
+    token = _current_session.set(holder)
+    try:
+        result = await client.fetch_html("https://www.tiktok.com/@scout2015")
+    finally:
+        _current_session.reset(token)
+    assert result == _HTML
+    assert holder.rotations == 0
+    assert holder.session.warm_calls == 1
+
+
+async def test_rotates_when_warm_fails_then_succeeds():
+    holder = _FakeHolder([_FakeSession(200, warms=False), _FakeSession(200, warms=True)])
+    token = _current_session.set(holder)
+    try:
+        result = await client.fetch_html("https://www.tiktok.com/@scout2015")
+    finally:
+        _current_session.reset(token)
+    assert result == _HTML
+    assert holder.rotations == 1
+
+
+async def test_404_returns_none_without_rotating():
+    holder = _FakeHolder([_FakeSession(404), _FakeSession(200)])
+    token = _current_session.set(holder)
+    try:
+        result = await client.fetch_html("https://www.tiktok.com/@missing")
+    finally:
+        _current_session.reset(token)
+    assert result is None
+    assert holder.rotations == 0
+
+
+async def test_rotates_and_rewarms_on_403():
+    holder = _FakeHolder([_FakeSession(403), _FakeSession(200, warms=True)])
+    token = _current_session.set(holder)
+    try:
+        result = await client.fetch_html("https://www.tiktok.com/@scout2015")
+    finally:
+        _current_session.reset(token)
+    assert result == _HTML
+    assert holder.rotations == 1
+    assert holder.session.warm_calls == 1
+
+
+async def test_persistent_403_raises_blocked(monkeypatch):
+    _no_sleep(monkeypatch)
+    holder = _FakeHolder(
+        [_FakeSession(403) for _ in range(client._MAX_ROTATIONS + 1)]
+    )
+    token = _current_session.set(holder)
+    try:
+        raised = False
+        try:
+            await client.fetch_html("https://www.tiktok.com/@scout2015")
+        except TikTokAccessBlockedError:
+            raised = True
+    finally:
+        _current_session.reset(token)
+    assert raised
+    assert holder.rotations == client._MAX_ROTATIONS