feat(tiktok): blob-first orchestrator + video flow

surfsense_backend/app/proprietary/platforms/tiktok/__init__.py

@@ -0,0 +1,19 @@
+"""Anonymous, blob-first TikTok scraper (public interface).
+
+The capability layer depends only on the names re-exported here: the input
+schema, the collector/generator, the video item shape, and the hard-block error.
+"""
+
+from __future__ import annotations
+
+from .orchestrator import iter_tiktok, scrape_tiktok
+from .schemas import TikTokScrapeInput, TikTokVideoItem
+from .session import TikTokAccessBlockedError
+
+__all__ = [
+    "TikTokAccessBlockedError",
+    "TikTokScrapeInput",
+    "TikTokVideoItem",
+    "iter_tiktok",
+    "scrape_tiktok",
+]

surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py

@@ -4,6 +4,13 @@ from __future__ import annotations
 
 from .author import parse_author
 from .hydration import extract_rehydration_data
+from .scopes import user_info, video_item_struct
 from .video import parse_video
 
-__all__ = ["extract_rehydration_data", "parse_author", "parse_video"]
+__all__ = [
+    "extract_rehydration_data",
+    "parse_author",
+    "parse_video",
+    "user_info",
+    "video_item_struct",
+]

surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py

@@ -0,0 +1,30 @@
+"""Navigate the rehydration blob to the scopes the flows consume."""
+
+from __future__ import annotations
+
+from typing import Any
+
+_DEFAULT = "__DEFAULT_SCOPE__"
+
+
+def _scope(data: dict[str, Any], name: str) -> dict[str, Any] | None:
+    scope = (data.get(_DEFAULT) or {}).get(name)
+    return scope if isinstance(scope, dict) else None
+
+
+def video_item_struct(data: dict[str, Any]) -> dict[str, Any] | None:
+    """The ``itemStruct`` of a video-detail page, or ``None``."""
+    scope = _scope(data, "webapp.video-detail")
+    if not scope:
+        return None
+    item = (scope.get("itemInfo") or {}).get("itemStruct")
+    return item if isinstance(item, dict) else None
+
+
+def user_info(data: dict[str, Any]) -> dict[str, Any] | None:
+    """The ``userInfo`` (``{user, stats}``) of a profile page, or ``None``."""
+    scope = _scope(data, "webapp.user-detail")
+    if not scope:
+        return None
+    info = scope.get("userInfo")
+    return info if isinstance(info, dict) else None

surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py

@@ -0,0 +1,8 @@
+"""Per-target scrape flows: resolved target -> normalized items."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Awaitable, Callable
+
+FetchFn = Callable[[str], Awaitable[str | None]]
+FlowResult = AsyncIterator[dict]

surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py

@@ -0,0 +1,26 @@
+"""Video-URL flow: fetch a post page, read its rehydration blob, emit one item."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from typing import Any
+
+from ..extraction import extract_rehydration_data, parse_video, video_item_struct
+from ..extraction.timestamps import now_iso
+from ..targets.types import TikTokTarget
+from . import FetchFn
+
+
+async def iter_video(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]:
+    html = await fetch(target.url)
+    if not html:
+        return
+    data = extract_rehydration_data(html)
+    if not data:
+        return
+    item = video_item_struct(data)
+    if item is None:
+        return
+    out = parse_video(item)
+    out["scrapedAt"] = now_iso()
+    yield out

surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py

@@ -0,0 +1,90 @@
+"""Resolve a :class:`TikTokScrapeInput` into targets and stream their items.
+
+Targets run sequentially on one warm sticky IP; ``limit`` is collector policy
+applied by :func:`scrape_tiktok`, never baked into a flow. Each kind routes to
+its flow via :func:`_dispatch`.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import AsyncIterator
+from typing import Any
+from urllib.parse import quote
+
+from .flows import FetchFn
+from .flows.video import iter_video
+from .schemas import TikTokScrapeInput
+from .session import fetch_html, proxy_session
+from .targets import resolve_target
+from .targets.types import TikTokTarget
+
+logger = logging.getLogger(__name__)
+
+_PROFILE_URL = "https://www.tiktok.com/@{name}"
+_HASHTAG_URL = "https://www.tiktok.com/tag/{tag}"
+_SEARCH_URL = "https://www.tiktok.com/search?q={query}"
+
+
+async def _empty() -> AsyncIterator[dict[str, Any]]:
+    for _ in ():
+        yield {}
+
+
+def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
+    """Build the target list from every input source, dropping unresolved URLs."""
+    targets: list[TikTokTarget] = []
+    for entry in input_model.startUrls:
+        resolved = resolve_target(entry.url)
+        if resolved is not None:
+            targets.append(resolved)
+    for url in input_model.postURLs:
+        resolved = resolve_target(url)
+        if resolved is not None:
+            targets.append(resolved)
+    for profile in input_model.profiles:
+        name = profile.lstrip("@")
+        targets.append(TikTokTarget("profile", name, _PROFILE_URL.format(name=name)))
+    for tag in input_model.hashtags:
+        targets.append(TikTokTarget("hashtag", tag, _HASHTAG_URL.format(tag=tag)))
+    for query in input_model.searchQueries:
+        targets.append(
+            TikTokTarget("search", query, _SEARCH_URL.format(query=quote(query)))
+        )
+    return targets
+
+
+def _dispatch(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]:
+    if target.kind == "video":
+        return iter_video(target, fetch=fetch)
+    # Listings come from the signed item_list API, not the blob.
+    logger.debug("[tiktok] no blob flow for %s target", target.kind)
+    return _empty()
+
+
+async def iter_tiktok(
+    input_model: TikTokScrapeInput, *, fetch: FetchFn = fetch_html
+) -> AsyncIterator[dict[str, Any]]:
+    """Yield normalized items for every resolved target, in order."""
+    async with proxy_session():
+        for target in _resolve_targets(input_model):
+            async for item in _dispatch(target, fetch=fetch):
+                yield item
+
+
+async def scrape_tiktok(
+    input_model: TikTokScrapeInput,
+    *,
+    limit: int | None = None,
+    fetch: FetchFn = fetch_html,
+) -> list[dict[str, Any]]:
+    """Collect :func:`iter_tiktok` into a list, honoring an optional ``limit``."""
+    from app.capabilities.core.progress import emit_progress
+
+    results: list[dict[str, Any]] = []
+    async for item in iter_tiktok(input_model, fetch=fetch):
+        results.append(item)
+        emit_progress("scraping", current=len(results), total=limit, unit="item")
+        if limit is not None and len(results) >= limit:
+            break
+    return results

surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py

@@ -1,12 +1,11 @@
 # ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API
 """Input surface for the TikTok scraper, shaped to the Clockworks actor.
 
-Anonymous only: no auth-shaped field exists here. Fields the Phase-1 blob-first
-path does not yet act on (media downloads, follower add-ons) are still accepted
-via ``extra="allow"`` for contract parity and land inert.
+Anonymous only: no auth-shaped field exists here. ``extra="allow"`` keeps the
+contract in parity with the actor for fields the scraper does not read.
 
-Caps (``resultsPerPage``) are per-target counts; the cross-target ceiling is
-caller policy applied by the collector, never baked into the flows.
+``resultsPerPage`` is a per-target count; the cross-target ceiling is caller
+policy applied by the collector, never baked into a flow.
 """
 
 from __future__ import annotations

surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py

@@ -6,6 +6,5 @@ from __future__ import annotations
 class TikTokAccessBlockedError(RuntimeError):
     """Raised when every rotated IP is refused anonymous access.
 
-    Anonymous-only: we cannot log in, so a hard block is surfaced loudly rather
-    than returning empty data. The route maps it to a 403.
+    Distinguishes a hard block from an empty result; the route maps it to 403.
     """

surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py

@@ -0,0 +1,73 @@
+"""End-to-end orchestration over a fake fetch (no network).
+
+Drives the public collector: input -> target -> blob-first flow -> items.
+"""
+
+from __future__ import annotations
+
+import json
+
+from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok
+
+
+def _video_page(video_id: str, username: str) -> str:
+    blob = {
+        "__DEFAULT_SCOPE__": {
+            "webapp.video-detail": {
+                "itemInfo": {
+                    "itemStruct": {
+                        "id": video_id,
+                        "desc": "hello",
+                        "author": {"uniqueId": username},
+                        "stats": {"diggCount": 5},
+                    }
+                }
+            }
+        }
+    }
+    return (
+        '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" '
+        f'type="application/json">{json.dumps(blob)}</script>'
+    )
+
+
+async def test_scrape_video_url_returns_parsed_item():
+    url = "https://www.tiktok.com/@scout2015/video/123"
+
+    async def fake_fetch(_url: str) -> str:
+        return _video_page("123", "scout2015")
+
+    items = await scrape_tiktok(TikTokScrapeInput(postURLs=[url]), fetch=fake_fetch)
+
+    assert len(items) == 1
+    assert items[0]["id"] == "123"
+    assert items[0]["diggCount"] == 5
+    assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@scout2015/video/123"
+    assert items[0]["scrapedAt"] is not None
+
+
+async def test_scrape_honors_limit_across_targets():
+    urls = [
+        "https://www.tiktok.com/@a/video/1",
+        "https://www.tiktok.com/@b/video/2",
+    ]
+
+    async def fake_fetch(url: str) -> str:
+        vid = url.rsplit("/", 1)[1]
+        user = url.split("@")[1].split("/")[0]
+        return _video_page(vid, user)
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(postURLs=urls), limit=1, fetch=fake_fetch
+    )
+    assert len(items) == 1
+
+
+async def test_scrape_skips_unrecognized_urls():
+    async def fake_fetch(_url: str) -> str:
+        return ""
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(postURLs=["https://example.com/x"]), fetch=fake_fetch
+    )
+    assert items == []

surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py

@@ -0,0 +1,30 @@
+"""Navigating the rehydration blob to its useful scopes (pure, no network)."""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.extraction import user_info, video_item_struct
+
+
+def test_video_item_struct_navigates_video_detail_scope():
+    data = {
+        "__DEFAULT_SCOPE__": {
+            "webapp.video-detail": {"itemInfo": {"itemStruct": {"id": "123"}}}
+        }
+    }
+    item = video_item_struct(data)
+    assert item == {"id": "123"}
+
+
+def test_user_info_navigates_user_detail_scope():
+    data = {
+        "__DEFAULT_SCOPE__": {
+            "webapp.user-detail": {"userInfo": {"user": {"uniqueId": "scout2015"}}}
+        }
+    }
+    info = user_info(data)
+    assert info == {"user": {"uniqueId": "scout2015"}}
+
+
+def test_scopes_return_none_when_absent():
+    assert video_item_struct({}) is None
+    assert user_info({"__DEFAULT_SCOPE__": {}}) is None