diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py index e69de29bb..27867df73 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py @@ -0,0 +1,19 @@ +"""Anonymous, blob-first TikTok scraper (public interface). + +The capability layer depends only on the names re-exported here: the input +schema, the collector/generator, the video item shape, and the hard-block error. +""" + +from __future__ import annotations + +from .orchestrator import iter_tiktok, scrape_tiktok +from .schemas import TikTokScrapeInput, TikTokVideoItem +from .session import TikTokAccessBlockedError + +__all__ = [ + "TikTokAccessBlockedError", + "TikTokScrapeInput", + "TikTokVideoItem", + "iter_tiktok", + "scrape_tiktok", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index 9b4f71c68..0e6b1c3b7 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -4,6 +4,13 @@ from __future__ import annotations from .author import parse_author from .hydration import extract_rehydration_data +from .scopes import user_info, video_item_struct from .video import parse_video -__all__ = ["extract_rehydration_data", "parse_author", "parse_video"] +__all__ = [ + "extract_rehydration_data", + "parse_author", + "parse_video", + "user_info", + "video_item_struct", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py new file mode 100644 index 000000000..a20a72b2b --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py @@ -0,0 +1,30 @@ +"""Navigate the rehydration blob to the scopes the flows consume.""" + +from __future__ import annotations + +from typing import Any + +_DEFAULT = "__DEFAULT_SCOPE__" + + +def _scope(data: dict[str, Any], name: str) -> dict[str, Any] | None: + scope = (data.get(_DEFAULT) or {}).get(name) + return scope if isinstance(scope, dict) else None + + +def video_item_struct(data: dict[str, Any]) -> dict[str, Any] | None: + """The ``itemStruct`` of a video-detail page, or ``None``.""" + scope = _scope(data, "webapp.video-detail") + if not scope: + return None + item = (scope.get("itemInfo") or {}).get("itemStruct") + return item if isinstance(item, dict) else None + + +def user_info(data: dict[str, Any]) -> dict[str, Any] | None: + """The ``userInfo`` (``{user, stats}``) of a profile page, or ``None``.""" + scope = _scope(data, "webapp.user-detail") + if not scope: + return None + info = scope.get("userInfo") + return info if isinstance(info, dict) else None diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py new file mode 100644 index 000000000..f2f4b67cc --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py @@ -0,0 +1,8 @@ +"""Per-target scrape flows: resolved target -> normalized items.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator, Awaitable, Callable + +FetchFn = Callable[[str], Awaitable[str | None]] +FlowResult = AsyncIterator[dict] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py new file mode 100644 index 000000000..48e558677 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py @@ -0,0 +1,26 @@ +"""Video-URL flow: fetch a post page, read its rehydration blob, emit one item.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from ..extraction import extract_rehydration_data, parse_video, video_item_struct +from ..extraction.timestamps import now_iso +from ..targets.types import TikTokTarget +from . import FetchFn + + +async def iter_video(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]: + html = await fetch(target.url) + if not html: + return + data = extract_rehydration_data(html) + if not data: + return + item = video_item_struct(data) + if item is None: + return + out = parse_video(item) + out["scrapedAt"] = now_iso() + yield out diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py new file mode 100644 index 000000000..d924e3a9e --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -0,0 +1,90 @@ +"""Resolve a :class:`TikTokScrapeInput` into targets and stream their items. + +Targets run sequentially on one warm sticky IP; ``limit`` is collector policy +applied by :func:`scrape_tiktok`, never baked into a flow. Each kind routes to +its flow via :func:`_dispatch`. +""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import quote + +from .flows import FetchFn +from .flows.video import iter_video +from .schemas import TikTokScrapeInput +from .session import fetch_html, proxy_session +from .targets import resolve_target +from .targets.types import TikTokTarget + +logger = logging.getLogger(__name__) + +_PROFILE_URL = "https://www.tiktok.com/@{name}" +_HASHTAG_URL = "https://www.tiktok.com/tag/{tag}" +_SEARCH_URL = "https://www.tiktok.com/search?q={query}" + + +async def _empty() -> AsyncIterator[dict[str, Any]]: + for _ in (): + yield {} + + +def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]: + """Build the target list from every input source, dropping unresolved URLs.""" + targets: list[TikTokTarget] = [] + for entry in input_model.startUrls: + resolved = resolve_target(entry.url) + if resolved is not None: + targets.append(resolved) + for url in input_model.postURLs: + resolved = resolve_target(url) + if resolved is not None: + targets.append(resolved) + for profile in input_model.profiles: + name = profile.lstrip("@") + targets.append(TikTokTarget("profile", name, _PROFILE_URL.format(name=name))) + for tag in input_model.hashtags: + targets.append(TikTokTarget("hashtag", tag, _HASHTAG_URL.format(tag=tag))) + for query in input_model.searchQueries: + targets.append( + TikTokTarget("search", query, _SEARCH_URL.format(query=quote(query))) + ) + return targets + + +def _dispatch(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]: + if target.kind == "video": + return iter_video(target, fetch=fetch) + # Listings come from the signed item_list API, not the blob. + logger.debug("[tiktok] no blob flow for %s target", target.kind) + return _empty() + + +async def iter_tiktok( + input_model: TikTokScrapeInput, *, fetch: FetchFn = fetch_html +) -> AsyncIterator[dict[str, Any]]: + """Yield normalized items for every resolved target, in order.""" + async with proxy_session(): + for target in _resolve_targets(input_model): + async for item in _dispatch(target, fetch=fetch): + yield item + + +async def scrape_tiktok( + input_model: TikTokScrapeInput, + *, + limit: int | None = None, + fetch: FetchFn = fetch_html, +) -> list[dict[str, Any]]: + """Collect :func:`iter_tiktok` into a list, honoring an optional ``limit``.""" + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + async for item in iter_tiktok(input_model, fetch=fetch): + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + break + return results diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py index fa87772fe..f710eed72 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py @@ -1,12 +1,11 @@ # ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API """Input surface for the TikTok scraper, shaped to the Clockworks actor. -Anonymous only: no auth-shaped field exists here. Fields the Phase-1 blob-first -path does not yet act on (media downloads, follower add-ons) are still accepted -via ``extra="allow"`` for contract parity and land inert. +Anonymous only: no auth-shaped field exists here. ``extra="allow"`` keeps the +contract in parity with the actor for fields the scraper does not read. -Caps (``resultsPerPage``) are per-target counts; the cross-target ceiling is -caller policy applied by the collector, never baked into the flows. +``resultsPerPage`` is a per-target count; the cross-target ceiling is caller +policy applied by the collector, never baked into a flow. """ from __future__ import annotations diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py index 4d9fc712d..708cb0212 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py @@ -6,6 +6,5 @@ from __future__ import annotations class TikTokAccessBlockedError(RuntimeError): """Raised when every rotated IP is refused anonymous access. - Anonymous-only: we cannot log in, so a hard block is surfaced loudly rather - than returning empty data. The route maps it to a 403. + Distinguishes a hard block from an empty result; the route maps it to 403. """ diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py new file mode 100644 index 000000000..85c601047 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py @@ -0,0 +1,73 @@ +"""End-to-end orchestration over a fake fetch (no network). + +Drives the public collector: input -> target -> blob-first flow -> items. +""" + +from __future__ import annotations + +import json + +from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok + + +def _video_page(video_id: str, username: str) -> str: + blob = { + "__DEFAULT_SCOPE__": { + "webapp.video-detail": { + "itemInfo": { + "itemStruct": { + "id": video_id, + "desc": "hello", + "author": {"uniqueId": username}, + "stats": {"diggCount": 5}, + } + } + } + } + } + return ( + '' + ) + + +async def test_scrape_video_url_returns_parsed_item(): + url = "https://www.tiktok.com/@scout2015/video/123" + + async def fake_fetch(_url: str) -> str: + return _video_page("123", "scout2015") + + items = await scrape_tiktok(TikTokScrapeInput(postURLs=[url]), fetch=fake_fetch) + + assert len(items) == 1 + assert items[0]["id"] == "123" + assert items[0]["diggCount"] == 5 + assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@scout2015/video/123" + assert items[0]["scrapedAt"] is not None + + +async def test_scrape_honors_limit_across_targets(): + urls = [ + "https://www.tiktok.com/@a/video/1", + "https://www.tiktok.com/@b/video/2", + ] + + async def fake_fetch(url: str) -> str: + vid = url.rsplit("/", 1)[1] + user = url.split("@")[1].split("/")[0] + return _video_page(vid, user) + + items = await scrape_tiktok( + TikTokScrapeInput(postURLs=urls), limit=1, fetch=fake_fetch + ) + assert len(items) == 1 + + +async def test_scrape_skips_unrecognized_urls(): + async def fake_fetch(_url: str) -> str: + return "" + + items = await scrape_tiktok( + TikTokScrapeInput(postURLs=["https://example.com/x"]), fetch=fake_fetch + ) + assert items == [] diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py b/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py new file mode 100644 index 000000000..7a52dd60d --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py @@ -0,0 +1,30 @@ +"""Navigating the rehydration blob to its useful scopes (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import user_info, video_item_struct + + +def test_video_item_struct_navigates_video_detail_scope(): + data = { + "__DEFAULT_SCOPE__": { + "webapp.video-detail": {"itemInfo": {"itemStruct": {"id": "123"}}} + } + } + item = video_item_struct(data) + assert item == {"id": "123"} + + +def test_user_info_navigates_user_detail_scope(): + data = { + "__DEFAULT_SCOPE__": { + "webapp.user-detail": {"userInfo": {"user": {"uniqueId": "scout2015"}}} + } + } + info = user_info(data) + assert info == {"user": {"uniqueId": "scout2015"}} + + +def test_scopes_return_none_when_absent(): + assert video_item_struct({}) is None + assert user_info({"__DEFAULT_SCOPE__": {}}) is None