diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py new file mode 100644 index 000000000..3bd6b160d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -0,0 +1,7 @@ +"""Turn raw TikTok page/API payloads into normalized items.""" + +from __future__ import annotations + +from .hydration import extract_rehydration_data + +__all__ = ["extract_rehydration_data"] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py new file mode 100644 index 000000000..4cc466849 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py @@ -0,0 +1,28 @@ +"""Extract the ``__UNIVERSAL_DATA_FOR_REHYDRATION__`` JSON embedded in page HTML. + +TikTok server-renders the first page of data into a single script tag; parsing +it yields page-one items (video/profile/hashtag) without any signed API call. +""" + +from __future__ import annotations + +import json +import re +from typing import Any + +_BLOB_RE = re.compile( + r']*id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>(.*?)', + re.DOTALL, +) + + +def extract_rehydration_data(html: str) -> dict[str, Any] | None: + """Return the parsed rehydration blob, or ``None`` if absent/unparseable.""" + match = _BLOB_RE.search(html) + if not match: + return None + try: + data = json.loads(match.group(1)) + except (json.JSONDecodeError, ValueError): + return None + return data if isinstance(data, dict) else None diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py b/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py new file mode 100644 index 000000000..cdca568f7 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py @@ -0,0 +1,32 @@ +"""Rehydration-blob extraction from TikTok page HTML (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import extract_rehydration_data + +_BLOB = ( + '" +) + + +def test_extracts_default_scope_from_blob(): + data = extract_rehydration_data(_BLOB) + assert data is not None + item = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"] + assert item["id"] == "123" + + +def test_returns_none_when_blob_absent(): + assert extract_rehydration_data("no blob here") is None + + +def test_returns_none_when_blob_json_malformed(): + broken = ( + '' + ) + assert extract_rehydration_data(broken) is None