apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-16 23:01:06 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(tiktok): rehydration blob extractor

Browse source
This commit is contained in:
CREDO23 2026-07-08 15:40:21 +02:00
parent b2d387f994
commit 8840fd2c53
3 changed files with 67 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

7
surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py Normal file
View file

@ -0,0 +1,7 @@
"""Turn raw TikTok page/API payloads into normalized items."""
from __future__ import annotations
from .hydration import extract_rehydration_data
__all__ = ["extract_rehydration_data"]

28
surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py Normal file
View file

@ -0,0 +1,28 @@
"""Extract the ``__UNIVERSAL_DATA_FOR_REHYDRATION__`` JSON embedded in page HTML.
TikTok server-renders the first page of data into a single script tag; parsing
it yields page-one items (video/profile/hashtag) without any signed API call.
"""
from __future__ import annotations
import json
import re
from typing import Any
_BLOB_RE = re.compile(
r'<script[^>]*id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>(.*?)</script>',
re.DOTALL,
)
def extract_rehydration_data(html: str) -> dict[str, Any] | None:
"""Return the parsed rehydration blob, or ``None`` if absent/unparseable."""
match = _BLOB_RE.search(html)
if not match:
return None
try:
data = json.loads(match.group(1))
except (json.JSONDecodeError, ValueError):
return None
return data if isinstance(data, dict) else None

32
surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py Normal file
View file

@ -0,0 +1,32 @@
"""Rehydration-blob extraction from TikTok page HTML (pure, no network)."""
from __future__ import annotations
from app.proprietary.platforms.tiktok.extraction import extract_rehydration_data
_BLOB = (
'<html><body><script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" '
'type="application/json">'
'{"__DEFAULT_SCOPE__":{"webapp.video-detail":{"itemInfo":{"itemStruct":'
'{"id":"123"}}}}}'
"</script></body></html>"
)
def test_extracts_default_scope_from_blob():
data = extract_rehydration_data(_BLOB)
assert data is not None
item = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]
assert item["id"] == "123"
def test_returns_none_when_blob_absent():
assert extract_rehydration_data("<html><body>no blob here</body></html>") is None
def test_returns_none_when_blob_json_malformed():
broken = (
'<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" '
'type="application/json">{not json}</script>'
)
assert extract_rehydration_data(broken) is None