mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-16 23:01:06 +02:00
feat(tiktok): rehydration blob extractor
This commit is contained in:
parent
b2d387f994
commit
8840fd2c53
3 changed files with 67 additions and 0 deletions
|
|
@ -0,0 +1,7 @@
|
|||
"""Turn raw TikTok page/API payloads into normalized items."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .hydration import extract_rehydration_data
|
||||
|
||||
__all__ = ["extract_rehydration_data"]
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
"""Extract the ``__UNIVERSAL_DATA_FOR_REHYDRATION__`` JSON embedded in page HTML.
|
||||
|
||||
TikTok server-renders the first page of data into a single script tag; parsing
|
||||
it yields page-one items (video/profile/hashtag) without any signed API call.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
_BLOB_RE = re.compile(
|
||||
r'<script[^>]*id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>(.*?)</script>',
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
def extract_rehydration_data(html: str) -> dict[str, Any] | None:
|
||||
"""Return the parsed rehydration blob, or ``None`` if absent/unparseable."""
|
||||
match = _BLOB_RE.search(html)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(match.group(1))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
Loading…
Add table
Add a link
Reference in a new issue