From ef66063bfc7e43bd329450fadca7e969df5aaf13 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:40:50 +0200 Subject: [PATCH] feat(tiktok): video normalizer and parser tests --- .../platforms/tiktok/extraction/__init__.py | 3 +- .../platforms/tiktok/extraction/timestamps.py | 18 ++++ .../platforms/tiktok/extraction/video.py | 73 +++++++++++++ .../unit/platforms/tiktok/test_parsers.py | 102 ++++++++++++++++++ 4 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index fa4df8731..9b4f71c68 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -4,5 +4,6 @@ from __future__ import annotations from .author import parse_author from .hydration import extract_rehydration_data +from .video import parse_video -__all__ = ["extract_rehydration_data", "parse_author"] +__all__ = ["extract_rehydration_data", "parse_author", "parse_video"] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py new file mode 100644 index 000000000..4b155c137 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py @@ -0,0 +1,18 @@ +"""Millisecond-ISO timestamps matching the actor's output shape.""" + +from __future__ import annotations + +from datetime import UTC, datetime + + +def epoch_to_iso(seconds: int | None) -> str | None: + """Convert a Unix-seconds timestamp to ``YYYY-MM-DDTHH:MM:SS.000Z``.""" + if not seconds: + return None + stamp = datetime.fromtimestamp(seconds, tz=UTC) + return stamp.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + +def now_iso() -> str: + """Current UTC time in the millisecond-ISO output shape.""" + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py new file mode 100644 index 000000000..1bfe66452 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py @@ -0,0 +1,73 @@ +"""Normalize a raw TikTok ``itemStruct`` into a :class:`TikTokVideoItem` dict.""" + +from __future__ import annotations + +from typing import Any + +from ..schemas.items import TikTokVideoItem +from .author import build_author_meta +from .timestamps import epoch_to_iso + +_VIDEO_URL = "https://www.tiktok.com/@{username}/video/{video_id}" + + +def _music_meta(music: dict[str, Any]) -> dict[str, Any]: + return { + "musicId": music.get("id"), + "musicName": music.get("title"), + "musicAuthor": music.get("authorName"), + "musicOriginal": music.get("original"), + "playUrl": music.get("playUrl"), + } + + +def _video_meta(video: dict[str, Any]) -> dict[str, Any]: + return { + "height": video.get("height"), + "width": video.get("width"), + "duration": video.get("duration"), + "coverUrl": video.get("cover"), + "format": video.get("format"), + "definition": video.get("definition"), + } + + +def _hashtags(challenges: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [ + {"id": c.get("id"), "name": c.get("title")} + for c in challenges + if isinstance(c, dict) + ] + + +def parse_video(item: dict[str, Any]) -> dict[str, Any]: + """Map an ``itemStruct`` to the flat output contract, filling known fields.""" + author = item.get("author") or {} + author_stats = item.get("authorStats") or {} + stats = item.get("stats") or {} + username = author.get("uniqueId") + video_id = item.get("id") + + web_url = ( + _VIDEO_URL.format(username=username, video_id=video_id) + if username and video_id + else None + ) + create_time = item.get("createTime") + + return TikTokVideoItem( + id=video_id, + text=item.get("desc"), + createTime=create_time, + createTimeISO=epoch_to_iso(create_time), + authorMeta=build_author_meta(author, author_stats), + musicMeta=_music_meta(item.get("music") or {}), + videoMeta=_video_meta(item.get("video") or {}), + webVideoUrl=web_url, + diggCount=stats.get("diggCount"), + shareCount=stats.get("shareCount"), + playCount=stats.get("playCount"), + collectCount=stats.get("collectCount"), + commentCount=stats.get("commentCount"), + hashtags=_hashtags(item.get("challenges") or []), + ).to_output() diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py b/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py new file mode 100644 index 000000000..03fa63133 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py @@ -0,0 +1,102 @@ +"""Raw TikTok payload -> normalized item mapping (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import parse_author, parse_video + +_ITEM_STRUCT = { + "id": "7534061113365859586", + "desc": "haha #comeramabanana", + "createTime": 1_700_000_000, + "author": { + "id": "6733", + "uniqueId": "bruniela_", + "nickname": "Bruni", + "verified": False, + "signature": "bio here", + "avatarLarger": "https://cdn/avatar.jpg", + }, + "authorStats": { + "followerCount": 51200, + "followingCount": 269, + "heartCount": 3_000_000, + "videoCount": 259, + }, + "stats": { + "diggCount": 5344, + "shareCount": 701, + "playCount": 55700, + "commentCount": 24, + "collectCount": 291, + }, + "music": { + "id": "7529", + "title": "som original", + "authorName": "fox_rus0", + "original": True, + "playUrl": "https://cdn/music.mp3", + }, + "video": { + "height": 1024, + "width": 576, + "duration": 16, + "cover": "https://cdn/cover.jpg", + "format": "mp4", + "definition": "540p", + }, + "challenges": [{"id": "4982299", "title": "comeramabanana"}], +} + + +def test_parse_video_maps_core_and_derived_fields(): + item = parse_video(_ITEM_STRUCT) + + assert item["id"] == "7534061113365859586" + assert item["text"] == "haha #comeramabanana" + assert item["createTimeISO"] == "2023-11-14T22:13:20.000Z" + + assert item["authorMeta"]["name"] == "bruniela_" + assert item["authorMeta"]["nickName"] == "Bruni" + assert item["authorMeta"]["profileUrl"] == "https://www.tiktok.com/@bruniela_" + assert item["authorMeta"]["fans"] == 51200 + + assert item["musicMeta"]["musicName"] == "som original" + assert item["videoMeta"]["duration"] == 16 + + assert item["diggCount"] == 5344 + assert item["playCount"] == 55700 + + assert item["hashtags"] == [{"id": "4982299", "name": "comeramabanana"}] + assert ( + item["webVideoUrl"] + == "https://www.tiktok.com/@bruniela_/video/7534061113365859586" + ) + + +_USER_INFO = { + "user": { + "id": "6733", + "uniqueId": "bruniela_", + "nickname": "Bruni", + "verified": True, + "signature": "bio here", + "avatarLarger": "https://cdn/avatar.jpg", + "privateAccount": False, + }, + "stats": { + "followerCount": 51200, + "followingCount": 269, + "heartCount": 3_000_000, + "videoCount": 259, + }, +} + + +def test_parse_author_maps_user_and_stats(): + author = parse_author(_USER_INFO) + assert author["name"] == "bruniela_" + assert author["nickName"] == "Bruni" + assert author["verified"] is True + assert author["profileUrl"] == "https://www.tiktok.com/@bruniela_" + assert author["fans"] == 51200 + assert author["video"] == 259