From b2d387f994d415a37383d6d2b1e4ff0755e13eae Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:40:07 +0200 Subject: [PATCH] feat(tiktok): URL target resolver --- .../platforms/tiktok/targets/__init__.py | 8 +++ .../platforms/tiktok/targets/resolver.py | 50 +++++++++++++++++++ .../platforms/tiktok/targets/types.py | 25 ++++++++++ .../tests/unit/platforms/tiktok/__init__.py | 0 .../platforms/tiktok/test_target_resolver.py | 47 +++++++++++++++++ 5 files changed, 130 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/__init__.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py new file mode 100644 index 000000000..ac7410154 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py @@ -0,0 +1,8 @@ +"""TikTok URL classification into scrape targets.""" + +from __future__ import annotations + +from .resolver import resolve_target +from .types import TargetKind, TikTokTarget + +__all__ = ["TargetKind", "TikTokTarget", "resolve_target"] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py new file mode 100644 index 000000000..ffd15ba2a --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py @@ -0,0 +1,50 @@ +"""Classify a TikTok URL into a :class:`TikTokTarget`, or ``None``.""" + +from __future__ import annotations + +from urllib.parse import parse_qs, unquote, urlparse + +from .types import SearchSection, TikTokTarget + +_TIKTOK_HOSTS = frozenset({"tiktok.com", "www.tiktok.com", "m.tiktok.com"}) +_SEARCH_SECTIONS: frozenset[SearchSection] = frozenset({"video", "user"}) + + +def _is_tiktok_host(hostname: str | None) -> bool: + return bool(hostname) and hostname.lower() in _TIKTOK_HOSTS + + +def resolve_target(url: str) -> TikTokTarget | None: + parsed = urlparse(url) + if not _is_tiktok_host(parsed.hostname): + return None + + segments = [s for s in (parsed.path or "").split("/") if s] + if not segments: + return None + + # Profile / video live under /@username[...]. + if segments[0].startswith("@"): + username = segments[0][1:] + if not username: + return None + if len(segments) >= 3 and segments[1] == "video" and segments[2]: + return TikTokTarget("video", segments[2], url, username=username) + return TikTokTarget("profile", username, url) + + if segments[0] == "tag" and len(segments) >= 2 and segments[1]: + return TikTokTarget("hashtag", unquote(segments[1]), url) + + if segments[0] == "search": + query = parse_qs(parsed.query).get("q", [None])[0] + if not query: + return None + section = segments[1] if len(segments) >= 2 else None + return TikTokTarget( + "search", + unquote(query), + url, + section=section if section in _SEARCH_SECTIONS else None, + ) + + return None diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py new file mode 100644 index 000000000..c53d5239d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py @@ -0,0 +1,25 @@ +"""Scrape-target value object produced by URL classification.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +TargetKind = Literal["video", "profile", "hashtag", "search"] +SearchSection = Literal["video", "user"] + + +@dataclass(frozen=True, slots=True) +class TikTokTarget: + """One classified scrape target. + + ``value`` holds the kind-specific identifier: video id, username, hashtag + name, or search query. ``username`` is set for videos (needed to build the + canonical post URL). ``section`` narrows a search to videos or users. + """ + + kind: TargetKind + value: str + url: str + username: str | None = None + section: SearchSection | None = None diff --git a/surfsense_backend/tests/unit/platforms/tiktok/__init__.py b/surfsense_backend/tests/unit/platforms/tiktok/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py b/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py new file mode 100644 index 000000000..c118f4f86 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py @@ -0,0 +1,47 @@ +"""URL classification for the TikTok scraper (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.targets import resolve_target + + +def test_resolve_video_carries_username_and_id(): + target = resolve_target("https://www.tiktok.com/@scout2015/video/6718335390845095173") + assert target is not None + assert target.kind == "video" + assert target.value == "6718335390845095173" + assert target.username == "scout2015" + + +def test_resolve_profile(): + target = resolve_target("https://www.tiktok.com/@scout2015") + assert target is not None + assert target.kind == "profile" + assert target.value == "scout2015" + + +def test_resolve_hashtag(): + target = resolve_target("https://www.tiktok.com/tag/funny") + assert target is not None + assert target.kind == "hashtag" + assert target.value == "funny" + + +def test_resolve_search_top_video_and_user_sections(): + top = resolve_target("https://www.tiktok.com/search?q=cats") + assert top is not None + assert top.kind == "search" + assert top.value == "cats" + assert top.section is None + + videos = resolve_target("https://www.tiktok.com/search/video?q=cats") + assert videos is not None and videos.section == "video" + + users = resolve_target("https://www.tiktok.com/search/user?q=cats") + assert users is not None and users.section == "user" + + +def test_resolve_rejects_non_tiktok_and_unknown_paths(): + assert resolve_target("https://example.com/@scout2015") is None + assert resolve_target("https://www.tiktok.com/") is None + assert resolve_target("https://www.tiktok.com/foundation") is None