diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py index 28326a544..3593f0e28 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations +from .input import StartUrl, TikTokScrapeInput from .items import ( AuthorMeta, CommentItem, @@ -16,6 +17,8 @@ __all__ = [ "CommentItem", "ErrorItem", "MusicMeta", + "StartUrl", + "TikTokScrapeInput", "TikTokVideoItem", "VideoMeta", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py new file mode 100644 index 000000000..fa87772fe --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py @@ -0,0 +1,60 @@ +# ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API +"""Input surface for the TikTok scraper, shaped to the Clockworks actor. + +Anonymous only: no auth-shaped field exists here. Fields the Phase-1 blob-first +path does not yet act on (media downloads, follower add-ons) are still accepted +via ``extra="allow"`` for contract parity and land inert. + +Caps (``resultsPerPage``) are per-target counts; the cross-target ceiling is +caller policy applied by the collector, never baked into the flows. +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +ProfileSorting = Literal["latest", "popular", "oldest"] +ProfileSection = Literal["videos", "reposts"] +SearchSection = Literal["", "/video", "/user"] + + +class StartUrl(BaseModel): + """A single direct URL entry (``{"url": ...}``; extra keys ignored).""" + + model_config = ConfigDict(extra="allow") + + url: str + + +class TikTokScrapeInput(BaseModel): + model_config = ConfigDict(extra="allow") + + # Discovery + startUrls: list[StartUrl] = Field(default_factory=list) + hashtags: list[str] = Field(default_factory=list) + profiles: list[str] = Field(default_factory=list) + searchQueries: list[str] = Field(default_factory=list) + postURLs: list[str] = Field(default_factory=list) + + # Per-target count + resultsPerPage: int = Field(default=1, ge=1) + + # Profile options + profileScrapeSections: list[ProfileSection] = Field( + default_factory=lambda: ["videos"] + ) + profileSorting: ProfileSorting = "latest" + excludePinnedPosts: bool = False + + # Search options + searchSection: SearchSection = "" + maxProfilesPerQuery: int = Field(default=10, ge=1) + + # Incremental filters (ISO date or relative " days" per the actor) + oldestPostDateUnified: str | None = None + newestPostDate: str | None = None + + # Proxy + proxyCountryCode: str = "None" diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_input.py b/surfsense_backend/tests/unit/platforms/tiktok/test_input.py new file mode 100644 index 000000000..c080ffec3 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_input.py @@ -0,0 +1,26 @@ +"""Input surface for the TikTok scraper (anonymous, Apify-shaped).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.schemas import TikTokScrapeInput + + +def test_input_has_no_auth_fields(): + forbidden = {"username", "password", "token", "login", "auth", "credentials"} + assert forbidden.isdisjoint(TikTokScrapeInput.model_fields) + + +def test_input_defaults(): + model = TikTokScrapeInput() + assert model.resultsPerPage == 1 + assert model.profileSorting == "latest" + assert model.proxyCountryCode == "None" + assert model.hashtags == [] + assert model.profiles == [] + assert model.searchQueries == [] + assert model.postURLs == [] + + +def test_input_allows_extra_inert_fields(): + model = TikTokScrapeInput(shouldDownloadVideos=True, videoKvStoreIdOrName="x") + assert model.model_dump().get("shouldDownloadVideos") is True