mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-16 23:01:06 +02:00
feat(tiktok): scrape input schema
This commit is contained in:
parent
ef66063bfc
commit
5688ab0678
3 changed files with 89 additions and 0 deletions
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from .input import StartUrl, TikTokScrapeInput
|
||||
from .items import (
|
||||
AuthorMeta,
|
||||
CommentItem,
|
||||
|
|
@ -16,6 +17,8 @@ __all__ = [
|
|||
"CommentItem",
|
||||
"ErrorItem",
|
||||
"MusicMeta",
|
||||
"StartUrl",
|
||||
"TikTokScrapeInput",
|
||||
"TikTokVideoItem",
|
||||
"VideoMeta",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
# ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API
|
||||
"""Input surface for the TikTok scraper, shaped to the Clockworks actor.
|
||||
|
||||
Anonymous only: no auth-shaped field exists here. Fields the Phase-1 blob-first
|
||||
path does not yet act on (media downloads, follower add-ons) are still accepted
|
||||
via ``extra="allow"`` for contract parity and land inert.
|
||||
|
||||
Caps (``resultsPerPage``) are per-target counts; the cross-target ceiling is
|
||||
caller policy applied by the collector, never baked into the flows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
ProfileSorting = Literal["latest", "popular", "oldest"]
|
||||
ProfileSection = Literal["videos", "reposts"]
|
||||
SearchSection = Literal["", "/video", "/user"]
|
||||
|
||||
|
||||
class StartUrl(BaseModel):
|
||||
"""A single direct URL entry (``{"url": ...}``; extra keys ignored)."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
url: str
|
||||
|
||||
|
||||
class TikTokScrapeInput(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
# Discovery
|
||||
startUrls: list[StartUrl] = Field(default_factory=list)
|
||||
hashtags: list[str] = Field(default_factory=list)
|
||||
profiles: list[str] = Field(default_factory=list)
|
||||
searchQueries: list[str] = Field(default_factory=list)
|
||||
postURLs: list[str] = Field(default_factory=list)
|
||||
|
||||
# Per-target count
|
||||
resultsPerPage: int = Field(default=1, ge=1)
|
||||
|
||||
# Profile options
|
||||
profileScrapeSections: list[ProfileSection] = Field(
|
||||
default_factory=lambda: ["videos"]
|
||||
)
|
||||
profileSorting: ProfileSorting = "latest"
|
||||
excludePinnedPosts: bool = False
|
||||
|
||||
# Search options
|
||||
searchSection: SearchSection = ""
|
||||
maxProfilesPerQuery: int = Field(default=10, ge=1)
|
||||
|
||||
# Incremental filters (ISO date or relative "<n> days" per the actor)
|
||||
oldestPostDateUnified: str | None = None
|
||||
newestPostDate: str | None = None
|
||||
|
||||
# Proxy
|
||||
proxyCountryCode: str = "None"
|
||||
26
surfsense_backend/tests/unit/platforms/tiktok/test_input.py
Normal file
26
surfsense_backend/tests/unit/platforms/tiktok/test_input.py
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
"""Input surface for the TikTok scraper (anonymous, Apify-shaped)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.proprietary.platforms.tiktok.schemas import TikTokScrapeInput
|
||||
|
||||
|
||||
def test_input_has_no_auth_fields():
|
||||
forbidden = {"username", "password", "token", "login", "auth", "credentials"}
|
||||
assert forbidden.isdisjoint(TikTokScrapeInput.model_fields)
|
||||
|
||||
|
||||
def test_input_defaults():
|
||||
model = TikTokScrapeInput()
|
||||
assert model.resultsPerPage == 1
|
||||
assert model.profileSorting == "latest"
|
||||
assert model.proxyCountryCode == "None"
|
||||
assert model.hashtags == []
|
||||
assert model.profiles == []
|
||||
assert model.searchQueries == []
|
||||
assert model.postURLs == []
|
||||
|
||||
|
||||
def test_input_allows_extra_inert_fields():
|
||||
model = TikTokScrapeInput(shouldDownloadVideos=True, videoKvStoreIdOrName="x")
|
||||
assert model.model_dump().get("shouldDownloadVideos") is True
|
||||
Loading…
Add table
Add a link
Reference in a new issue