mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-18 23:11:12 +02:00
feat(tiktok): scrape input schema
This commit is contained in:
parent
ef66063bfc
commit
5688ab0678
3 changed files with 89 additions and 0 deletions
|
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from .input import StartUrl, TikTokScrapeInput
|
||||||
from .items import (
|
from .items import (
|
||||||
AuthorMeta,
|
AuthorMeta,
|
||||||
CommentItem,
|
CommentItem,
|
||||||
|
|
@ -16,6 +17,8 @@ __all__ = [
|
||||||
"CommentItem",
|
"CommentItem",
|
||||||
"ErrorItem",
|
"ErrorItem",
|
||||||
"MusicMeta",
|
"MusicMeta",
|
||||||
|
"StartUrl",
|
||||||
|
"TikTokScrapeInput",
|
||||||
"TikTokVideoItem",
|
"TikTokVideoItem",
|
||||||
"VideoMeta",
|
"VideoMeta",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,60 @@
|
||||||
|
# ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API
|
||||||
|
"""Input surface for the TikTok scraper, shaped to the Clockworks actor.
|
||||||
|
|
||||||
|
Anonymous only: no auth-shaped field exists here. Fields the Phase-1 blob-first
|
||||||
|
path does not yet act on (media downloads, follower add-ons) are still accepted
|
||||||
|
via ``extra="allow"`` for contract parity and land inert.
|
||||||
|
|
||||||
|
Caps (``resultsPerPage``) are per-target counts; the cross-target ceiling is
|
||||||
|
caller policy applied by the collector, never baked into the flows.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
|
|
||||||
|
ProfileSorting = Literal["latest", "popular", "oldest"]
|
||||||
|
ProfileSection = Literal["videos", "reposts"]
|
||||||
|
SearchSection = Literal["", "/video", "/user"]
|
||||||
|
|
||||||
|
|
||||||
|
class StartUrl(BaseModel):
|
||||||
|
"""A single direct URL entry (``{"url": ...}``; extra keys ignored)."""
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="allow")
|
||||||
|
|
||||||
|
url: str
|
||||||
|
|
||||||
|
|
||||||
|
class TikTokScrapeInput(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="allow")
|
||||||
|
|
||||||
|
# Discovery
|
||||||
|
startUrls: list[StartUrl] = Field(default_factory=list)
|
||||||
|
hashtags: list[str] = Field(default_factory=list)
|
||||||
|
profiles: list[str] = Field(default_factory=list)
|
||||||
|
searchQueries: list[str] = Field(default_factory=list)
|
||||||
|
postURLs: list[str] = Field(default_factory=list)
|
||||||
|
|
||||||
|
# Per-target count
|
||||||
|
resultsPerPage: int = Field(default=1, ge=1)
|
||||||
|
|
||||||
|
# Profile options
|
||||||
|
profileScrapeSections: list[ProfileSection] = Field(
|
||||||
|
default_factory=lambda: ["videos"]
|
||||||
|
)
|
||||||
|
profileSorting: ProfileSorting = "latest"
|
||||||
|
excludePinnedPosts: bool = False
|
||||||
|
|
||||||
|
# Search options
|
||||||
|
searchSection: SearchSection = ""
|
||||||
|
maxProfilesPerQuery: int = Field(default=10, ge=1)
|
||||||
|
|
||||||
|
# Incremental filters (ISO date or relative "<n> days" per the actor)
|
||||||
|
oldestPostDateUnified: str | None = None
|
||||||
|
newestPostDate: str | None = None
|
||||||
|
|
||||||
|
# Proxy
|
||||||
|
proxyCountryCode: str = "None"
|
||||||
26
surfsense_backend/tests/unit/platforms/tiktok/test_input.py
Normal file
26
surfsense_backend/tests/unit/platforms/tiktok/test_input.py
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
"""Input surface for the TikTok scraper (anonymous, Apify-shaped)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.proprietary.platforms.tiktok.schemas import TikTokScrapeInput
|
||||||
|
|
||||||
|
|
||||||
|
def test_input_has_no_auth_fields():
|
||||||
|
forbidden = {"username", "password", "token", "login", "auth", "credentials"}
|
||||||
|
assert forbidden.isdisjoint(TikTokScrapeInput.model_fields)
|
||||||
|
|
||||||
|
|
||||||
|
def test_input_defaults():
|
||||||
|
model = TikTokScrapeInput()
|
||||||
|
assert model.resultsPerPage == 1
|
||||||
|
assert model.profileSorting == "latest"
|
||||||
|
assert model.proxyCountryCode == "None"
|
||||||
|
assert model.hashtags == []
|
||||||
|
assert model.profiles == []
|
||||||
|
assert model.searchQueries == []
|
||||||
|
assert model.postURLs == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_input_allows_extra_inert_fields():
|
||||||
|
model = TikTokScrapeInput(shouldDownloadVideos=True, videoKvStoreIdOrName="x")
|
||||||
|
assert model.model_dump().get("shouldDownloadVideos") is True
|
||||||
Loading…
Add table
Add a link
Reference in a new issue