diff --git a/docker/.env.example b/docker/.env.example index beca9d5f6..2aa0806a8 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -454,6 +454,9 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # GOOGLE_MAPS_MICROS_PER_REVIEW=1500 # YOUTUBE_MICROS_PER_VIDEO=2500 # YOUTUBE_MICROS_PER_COMMENT=1500 +# TIKTOK_MICROS_PER_VIDEO=3500 +# TIKTOK_MICROS_PER_USER=2500 +# TIKTOK_MICROS_PER_COMMENT=1500 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index a030b75aa..2c0cf38ff 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -292,6 +292,12 @@ MICROS_PER_PAGE=1000 # YOUTUBE_MICROS_PER_COMMENT=1500 # INSTAGRAM_SCRAPE_MICROS_PER_ITEM=3500 # INSTAGRAM_SCRAPE_MICROS_PER_COMMENT=1500 +# TIKTOK_MICROS_PER_VIDEO=3500 +# TIKTOK_MICROS_PER_USER=2500 +# TIKTOK_MICROS_PER_COMMENT=1500 +# Browser-listing retries when a feed is empty (profile feed is withheld from +# flagged IPs; each retry draws a fresh rotating exit IP). Set to 1 for a static IP. +# TIKTOK_LISTING_MAX_ATTEMPTS=3 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. CREDIT_LOW_BALANCE_WARNING_MICROS=500000 @@ -400,6 +406,9 @@ TURNSTILE_SECRET_KEY= # Route DNS via Cloudflare DoH (anti DNS-leak). Adds a DNS round-trip => off by # default to avoid any speed regression; enable for leak-safety-first setups. # CRAWL_DNS_OVER_HTTPS=FALSE +# Run the browser headful on Xvfb — required for TikTok's profile video feed +# (empty to headless Chromium). Entrypoint starts Xvfb when TRUE. +# CRAWL_HEADED_XVFB_ENABLED=FALSE # File Parser Service ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING diff --git a/surfsense_backend/Dockerfile b/surfsense_backend/Dockerfile index fffe2e45d..7140f8809 100644 --- a/surfsense_backend/Dockerfile +++ b/surfsense_backend/Dockerfile @@ -31,11 +31,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ dos2unix \ git \ # ── Phase 3e stealth hardening ────────────────────────────────────────── - # Xvfb: virtual framebuffer so the stealth browser can run headful - # (headless=False) without a real display — many WAFs flag headless Chromium. - # Gated at runtime by CRAWL_HEADED_XVFB_ENABLED (Slice B); installed now so - # the image is ready. Real font packages make canvas/emoji/font-enumeration - # fingerprints resemble a real desktop (set proven against Kasada/Akamai). + # Xvfb: virtual display so the browser can run headful without real hardware + # (entrypoint starts it when CRAWL_HEADED_XVFB_ENABLED; used by TikTok's profile + # feed). Real fonts make canvas/emoji/font fingerprints look like a real desktop. xvfb \ fonts-noto-color-emoji \ fonts-unifont \ diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py index 40e8dd6d5..6bbd808fb 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py @@ -37,6 +37,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = { "google_search": frozenset(), "reddit": frozenset(), "instagram": frozenset(), + "tiktok": frozenset(), "mcp_discovery": frozenset( { "SLACK_CONNECTOR", diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md index 378c052d4..0c7e4071c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md @@ -6,9 +6,9 @@ reviews are moving, and what is being said across the open web — and to put that intelligence to work alongside their own knowledge base. You do this by dispatching **specialist subagents** via the `task` tool: -- **Live market data** — Reddit, YouTube, Google Maps, Google Search, and the - web crawler return structured, current platform data (posts, comments, - transcripts, reviews, SERPs, full page content). +- **Live market data** — Reddit, YouTube, TikTok, Google Maps, Google Search, + and the web crawler return structured, current platform data (posts, + comments, transcripts, videos, reviews, SERPs, full page content). - **The user's own context** — their knowledge base, connected apps, and persistent memory. - **Deliverables** — reports, podcasts, and presentations built from what the diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md index 2765c06b7..6c1076a89 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md @@ -6,9 +6,9 @@ reviews are moving, and what is being said across the open web — and to put that intelligence to work alongside the team's shared knowledge base. You do this by dispatching **specialist subagents** via the `task` tool: -- **Live market data** — Reddit, YouTube, Google Maps, Google Search, and the - web crawler return structured, current platform data (posts, comments, - transcripts, reviews, SERPs, full page content). +- **Live market data** — Reddit, YouTube, TikTok, Google Maps, Google Search, + and the web crawler return structured, current platform data (posts, + comments, transcripts, videos, reviews, SERPs, full page content). - **The team's own context** — its shared knowledge base, connected apps, and persistent team memory. - **Deliverables** — reports, podcasts, and presentations built from what the diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md index c69ed400c..ef753b93c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md @@ -1,8 +1,9 @@ CRITICAL — ground factual answers in what you actually receive this turn: - **live platform data** via the market specialists — - `task(reddit, ...)`, `task(youtube, ...)`, `task(google_maps, ...)`, - `task(google_search, ...)`, `task(web_crawler, ...)`. Anything about + `task(reddit, ...)`, `task(youtube, ...)`, `task(tiktok, ...)`, + `task(google_maps, ...)`, `task(google_search, ...)`, + `task(web_crawler, ...)`. Anything about competitors, markets, rankings, reviews, or audience sentiment is answered from what these return **this turn**, never from your training data: your general knowledge of companies, prices, and rankings is stale by definition, diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md index eec860f3c..fb818cabc 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md @@ -31,6 +31,7 @@ bounded fan-out (≤20 sites) the user already requested. about a brand, product, or topic is answered from the platform where they say it — `task(reddit, …)` for community discussion and threads, `task(youtube, …)` for video content, transcripts, and comment sections, +`task(tiktok, …)` for short-form video trends by hashtag or search, `task(google_maps, …)` for customer reviews of physical businesses. Web search only finds articles *about* the conversation; the platform specialists return the conversation itself, structured and current. For diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py new file mode 100644 index 000000000..ec7d5955f --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py @@ -0,0 +1 @@ +"""``tiktok`` builtin subagent: structured public TikTok videos and listings.""" diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py new file mode 100644 index 000000000..2c7dd014b --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py @@ -0,0 +1,43 @@ +"""``tiktok`` route: ``SurfSenseSubagentSpec`` builder for deepagents.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.language_models import BaseChatModel +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) + +from .tools.index import NAME, RULESET, load_tools + + +def build_subagent( + *, + dependencies: dict[str, Any], + model: BaseChatModel | None = None, + middleware_stack: dict[str, Any] | None = None, + mcp_tools: list[BaseTool] | None = None, +) -> SurfSenseSubagentSpec: + tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] + description = ( + read_md_file(__package__, "description").strip() + or "Pulls structured data from public TikTok videos, hashtags, and searches." + ) + system_prompt = read_md_file(__package__, "system_prompt").strip() + return pack_subagent( + name=NAME, + description=description, + system_prompt=system_prompt, + tools=tools, + ruleset=RULESET, + dependencies=dependencies, + model=model, + middleware_stack=middleware_stack, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md new file mode 100644 index 000000000..750979348 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md @@ -0,0 +1,2 @@ +TikTok specialist: pulls structured public TikTok data — videos (caption/text, author, play/like/comment/share counts, music, hashtags, timestamps, web URL) from a hashtag feed, a creator profile, or a known video URL; a video's public comment thread; accounts found by keyword; and the current Explore trending-video feed. Also compares fresh TikTok results against earlier findings in this chat. +Use whenever the task is to find what is trending or being said on TikTok about a topic, gather a creator's or hashtag's videos, read a video's comments, discover accounts by keyword, or scrape a specific video URL. Triggers include "search TikTok for X", "what's trending on TikTok", "videos with #X", "comments on this TikTok", "find TikTok accounts about X", and "scrape this TikTok video". Not for general web pages (use the web crawling specialist), Google results (use the Google Search specialist), Reddit (use the Reddit specialist), or YouTube (use the YouTube specialist). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md new file mode 100644 index 000000000..03114cd12 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md @@ -0,0 +1,70 @@ +You are the SurfSense TikTok sub-agent. +You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis. + + +Answer the delegated question from live TikTok data gathered with your verb, comparing against earlier results already in this conversation when the task calls for it. + + + +- `tiktok_scrape` — videos from a hashtag, a profile, or a TikTok URL +- `tiktok_comments` — a video's comment thread, from `video_urls` +- `tiktok_user_search` — find accounts by keyword, from `queries` +- `tiktok_trending` — the current Explore trending-video feed +- `read_run` / `search_run` (free readers for stored scrape output) + + + +- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#'), or pass a TikTok URL in `urls`. +- Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`. +- Profiles: a creator's `profiles` feed returns the account's metadata (name, followers, bio, verification) reliably, but its video list is often withheld by TikTok — treat an empty video list as a known limit, not a failure to retry endlessly. Prefer `hashtags` or a direct video URL for videos. +- Comments on a video: call `tiktok_comments` with the video URL(s) in `video_urls`. +- Finding accounts by keyword: call `tiktok_user_search` with `queries`. Keyword search returns no videos, so do not use `search_queries` for videos — use `hashtags` or a video URL. +- "What's trending now": call `tiktok_trending` (no query needed); set `max_items` for how many. +- Controlling volume: use `max_items` for the total cap and `results_per_page` per target (per-verb equivalents: `comments_per_video`, `results_per_query`). +- Requested counts: `max_items` defaults low — when the task asks for N items, set `max_items` (and the per-target count) above N. A call that caps below the target can never satisfy it. +- Batch multiple hashtags, queries, or video URLs into one call rather than many single-item calls. + +- Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, count changes). + + + +- Use only tools in ``. +- Report only results present in the tool output. Never invent captions, URLs, authors, or counts. + + + +- Do not read arbitrary web pages — that belongs to the web crawling specialist. +- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on. +- Reddit belongs to the Reddit specialist; YouTube belongs to the YouTube specialist; Google results belong to the Google Search specialist. + + + +- Report uncertainty explicitly when evidence is incomplete or conflicting. +- Never present unverified claims as facts. + + + +- Underspecified request — no usable hashtag, query, or URL — return `status=blocked` with the missing fields. +- Tool failure: return `status=error` with a concise recovery `next_step`. +- No useful evidence: return `status=blocked` with a narrower query or the scope you still need. + + + +Return **only** one JSON object (no markdown/prose): +{ + "status": "success" | "partial" | "blocked" | "error", + "action_summary": string, + "evidence": { + "findings": string[], + "sources": string[], + "confidence": "high" | "medium" | "low" + }, + "next_step": string | null, + "missing_fields": string[] | null, + "assumptions": string[] | null +} + +Route-specific rules: +- `evidence.findings`: one entry per distinct result (video, comment, or account) or delta — a single sentence each; do not paste raw payloads. Max 10 entries, unless the delegated task asks for N items: then return up to N (each backed by a real scraped result, never padded). +- `evidence.sources`: one TikTok URL per finding when applicable, same cap as findings. List each URL once. + diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py new file mode 100644 index 000000000..f1bec7fc2 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py @@ -0,0 +1,30 @@ +"""``tiktok`` sub-agent tools: scrape, comments, user-search, and trending verbs.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset +from app.capabilities.core.access.agent import build_capability_tools +from app.capabilities.tiktok.comments.definition import TIKTOK_COMMENTS +from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE +from app.capabilities.tiktok.trending.definition import TIKTOK_TRENDING +from app.capabilities.tiktok.user_search.definition import TIKTOK_USER_SEARCH + +NAME = "tiktok" + +RULESET = Ruleset(origin=NAME, rules=[]) + +_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_COMMENTS, TIKTOK_USER_SEARCH, TIKTOK_TRENDING] + + +def load_tools( + *, dependencies: dict[str, Any] | None = None, **kwargs: Any +) -> list[BaseTool]: + d = {**(dependencies or {}), **kwargs} + return build_capability_tools( + workspace_id=d.get("workspace_id"), + capabilities=_CI_VERBS, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index 7a9b5b1f7..de37edfb9 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -36,6 +36,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.memory.agent import ( from app.agents.chat.multi_agent_chat.subagents.builtins.reddit.agent import ( build_subagent as build_reddit_subagent, ) +from app.agents.chat.multi_agent_chat.subagents.builtins.tiktok.agent import ( + build_subagent as build_tiktok_subagent, +) from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import ( build_subagent as build_web_crawler_subagent, ) @@ -88,6 +91,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { "memory": build_memory_subagent, "onedrive": build_onedrive_subagent, "reddit": build_reddit_subagent, + "tiktok": build_tiktok_subagent, "web_crawler": build_web_crawler_subagent, "youtube": build_youtube_subagent, } diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 7c2fbba9f..67abf4164 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -37,6 +37,9 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT", BillingUnit.INSTAGRAM_ITEM: "INSTAGRAM_SCRAPE_MICROS_PER_ITEM", BillingUnit.INSTAGRAM_COMMENT: "INSTAGRAM_SCRAPE_MICROS_PER_COMMENT", + BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO", + BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER", + BillingUnit.TIKTOK_COMMENT: "TIKTOK_MICROS_PER_COMMENT", } @@ -55,6 +58,9 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.YOUTUBE_COMMENT: "comment", BillingUnit.INSTAGRAM_ITEM: "item", BillingUnit.INSTAGRAM_COMMENT: "comment", + BillingUnit.TIKTOK_VIDEO: "video", + BillingUnit.TIKTOK_USER: "profile", + BillingUnit.TIKTOK_COMMENT: "comment", } diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index e0fef5379..67b9175fc 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -27,6 +27,9 @@ class BillingUnit(StrEnum): YOUTUBE_COMMENT = "youtube_comment" INSTAGRAM_ITEM = "instagram_item" INSTAGRAM_COMMENT = "instagram_comment" + TIKTOK_VIDEO = "tiktok_video" + TIKTOK_USER = "tiktok_user" + TIKTOK_COMMENT = "tiktok_comment" class BillableInput(Protocol): diff --git a/surfsense_backend/app/capabilities/tiktok/__init__.py b/surfsense_backend/app/capabilities/tiktok/__init__.py new file mode 100644 index 000000000..4962d3162 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/__init__.py @@ -0,0 +1,8 @@ +"""``tiktok.*`` namespace: platform-native TikTok data verbs.""" + +from __future__ import annotations + +from app.capabilities.tiktok.comments import definition as _comments # noqa: F401 +from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401 +from app.capabilities.tiktok.trending import definition as _trending # noqa: F401 +from app.capabilities.tiktok.user_search import definition as _user_search # noqa: F401 diff --git a/surfsense_backend/app/capabilities/tiktok/comments/__init__.py b/surfsense_backend/app/capabilities/tiktok/comments/__init__.py new file mode 100644 index 000000000..f3bd1db88 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/__init__.py @@ -0,0 +1,3 @@ +"""``tiktok.comments``: scrape the public comment thread of TikTok videos.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/tiktok/comments/definition.py b/surfsense_backend/app/capabilities/tiktok/comments/definition.py new file mode 100644 index 000000000..1e5917fa2 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/definition.py @@ -0,0 +1,23 @@ +"""``tiktok.comments`` capability registration (billed per comment; see config +``TIKTOK_MICROS_PER_COMMENT``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.comments.executor import build_comments_executor +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput + +TIKTOK_COMMENTS = Capability( + name="tiktok.comments", + description=( + "Scrape the public comments of TikTok videos. Provide video URLs; " + "returns comment text, author, likes, and reply counts." + ), + input_schema=CommentsInput, + output_schema=CommentsOutput, + executor=build_comments_executor(), + billing_unit=BillingUnit.TIKTOK_COMMENT, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_COMMENTS) diff --git a/surfsense_backend/app/capabilities/tiktok/comments/executor.py b/surfsense_backend/app/capabilities/tiktok/comments/executor.py new file mode 100644 index 000000000..c8bbc776f --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/executor.py @@ -0,0 +1,36 @@ +"""``tiktok.comments`` executor: video URLs -> scraper -> TikTok comment items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput +from app.proprietary.platforms.tiktok import scrape_tiktok_comments + +CommentsFn = Callable[..., Awaitable[list[dict]]] + + +def build_comments_executor(comments_fn: CommentsFn | None = None) -> Executor: + """Bind the executor to a comments fn (defaults to the proprietary actor).""" + comments_fn = comments_fn or scrape_tiktok_comments + + async def execute(payload: CommentsInput) -> CommentsOutput: + emit_progress( + "starting", + "Scraping TikTok comments", + total=payload.max_items, + unit="item", + ) + items = await comments_fn( + payload.video_urls, + per_video=payload.comments_per_video, + limit=payload.max_items, + ) + emit_progress( + "done", f"Scraped {len(items)} comment(s)", current=len(items), unit="item" + ) + return CommentsOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/comments/schemas.py b/surfsense_backend/app/capabilities/tiktok/comments/schemas.py new file mode 100644 index 000000000..f3c68d445 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/schemas.py @@ -0,0 +1,56 @@ +"""``tiktok.comments`` I/O contracts. + +URL-only: given TikTok video URLs, return each video's public comment thread. +Unlike profile-video/general-search feeds, ``/api/comment/list`` is served to +anonymous sessions once the comments panel opens, so this verb is reliable. Each +result is a :class:`CommentItem` (top-level comments; replies carry ``repliesToId``). +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.capabilities.tiktok.scrape.schemas import ( + MAX_TIKTOK_ITEMS, + MAX_TIKTOK_SOURCES, +) +from app.proprietary.platforms.tiktok import CommentItem + + +class CommentsInput(BaseModel): + video_urls: list[str] = Field( + min_length=1, + max_length=MAX_TIKTOK_SOURCES, + description="TikTok video URLs (/@/video/) to pull comments from.", + ) + comments_per_video: int = Field( + default=20, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max comments to return per video.", + ) + max_items: int = Field( + default=20, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max total comments to return across all videos.", + ) + + @property + def estimated_units(self) -> int: + """Worst-case billable comments for the pre-flight gate: ``max_items`` is a + hard cross-video ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class CommentsOutput(BaseModel): + items: list[CommentItem] = Field( + default_factory=list, + description="One item per comment returned, in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned comment = one billable unit; ErrorItems (``errorCode`` set, + for bad URLs or empty/withheld videos) are surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py b/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py new file mode 100644 index 000000000..9e5acc165 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py @@ -0,0 +1 @@ +"""``tiktok.scrape``: public TikTok videos over the browser-driven scraper.""" diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/definition.py b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py new file mode 100644 index 000000000..9f8632f3b --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py @@ -0,0 +1,23 @@ +"""``tiktok.scrape`` capability registration (billed per video; see config +``TIKTOK_MICROS_PER_VIDEO``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.scrape.executor import build_scrape_executor +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput + +TIKTOK_SCRAPE = Capability( + name="tiktok.scrape", + description=( + "Scrape public TikTok videos. Use urls, profiles, hashtags, or " + "search_queries." + ), + input_schema=ScrapeInput, + output_schema=ScrapeOutput, + executor=build_scrape_executor(), + billing_unit=BillingUnit.TIKTOK_VIDEO, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_SCRAPE) diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/executor.py b/surfsense_backend/app/capabilities/tiktok/scrape/executor.py new file mode 100644 index 000000000..7ef71486d --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/executor.py @@ -0,0 +1,48 @@ +"""``tiktok.scrape`` executor: verb input → scraper → TikTok video items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.tiktok import ( + TikTokAccessBlockedError, + TikTokScrapeInput, + scrape_tiktok, +) + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the executor to a scraper fn (defaults to the proprietary actor).""" + scrape_fn = scrape_fn or scrape_tiktok + + async def execute(payload: ScrapeInput) -> ScrapeOutput: + actor_input = TikTokScrapeInput( + startUrls=[{"url": url} for url in payload.urls], + profiles=payload.profiles, + hashtags=payload.hashtags, + searchQueries=payload.search_queries, + resultsPerPage=payload.results_per_page, + ) + emit_progress( + "starting", "Resolving TikTok targets", total=payload.max_items, unit="item" + ) + try: + items = await scrape_fn(actor_input, limit=payload.max_items) + except TikTokAccessBlockedError as exc: + # Anonymous-only scraper; a hard block can't be retried with creds. + raise ForbiddenError( + f"TikTok refused anonymous access: {exc}", + code="TIKTOK_ACCESS_BLOCKED", + ) from exc + emit_progress( + "done", f"Scraped {len(items)} item(s)", current=len(items), unit="item" + ) + return ScrapeOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py new file mode 100644 index 000000000..2f5005353 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py @@ -0,0 +1,87 @@ +"""``tiktok.scrape`` I/O contracts. + +A lean, agent-friendly surface over ``TikTokScrapeInput`` +(``app/proprietary/platforms/tiktok``). The executor maps this to the full +scraper input; the scraper's ``TikTokVideoItem`` is reused verbatim as the +output element. Any TikTok URL kind (video, profile, hashtag, search) goes in +``urls``; ``profiles``/``hashtags``/``search_queries`` are typed shortcuts. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field, model_validator + +from app.proprietary.platforms.tiktok import TikTokVideoItem + +MAX_TIKTOK_SOURCES = 20 +"""Per-call cap on each source list: bounds a synchronous request's fan-out.""" + +MAX_TIKTOK_ITEMS = 100 +"""Hard ceiling on items returned per call, regardless of the per-target count.""" + + +class ScrapeInput(BaseModel): + urls: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description=( + "TikTok URLs to scrape: a video, a profile (/@), a hashtag " + "(/tag/), or a search URL. Provide these OR profiles/hashtags/" + "search_queries (at least one source is required)." + ), + ) + profiles: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description="Profile usernames (with or without a leading '@').", + ) + hashtags: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description="Hashtag names to scrape, without the leading '#'.", + ) + search_queries: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description="Search terms to run on TikTok.", + ) + results_per_page: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max videos to pull per profile/hashtag/search target.", + ) + max_items: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max total items to return across all sources.", + ) + + @model_validator(mode="after") + def _require_a_source(self) -> ScrapeInput: + if not any((self.urls, self.profiles, self.hashtags, self.search_queries)): + raise ValueError( + "Provide at least one of 'urls', 'profiles', 'hashtags', or " + "'search_queries'." + ) + return self + + @property + def estimated_units(self) -> int: + """Worst-case billable items for the pre-flight gate: ``max_items`` is a + hard cross-source ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class ScrapeOutput(BaseModel): + items: list[TikTokVideoItem] = Field( + default_factory=list, + description="One item per video returned, in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned video = one billable unit; ErrorItems (``errorCode`` set, + for blocked/empty targets) are surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/capabilities/tiktok/trending/__init__.py b/surfsense_backend/app/capabilities/tiktok/trending/__init__.py new file mode 100644 index 000000000..c335258b3 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/trending/__init__.py @@ -0,0 +1,3 @@ +"""``tiktok.trending``: pull the current trending videos from the Explore feed.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/tiktok/trending/definition.py b/surfsense_backend/app/capabilities/tiktok/trending/definition.py new file mode 100644 index 000000000..7f1b006ce --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/trending/definition.py @@ -0,0 +1,23 @@ +"""``tiktok.trending`` capability registration (billed per video on the shared +``TIKTOK_MICROS_PER_VIDEO`` meter).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.trending.executor import build_trending_executor +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput + +TIKTOK_TRENDING = Capability( + name="tiktok.trending", + description=( + "Get the current trending TikTok videos from the Explore feed. No input " + "needed beyond how many to return." + ), + input_schema=TrendingInput, + output_schema=TrendingOutput, + executor=build_trending_executor(), + billing_unit=BillingUnit.TIKTOK_VIDEO, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_TRENDING) diff --git a/surfsense_backend/app/capabilities/tiktok/trending/executor.py b/surfsense_backend/app/capabilities/tiktok/trending/executor.py new file mode 100644 index 000000000..9551f4b6a --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/trending/executor.py @@ -0,0 +1,32 @@ +"""``tiktok.trending`` executor: Explore feed -> scraper -> TikTok video items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput +from app.proprietary.platforms.tiktok import scrape_tiktok_trending + +TrendingFn = Callable[..., Awaitable[list[dict]]] + + +def build_trending_executor(trending_fn: TrendingFn | None = None) -> Executor: + """Bind the executor to a trending fn (defaults to the proprietary actor).""" + trending_fn = trending_fn or scrape_tiktok_trending + + async def execute(payload: TrendingInput) -> TrendingOutput: + emit_progress( + "starting", + "Fetching TikTok trending videos", + total=payload.max_items, + unit="item", + ) + items = await trending_fn(count=payload.max_items) + emit_progress( + "done", f"Fetched {len(items)} video(s)", current=len(items), unit="item" + ) + return TrendingOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/trending/schemas.py b/surfsense_backend/app/capabilities/tiktok/trending/schemas.py new file mode 100644 index 000000000..879206907 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/trending/schemas.py @@ -0,0 +1,41 @@ +"""``tiktok.trending`` I/O contracts. + +The Explore feed (``/api/explore/item_list``) is a single global feed of trending +videos, served to anonymous sessions. No source input is needed — just how many +to return. Each result reuses :class:`TikTokVideoItem`, so trending videos bill on +the same per-video meter as ``tiktok.scrape``. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS +from app.proprietary.platforms.tiktok import TikTokVideoItem + + +class TrendingInput(BaseModel): + max_items: int = Field( + default=20, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max trending videos to return from the Explore feed.", + ) + + @property + def estimated_units(self) -> int: + """Worst-case billable videos for the pre-flight gate (le=100 ceiling).""" + return self.max_items + + +class TrendingOutput(BaseModel): + items: list[TikTokVideoItem] = Field( + default_factory=list, + description="One item per trending video returned, in feed order.", + ) + + @property + def billable_units(self) -> int: + """One returned video = one billable unit; an ErrorItem (``errorCode`` set, + for an empty/withheld feed) is surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/__init__.py b/surfsense_backend/app/capabilities/tiktok/user_search/__init__.py new file mode 100644 index 000000000..d4344968c --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/__init__.py @@ -0,0 +1,3 @@ +"""``tiktok.user_search``: find public TikTok accounts by keyword.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/definition.py b/surfsense_backend/app/capabilities/tiktok/user_search/definition.py new file mode 100644 index 000000000..6a37f879f --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/definition.py @@ -0,0 +1,26 @@ +"""``tiktok.user_search`` capability registration (billed per account; see config +``TIKTOK_MICROS_PER_USER``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.user_search.executor import build_user_search_executor +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) + +TIKTOK_USER_SEARCH = Capability( + name="tiktok.user_search", + description=( + "Find public TikTok accounts by keyword. Returns profile metadata " + "(name, followers, bio, verification) per matching account." + ), + input_schema=UserSearchInput, + output_schema=UserSearchOutput, + executor=build_user_search_executor(), + billing_unit=BillingUnit.TIKTOK_USER, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_USER_SEARCH) diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/executor.py b/surfsense_backend/app/capabilities/tiktok/user_search/executor.py new file mode 100644 index 000000000..e83f396bd --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/executor.py @@ -0,0 +1,39 @@ +"""``tiktok.user_search`` executor: queries -> scraper -> TikTok profile items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) +from app.proprietary.platforms.tiktok import search_tiktok_users + +SearchFn = Callable[..., Awaitable[list[dict]]] + + +def build_user_search_executor(search_fn: SearchFn | None = None) -> Executor: + """Bind the executor to a search fn (defaults to the proprietary actor).""" + search_fn = search_fn or search_tiktok_users + + async def execute(payload: UserSearchInput) -> UserSearchOutput: + emit_progress( + "starting", + "Searching TikTok accounts", + total=payload.max_items, + unit="item", + ) + items = await search_fn( + payload.queries, + per_query=payload.results_per_query, + limit=payload.max_items, + ) + emit_progress( + "done", f"Found {len(items)} account(s)", current=len(items), unit="item" + ) + return UserSearchOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/schemas.py b/surfsense_backend/app/capabilities/tiktok/user_search/schemas.py new file mode 100644 index 000000000..7cb2aecbb --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/schemas.py @@ -0,0 +1,56 @@ +"""``tiktok.user_search`` I/O contracts. + +Account discovery over ``TikTok``'s Users tab. Where video/general search is +login-walled for anonymous sessions, ``/api/search/user`` returns public account +records, so this verb exposes the one reliably-unblocked search path. Each result +is a :class:`TikTokProfileItem` (the same shape the profile verb emits). +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.capabilities.tiktok.scrape.schemas import ( + MAX_TIKTOK_ITEMS, + MAX_TIKTOK_SOURCES, +) +from app.proprietary.platforms.tiktok import TikTokProfileItem + + +class UserSearchInput(BaseModel): + queries: list[str] = Field( + min_length=1, + max_length=MAX_TIKTOK_SOURCES, + description="Keywords to search for TikTok accounts (e.g. names, brands).", + ) + results_per_query: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max accounts to return per query.", + ) + max_items: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max total accounts to return across all queries.", + ) + + @property + def estimated_units(self) -> int: + """Worst-case billable accounts for the pre-flight gate: ``max_items`` is a + hard cross-query ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class UserSearchOutput(BaseModel): + items: list[TikTokProfileItem] = Field( + default_factory=list, + description="One item per account found, in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned account = one billable unit; ErrorItems (``errorCode`` set, + for empty/withheld queries) are surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 23411d979..92e448d9a 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -727,6 +727,18 @@ class Config: INSTAGRAM_SCRAPE_MICROS_PER_COMMENT = int( os.getenv("INSTAGRAM_SCRAPE_MICROS_PER_COMMENT", "1500") ) + # Browser-driven listings make TikTok heavier per item than the API-backed + # video meter, so it sits a touch above YouTube's video rate. + TIKTOK_MICROS_PER_VIDEO = int(os.getenv("TIKTOK_MICROS_PER_VIDEO", "3500")) + # User search returns lighter account records (name/followers/bio), priced + # below the video meter to mirror the cheaper account-discovery market. + TIKTOK_MICROS_PER_USER = int(os.getenv("TIKTOK_MICROS_PER_USER", "2500")) + # Comments are the cheapest per-item TikTok data, matching the per-comment + # market (and YouTube's comment meter). + TIKTOK_MICROS_PER_COMMENT = int(os.getenv("TIKTOK_MICROS_PER_COMMENT", "1500")) + # Retry an empty listing draw on a fresh rotating IP. Set to 1 for a static + # proxy, where every retry re-hits the same exit. + TIKTOK_LISTING_MAX_ATTEMPTS = int(os.getenv("TIKTOK_LISTING_MAX_ATTEMPTS", "3")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. @@ -1146,6 +1158,11 @@ class Config: # round-trip => default FALSE to honor the "no speed regression" bar; flip on # when leak-safety outweighs the marginal latency. CRAWL_DNS_OVER_HTTPS = os.getenv("CRAWL_DNS_OVER_HTTPS", "FALSE").upper() == "TRUE" + # Promises an Xvfb display so the browser can run headful (TikTok's profile + # feed is empty to headless Chromium). Off keeps every browser headless. + CRAWL_HEADED_XVFB_ENABLED = ( + os.getenv("CRAWL_HEADED_XVFB_ENABLED", "FALSE").upper() == "TRUE" + ) # Litellm TTS Configuration TTS_SERVICE = os.getenv("TTS_SERVICE") diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py new file mode 100644 index 000000000..91f716d1f --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py @@ -0,0 +1,35 @@ +"""Anonymous, blob-first TikTok scraper (public interface). + +The capability layer depends only on the names re-exported here: the input +schema, the collector/generator, the video item shape, and the hard-block error. +""" + +from __future__ import annotations + +from .orchestrator import ( + iter_tiktok, + scrape_tiktok, + scrape_tiktok_comments, + scrape_tiktok_trending, + search_tiktok_users, +) +from .schemas import ( + CommentItem, + TikTokProfileItem, + TikTokScrapeInput, + TikTokVideoItem, +) +from .session import TikTokAccessBlockedError + +__all__ = [ + "CommentItem", + "TikTokAccessBlockedError", + "TikTokProfileItem", + "TikTokScrapeInput", + "TikTokVideoItem", + "iter_tiktok", + "scrape_tiktok", + "scrape_tiktok_comments", + "scrape_tiktok_trending", + "search_tiktok_users", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py new file mode 100644 index 000000000..3afd536f2 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -0,0 +1,25 @@ +"""Turn raw TikTok page/API payloads into normalized items.""" + +from __future__ import annotations + +from .author import parse_author, parse_profile +from .comments import comments_from_response, parse_comment +from .hydration import extract_rehydration_data +from .item_list import items_from_response +from .scopes import user_info, video_item_struct +from .user_search import parse_search_user, users_from_response +from .video import parse_video + +__all__ = [ + "comments_from_response", + "extract_rehydration_data", + "items_from_response", + "parse_author", + "parse_comment", + "parse_profile", + "parse_search_user", + "parse_video", + "user_info", + "users_from_response", + "video_item_struct", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py new file mode 100644 index 000000000..7dc1783e4 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py @@ -0,0 +1,38 @@ +"""Normalize TikTok author/profile payloads into an ``authorMeta`` dict.""" + +from __future__ import annotations + +from typing import Any + +_PROFILE_URL = "https://www.tiktok.com/@{username}" + + +def build_author_meta(author: dict[str, Any], stats: dict[str, Any]) -> dict[str, Any]: + """Map an author object + its stats to the ``authorMeta`` output shape.""" + username = author.get("uniqueId") + return { + "id": author.get("id"), + "name": username, + "nickName": author.get("nickname"), + "profileUrl": _PROFILE_URL.format(username=username) if username else None, + "verified": author.get("verified"), + "signature": author.get("signature"), + "avatar": author.get("avatarLarger") or author.get("avatarMedium"), + "privateAccount": author.get("privateAccount"), + "fans": stats.get("followerCount"), + "following": stats.get("followingCount"), + "heart": stats.get("heartCount"), + "video": stats.get("videoCount"), + } + + +def parse_author(user_info: dict[str, Any]) -> dict[str, Any]: + """Map a ``webapp.user-detail`` ``userInfo`` (``{user, stats}``) to authorMeta.""" + return build_author_meta(user_info.get("user") or {}, user_info.get("stats") or {}) + + +def parse_profile(user_info: dict[str, Any]) -> dict[str, Any]: + """Map a ``userInfo`` to a standalone :class:`TikTokProfileItem` output dict.""" + from ..schemas.items import TikTokProfileItem + + return TikTokProfileItem(**parse_author(user_info)).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/comments.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/comments.py new file mode 100644 index 000000000..8ca96cad3 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/comments.py @@ -0,0 +1,55 @@ +"""Parse a captured ``/api/comment/list`` response into comment items. + +The comment API returns ``{"comments": [...]}`` where each entry uses the +mobile-API snake_case shape (``cid``, ``digg_count``, ``reply_comment_total``, +``create_time``, and a nested ``user`` with ``uid``/``unique_id``/``nickname``/ +``avatar_thumb``). ``reply_id != "0"`` marks a reply to a parent comment. +""" + +from __future__ import annotations + +from typing import Any + +from .timestamps import epoch_to_iso + + +def comments_from_response(body: Any) -> list[dict[str, Any]]: + """Return the raw comment records carried by one API response, or ``[]``.""" + if not isinstance(body, dict): + return [] + comments = body.get("comments") + if not isinstance(comments, list): + return [] + return [c for c in comments if isinstance(c, dict)] + + +def _avatar(user: dict[str, Any]) -> str | None: + thumb = user.get("avatar_thumb") + if isinstance(thumb, dict): + urls = thumb.get("url_list") + if isinstance(urls, list) and urls: + return urls[0] + return None + + +def parse_comment(raw: dict[str, Any], video_url: str) -> dict[str, Any]: + """Map a raw comment record to a :class:`CommentItem` output dict.""" + from ..schemas.items import CommentItem + + user = raw.get("user") if isinstance(raw.get("user"), dict) else {} + reply_id = raw.get("reply_id") + create_time = raw.get("create_time") + return CommentItem( + id=raw.get("cid"), + text=raw.get("text"), + videoWebUrl=video_url, + diggCount=raw.get("digg_count"), + replyCommentTotal=raw.get("reply_comment_total"), + createTime=create_time, + createTimeISO=epoch_to_iso(create_time), + uid=user.get("uid"), + uniqueId=user.get("unique_id"), + nickName=user.get("nickname"), + avatar=_avatar(user), + repliesToId=reply_id if reply_id and reply_id != "0" else None, + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py new file mode 100644 index 000000000..4cc466849 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py @@ -0,0 +1,28 @@ +"""Extract the ``__UNIVERSAL_DATA_FOR_REHYDRATION__`` JSON embedded in page HTML. + +TikTok server-renders the first page of data into a single script tag; parsing +it yields page-one items (video/profile/hashtag) without any signed API call. +""" + +from __future__ import annotations + +import json +import re +from typing import Any + +_BLOB_RE = re.compile( + r']*id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>(.*?)', + re.DOTALL, +) + + +def extract_rehydration_data(html: str) -> dict[str, Any] | None: + """Return the parsed rehydration blob, or ``None`` if absent/unparseable.""" + match = _BLOB_RE.search(html) + if not match: + return None + try: + data = json.loads(match.group(1)) + except (json.JSONDecodeError, ValueError): + return None + return data if isinstance(data, dict) else None diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py new file mode 100644 index 000000000..7dddf891f --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py @@ -0,0 +1,30 @@ +"""Item structs from a captured ``item_list`` / search API response body. + +Profile and hashtag listings return ``{"itemList": [...]}``; search returns +``{"data": [{"item": {...}}]}``. Both element shapes are the same itemStruct +:func:`parse_video` already consumes. +""" + +from __future__ import annotations + +from typing import Any + + +def items_from_response(body: Any) -> list[dict[str, Any]]: + """Return the itemStructs carried by one API response, or ``[]``.""" + if not isinstance(body, dict): + return [] + + item_list = body.get("itemList") + if isinstance(item_list, list): + return [i for i in item_list if isinstance(i, dict)] + + data = body.get("data") + if isinstance(data, list): + return [ + entry["item"] + for entry in data + if isinstance(entry, dict) and isinstance(entry.get("item"), dict) + ] + + return [] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py new file mode 100644 index 000000000..a20a72b2b --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py @@ -0,0 +1,30 @@ +"""Navigate the rehydration blob to the scopes the flows consume.""" + +from __future__ import annotations + +from typing import Any + +_DEFAULT = "__DEFAULT_SCOPE__" + + +def _scope(data: dict[str, Any], name: str) -> dict[str, Any] | None: + scope = (data.get(_DEFAULT) or {}).get(name) + return scope if isinstance(scope, dict) else None + + +def video_item_struct(data: dict[str, Any]) -> dict[str, Any] | None: + """The ``itemStruct`` of a video-detail page, or ``None``.""" + scope = _scope(data, "webapp.video-detail") + if not scope: + return None + item = (scope.get("itemInfo") or {}).get("itemStruct") + return item if isinstance(item, dict) else None + + +def user_info(data: dict[str, Any]) -> dict[str, Any] | None: + """The ``userInfo`` (``{user, stats}``) of a profile page, or ``None``.""" + scope = _scope(data, "webapp.user-detail") + if not scope: + return None + info = scope.get("userInfo") + return info if isinstance(info, dict) else None diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py new file mode 100644 index 000000000..4b155c137 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py @@ -0,0 +1,18 @@ +"""Millisecond-ISO timestamps matching the actor's output shape.""" + +from __future__ import annotations + +from datetime import UTC, datetime + + +def epoch_to_iso(seconds: int | None) -> str | None: + """Convert a Unix-seconds timestamp to ``YYYY-MM-DDTHH:MM:SS.000Z``.""" + if not seconds: + return None + stamp = datetime.fromtimestamp(seconds, tz=UTC) + return stamp.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + +def now_iso() -> str: + """Current UTC time in the millisecond-ISO output shape.""" + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/user_search.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/user_search.py new file mode 100644 index 000000000..5ffc2cd4d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/user_search.py @@ -0,0 +1,56 @@ +"""Parse the ``/api/search/user`` response into profile items. + +User search returns ``{"user_list": [{"user_info": {...}}, ...]}`` where each +``user_info`` uses the mobile-API snake_case shape (``uid``, ``unique_id``, +``follower_count``, ``total_favorited``, ``avatar_thumb.url_list``) — distinct +from the camelCase ``webapp.user-detail`` blob the profile flow reads, so it gets +its own mapping into the shared :class:`TikTokProfileItem` output contract. +""" + +from __future__ import annotations + +from typing import Any + +_PROFILE_URL = "https://www.tiktok.com/@{username}" + + +def users_from_response(body: Any) -> list[dict[str, Any]]: + """Return the ``user_info`` objects carried by one search response, or ``[]``.""" + if not isinstance(body, dict): + return [] + user_list = body.get("user_list") + if not isinstance(user_list, list): + return [] + return [ + entry["user_info"] + for entry in user_list + if isinstance(entry, dict) and isinstance(entry.get("user_info"), dict) + ] + + +def _avatar(user_info: dict[str, Any]) -> str | None: + thumb = user_info.get("avatar_thumb") + if isinstance(thumb, dict): + urls = thumb.get("url_list") + if isinstance(urls, list) and urls: + return urls[0] + return None + + +def parse_search_user(user_info: dict[str, Any]) -> dict[str, Any]: + """Map a search ``user_info`` to a :class:`TikTokProfileItem` output dict.""" + from ..schemas.items import TikTokProfileItem + + username = user_info.get("unique_id") + return TikTokProfileItem( + id=user_info.get("uid"), + name=username, + nickName=user_info.get("nickname"), + profileUrl=_PROFILE_URL.format(username=username) if username else None, + verified=bool(user_info.get("enterprise_verify_reason")), + signature=user_info.get("signature"), + avatar=_avatar(user_info), + fans=user_info.get("follower_count"), + heart=user_info.get("total_favorited"), + secUid=user_info.get("sec_uid"), + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py new file mode 100644 index 000000000..1bfe66452 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py @@ -0,0 +1,73 @@ +"""Normalize a raw TikTok ``itemStruct`` into a :class:`TikTokVideoItem` dict.""" + +from __future__ import annotations + +from typing import Any + +from ..schemas.items import TikTokVideoItem +from .author import build_author_meta +from .timestamps import epoch_to_iso + +_VIDEO_URL = "https://www.tiktok.com/@{username}/video/{video_id}" + + +def _music_meta(music: dict[str, Any]) -> dict[str, Any]: + return { + "musicId": music.get("id"), + "musicName": music.get("title"), + "musicAuthor": music.get("authorName"), + "musicOriginal": music.get("original"), + "playUrl": music.get("playUrl"), + } + + +def _video_meta(video: dict[str, Any]) -> dict[str, Any]: + return { + "height": video.get("height"), + "width": video.get("width"), + "duration": video.get("duration"), + "coverUrl": video.get("cover"), + "format": video.get("format"), + "definition": video.get("definition"), + } + + +def _hashtags(challenges: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [ + {"id": c.get("id"), "name": c.get("title")} + for c in challenges + if isinstance(c, dict) + ] + + +def parse_video(item: dict[str, Any]) -> dict[str, Any]: + """Map an ``itemStruct`` to the flat output contract, filling known fields.""" + author = item.get("author") or {} + author_stats = item.get("authorStats") or {} + stats = item.get("stats") or {} + username = author.get("uniqueId") + video_id = item.get("id") + + web_url = ( + _VIDEO_URL.format(username=username, video_id=video_id) + if username and video_id + else None + ) + create_time = item.get("createTime") + + return TikTokVideoItem( + id=video_id, + text=item.get("desc"), + createTime=create_time, + createTimeISO=epoch_to_iso(create_time), + authorMeta=build_author_meta(author, author_stats), + musicMeta=_music_meta(item.get("music") or {}), + videoMeta=_video_meta(item.get("video") or {}), + webVideoUrl=web_url, + diggCount=stats.get("diggCount"), + shareCount=stats.get("shareCount"), + playCount=stats.get("playCount"), + collectCount=stats.get("collectCount"), + commentCount=stats.get("commentCount"), + hashtags=_hashtags(item.get("challenges") or []), + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py new file mode 100644 index 000000000..a7e38eba1 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py @@ -0,0 +1,19 @@ +"""Per-target scrape flows: resolved target -> normalized items.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator, Awaitable, Callable + +FetchFn = Callable[[str], Awaitable[str | None]] +"""Fetch a page's HTML by URL (blob-first video flow).""" + +FetchListingFn = Callable[[str, int], Awaitable[list[dict]]] +"""Load a listing page and return up to ``count`` captured itemStructs.""" + +FetchUsersFn = Callable[[str, int], Awaitable[list[dict]]] +"""Load a user-search page and return up to ``count`` captured ``user_info`` records.""" + +FetchCommentsFn = Callable[[str, int], Awaitable[list[dict]]] +"""Load a video page and return up to ``count`` captured raw comment records.""" + +FlowResult = AsyncIterator[dict] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/comments.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/comments.py new file mode 100644 index 000000000..28a7772a2 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/comments.py @@ -0,0 +1,52 @@ +"""Comments flow: a video URL -> its public comment thread. + +Comments load over a signed ``/api/comment/list`` XHR that TikTok *does* serve to +anonymous sessions once the comments panel is opened (unlike profile-video/search +feeds). Records are deduped by comment id, capped, and — when a video has none or +withholds them — degraded to one ErrorItem, mirroring the listing flow. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from ..extraction import parse_comment +from ..extraction.timestamps import now_iso +from ..schemas import ErrorItem +from ..targets.types import TikTokTarget +from . import FetchCommentsFn + +_EMPTY_MESSAGE = ( + "No comments returned. The video may have none, comments disabled, or TikTok " + "withheld them from anonymous access." +) + + +async def iter_comments( + target: TikTokTarget, *, cap: int, fetch_comments: FetchCommentsFn +) -> AsyncIterator[dict[str, Any]]: + if cap <= 0: + return + seen: set[str] = set() + emitted = 0 + for raw in await fetch_comments(target.url, cap): + out = parse_comment(raw, target.url) + cid = out.get("id") + if cid is not None: + if cid in seen: + continue + seen.add(cid) + out["scrapedAt"] = now_iso() + yield out + emitted += 1 + if emitted >= cap: + return + if emitted == 0: + yield ErrorItem( + url=target.url, + input=target.value, + error=_EMPTY_MESSAGE, + errorCode="no_comments", + scrapedAt=now_iso(), + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py new file mode 100644 index 000000000..3e930271f --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py @@ -0,0 +1,55 @@ +"""Listing flow shared by profile, hashtag, and search targets. + +The browser seam returns raw itemStructs captured from the signed ``item_list`` +XHRs; this maps each to the output contract, drops duplicate video ids, and +stops at the per-target ``cap``. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from ..extraction import parse_video +from ..extraction.timestamps import now_iso +from ..schemas import ErrorItem +from ..targets.types import TikTokTarget +from . import FetchListingFn + +# Profile and search feeds are trust-gated: an anonymous headless session gets an +# empty body (profile) or no results XHR (search), while hashtag feeds load. We +# can't tell "genuinely empty" from "blocked" here, so a zero-item listing emits +# one honest ErrorItem instead of vanishing silently. +_EMPTY_LISTING_MESSAGE = ( + "No videos returned. The target may be empty/private/removed, or TikTok " + "withheld this feed from anonymous access (common for profiles and search)." +) + + +async def iter_listing( + target: TikTokTarget, *, cap: int, fetch_listing: FetchListingFn +) -> AsyncIterator[dict[str, Any]]: + if cap <= 0: + return + seen: set[str] = set() + emitted = 0 + for item in await fetch_listing(target.url, cap): + out = parse_video(item) + video_id = out.get("id") + if video_id is not None: + if video_id in seen: + continue + seen.add(video_id) + out["scrapedAt"] = now_iso() + yield out + emitted += 1 + if emitted >= cap: + return + if emitted == 0: + yield ErrorItem( + url=target.url, + input=target.value, + error=_EMPTY_LISTING_MESSAGE, + errorCode="no_items", + scrapedAt=now_iso(), + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/profile.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/profile.py new file mode 100644 index 000000000..feebd3c0d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/profile.py @@ -0,0 +1,37 @@ +"""Profile flow: reliable blob metadata first, then the (gated) video listing. + +A profile's account data (name, followers, bio, verification) lives in the page's +rehydration blob and loads over plain HTTP without a signed request, so we emit it +first and always. The video listing needs a signed ``item_list`` XHR that TikTok +withholds from anonymous sessions, so it is best-effort: it streams videos when it +loads and degrades to an ErrorItem (via :func:`iter_listing`) when withheld. The +metadata item therefore survives even when the videos are blocked. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from ..extraction import extract_rehydration_data, parse_profile, user_info +from ..extraction.timestamps import now_iso +from ..targets.types import TikTokTarget +from . import FetchFn, FetchListingFn +from .listing import iter_listing + + +async def iter_profile( + target: TikTokTarget, + *, + cap: int, + fetch: FetchFn, + fetch_listing: FetchListingFn, +) -> AsyncIterator[dict[str, Any]]: + html = await fetch(target.url) + info = user_info(extract_rehydration_data(html) or {}) if html else None + if info: + item = parse_profile(info) + item["scrapedAt"] = now_iso() + yield item + async for out in iter_listing(target, cap=cap, fetch_listing=fetch_listing): + yield out diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/user_search.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/user_search.py new file mode 100644 index 000000000..872212e80 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/user_search.py @@ -0,0 +1,54 @@ +"""User-search flow: keyword -> public account records. + +Unlike video/general search (login-walled for anonymous sessions), the Users tab +hits ``/api/search/user`` and returns account records without a redirect. Each +query's results are deduped by uid, capped, and — when a query returns nothing — +degraded to one ErrorItem, mirroring the listing flow. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import quote + +from ..extraction import parse_search_user +from ..extraction.timestamps import now_iso +from ..schemas import ErrorItem +from . import FetchUsersFn + +_USER_SEARCH_URL = "https://www.tiktok.com/search/user?q={query}" +_EMPTY_MESSAGE = ( + "No accounts returned for this query. It may have no matches, or TikTok " + "withheld the results from anonymous access." +) + + +async def iter_user_search( + query: str, *, cap: int, fetch_users: FetchUsersFn +) -> AsyncIterator[dict[str, Any]]: + if cap <= 0: + return + url = _USER_SEARCH_URL.format(query=quote(query)) + seen: set[str] = set() + emitted = 0 + for user_info in await fetch_users(url, cap): + out = parse_search_user(user_info) + uid = out.get("id") + if uid is not None: + if uid in seen: + continue + seen.add(uid) + out["scrapedAt"] = now_iso() + yield out + emitted += 1 + if emitted >= cap: + return + if emitted == 0: + yield ErrorItem( + url=url, + input=query, + error=_EMPTY_MESSAGE, + errorCode="no_users", + scrapedAt=now_iso(), + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py new file mode 100644 index 000000000..48e558677 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py @@ -0,0 +1,26 @@ +"""Video-URL flow: fetch a post page, read its rehydration blob, emit one item.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from ..extraction import extract_rehydration_data, parse_video, video_item_struct +from ..extraction.timestamps import now_iso +from ..targets.types import TikTokTarget +from . import FetchFn + + +async def iter_video(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]: + html = await fetch(target.url) + if not html: + return + data = extract_rehydration_data(html) + if not data: + return + item = video_item_struct(data) + if item is None: + return + out = parse_video(item) + out["scrapedAt"] = now_iso() + yield out diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py new file mode 100644 index 000000000..5d6a24545 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -0,0 +1,195 @@ +"""Resolve a :class:`TikTokScrapeInput` into targets and stream their items. + +Targets run sequentially on one warm sticky IP; ``limit`` is collector policy +applied by :func:`scrape_tiktok`, never baked into a flow. Each kind routes to +its flow via :func:`_dispatch`: video URLs read the rehydration blob over HTTP, +listings capture signed item_list XHRs through the stealth browser. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import quote + +from .extraction.timestamps import now_iso +from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn +from .flows.comments import iter_comments +from .flows.listing import iter_listing +from .flows.profile import iter_profile +from .flows.user_search import iter_user_search +from .flows.video import iter_video +from .schemas import ErrorItem, TikTokScrapeInput +from .session import ( + fetch_comments, + fetch_html, + fetch_item_list, + fetch_trending, + fetch_user_search, +) +from .targets import resolve_target +from .targets.types import TikTokTarget + +_PROFILE_URL = "https://www.tiktok.com/@{name}" +_HASHTAG_URL = "https://www.tiktok.com/tag/{tag}" +_SEARCH_URL = "https://www.tiktok.com/search?q={query}" +_EXPLORE_URL = "https://www.tiktok.com/explore" + + +def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]: + """Build the target list from every input source, dropping unresolved URLs.""" + targets: list[TikTokTarget] = [] + for entry in input_model.startUrls: + resolved = resolve_target(entry.url) + if resolved is not None: + targets.append(resolved) + for url in input_model.postURLs: + resolved = resolve_target(url) + if resolved is not None: + targets.append(resolved) + for profile in input_model.profiles: + name = profile.lstrip("@") + targets.append(TikTokTarget("profile", name, _PROFILE_URL.format(name=name))) + for tag in input_model.hashtags: + targets.append(TikTokTarget("hashtag", tag, _HASHTAG_URL.format(tag=tag))) + for query in input_model.searchQueries: + targets.append( + TikTokTarget("search", query, _SEARCH_URL.format(query=quote(query))) + ) + return targets + + +def _dispatch( + target: TikTokTarget, + *, + cap: int, + fetch: FetchFn, + fetch_listing: FetchListingFn, +) -> AsyncIterator[dict[str, Any]]: + if target.kind == "video": + return iter_video(target, fetch=fetch) + if target.kind == "profile": + return iter_profile(target, cap=cap, fetch=fetch, fetch_listing=fetch_listing) + return iter_listing(target, cap=cap, fetch_listing=fetch_listing) + + +async def iter_tiktok( + input_model: TikTokScrapeInput, + *, + fetch: FetchFn = fetch_html, + fetch_listing: FetchListingFn = fetch_item_list, +) -> AsyncIterator[dict[str, Any]]: + """Yield normalized items for every resolved target, in order. + + The video flow's ``fetch_html`` opens its own warmed proxy session per call + when none is bound; the listing flow drives its own browser. Neither binds a + ContextVar across these ``yield``s, so the generator stays context-safe. + """ + cap = input_model.resultsPerPage + for target in _resolve_targets(input_model): + async for item in _dispatch( + target, cap=cap, fetch=fetch, fetch_listing=fetch_listing + ): + yield item + + +async def scrape_tiktok( + input_model: TikTokScrapeInput, + *, + limit: int | None = None, + fetch: FetchFn = fetch_html, + fetch_listing: FetchListingFn = fetch_item_list, +) -> list[dict[str, Any]]: + """Collect :func:`iter_tiktok` into a list, honoring an optional ``limit``.""" + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + async for item in iter_tiktok(input_model, fetch=fetch, fetch_listing=fetch_listing): + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + break + return results + + +async def search_tiktok_users( + queries: list[str], + *, + per_query: int, + limit: int | None = None, + fetch_users: FetchUsersFn = fetch_user_search, +) -> list[dict[str, Any]]: + """Collect user-search account records across queries, honoring ``limit``.""" + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + for query in queries: + async for item in iter_user_search( + query, cap=per_query, fetch_users=fetch_users + ): + results.append(item) + emit_progress("searching", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + return results + return results + + +async def scrape_tiktok_trending( + *, + count: int, + fetch_trending_fn: FetchListingFn = fetch_trending, +) -> list[dict[str, Any]]: + """Collect up to ``count`` trending videos from the Explore feed. + + A single global feed, so it reuses the listing flow (parse/dedupe/cap/empty- + ErrorItem) over a synthetic target — no user input to resolve. + """ + from app.capabilities.core.progress import emit_progress + + target = TikTokTarget(kind="trending", value="explore", url=_EXPLORE_URL) + results: list[dict[str, Any]] = [] + async for item in iter_listing(target, cap=count, fetch_listing=fetch_trending_fn): + results.append(item) + emit_progress("scraping", current=len(results), total=count, unit="item") + return results + + +async def scrape_tiktok_comments( + video_urls: list[str], + *, + per_video: int, + limit: int | None = None, + fetch_comments_fn: FetchCommentsFn = fetch_comments, +) -> list[dict[str, Any]]: + """Collect comments across video URLs, honoring ``limit``. + + A non-video URL yields one ``bad_url`` ErrorItem (never a silent drop) so the + caller can tell "wrong input" from "video has no comments". + """ + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + for url in video_urls: + target = resolve_target(url) + if target is None or target.kind != "video": + results.append( + ErrorItem( + url=url, + input=url, + error="Not a TikTok video URL.", + errorCode="bad_url", + scrapedAt=now_iso(), + ).to_output() + ) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + return results + continue + async for item in iter_comments( + target, cap=per_video, fetch_comments=fetch_comments_fn + ): + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + return results + return results diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py new file mode 100644 index 000000000..a52400a58 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py @@ -0,0 +1,26 @@ +"""Apify-compatible input and output contracts for the TikTok scraper.""" + +from __future__ import annotations + +from .input import StartUrl, TikTokScrapeInput +from .items import ( + AuthorMeta, + CommentItem, + ErrorItem, + MusicMeta, + TikTokProfileItem, + TikTokVideoItem, + VideoMeta, +) + +__all__ = [ + "AuthorMeta", + "CommentItem", + "ErrorItem", + "MusicMeta", + "StartUrl", + "TikTokProfileItem", + "TikTokScrapeInput", + "TikTokVideoItem", + "VideoMeta", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py new file mode 100644 index 000000000..f710eed72 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py @@ -0,0 +1,59 @@ +# ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API +"""Input surface for the TikTok scraper, shaped to the Clockworks actor. + +Anonymous only: no auth-shaped field exists here. ``extra="allow"`` keeps the +contract in parity with the actor for fields the scraper does not read. + +``resultsPerPage`` is a per-target count; the cross-target ceiling is caller +policy applied by the collector, never baked into a flow. +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +ProfileSorting = Literal["latest", "popular", "oldest"] +ProfileSection = Literal["videos", "reposts"] +SearchSection = Literal["", "/video", "/user"] + + +class StartUrl(BaseModel): + """A single direct URL entry (``{"url": ...}``; extra keys ignored).""" + + model_config = ConfigDict(extra="allow") + + url: str + + +class TikTokScrapeInput(BaseModel): + model_config = ConfigDict(extra="allow") + + # Discovery + startUrls: list[StartUrl] = Field(default_factory=list) + hashtags: list[str] = Field(default_factory=list) + profiles: list[str] = Field(default_factory=list) + searchQueries: list[str] = Field(default_factory=list) + postURLs: list[str] = Field(default_factory=list) + + # Per-target count + resultsPerPage: int = Field(default=1, ge=1) + + # Profile options + profileScrapeSections: list[ProfileSection] = Field( + default_factory=lambda: ["videos"] + ) + profileSorting: ProfileSorting = "latest" + excludePinnedPosts: bool = False + + # Search options + searchSection: SearchSection = "" + maxProfilesPerQuery: int = Field(default=10, ge=1) + + # Incremental filters (ISO date or relative " days" per the actor) + oldestPostDateUnified: str | None = None + newestPostDate: str | None = None + + # Proxy + proxyCountryCode: str = "None" diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py new file mode 100644 index 000000000..50a5c4293 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py @@ -0,0 +1,147 @@ +# ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API +"""Output items keyed to the Clockworks TikTok actor's shape. + +Every model is open (``extra="allow"``) and defaults unsourced fields to +``None``/``[]`` so the MVP can populate a reliable subset and expand the +contract additively without breaking consumers. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class AuthorMeta(BaseModel): + model_config = ConfigDict(extra="allow") + + id: str | None = None + name: str | None = None + nickName: str | None = None + profileUrl: str | None = None + verified: bool | None = None + signature: str | None = None + avatar: str | None = None + privateAccount: bool | None = None + fans: int | None = None + following: int | None = None + heart: int | None = None + video: int | None = None + + +class TikTokProfileItem(AuthorMeta): + """A profile's public metadata, read from the page's rehydration blob. + + Emitted even when the video listing is withheld from anonymous access, so a + blocked profile still yields its account data (name, followers, bio, + verification) instead of only an ErrorItem. Distinguishable from a video item + by the absence of ``webVideoUrl``/``text``. + """ + + scrapedAt: str | None = None + + def to_output(self) -> dict[str, Any]: + return self.model_dump(exclude_none=False) + + +class MusicMeta(BaseModel): + model_config = ConfigDict(extra="allow") + + musicId: str | None = None + musicName: str | None = None + musicAuthor: str | None = None + musicOriginal: bool | None = None + playUrl: str | None = None + + +class VideoMeta(BaseModel): + model_config = ConfigDict(extra="allow") + + height: int | None = None + width: int | None = None + duration: int | None = None + coverUrl: str | None = None + format: str | None = None + definition: str | None = None + + +class TikTokVideoItem(BaseModel): + """A single scraped post (video or slideshow).""" + + model_config = ConfigDict(extra="allow") + + id: str | None = None + text: str | None = None + textLanguage: str | None = None + createTime: int | None = None + createTimeISO: str | None = None + isAd: bool | None = None + + authorMeta: AuthorMeta = Field(default_factory=AuthorMeta) + musicMeta: MusicMeta = Field(default_factory=MusicMeta) + videoMeta: VideoMeta = Field(default_factory=VideoMeta) + + webVideoUrl: str | None = None + mediaUrls: list[str] = Field(default_factory=list) + + diggCount: int | None = None + shareCount: int | None = None + playCount: int | None = None + collectCount: int | None = None + commentCount: int | None = None + + hashtags: list[dict[str, Any]] = Field(default_factory=list) + mentions: list[str] = Field(default_factory=list) + + isSlideshow: bool | None = None + isPinned: bool | None = None + isSponsored: bool | None = None + + scrapedAt: str | None = None + + def to_output(self) -> dict[str, Any]: + return self.model_dump(exclude_none=False) + + +class CommentItem(BaseModel): + """A single comment or reply under a post.""" + + model_config = ConfigDict(extra="allow") + + id: str | None = None + text: str | None = None + videoWebUrl: str | None = None + diggCount: int | None = None + replyCommentTotal: int | None = None + createTime: int | None = None + createTimeISO: str | None = None + + uid: str | None = None + uniqueId: str | None = None + nickName: str | None = None + avatar: str | None = None + + repliesToId: str | None = None + scrapedAt: str | None = None + + def to_output(self) -> dict[str, Any]: + return self.model_dump(exclude_none=False) + + +class ErrorItem(BaseModel): + """Per-input failure, distinguished from normal items by ``errorCode``. + + Mirrors the actor's convention so a private/deleted/empty target surfaces + as an item instead of silently vanishing from the results. + """ + + model_config = ConfigDict(extra="allow") + + url: str | None = None + input: str | None = None + error: str | None = None + errorCode: str | None = None + + def to_output(self) -> dict[str, Any]: + return self.model_dump(exclude_none=False) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py new file mode 100644 index 000000000..6fa1c1b45 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py @@ -0,0 +1,25 @@ +"""Cookie-warmed, rotate-on-block proxy session and page-fetch seam.""" + +from __future__ import annotations + +from .client import fetch_html +from .errors import TikTokAccessBlockedError +from .listing import ( + fetch_comments, + fetch_item_list, + fetch_trending, + fetch_user_search, +) +from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session + +__all__ = [ + "TikTokAccessBlockedError", + "bind_proxy_holder", + "fetch_comments", + "fetch_html", + "fetch_item_list", + "fetch_trending", + "fetch_user_search", + "open_proxy_holder", + "proxy_session", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py new file mode 100644 index 000000000..e5be1e419 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py @@ -0,0 +1,130 @@ +"""GET TikTok page HTML through a cookie-warmed, sticky-IP proxy session. + +The warm-up mints TikTok's anonymous device cookie (``ttwid``) on the first +homepage hit; the target page then server-renders its rehydration blob. Rotates +the residential IP and re-warms on 403, backs off on 429, and raises +:class:`TikTokAccessBlockedError` only when every rotated IP refuses access. +""" + +from __future__ import annotations + +import asyncio +import logging +import random +from contextlib import suppress +from typing import Any + +from scrapling.fetchers import AsyncFetcher + +from app.utils.proxy import get_proxy_url + +from .errors import TikTokAccessBlockedError +from .proxy import _REQUEST_TIMEOUT_S, _current_session, proxy_session + +logger = logging.getLogger(__name__) + +# 403 => IP blocked; rotate and re-warm. 429 => rate limited; back off same IP. +_ROTATE_STATUS = 403 +_BACKOFF_STATUS = 429 +_MAX_ROTATIONS = 3 +_MAX_BACKOFFS = 4 +_BACKOFF_BASE_S = 5.0 + +_HOME_URL = "https://www.tiktok.com/" +_TTWID_COOKIE = "ttwid" +_HEADERS = {"Accept-Language": "en-US,en;q=0.9"} + + +def _response_cookie_names(page: Any) -> set[str]: + cookies = getattr(page, "cookies", None) + return set(cookies.keys()) if isinstance(cookies, dict) else set() + + +def _page_html(page: Any) -> str | None: + for attr in ("text", "body"): + val = getattr(page, attr, None) + if isinstance(val, bytes): + val = val.decode("utf-8", "replace") + if isinstance(val, str) and val.strip(): + return val + return None + + +async def warm_session(session: Any) -> bool: + """Mint an anonymous ``ttwid`` cookie; ``True`` if the session can now fetch.""" + with suppress(Exception): + page = await session.get(_HOME_URL, headers=_HEADERS) + if _TTWID_COOKIE in _response_cookie_names(page): + return True + return False + + +async def _get_page(session: Any, url: str) -> Any: + if session is not None: + return await session.get(url, headers=_HEADERS) + return await AsyncFetcher.get( + url, + headers=_HEADERS, + proxy=get_proxy_url(), + stealthy_headers=True, + timeout=_REQUEST_TIMEOUT_S, + ) + + +async def fetch_html(url: str) -> str | None: + """Return page HTML, or ``None`` on 404 / non-block failure.""" + holder = _current_session.get() + if holder is None: + async with proxy_session(): + return await fetch_html(url) + + attempt = 0 + backoffs = 0 + while True: + session = holder.session + try: + if session is not None and not holder.warmed: + warmed_ok = await warm_session(session) + holder.warmed = True + if not warmed_ok: + if attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + raise TikTokAccessBlockedError( + f"could not warm session after {attempt} IP rotations: {url}" + ) + + await holder.pace() + page = await _get_page(session, url) + status = page.status + + if status == 200: + return _page_html(page) + if status == 404: + return None + if status == _BACKOFF_STATUS and backoffs < _MAX_BACKOFFS: + backoffs += 1 + delay = _BACKOFF_BASE_S * (2 ** (backoffs - 1)) + logger.warning("[tiktok] 429 on %s; backing off %.1fs", url, delay) + await asyncio.sleep(delay + random.uniform(0, 1)) + continue + if status == _ROTATE_STATUS and attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + if status == _ROTATE_STATUS: + raise TikTokAccessBlockedError( + f"TikTok refused {url} on {attempt} rotated IPs (403)" + ) + logger.warning("[tiktok] GET %s returned %s", url, status) + return None + except TikTokAccessBlockedError: + raise + except Exception as e: + logger.warning("[tiktok] GET %s failed: %s", url, e) + if attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + return None diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py new file mode 100644 index 000000000..708cb0212 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py @@ -0,0 +1,10 @@ +"""Fetch-seam errors surfaced to the capability layer.""" + +from __future__ import annotations + + +class TikTokAccessBlockedError(RuntimeError): + """Raised when every rotated IP is refused anonymous access. + + Distinguishes a hard block from an empty result; the route maps it to 403. + """ diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py new file mode 100644 index 000000000..bc8af56ff --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -0,0 +1,314 @@ +"""Browser-driven listing fetch: let TikTok sign its own ``item_list`` XHRs. + +Profile/hashtag/search listings need signed requests (``X-Gnarly``) whose +algorithm rev's monthly and reads a browser canvas fingerprint. Rather than port +and chase that signer, we load the page in the stealth browser we already run +(patchright-Chromium, via the web-crawler tier) and capture the itemStruct JSON +the page's own scripts fetch while scrolling. The browser is the client, so it +signs correctly for whatever version TikTok ships. + +The pure response-shape parsing lives in :func:`items_from_response`; this module +is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). + +Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile +feed returns an empty 200 to headless sessions, so :func:`fetch_item_list` goes +headful only when ``CRAWL_HEADED_XVFB_ENABLED`` promises an Xvfb display — else it +stays headless and degrades to an ``ErrorItem`` instead of crashing on launch. +""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Callable +from typing import Any + +from scrapling.fetchers import StealthyFetcher + +from app.config import config +from app.proprietary.web_crawler.stealth import ( + build_stealthy_kwargs, + get_stealth_config, +) +from app.utils.proxy import get_proxy_url + +from ..extraction import ( + comments_from_response, + items_from_response, + users_from_response, +) + +logger = logging.getLogger(__name__) + +ExtractFn = Callable[[Any], list[dict[str, Any]]] +# Drives the page after navigation to trigger/paginate the target XHRs, filling +# ``collected`` until it reaches ``target_count`` (or the interaction gives up). +InteractFn = Callable[[Any, list[dict[str, Any]], int], None] + +# XHR paths that carry itemStructs for the three listing kinds. +_ITEM_LIST_MARKERS = ( + "/api/post/item_list", + "/api/challenge/item_list", + "/api/search/", +) +# The user-search XHR carries account records (user_list), not itemStructs. +_USER_SEARCH_MARKERS = ("/api/search/user",) +# The Explore feed's trending videos arrive as ordinary itemStructs. +_EXPLORE_MARKERS = ("/api/explore/item_list",) +# The comment feed fires only after the comments panel is opened. +_COMMENT_MARKERS = ("/api/comment/list",) +_COMMENT_ICON_SELECTORS = ( + '[data-e2e="comment-icon"]', + '[data-e2e="browse-comment"]', +) +# The comment icon hydrates a beat after DOM-ready; wait for it before clicking. +_COMMENT_ICON_WAIT_MS = 8000 +# First comment page lands shortly after the click — don't declare "empty" early. +_COMMENT_FIRST_PAGE_MS = 3500 +_HOME_URL = "https://www.tiktok.com/" +_MSTOKEN_COOKIE = "msToken" +# Bounded scroll: a dead page can't loop forever, and a live one stops early +# once enough items are captured. +_SCROLL_MAX_ROUNDS = 20 +_SCROLL_SETTLE_MS = 1500 +# Warm-up poll for the anonymous msToken cookie the item_list API requires. +_WARM_POLLS = 8 +_WARM_POLL_MS = 500 + + +def _has_mstoken(page: Any) -> bool: + try: + return any(c.get("name") == _MSTOKEN_COOKIE for c in page.context.cookies()) + except Exception: + return False + + +def _dismiss_login_modal(page: Any) -> None: + """Close the login modal that blocks scrolling; Escape as fallback. + + Only the modal's own close button, never a generic dialog button (avoids + clicking "Log in"). + """ + try: + closed = page.evaluate( + """() => { + const btn = document.querySelector('[data-e2e="modal-close-inner-button"]'); + if (btn) { btn.click(); return true; } + return false; + }""" + ) + if not closed: + page.keyboard.press("Escape") + except Exception: + pass + + +def _scroll_page(page: Any, collected: list[dict[str, Any]], target_count: int) -> None: + """Page down a listing feed until enough items are captured or it stops growing.""" + last_height = 0 + for _ in range(_SCROLL_MAX_ROUNDS): + if len(collected) >= target_count: + break + _dismiss_login_modal(page) + page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + page.wait_for_timeout(_SCROLL_SETTLE_MS) + height = page.evaluate("document.body.scrollHeight") + if not height or height <= last_height: + break + last_height = height + + +def _open_comments(page: Any) -> None: + """Click the comment icon so the first ``/api/comment/list`` XHR fires. + + The icon must be present and interactive first (the SPA hydrates it a beat + after DOM-ready), so we wait for it, then fall back to a JS click if the + normal click is intercepted (cookie banner / overlay). + """ + for selector in _COMMENT_ICON_SELECTORS: + try: + page.wait_for_selector(selector, timeout=_COMMENT_ICON_WAIT_MS) + except Exception: + continue + try: + page.click(selector, timeout=_COMMENT_ICON_WAIT_MS) + return + except Exception: + try: + page.eval_on_selector(selector, "el => el.click()") + return + except Exception: + continue + + +def _scroll_comments( + page: Any, collected: list[dict[str, Any]], target_count: int +) -> None: + """Open the comments panel, then scroll its last comment into view to paginate. + + Comment XHRs fire only after the panel is opened, and paging must scroll the + panel (not the page, which would advance the video feed), so we anchor on the + last ``comment-level-1`` element. ponytail: naive scroll-to-last paging, + bounded by ``_SCROLL_MAX_ROUNDS``; upgrade to container-height tracking if + deep threads under-fetch. + """ + _open_comments(page) + # The panel's first page lands a beat after the click; give it room before + # we decide there are no comments to page through. + page.wait_for_timeout(_COMMENT_FIRST_PAGE_MS) + for _ in range(_SCROLL_MAX_ROUNDS): + if len(collected) >= target_count: + break + moved = page.evaluate( + """() => { + const items = document.querySelectorAll('[data-e2e="comment-level-1"]'); + if (!items.length) return false; + items[items.length - 1].scrollIntoView({block: 'end'}); + return true; + }""" + ) + page.wait_for_timeout(_SCROLL_SETTLE_MS) + if not moved: + break + + +def _build_page_action( + collected: list[dict[str, Any]], + url: str, + target_count: int, + markers: tuple[str, ...], + extract: ExtractFn, + interact: InteractFn, +): + """A sync ``page_action`` that warms the session then captures matching XHRs. + + A cold context returns an empty body, so we first mint the anonymous + ``msToken`` (homepage hit), then navigate to the target with the listener + already attached so page-one fires into it; ``interact`` pages the rest. + ``markers``/``extract`` select which XHRs to keep and how to unwrap them. + """ + + def _on_response(response: Any) -> None: + response_url = getattr(response, "url", "") + if not any(marker in response_url for marker in markers): + return + try: + body = response.json() + except Exception: + # An empty 200 (TikTok soft-block) or a body evicted before read. + return + collected.extend(extract(body)) + + def _warm(page: Any) -> None: + if _has_mstoken(page): + return + page.goto(_HOME_URL, wait_until="domcontentloaded") + for _ in range(_WARM_POLLS): + page.wait_for_timeout(_WARM_POLL_MS) + if _has_mstoken(page): + break + + def page_action(page: Any) -> Any: + page.on("response", _on_response) + try: + _warm(page) + # Navigate (back) to the target with the listener attached and a + # token in hand, so the page-one XHR fires into the capture. + page.goto(url, wait_until="domcontentloaded") + page.wait_for_timeout(_SCROLL_SETTLE_MS) + interact(page, collected, target_count) + except Exception as exc: + logger.debug("[tiktok] capture interaction aborted: %s", exc) + return page + + return page_action + + +def _fetch_sync( + url: str, + target_count: int, + markers: tuple[str, ...], + extract: ExtractFn, + interact: InteractFn, + *, + headless: bool = True, +) -> list[dict[str, Any]]: + collected: list[dict[str, Any]] = [] + kwargs = build_stealthy_kwargs(get_stealth_config()) + StealthyFetcher.fetch( + url, + headless=headless, + network_idle=False, + proxy=get_proxy_url(), + page_action=_build_page_action( + collected, url, target_count, markers, extract, interact + ), + **kwargs, + ) + return collected[:target_count] + + +async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` itemStructs from a listing page's XHRs. + + Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display (the profile feed + is empty to headless sessions); headless otherwise so launch never fails. + Retries an empty draw up to ``TIKTOK_LISTING_MAX_ATTEMPTS`` for a fresh exit IP. + """ + headless = not config.CRAWL_HEADED_XVFB_ENABLED + attempts = max(1, config.TIKTOK_LISTING_MAX_ATTEMPTS) + for attempt in range(1, attempts + 1): + items = await asyncio.to_thread( + _fetch_sync, + page_url, + target_count, + _ITEM_LIST_MARKERS, + items_from_response, + _scroll_page, + headless=headless, + ) + if items or attempt == attempts: + return items + logger.info( + "[tiktok] empty item_list for %s (attempt %d/%d); retrying on a fresh exit IP", + page_url, + attempt, + attempts, + ) + return [] + + +async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` ``user_info`` records from a user-search page.""" + return await asyncio.to_thread( + _fetch_sync, + page_url, + target_count, + _USER_SEARCH_MARKERS, + users_from_response, + _scroll_page, + ) + + +async def fetch_comments(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` raw comment records from a video page's XHRs.""" + return await asyncio.to_thread( + _fetch_sync, + page_url, + target_count, + _COMMENT_MARKERS, + comments_from_response, + _scroll_comments, + ) + + +async def fetch_trending(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` trending itemStructs from the Explore feed.""" + return await asyncio.to_thread( + _fetch_sync, + page_url, + target_count, + _EXPLORE_MARKERS, + items_from_response, + _scroll_page, + ) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py new file mode 100644 index 000000000..dca6c776b --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py @@ -0,0 +1,113 @@ +"""Rotate-on-block sticky proxy session, bound per-flow via a ContextVar. + +Reusing one keep-alive connection pins a single residential exit IP so the +warmed cookie jar (``ttwid``/``msToken``, bound to that IP) stays valid across +the warm-up and every subsequent fetch. Ported from the Reddit sibling; the +TikTok-specific warm-up lives in :mod:`client`. +""" + +from __future__ import annotations + +import asyncio +import logging +import random +import time +from contextlib import asynccontextmanager, suppress +from contextvars import ContextVar +from typing import Any + +from scrapling.fetchers import FetcherSession + +from app.utils.proxy import get_proxy_url + +logger = logging.getLogger(__name__) + +# Pace each sticky IP so a fast exit can't burst past TikTok's per-IP threshold. +_MIN_INTERVAL_S = 0.5 +_PACE_JITTER_S = 0.25 +# A healthy fetch lands in ~1-2s; cap a dead IP at one bounded wait before it +# falls through to a rotation. +_REQUEST_TIMEOUT_S = 15.0 + +_current_session: ContextVar[_RotatingSession | None] = ContextVar( + "tiktok_proxy_session", default=None +) + + +class _RotatingSession: + """Owns one live ``FetcherSession`` (sticky IP); ``rotate()`` swaps the IP. + + Used sequentially within a single flow (never shared across concurrent + tasks), so no locking is needed. ``session`` is ``None`` only when no proxy + is configured. + """ + + def __init__(self) -> None: + self._cm: Any | None = None + self.session: Any | None = None + self.rotations = 0 + self.warmed = False + self._last_at = 0.0 + + async def _open(self) -> None: + proxy = get_proxy_url() + self.warmed = False + if proxy is None: + self._cm = self.session = None + return + self._cm = FetcherSession( + proxy=proxy, + stealthy_headers=True, + impersonate="chrome", + timeout=_REQUEST_TIMEOUT_S, + ) + self.session = await self._cm.__aenter__() + + async def close(self) -> None: + if self._cm is not None: + with suppress(Exception): + await self._cm.__aexit__(None, None, None) + self._cm = self.session = None + + async def rotate(self) -> Any | None: + """Drop the current IP and connect through a fresh one.""" + await self.close() + self.rotations += 1 + await self._open() + logger.info("[tiktok] rotated proxy session (rotation #%d)", self.rotations) + return self.session + + async def pace(self) -> None: + """Sleep to hold this sticky IP under TikTok's per-IP rate threshold.""" + wait = _MIN_INTERVAL_S - (time.monotonic() - self._last_at) + if wait > 0: + await asyncio.sleep(wait + random.uniform(0, _PACE_JITTER_S)) + self._last_at = time.monotonic() + + +async def open_proxy_holder() -> _RotatingSession: + """Open a warm rotate-on-block session holder (caller owns ``close()``).""" + holder = _RotatingSession() + await holder._open() + return holder + + +@asynccontextmanager +async def bind_proxy_holder(holder: _RotatingSession): + """Route this task's fetches through ``holder`` for the enclosed block.""" + token = _current_session.set(holder) + try: + yield holder + finally: + _current_session.reset(token) + + +@asynccontextmanager +async def proxy_session(): + """Open one reused, rotate-on-block proxy session for a continuation chain.""" + holder = await open_proxy_holder() + try: + async with bind_proxy_holder(holder): + yield holder + finally: + await holder.close() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py new file mode 100644 index 000000000..ac7410154 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py @@ -0,0 +1,8 @@ +"""TikTok URL classification into scrape targets.""" + +from __future__ import annotations + +from .resolver import resolve_target +from .types import TargetKind, TikTokTarget + +__all__ = ["TargetKind", "TikTokTarget", "resolve_target"] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py new file mode 100644 index 000000000..ffd15ba2a --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py @@ -0,0 +1,50 @@ +"""Classify a TikTok URL into a :class:`TikTokTarget`, or ``None``.""" + +from __future__ import annotations + +from urllib.parse import parse_qs, unquote, urlparse + +from .types import SearchSection, TikTokTarget + +_TIKTOK_HOSTS = frozenset({"tiktok.com", "www.tiktok.com", "m.tiktok.com"}) +_SEARCH_SECTIONS: frozenset[SearchSection] = frozenset({"video", "user"}) + + +def _is_tiktok_host(hostname: str | None) -> bool: + return bool(hostname) and hostname.lower() in _TIKTOK_HOSTS + + +def resolve_target(url: str) -> TikTokTarget | None: + parsed = urlparse(url) + if not _is_tiktok_host(parsed.hostname): + return None + + segments = [s for s in (parsed.path or "").split("/") if s] + if not segments: + return None + + # Profile / video live under /@username[...]. + if segments[0].startswith("@"): + username = segments[0][1:] + if not username: + return None + if len(segments) >= 3 and segments[1] == "video" and segments[2]: + return TikTokTarget("video", segments[2], url, username=username) + return TikTokTarget("profile", username, url) + + if segments[0] == "tag" and len(segments) >= 2 and segments[1]: + return TikTokTarget("hashtag", unquote(segments[1]), url) + + if segments[0] == "search": + query = parse_qs(parsed.query).get("q", [None])[0] + if not query: + return None + section = segments[1] if len(segments) >= 2 else None + return TikTokTarget( + "search", + unquote(query), + url, + section=section if section in _SEARCH_SECTIONS else None, + ) + + return None diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py new file mode 100644 index 000000000..417d27741 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py @@ -0,0 +1,25 @@ +"""Scrape-target value object produced by URL classification.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +TargetKind = Literal["video", "profile", "hashtag", "search", "trending"] +SearchSection = Literal["video", "user"] + + +@dataclass(frozen=True, slots=True) +class TikTokTarget: + """One classified scrape target. + + ``value`` holds the kind-specific identifier: video id, username, hashtag + name, or search query. ``username`` is set for videos (needed to build the + canonical post URL). ``section`` narrows a search to videos or users. + """ + + kind: TargetKind + value: str + url: str + username: str | None = None + section: SearchSection | None = None diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index d15cfdc40..85450d4b7 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -5,6 +5,7 @@ import app.capabilities.google_maps import app.capabilities.google_search import app.capabilities.instagram import app.capabilities.reddit +import app.capabilities.tiktok import app.capabilities.web import app.capabilities.youtube # noqa: F401 from app.automations.api import router as automations_router diff --git a/surfsense_backend/scripts/docker/entrypoint.sh b/surfsense_backend/scripts/docker/entrypoint.sh index 2efd78f94..4e46d184a 100644 --- a/surfsense_backend/scripts/docker/entrypoint.sh +++ b/surfsense_backend/scripts/docker/entrypoint.sh @@ -121,7 +121,28 @@ start_beat() { echo " Celery Beat PID=${PIDS[-1]}" } +# ── Headful browser display ────────────────────────────────── +# Give the stealth browser a virtual display so it can run headful (TikTok's +# profile feed is empty to headless Chromium). Off => every browser stays headless. +start_xvfb() { + if [ "$(echo "${CRAWL_HEADED_XVFB_ENABLED:-false}" | tr '[:upper:]' '[:lower:]')" != "true" ]; then + return + fi + local display="${XVFB_DISPLAY:-:99}" + echo "Starting Xvfb on ${display} (CRAWL_HEADED_XVFB_ENABLED=true)..." + Xvfb "${display}" -screen 0 1920x1080x24 -nolisten tcp >/dev/null 2>&1 & + PIDS+=($!) + export DISPLAY="${display}" + sleep 1 # let the X server accept connections before a browser starts + echo " Xvfb PID=${PIDS[-1]} DISPLAY=${DISPLAY}" +} + # ── Main: run based on role ────────────────────────────────── +# migrate never launches a browser; every other role might, so start the display. +if [ "${SERVICE_ROLE}" != "migrate" ]; then + start_xvfb +fi + case "${SERVICE_ROLE}" in migrate) run_migrations diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py new file mode 100644 index 000000000..b399ca3fb --- /dev/null +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -0,0 +1,299 @@ +"""Manual functional e2e for the TikTok scraper (blob + browser-listing seams). + +Run from the backend directory: + cd surfsense_backend + uv run python scripts/e2e_tiktok_scrape.py + +What it exercises (everything REAL — live network, live proxy, live browser): + + Stage 1 — proxy egress proof (informational). + Stage 2 — profile via the full pipeline: real videos when headful + (CRAWL_HEADED_XVFB_ENABLED), else one ErrorItem — never silent empty. + Stage 3 — blob video path over HTTP (URL taken from a captured hashtag struct). + Stage 4 — hashtag listing via the stealth browser (captures item_list XHRs). + Stage 5 — full scrape_tiktok() pipeline on a hashtag. + Stage 6 — search via the full pipeline: same graceful-degrade contract as + profile (results feed doesn't load for anonymous sessions). + Stage 7 — comments on a real video URL (served anonymously once the panel + opens): real comments OR a single honest ErrorItem. + Stage 8 — user search: the account-discovery XHR that DOES serve anonymous + headless sessions — asserts real account records come back. + Stage 9 — trending: the Explore feed of trending videos — asserts real, + normalized video items come back. + +On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser +suites can pin against real-shaped data without network. + +This is NOT a pytest test (it needs a live stack + proxy + a real browser). It is +the manual counterpart to the unit suites and the truth check for +``session/listing.py``, which is otherwise unverified. +""" + +import asyncio +import json +import sys +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +from dotenv import load_dotenv + +# --- bootstrap: load .env and put the backend root on sys.path before app.* --- +_BACKEND_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_BACKEND_ROOT)) +for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): + if _candidate.exists(): + load_dotenv(_candidate) + break + +_FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok" + +# Evergreen public targets: a regular high-volume creator, a broad hashtag, and +# a common search term. +_PROFILE = "nasa" +_HASHTAG = "food" +_SEARCH = "meal prep" +_COUNT = 5 + + +def _mask(url: str | None) -> str: + if not url: + return "" + p = urlsplit(url) + creds = "***@" if p.username else "" + port = f":{p.port}" if p.port else "" + return f"{p.scheme}://{creds}{p.hostname or '?'}{port}" + + +def _hr(title: str) -> None: + print(f"\n{'=' * 70}\n{title}\n{'=' * 70}") + + +def _check(label: str, ok: bool, detail: str = "") -> bool: + print(f" [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}") + return ok + + +def _dump_fixture(name: str, data: Any) -> None: + _FIXTURES.mkdir(parents=True, exist_ok=True) + path = _FIXTURES / name + path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + print(f" [fixture] wrote {path.relative_to(_BACKEND_ROOT)}") + + +def _url_from_struct(struct: dict[str, Any]) -> str | None: + """Build a canonical video URL from a captured itemStruct.""" + author = (struct.get("author") or {}).get("uniqueId") + vid = struct.get("id") + return f"https://www.tiktok.com/@{author}/video/{vid}" if author and vid else None + + +async def stage_proxy() -> bool: + _hr("STAGE 1 — proxy egress (informational)") + from app.utils.proxy import get_active_provider, get_proxy_url, is_pool_backed + + provider = get_active_provider() + proxy_url = get_proxy_url() + print(f" active proxy provider : {provider.name}") + print(f" proxy url : {_mask(proxy_url)}") + print(f" pool-backed (rotates) : {is_pool_backed()}") + if not proxy_url: + print(" [INFO] no proxy configured — TikTok may block anonymous access") + return True + + +async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]: + _hr(f"STAGE 2 — profile listing graceful-degrade: @{_PROFILE}") + from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok + + # Accept videos OR an ErrorItem (never a silent empty): the feed degrades + # gracefully when the exit IP is flagged. + items = await scrape_tiktok( + TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=_COUNT), limit=_COUNT + ) + has_video = any(it.get("id") and not it.get("errorCode") for it in items) + has_error = any(it.get("errorCode") == "no_items" for it in items) + ok = _check( + "profile yields videos or a graceful ErrorItem (never silent empty)", + has_video or has_error, + f"{len(items)} item(s); video={has_video} error={has_error}", + ) + return ok, items + + +async def stage_blob_video(video_url: str) -> tuple[bool, dict[str, Any] | None]: + _hr("STAGE 3 — blob video path (HTTP)") + print(f" target: {video_url}") + from app.proprietary.platforms.tiktok.extraction import ( + extract_rehydration_data, + video_item_struct, + ) + from app.proprietary.platforms.tiktok.session import fetch_html + + html = await fetch_html(video_url) + if not _check("fetched page HTML", bool(html), f"{len(html or '')} chars"): + return False, None + data = extract_rehydration_data(html or "") + if not _check("extracted rehydration blob", data is not None): + return False, None + raw = video_item_struct(data or {}) + ok = _check( + "blob carries the video itemStruct", + raw is not None and bool(raw.get("id")), + f"id={None if raw is None else raw.get('id')}", + ) + if ok and raw is not None: + _dump_fixture("video_item_struct.json", raw) + return ok, raw + + +async def stage_hashtag_listing() -> tuple[bool, list[dict[str, Any]]]: + _hr(f"STAGE 4 — hashtag listing (browser): #{_HASHTAG}") + from app.proprietary.platforms.tiktok.session import fetch_item_list + + url = f"https://www.tiktok.com/tag/{_HASHTAG}" + raw = await fetch_item_list(url, _COUNT) + ok = _check( + "captured itemStructs for hashtag", + len(raw) > 0 and bool(raw[0].get("id")), + f"{len(raw)} struct(s)", + ) + if ok: + _dump_fixture("listing_item.json", raw[0]) + return ok, raw + + +async def stage_pipeline() -> bool: + _hr("STAGE 5 — full scrape_tiktok() pipeline") + from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok + + items = await scrape_tiktok( + TikTokScrapeInput(hashtags=[_HASHTAG], resultsPerPage=3), limit=3 + ) + ok = _check( + "pipeline returns normalized video items", + len(items) > 0 + and bool(items[0].get("id")) + and bool(items[0].get("webVideoUrl")), + f"{len(items)} item(s)", + ) + if items: + print(f" sample: {items[0].get('webVideoUrl')} — {items[0].get('text', '')[:60]!r}") + return ok + + +async def stage_search_listing() -> tuple[bool, list[dict[str, Any]]]: + _hr(f"STAGE 6 — search listing graceful-degrade: {_SEARCH!r}") + from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok + + # The search results feed doesn't load for anonymous headless sessions + # (results XHR never fires on a cold deep-link). Same contract as profile: + # verify a graceful ErrorItem instead of a silent empty. + items = await scrape_tiktok( + TikTokScrapeInput(searchQueries=[_SEARCH], resultsPerPage=_COUNT), limit=_COUNT + ) + has_video = any(it.get("id") and not it.get("errorCode") for it in items) + has_error = any(it.get("errorCode") == "no_items" for it in items) + ok = _check( + "search yields videos or a graceful ErrorItem (never silent empty)", + has_video or has_error, + f"{len(items)} item(s); video={has_video} error={has_error}", + ) + return ok, items + + +async def stage_comments(video_url: str) -> tuple[bool, list[dict[str, Any]]]: + _hr("STAGE 7 — comments graceful-degrade") + print(f" target: {video_url}") + from app.proprietary.platforms.tiktok import scrape_tiktok_comments + + # Comments load over a signed /api/comment/list XHR that TikTok serves to + # anonymous sessions once the panel opens. Pass if real comments come back + # OR a graceful ErrorItem (video has none / disabled / withheld). + items = await scrape_tiktok_comments( + [video_url], per_video=_COUNT, limit=_COUNT + ) + has_comment = any(it.get("id") and not it.get("errorCode") for it in items) + has_error = any(it.get("errorCode") == "no_comments" for it in items) + ok = _check( + "comments yield records or a graceful ErrorItem (never silent empty)", + has_comment or has_error, + f"{len(items)} item(s); comment={has_comment} error={has_error}", + ) + return ok, items + + +async def stage_user_search() -> tuple[bool, list[dict[str, Any]]]: + _hr(f"STAGE 8 — user search (browser): {_PROFILE!r}") + from app.proprietary.platforms.tiktok import search_tiktok_users + + # Unlike keyword *video* search, the account-search XHR serves anonymous + # headless sessions — so this asserts real records, not just degradation. + items = await search_tiktok_users([_PROFILE], per_query=_COUNT, limit=_COUNT) + real = [it for it in items if not it.get("errorCode")] + ok = _check( + "user search returns account records", + bool(real) and bool(real[0].get("uniqueId") or real[0].get("name")), + f"{len(items)} item(s); accounts={len(real)}", + ) + if real: + print(f" sample: @{real[0].get('uniqueId') or real[0].get('name')}") + return ok, items + + +async def stage_trending() -> tuple[bool, list[dict[str, Any]]]: + _hr("STAGE 9 — trending (browser): Explore feed") + from app.proprietary.platforms.tiktok import scrape_tiktok_trending + + items = await scrape_tiktok_trending(count=_COUNT) + real = [it for it in items if not it.get("errorCode")] + ok = _check( + "trending returns normalized video items", + bool(real) and bool(real[0].get("id")) and bool(real[0].get("webVideoUrl")), + f"{len(items)} item(s); videos={len(real)}", + ) + if real: + print(f" sample: {real[0].get('webVideoUrl')} — {real[0].get('text', '')[:60]!r}") + return ok, items + + +async def main() -> int: + print("TikTok scraper functional e2e — live network + proxy + browser") + results: dict[str, bool] = {} + + results["Stage 1 proxy"] = await stage_proxy() + + # Hashtag listing is the reliable browser path; use one of its captured + # structs to build a real video URL for the HTTP blob path. + ok_tag, tag_structs = await stage_hashtag_listing() + results["Stage 4 hashtag listing"] = ok_tag + + video_url = _url_from_struct(tag_structs[0]) if tag_structs else None + if video_url: + ok_video, _ = await stage_blob_video(video_url) + results["Stage 3 blob video"] = ok_video + ok_comments, _ = await stage_comments(video_url) + results["Stage 7 comments"] = ok_comments + else: + print("\n [SKIP] Stage 3/7 — no captured struct to build a video URL") + + ok_search, _ = await stage_search_listing() + results["Stage 6 search listing"] = ok_search + + ok_profile, _ = await stage_profile_listing() + results["Stage 2 profile listing"] = ok_profile + results["Stage 5 pipeline"] = await stage_pipeline() + + ok_users, _ = await stage_user_search() + results["Stage 8 user search"] = ok_users + ok_trending, _ = await stage_trending() + results["Stage 9 trending"] = ok_trending + + _hr("SUMMARY") + for name, ok in results.items(): + print(f" {'PASS' if ok else 'FAIL/SKIP'} — {name}") + return 0 if all(results.values()) else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/surfsense_backend/tests/fixtures/tiktok/listing_item.json b/surfsense_backend/tests/fixtures/tiktok/listing_item.json new file mode 100644 index 000000000..374a9bf67 --- /dev/null +++ b/surfsense_backend/tests/fixtures/tiktok/listing_item.json @@ -0,0 +1,559 @@ +{ + "AIGCDescription": "", + "CategoryType": 111, + "IsHDBitrate": false, + "anchors": [ + { + "description": "CapCut · Video Editor", + "extraInfo": { + "subtype": "" + }, + "icon": { + "urlList": [ + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=44xwWHkyYJdE%2FIwqS3zAAjU4XQ4%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=lMwgk8e0wLaM3WkpnsMqYcxZN6w%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=1EMaRlc0Snbs9Ltwr0s95%2BDvpFk%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5" + ] + }, + "id": "0", + "keyword": "CapCut · Editing made easy", + "logExtra": "{\"anchor_id\":0,\"anchor_name\":\"CapCut · Editing made easy\",\"anchor_title_detail\":\"None\",\"anchor_title_id\":21502485763,\"anchor_type\":\"TT_CAPCUT\",\"capability_key\":\"default\",\"ccep_vip\":0,\"if_race_trigger\":0,\"is_two_line\":0,\"landing_page_style_id\":21502486275,\"maker_source\":\"\",\"publish_key\":\"default\",\"template_id\":\"none\",\"tutorial_id\":\"none\",\"viamaker_anchor_capability_key_weight\":1,\"viamaker_anchor_style_idx\":-1,\"viamaker_anchor_style_source\":3,\"video_source\":0,\"video_type_id\":1}", + "schema": "https://www.capcut.com/editor?scenario=tiktok&utm_source=tiktok_anchor_tool&utm_medium=tiktok_anchor&utm_campaign=70361230", + "thumbnail": { + "height": 64, + "urlList": [ + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=GOZUldk%2BrxINygKT0jwxeJz9VLQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=SJ6q86bjtCoRArlVNS1DjDDFWoc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=c7mUcXdyEOFc0SMcLAkkib3v9RU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5" + ], + "width": 64 + }, + "type": 54 + } + ], + "author": { + "avatarLarger": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "avatarMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "avatarThumb": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=8c557171&x-expires=1783695600&x-signature=VaAysM3zFkbl%2FOT74d0h2kdkLqo%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "commentSetting": 0, + "downloadSetting": 3, + "duetSetting": 0, + "ftc": false, + "id": "7501138183701103662", + "isADVirtual": false, + "isEmbedBanned": false, + "nickname": "Cookwithhali", + "openFavorite": false, + "privateAccount": false, + "relation": 0, + "secUid": "MS4wLjABAAAAywZ6lCXK3u4zKz8eQxxcjVblLiugz6zMysU98G7dyKVxu6IvMOJ97mpIfpDxUL4q", + "secret": false, + "shortDramaCreator": {}, + "signature": "From my kitchen to your screen! 🍞\nFollow for more ✨\nIG: halikit25\n\n📩 Halikitchen25@gmail.com", + "stitchSetting": 0, + "uniqueId": "cookwithhali", + "verified": false + }, + "authorStats": { + "diggCount": 223, + "followerCount": 372600, + "followingCount": 0, + "friendCount": 0, + "heart": 8000000, + "heartCount": 8000000, + "videoCount": 279 + }, + "authorStatsV2": { + "diggCount": "223", + "followerCount": "372600", + "followingCount": "0", + "friendCount": "0", + "heart": "8000000", + "heartCount": "8000000", + "videoCount": "279" + }, + "backendSourceEventTracking": "", + "challenges": [ + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "", + "id": "6983", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "title": "bread" + }, + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "", + "id": "285169", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "title": "garlicbread" + }, + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "", + "id": "1643317566954502", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "title": "breadtok" + }, + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "Whether you're baking the perfect pie or just searching for a recipe, you've come to the right place for all things #Baking.", + "id": "7250", + "profileLarger": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "profileMedium": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "profileThumb": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "title": "baking" + }, + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "", + "id": "10488587", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "title": "homemadebread" + } + ], + "collected": false, + "contents": [ + { + "desc": "Pull Apart Garlic Bread " + }, + { + "desc": "" + }, + { + "desc": "Ingredients" + }, + { + "desc": "" + }, + { + "desc": "* 1 tsp yeast" + }, + { + "desc": "* 1 tbsp sugar" + }, + { + "desc": "* 270g milk (about 1 cup + 2 tbsp)" + }, + { + "desc": "* 380g flour (about 3 cups)" + }, + { + "desc": "* 1 tsp salt" + }, + { + "desc": "* 43g softened butter (about 3 tbsp)" + }, + { + "desc": "" + }, + { + "desc": "Butter Mixture" + }, + { + "desc": "" + }, + { + "desc": "* 100g butter, softened" + }, + { + "desc": "* 1 tbsp chopped cilantro" + }, + { + "desc": "* 1 garlic clove, minced" + }, + { + "desc": "* Shredded cheese" + }, + { + "desc": "" + }, + { + "desc": "Instructions" + }, + { + "desc": "" + }, + { + "desc": "1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined." + }, + { + "desc": "2. Add the salt and softened butter, then coil fold until the butter is fully incorporated." + }, + { + "desc": "3. Cover and let the dough rest for 2 hours or until doubled." + }, + { + "desc": "4. Mix the softened butter with the cilantro and minced garlic." + }, + { + "desc": "5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese." + }, + { + "desc": "6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan." + }, + { + "desc": "7. Cover and let rest for 20 minutes." + }, + { + "desc": "8. Bake at 370°F for 30 minutes, or until golden brown." + }, + { + "desc": "9. Brush with the remaining garlic butter mixture while still warm." + }, + { + "desc": "" + }, + { + "desc": "#bread #garlicbread #breadtok #baking #homemadebread ", + "textExtra": [ + { + "awemeId": "", + "end": 6, + "hashtagName": "bread", + "isCommerce": false, + "start": 0, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 19, + "hashtagName": "garlicbread", + "isCommerce": false, + "start": 7, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 29, + "hashtagName": "breadtok", + "isCommerce": false, + "start": 20, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 37, + "hashtagName": "baking", + "isCommerce": false, + "start": 30, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 52, + "hashtagName": "homemadebread", + "isCommerce": false, + "start": 38, + "subType": 0, + "type": 1 + } + ] + } + ], + "createTime": 1782509863, + "desc": "Pull Apart Garlic Bread Ingredients * 1 tsp yeast * 1 tbsp sugar * 270g milk (about 1 cup + 2 tbsp) * 380g flour (about 3 cups) * 1 tsp salt * 43g softened butter (about 3 tbsp) Butter Mixture * 100g butter, softened * 1 tbsp chopped cilantro * 1 garlic clove, minced * Shredded cheese Instructions 1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined. 2. Add the salt and softened butter, then coil fold until the butter is fully incorporated. 3. Cover and let the dough rest for 2 hours or until doubled. 4. Mix the softened butter with the cilantro and minced garlic. 5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese. 6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan. 7. Cover and let rest for 20 minutes. 8. Bake at 370°F for 30 minutes, or until golden brown. 9. Brush with the remaining garlic butter mixture while still warm. #bread #garlicbread #breadtok #baking #homemadebread ", + "digged": false, + "diversificationId": 10040, + "duetDisplay": 0, + "duetEnabled": true, + "forFriend": false, + "id": "7655821461772406047", + "isAd": false, + "isReviewing": false, + "itemCommentStatus": 0, + "item_control": { + "can_repost": true + }, + "music": { + "authorName": "Cookwithhali", + "coverLarge": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "coverMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "coverThumb": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=8c557171&x-expires=1783695600&x-signature=VaAysM3zFkbl%2FOT74d0h2kdkLqo%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "duration": 62, + "id": "7655821492566919966", + "isCopyrighted": false, + "is_commerce_music": true, + "is_unlimited_music": false, + "original": true, + "playUrl": "https://v16m.tiktokcdn-us.com/ecd83422a3e7bf5450bd92ebf07e6fce/6a4ec449/video/tos/useast8/tos-useast8-v-27dcd7-tx2/ocBIhANWILSemeAAINEGoTTPe58AJaRegg6eKS/?a=1233&bti=ODszNWYuMDE6&&bt=125&ft=GSDrKInz7Th2sbSGXq8Zmo&mime_type=audio_mpeg&rc=anY5cXA5cmRpOzMzaTU8NEBpanY5cXA5cmRpOzMzaTU8NEBpYzZrMmRjL3NhLS1kMTJzYSNpYzZrMmRjL3NhLS1kMTJzcw%3D%3D&vvpl=1&l=2026070815413036F7D0D0506FB569C78C&btag=e00050000", + "private": false, + "shoot_duration": 62, + "title": "original sound", + "tt2dsp": {} + }, + "officalItem": false, + "originalItem": false, + "privateItem": false, + "secret": false, + "shareEnabled": true, + "stats": { + "collectCount": 396400, + "commentCount": 3720, + "diggCount": 754100, + "playCount": 8300000, + "shareCount": 144500 + }, + "statsV2": { + "collectCount": "396397", + "commentCount": "3720", + "diggCount": "754100", + "playCount": "8300000", + "repostCount": "0", + "shareCount": "144500" + }, + "stickersOnItem": [ + { + "stickerText": [ + "Cheesy Pull Apart Garlic Bread! 🧄🧀\n", + "(Bakery-Style & No-Knead)" + ], + "stickerType": 4 + } + ], + "stitchDisplay": 0, + "stitchEnabled": true, + "textExtra": [ + { + "awemeId": "", + "end": 1030, + "hashtagName": "bread", + "isCommerce": false, + "start": 1024, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 1043, + "hashtagName": "garlicbread", + "isCommerce": false, + "start": 1031, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 1053, + "hashtagName": "breadtok", + "isCommerce": false, + "start": 1044, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 1061, + "hashtagName": "baking", + "isCommerce": false, + "start": 1054, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 1076, + "hashtagName": "homemadebread", + "isCommerce": false, + "start": 1062, + "subType": 0, + "type": 1 + } + ], + "textLanguage": "en", + "textTranslatable": true, + "video": { + "PlayAddrStruct": { + "DataSize": 13761060, + "FileCs": "c:0-51456-f8d1", + "FileHash": "485174c8492db027adbbf6dca7c7c201", + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 720 + }, + "VQScore": "69.69", + "bitrate": 1773703, + "bitrateInfo": [ + { + "Bitrate": 1836180, + "BitrateFPS": 29, + "CodecType": "h265_hvc1", + "Format": "mp4", + "GearName": "adapt_lowest_1080_1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 96.83, \"v960\": 97.64, \"v864\": 98.256, \"v720\": 98.838}, \"ori\": {\"v1080\": 91.834, \"v960\": 93.423, \"v864\": 94.54, \"v720\": 95.872}}}", + "PlayAddr": { + "DataSize": 14245777, + "FileCs": "c:0-52754-ed06", + "FileHash": "38996a98cfeb2714fca1b94768090c37", + "Height": 1920, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_1080p_1836180", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=db26af0a98ff05e504d7121b25048527&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=db26af0a98ff05e504d7121b25048527&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=5de0e21d367c4e2bb42b386ed1aca6fe&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjAzMmU1YzJlMTAwMDJiYjIwMmFiN2RhNTIzOWVhYWRk&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 1080 + }, + "QualityType": 2, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 385342}, {\\\"time\\\": 2, \\\"offset\\\": 561014}, {\\\"time\\\": 3, \\\"offset\\\": 703814}, {\\\"time\\\": 4, \\\"offset\\\": 855867}, {\\\"time\\\": 5, \\\"offset\\\": 1019333}, {\\\"time\\\": 10, \\\"offset\\\": 2028412}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 96.83, \\\"v960\\\": 97.64, \\\"v864\\\": 98.256, \\\"v720\\\": 98.838}, \\\"ori\\\": {\\\"v1080\\\": 91.834, \\\"v960\\\": 93.423, \\\"v864\\\": 94.54, \\\"v720\\\": 95.872}}}\",\"ufq\":\"{\\\"enh\\\":89.317,\\\"playback\\\":{\\\"ori\\\":85.151,\\\"srv1\\\":90.147},\\\"src\\\":85.957,\\\"trans\\\":85.151,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"11b0345f3bf185ddbe7c48bd7301980f\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwCWAACdzkgS\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwCWAACgAhyAHgWWdzkySUQs0RJJJJify4d/1+bPl/1+M4gQ5qAgICCAAAADAIAAAA8E\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 1773703, + "BitrateFPS": 29, + "CodecType": "h264", + "Format": "mp4", + "GearName": "normal_720_0", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}, \"ori\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}}}", + "PlayAddr": { + "DataSize": 13761060, + "FileCs": "c:0-51456-f8d1", + "FileHash": "485174c8492db027adbbf6dca7c7c201", + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 720 + }, + "QualityType": 10, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 400712}, {\\\"time\\\": 2, \\\"offset\\\": 659243}, {\\\"time\\\": 3, \\\"offset\\\": 886968}, {\\\"time\\\": 4, \\\"offset\\\": 1101663}, {\\\"time\\\": 5, \\\"offset\\\": 1368565}, {\\\"time\\\": 10, \\\"offset\\\": 2375704}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}, \\\"ori\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"65b88c41c0963253ad31b6f68981a242\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":64193}" + }, + { + "Bitrate": 966380, + "BitrateFPS": 29, + "CodecType": "h265_hvc1", + "Format": "mp4", + "GearName": "adapt_lower_720_1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 90.05, \"v960\": 91.958, \"v864\": 93.608, \"v720\": 95.869}, \"ori\": {\"v1080\": 82.259, \"v960\": 85.339, \"v864\": 87.288, \"v720\": 90.851}}}", + "PlayAddr": { + "DataSize": 7497540, + "FileCs": "c:0-52786-7f65", + "FileHash": "1376df24c917669727b27273697fb826", + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_720p_966380", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=e3690d67b2ecae289320f5db6c338d05&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=e3690d67b2ecae289320f5db6c338d05&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=87b87e594a2b40ec86b4c3d6e1ca50a6&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRjNWNmZGVmMmFkNTIyNWQ1ZTA1NzExM2NjOTVlNzMy&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 720 + }, + "QualityType": 14, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 284307}, {\\\"time\\\": 2, \\\"offset\\\": 357012}, {\\\"time\\\": 3, \\\"offset\\\": 438159}, {\\\"time\\\": 4, \\\"offset\\\": 532720}, {\\\"time\\\": 5, \\\"offset\\\": 636824}, {\\\"time\\\": 10, \\\"offset\\\": 1142427}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 90.05, \\\"v960\\\": 91.958, \\\"v864\\\": 93.608, \\\"v720\\\": 95.869}, \\\"ori\\\": {\\\"v1080\\\": 82.259, \\\"v960\\\": 85.339, \\\"v864\\\": 87.288, \\\"v720\\\": 90.851}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":76.711,\\\"srv1\\\":84.502},\\\"src\\\":85.957,\\\"trans\\\":76.711,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"97eb57dffd9e7a0333979f6ca6c9418d\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 91.625, \\\"sr20\\\": 97.355}}\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBaIAUBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 799620, + "BitrateFPS": 29, + "CodecType": "h265_hvc1", + "Format": "mp4", + "GearName": "adapt_540_1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 87.061, \"v960\": 89.844, \"v864\": 91.813, \"v720\": 94.499}, \"ori\": {\"v1080\": 77.717, \"v960\": 80.972, \"v864\": 83.632, \"v720\": 87.945}}}", + "PlayAddr": { + "DataSize": 6203756, + "FileCs": "c:0-52786-7f6d", + "FileHash": "41f62f740ad13045000450f28b9ff2cb", + "Height": 1024, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_540p_799620", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=05aaeb04a6cf8d012bba46bb131f00af&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=05aaeb04a6cf8d012bba46bb131f00af&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=ccd4b0e63fe840d587420c7a116a7552&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRlMWFkM2IxY2JkODUxODVlZDJjMDFjYTJmZWE0NTEx&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 576 + }, + "QualityType": 28, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 254747}, {\\\"time\\\": 2, \\\"offset\\\": 310409}, {\\\"time\\\": 3, \\\"offset\\\": 378126}, {\\\"time\\\": 4, \\\"offset\\\": 458158}, {\\\"time\\\": 5, \\\"offset\\\": 546764}, {\\\"time\\\": 10, \\\"offset\\\": 946117}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 87.061, \\\"v960\\\": 89.844, \\\"v864\\\": 91.813, \\\"v720\\\": 94.499}, \\\"ori\\\": {\\\"v1080\\\": 77.717, \\\"v960\\\": 80.972, \\\"v864\\\": 83.632, \\\"v720\\\": 87.945}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":73.304,\\\"srv1\\\":82.648},\\\"src\\\":85.957,\\\"trans\\\":73.304,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"86fda13c754bde67159dea14b7d55d6a\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 87.963, \\\"sr20\\\": 95.722}}\",\"audio_bit_rate\":64143,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBIIAQBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 622546, + "BitrateFPS": 29, + "CodecType": "h264", + "Format": "mp4", + "GearName": "lowest_540_0", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 72.483, \"v960\": 75.424, \"v864\": 79.843, \"v720\": 84.897}, \"ori\": {\"v1080\": 64.527, \"v960\": 67.93, \"v864\": 72.279, \"v720\": 74.495}}}", + "PlayAddr": { + "DataSize": 4829948, + "FileCs": "c:0-51513-3d83", + "FileHash": "a74fd8fabed8b36131a8e31bf78e9d1d", + "Height": 1024, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_540p_622546", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=fa8ddc2bcbad8ebb4394c4a8b55b28df&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=fa8ddc2bcbad8ebb4394c4a8b55b28df&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=1ce99a90ac5e412b94fb5981ce7e9e8e&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjU0ZWZiNDU4NWVmYzFlY2IyNmEyODAyMGY0ZWJjMGZj&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 576 + }, + "QualityType": 25, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 176720}, {\\\"time\\\": 2, \\\"offset\\\": 268125}, {\\\"time\\\": 3, \\\"offset\\\": 342814}, {\\\"time\\\": 4, \\\"offset\\\": 414132}, {\\\"time\\\": 5, \\\"offset\\\": 511717}, {\\\"time\\\": 10, \\\"offset\\\": 841405}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 72.483, \\\"v960\\\": 75.424, \\\"v864\\\": 79.843, \\\"v720\\\": 84.897}, \\\"ori\\\": {\\\"v1080\\\": 64.527, \\\"v960\\\": 67.93, \\\"v864\\\": 72.279, \\\"v720\\\": 74.495}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"25b3faebdda74a9e438e851988a6efcc\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":32099}" + } + ], + "claInfo": { + "enableAutoCaption": true, + "hasOriginalAudio": true, + "noCaptionReason": 3 + }, + "codecType": "h264", + "cover": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=ZIp3Rdhs6GNzMatSMsNWHFZX%2F7s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "definition": "720p", + "duration": 62, + "dynamicCover": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoPG0RxQTGE89EEqvAuADpFvfGABVMxQBeza~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=JWF%2F5g2txpFXhNhJzOwleSJgKik%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "encodeUserTag": "", + "encodedType": "normal", + "format": "mp4", + "height": 1280, + "id": "7655821461772406047", + "originCover": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoQGFsU9VvLxieEDDAQAPpRTqA8B5EzvEGfF~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=yHDGczHGo3joyDeerZP6Dp4S1Jc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "playAddr": "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "ratio": "720p", + "size": 13761060, + "videoID": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "videoQuality": "normal", + "volumeInfo": { + "Loudness": -48.5, + "Peak": 0.18408 + }, + "width": 720, + "zoomCover": { + "240": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:240:240.avif?dr=9616&x-expires=1783695600&x-signature=ZONw1V50ZmupszDX0NiC%2Bf8BimU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "480": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:480:480.avif?dr=9616&x-expires=1783695600&x-signature=LztWYGkvUC4%2BvhtQb0TO0l1ky0s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "720": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:720:720.avif?dr=9616&x-expires=1783695600&x-signature=X%2F3HQ%2BkiKqEAxnuk%2BdFfBfeOsko%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "960": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:960:960.avif?dr=9616&x-expires=1783695600&x-signature=aJaYG%2BDAo%2BhO7nZ8t8XP7g0DArU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1" + } + } +} \ No newline at end of file diff --git a/surfsense_backend/tests/fixtures/tiktok/video_item_struct.json b/surfsense_backend/tests/fixtures/tiktok/video_item_struct.json new file mode 100644 index 000000000..af3bd651b --- /dev/null +++ b/surfsense_backend/tests/fixtures/tiktok/video_item_struct.json @@ -0,0 +1,660 @@ +{ + "id": "7655821461772406047", + "desc": "Pull Apart Garlic Bread Ingredients * 1 tsp yeast * 1 tbsp sugar * 270g milk (about 1 cup + 2 tbsp) * 380g flour (about 3 cups) * 1 tsp salt * 43g softened butter (about 3 tbsp) Butter Mixture * 100g butter, softened * 1 tbsp chopped cilantro * 1 garlic clove, minced * Shredded cheese Instructions 1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined. 2. Add the salt and softened butter, then coil fold until the butter is fully incorporated. 3. Cover and let the dough rest for 2 hours or until doubled. 4. Mix the softened butter with the cilantro and minced garlic. 5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese. 6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan. 7. Cover and let rest for 20 minutes. 8. Bake at 370°F for 30 minutes, or until golden brown. 9. Brush with the remaining garlic butter mixture while still warm. #bread #garlicbread #breadtok #baking #homemadebread ", + "createTime": "1782509863", + "scheduleTime": 0, + "video": { + "id": "7655821461772406047", + "height": 1280, + "width": 720, + "duration": 62, + "ratio": "720p", + "cover": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=ZIp3Rdhs6GNzMatSMsNWHFZX%2F7s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "originCover": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoQGFsU9VvLxieEDDAQAPpRTqA8B5EzvEGfF~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=yHDGczHGo3joyDeerZP6Dp4S1Jc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "dynamicCover": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoPG0RxQTGE89EEqvAuADpFvfGABVMxQBeza~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=Jz%2BPtpSXdbjTV4qFbG4H7JFD0f8%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "playAddr": "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "downloadAddr": "", + "shareCover": [ + "" + ], + "reflowCover": "", + "bitrate": 1773703, + "encodedType": "normal", + "format": "mp4", + "videoQuality": "normal", + "encodeUserTag": "", + "codecType": "h264", + "definition": "720p", + "subtitleInfos": [], + "zoomCover": { + "240": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:240:240.avif?dr=9616&x-expires=1783695600&x-signature=cBXoX16WOKJyUth9fnaohYv%2B%2BBs%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "480": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:480:480.avif?dr=9616&x-expires=1783695600&x-signature=Ff5B1aV4teSPQc6JXMoidMXqJWk%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "720": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:720:720.avif?dr=9616&x-expires=1783695600&x-signature=X%2F3HQ%2BkiKqEAxnuk%2BdFfBfeOsko%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "960": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:960:960.avif?dr=9616&x-expires=1783695600&x-signature=aJaYG%2BDAo%2BhO7nZ8t8XP7g0DArU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1" + }, + "volumeInfo": { + "Loudness": -48.5, + "Peak": 0.18408 + }, + "bitrateInfo": [ + { + "Bitrate": 1836180, + "QualityType": 2, + "BitrateFPS": 29, + "GearName": "adapt_lowest_1080_1", + "PlayAddr": { + "DataSize": "14245777", + "Width": 1080, + "Height": 1920, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=0648319cea95ee08fd7b4f93a95cb14c&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=0648319cea95ee08fd7b4f93a95cb14c&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=5de0e21d367c4e2bb42b386ed1aca6fe&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjAzMmU1YzJlMTAwMDJiYjIwMmFiN2RhNTIzOWVhYWRk&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_1080p_1836180", + "FileHash": "38996a98cfeb2714fca1b94768090c37", + "FileCs": "c:0-52754-ed06" + }, + "CodecType": "h265_hvc1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 96.83, \"v960\": 97.64, \"v864\": 98.256, \"v720\": 98.838}, \"ori\": {\"v1080\": 91.834, \"v960\": 93.423, \"v864\": 94.54, \"v720\": 95.872}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 385342}, {\\\"time\\\": 2, \\\"offset\\\": 561014}, {\\\"time\\\": 3, \\\"offset\\\": 703814}, {\\\"time\\\": 4, \\\"offset\\\": 855867}, {\\\"time\\\": 5, \\\"offset\\\": 1019333}, {\\\"time\\\": 10, \\\"offset\\\": 2028412}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 96.83, \\\"v960\\\": 97.64, \\\"v864\\\": 98.256, \\\"v720\\\": 98.838}, \\\"ori\\\": {\\\"v1080\\\": 91.834, \\\"v960\\\": 93.423, \\\"v864\\\": 94.54, \\\"v720\\\": 95.872}}}\",\"ufq\":\"{\\\"enh\\\":89.317,\\\"playback\\\":{\\\"ori\\\":85.151,\\\"srv1\\\":90.147},\\\"src\\\":85.957,\\\"trans\\\":85.151,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"11b0345f3bf185ddbe7c48bd7301980f\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwCWAACdzkgS\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwCWAACgAhyAHgWWdzkySUQs0RJJJJify4d/1+bPl/1+M4gQ5qAgICCAAAADAIAAAA8E\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 1773703, + "QualityType": 10, + "BitrateFPS": 29, + "GearName": "normal_720_0", + "PlayAddr": { + "DataSize": "13761060", + "Width": 720, + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703", + "FileHash": "485174c8492db027adbbf6dca7c7c201", + "FileCs": "c:0-51456-f8d1" + }, + "CodecType": "h264", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}, \"ori\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 400712}, {\\\"time\\\": 2, \\\"offset\\\": 659243}, {\\\"time\\\": 3, \\\"offset\\\": 886968}, {\\\"time\\\": 4, \\\"offset\\\": 1101663}, {\\\"time\\\": 5, \\\"offset\\\": 1368565}, {\\\"time\\\": 10, \\\"offset\\\": 2375704}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}, \\\"ori\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"65b88c41c0963253ad31b6f68981a242\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":64193}" + }, + { + "Bitrate": 966380, + "QualityType": 14, + "BitrateFPS": 29, + "GearName": "adapt_lower_720_1", + "PlayAddr": { + "DataSize": "7497540", + "Width": 720, + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=3a0c7313f1a5d9908cc2f5b0dd24d048&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=3a0c7313f1a5d9908cc2f5b0dd24d048&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=87b87e594a2b40ec86b4c3d6e1ca50a6&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRjNWNmZGVmMmFkNTIyNWQ1ZTA1NzExM2NjOTVlNzMy&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_720p_966380", + "FileHash": "1376df24c917669727b27273697fb826", + "FileCs": "c:0-52786-7f65" + }, + "CodecType": "h265_hvc1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 90.05, \"v960\": 91.958, \"v864\": 93.608, \"v720\": 95.869}, \"ori\": {\"v1080\": 82.259, \"v960\": 85.339, \"v864\": 87.288, \"v720\": 90.851}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 284307}, {\\\"time\\\": 2, \\\"offset\\\": 357012}, {\\\"time\\\": 3, \\\"offset\\\": 438159}, {\\\"time\\\": 4, \\\"offset\\\": 532720}, {\\\"time\\\": 5, \\\"offset\\\": 636824}, {\\\"time\\\": 10, \\\"offset\\\": 1142427}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 90.05, \\\"v960\\\": 91.958, \\\"v864\\\": 93.608, \\\"v720\\\": 95.869}, \\\"ori\\\": {\\\"v1080\\\": 82.259, \\\"v960\\\": 85.339, \\\"v864\\\": 87.288, \\\"v720\\\": 90.851}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":76.711,\\\"srv1\\\":84.502},\\\"src\\\":85.957,\\\"trans\\\":76.711,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"97eb57dffd9e7a0333979f6ca6c9418d\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 91.625, \\\"sr20\\\": 97.355}}\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBaIAUBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 799620, + "QualityType": 28, + "BitrateFPS": 29, + "GearName": "adapt_540_1", + "PlayAddr": { + "DataSize": "6203756", + "Width": 576, + "Height": 1024, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=a8516d4d8e79814753dc2cd0fde9f7bd&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=a8516d4d8e79814753dc2cd0fde9f7bd&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=ccd4b0e63fe840d587420c7a116a7552&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRlMWFkM2IxY2JkODUxODVlZDJjMDFjYTJmZWE0NTEx&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_540p_799620", + "FileHash": "41f62f740ad13045000450f28b9ff2cb", + "FileCs": "c:0-52786-7f6d" + }, + "CodecType": "h265_hvc1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 87.061, \"v960\": 89.844, \"v864\": 91.813, \"v720\": 94.499}, \"ori\": {\"v1080\": 77.717, \"v960\": 80.972, \"v864\": 83.632, \"v720\": 87.945}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 254747}, {\\\"time\\\": 2, \\\"offset\\\": 310409}, {\\\"time\\\": 3, \\\"offset\\\": 378126}, {\\\"time\\\": 4, \\\"offset\\\": 458158}, {\\\"time\\\": 5, \\\"offset\\\": 546764}, {\\\"time\\\": 10, \\\"offset\\\": 946117}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 87.061, \\\"v960\\\": 89.844, \\\"v864\\\": 91.813, \\\"v720\\\": 94.499}, \\\"ori\\\": {\\\"v1080\\\": 77.717, \\\"v960\\\": 80.972, \\\"v864\\\": 83.632, \\\"v720\\\": 87.945}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":73.304,\\\"srv1\\\":82.648},\\\"src\\\":85.957,\\\"trans\\\":73.304,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"86fda13c754bde67159dea14b7d55d6a\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 87.963, \\\"sr20\\\": 95.722}}\",\"audio_bit_rate\":64143,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBIIAQBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 622546, + "QualityType": 25, + "BitrateFPS": 29, + "GearName": "lowest_540_0", + "PlayAddr": { + "DataSize": "4829948", + "Width": 576, + "Height": 1024, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=f44ac110e4573a7bc11c73155b9141d0&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=f44ac110e4573a7bc11c73155b9141d0&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=1ce99a90ac5e412b94fb5981ce7e9e8e&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjU0ZWZiNDU4NWVmYzFlY2IyNmEyODAyMGY0ZWJjMGZj&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_540p_622546", + "FileHash": "a74fd8fabed8b36131a8e31bf78e9d1d", + "FileCs": "c:0-51513-3d83" + }, + "CodecType": "h264", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 72.483, \"v960\": 75.424, \"v864\": 79.843, \"v720\": 84.897}, \"ori\": {\"v1080\": 64.527, \"v960\": 67.93, \"v864\": 72.279, \"v720\": 74.495}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 176720}, {\\\"time\\\": 2, \\\"offset\\\": 268125}, {\\\"time\\\": 3, \\\"offset\\\": 342814}, {\\\"time\\\": 4, \\\"offset\\\": 414132}, {\\\"time\\\": 5, \\\"offset\\\": 511717}, {\\\"time\\\": 10, \\\"offset\\\": 841405}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 72.483, \\\"v960\\\": 75.424, \\\"v864\\\": 79.843, \\\"v720\\\": 84.897}, \\\"ori\\\": {\\\"v1080\\\": 64.527, \\\"v960\\\": 67.93, \\\"v864\\\": 72.279, \\\"v720\\\": 74.495}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"25b3faebdda74a9e438e851988a6efcc\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":32099}" + } + ], + "size": "13761060", + "VQScore": "69.69", + "claInfo": { + "hasOriginalAudio": true, + "enableAutoCaption": true, + "captionInfos": [], + "noCaptionReason": 3 + }, + "videoID": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "PlayAddrStruct": { + "DataSize": "13761060", + "Width": 720, + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703", + "FileHash": "485174c8492db027adbbf6dca7c7c201", + "FileCs": "c:0-51456-f8d1" + } + }, + "author": { + "id": "7501138183701103662", + "shortId": "", + "uniqueId": "cookwithhali", + "nickname": "Cookwithhali", + "avatarLarger": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "avatarMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "avatarThumb": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=84bba9f5&x-expires=1783695600&x-signature=Et06On6iQeXFbtycXoVodzl0fdM%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "signature": "From my kitchen to your screen! 🍞\nFollow for more ✨\nIG: halikit25\n\n📩 Halikitchen25@gmail.com", + "createTime": 1746494976, + "verified": false, + "secUid": "MS4wLjABAAAAywZ6lCXK3u4zKz8eQxxcjVblLiugz6zMysU98G7dyKVxu6IvMOJ97mpIfpDxUL4q", + "ftc": false, + "relation": 0, + "openFavorite": false, + "commentSetting": 0, + "duetSetting": 0, + "stitchSetting": 0, + "privateAccount": false, + "secret": false, + "isADVirtual": false, + "roomId": "", + "uniqueIdModifyTime": 0, + "ttSeller": false, + "downloadSetting": 3, + "recommendReason": "", + "nowInvitationCardUrl": "", + "nickNameModifyTime": 0, + "isEmbedBanned": false, + "canExpPlaylist": false, + "suggestAccountBind": false, + "UserStoryStatus": 0, + "shortDramaCreator": {} + }, + "music": { + "id": "7655821492566919966", + "title": "original sound", + "playUrl": "https://v16m.tiktokcdn-us.com/84ccc1629d7199ad68f976d419148401/6a4ec44e/video/tos/useast8/tos-useast8-v-27dcd7-tx2/ocBIhANWILSemeAAINEGoTTPe58AJaRegg6eKS/?a=1233&bti=ODszNWYuMDE6&&bt=125&ft=GSDrKInz7ThasbSGXq8Zmo&mime_type=audio_mpeg&rc=anY5cXA5cmRpOzMzaTU8NEBpanY5cXA5cmRpOzMzaTU8NEBpYzZrMmRjL3NhLS1kMTJzYSNpYzZrMmRjL3NhLS1kMTJzcw%3D%3D&vvpl=1&l=20260708154135271AF032BAC01F42EF1E&btag=e00050000", + "coverLarge": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "coverMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "coverThumb": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=84bba9f5&x-expires=1783695600&x-signature=Et06On6iQeXFbtycXoVodzl0fdM%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "authorName": "Cookwithhali", + "original": true, + "private": false, + "duration": 62, + "scheduleSearchTime": 0, + "collected": false, + "preciseDuration": { + "preciseDuration": 62.093063, + "preciseShootDuration": 62.093063, + "preciseAuditionDuration": 62.093063, + "preciseVideoDuration": 62.093063 + }, + "isCopyrighted": false, + "tt2dsp": { + "tt_to_dsp_song_infos": [] + }, + "is_unlimited_music": false, + "is_commerce_music": true, + "shoot_duration": 62 + }, + "challenges": [ + { + "id": "6983", + "title": "bread", + "desc": "", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + }, + { + "id": "285169", + "title": "garlicbread", + "desc": "", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + }, + { + "id": "1643317566954502", + "title": "breadtok", + "desc": "", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + }, + { + "id": "7250", + "title": "baking", + "desc": "Whether you're baking the perfect pie or just searching for a recipe, you've come to the right place for all things #Baking.", + "profileLarger": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "profileMedium": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "profileThumb": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + }, + { + "id": "10488587", + "title": "homemadebread", + "desc": "", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + } + ], + "stats": { + "diggCount": 754100, + "shareCount": 144500, + "commentCount": 3720, + "playCount": 8300000, + "collectCount": "396399" + }, + "statsV2": { + "diggCount": "754100", + "shareCount": "144500", + "commentCount": "3720", + "playCount": "8300000", + "collectCount": "396399", + "repostCount": "0" + }, + "warnInfo": [], + "originalItem": false, + "officalItem": false, + "textExtra": [ + { + "awemeId": "", + "start": 1024, + "end": 1030, + "hashtagId": "6983", + "hashtagName": "bread", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 1031, + "end": 1043, + "hashtagId": "285169", + "hashtagName": "garlicbread", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 1044, + "end": 1053, + "hashtagId": "1643317566954502", + "hashtagName": "breadtok", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 1054, + "end": 1061, + "hashtagId": "7250", + "hashtagName": "baking", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 1062, + "end": 1076, + "hashtagId": "10488587", + "hashtagName": "homemadebread", + "type": 1, + "subType": 0, + "isCommerce": false + } + ], + "secret": false, + "forFriend": false, + "digged": false, + "itemCommentStatus": 0, + "takeDown": 0, + "effectStickers": [], + "authorStats": { + "followerCount": 372600, + "followingCount": 0, + "heart": 8000000, + "heartCount": 8000000, + "videoCount": 279, + "diggCount": 223, + "friendCount": 0 + }, + "privateItem": false, + "duetEnabled": true, + "stitchEnabled": true, + "stickersOnItem": [ + { + "stickerText": [ + "Cheesy Pull Apart Garlic Bread! 🧄🧀\n", + "(Bakery-Style & No-Knead)" + ], + "stickerType": 4 + } + ], + "isAd": false, + "shareEnabled": true, + "comments": [], + "duetDisplay": 0, + "stitchDisplay": 0, + "indexEnabled": true, + "diversificationLabels": [ + "Cooking", + "Food & Drink", + "Lifestyle" + ], + "locationCreated": "US", + "suggestedWords": [ + "Garlic Bread", + "garlic bread recipe", + "pull apart bread", + "Sourdough Bread", + "How To Make Garlic Bread", + "Baking", + "food", + "bread", + "Baking Recipes", + "Garlic Pull Apart Bread Recipe" + ], + "contents": [ + { + "desc": "Pull Apart Garlic Bread ", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "Ingredients", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "* 1 tsp yeast", + "textExtra": [] + }, + { + "desc": "* 1 tbsp sugar", + "textExtra": [] + }, + { + "desc": "* 270g milk (about 1 cup + 2 tbsp)", + "textExtra": [] + }, + { + "desc": "* 380g flour (about 3 cups)", + "textExtra": [] + }, + { + "desc": "* 1 tsp salt", + "textExtra": [] + }, + { + "desc": "* 43g softened butter (about 3 tbsp)", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "Butter Mixture", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "* 100g butter, softened", + "textExtra": [] + }, + { + "desc": "* 1 tbsp chopped cilantro", + "textExtra": [] + }, + { + "desc": "* 1 garlic clove, minced", + "textExtra": [] + }, + { + "desc": "* Shredded cheese", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "Instructions", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined.", + "textExtra": [] + }, + { + "desc": "2. Add the salt and softened butter, then coil fold until the butter is fully incorporated.", + "textExtra": [] + }, + { + "desc": "3. Cover and let the dough rest for 2 hours or until doubled.", + "textExtra": [] + }, + { + "desc": "4. Mix the softened butter with the cilantro and minced garlic.", + "textExtra": [] + }, + { + "desc": "5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese.", + "textExtra": [] + }, + { + "desc": "6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan.", + "textExtra": [] + }, + { + "desc": "7. Cover and let rest for 20 minutes.", + "textExtra": [] + }, + { + "desc": "8. Bake at 370°F for 30 minutes, or until golden brown.", + "textExtra": [] + }, + { + "desc": "9. Brush with the remaining garlic butter mixture while still warm.", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "#bread #garlicbread #breadtok #baking #homemadebread ", + "textExtra": [ + { + "awemeId": "", + "start": 0, + "end": 6, + "hashtagId": "6983", + "hashtagName": "bread", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 7, + "end": 19, + "hashtagId": "285169", + "hashtagName": "garlicbread", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 20, + "end": 29, + "hashtagId": "1643317566954502", + "hashtagName": "breadtok", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 30, + "end": 37, + "hashtagId": "7250", + "hashtagName": "baking", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 38, + "end": 52, + "hashtagId": "10488587", + "hashtagName": "homemadebread", + "type": 1, + "subType": 0, + "isCommerce": false + } + ] + } + ], + "diversificationId": 10040, + "anchors": [ + { + "id": "0", + "type": 54, + "keyword": "CapCut · Editing made easy", + "icon": { + "uri": "tiktok-obj/capcut_logo_64px_bk.png", + "urlList": [ + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=lMwgk8e0wLaM3WkpnsMqYcxZN6w%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=44xwWHkyYJdE%2FIwqS3zAAjU4XQ4%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=lzVxquDTAbut%2BvshGEhsFsYQc2s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5" + ] + }, + "schema": "https://www.capcut.com/tools/desktop-video-editor?channel=capcutpc_ttweb_anchor_edit1&download_channel=capcutpc_ttweb_anchor_edit1&enter_from=capcutpc_ttweb_anchor_edit1&force_dp=1&from_page=capcutpc_ttweb_anchor_edit1&type=tools", + "logExtra": "{\"anchor_id\":0,\"anchor_name\":\"CapCut · Editing made easy\",\"anchor_title_detail\":\"None\",\"anchor_title_id\":21502485763,\"anchor_type\":\"TT_CAPCUT\",\"capability_key\":\"default\",\"ccep_vip\":0,\"if_race_trigger\":0,\"is_two_line\":0,\"landing_page_style_id\":21502486275,\"maker_source\":\"\",\"publish_key\":\"default\",\"template_id\":\"none\",\"tutorial_id\":\"none\",\"viamaker_anchor_capability_key_weight\":1,\"viamaker_anchor_style_idx\":-1,\"viamaker_anchor_style_source\":3,\"video_source\":0,\"video_type_id\":1}", + "description": "CapCut · Video Editor", + "thumbnail": { + "uri": "tiktok-obj/64x_Capcut3x.png", + "urlList": [ + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=SJ6q86bjtCoRArlVNS1DjDDFWoc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=GOZUldk%2BrxINygKT0jwxeJz9VLQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=NBemEand1jjAoV81cQ1ANQzVqqc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5" + ], + "width": 64, + "height": 64 + }, + "extraInfo": { + "subtype": "" + } + } + ], + "collected": false, + "channelTags": [], + "item_control": { + "can_repost": true + }, + "IsAigc": false, + "AIGCDescription": "", + "backendSourceEventTracking": "", + "CategoryType": 111, + "textLanguage": "en", + "textTranslatable": true, + "authorStatsV2": { + "followerCount": "372600", + "followingCount": "0", + "heart": "8000000", + "heartCount": "8000000", + "videoCount": "279", + "diggCount": "223", + "friendCount": "0" + }, + "isReviewing": false, + "creatorAIComment": { + "hasAITopic": false, + "categoryList": [], + "eligibleVideo": false, + "notEligibleReason": 101 + } +} \ No newline at end of file diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py index 1d04f2ebf..ee889cd17 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py @@ -38,6 +38,7 @@ _EXPECTED_SUBAGENTS = frozenset( "memory", "onedrive", "reddit", + "tiktok", "web_crawler", "youtube", } diff --git a/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py b/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py index b2173292f..cdab68bc0 100644 --- a/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py +++ b/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py @@ -78,8 +78,14 @@ async def test_registry_becomes_one_tool_per_verb_plus_readers(isolate): tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=caps) by_name = {t.name: t for t in tools} - # One tool per verb, plus the two shared run-reader tools. - assert set(by_name) == {"web_scrape", "web_discover", "read_run", "search_run"} + # One tool per verb, plus the shared run-reader tools. + assert set(by_name) == { + "web_scrape", + "web_discover", + "read_run", + "search_run", + "export_run", + } assert by_name["web_scrape"].description == "web.scrape does a thing." assert by_name["web_scrape"].args_schema is _EchoInput diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/comments/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/comments/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py new file mode 100644 index 000000000..b23149420 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py @@ -0,0 +1,48 @@ +"""``tiktok.comments`` executor: verb input → scraper args → typed comment items. + +Boundary mocked: the proprietary comments actor (injected fake). NOT mocked: the +verb's own payload→args forwarding and the dict→CommentItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.comments.executor import build_comments_executor +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput + +pytestmark = pytest.mark.unit + +_VIDEO = "https://www.tiktok.com/@bob/video/123" + + +class _FakeComments: + """Records the urls + kwargs it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[tuple[list[str], int, int | None]] = [] + + async def __call__( + self, video_urls: list[str], *, per_video: int, limit: int | None = None + ) -> list[dict]: + self.calls.append((video_urls, per_video, limit)) + return self._items + + +async def test_forwards_urls_and_limits_and_wraps_items(): + comments = _FakeComments([{"id": "1", "text": "hi"}]) + execute = build_comments_executor(comments_fn=comments) + + out = await execute( + CommentsInput(video_urls=[_VIDEO], comments_per_video=15, max_items=40) + ) + + assert isinstance(out, CommentsOutput) + assert len(out.items) == 1 + assert out.items[0].text == "hi" + + (urls, per_video, limit) = comments.calls[0] + assert urls == [_VIDEO] + assert per_video == 15 + assert limit == 40 diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py new file mode 100644 index 000000000..cdcbc201e --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py @@ -0,0 +1,48 @@ +"""``tiktok.comments`` input guards and billing: a video URL is required, bounded, +and ErrorItems are surfaced free.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput +from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES + +pytestmark = pytest.mark.unit + +_VIDEO = "https://www.tiktok.com/@bob/video/123" + + +def test_rejects_input_with_no_url(): + with pytest.raises(ValidationError): + CommentsInput(video_urls=[]) + + +def test_defaults_and_bounds(): + payload = CommentsInput(video_urls=[_VIDEO]) + assert payload.max_items == 20 + assert payload.comments_per_video == 20 + assert payload.estimated_units == 20 + with pytest.raises(ValidationError): + CommentsInput(video_urls=[_VIDEO], max_items=0) + with pytest.raises(ValidationError): + CommentsInput(video_urls=[_VIDEO], max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_rejects_more_urls_than_the_cap(): + too_many = [f"{_VIDEO}{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] + with pytest.raises(ValidationError): + CommentsInput(video_urls=too_many) + + +def test_error_items_are_not_billed(): + # Real comments count; ErrorItems (bad URL / empty video) are surfaced free. + out = CommentsOutput( + items=[ + {"id": "1", "text": "hi"}, + {"errorCode": "no_comments", "input": "123", "error": "empty"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py new file mode 100644 index 000000000..1bf07b0c3 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py @@ -0,0 +1,79 @@ +"""``tiktok.scrape`` executor: verb input → actor input mapping → typed items. + +Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's +own payload→TikTokScrapeInput mapping and the dict→TikTokVideoItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.scrape.executor import build_scrape_executor +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.tiktok import TikTokAccessBlockedError, TikTokScrapeInput + +pytestmark = pytest.mark.unit + + +class _FakeScraper: + """Records the actor input + limit it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[tuple[TikTokScrapeInput, int | None]] = [] + + async def __call__( + self, actor_input: TikTokScrapeInput, *, limit: int | None = None + ) -> list[dict]: + self.calls.append((actor_input, limit)) + return self._items + + +async def test_maps_urls_to_start_urls_and_wraps_items(): + scraper = _FakeScraper([{"id": "123", "text": "hello"}]) + execute = build_scrape_executor(scrape_fn=scraper) + + out = await execute(ScrapeInput(urls=["https://www.tiktok.com/@nasa/video/123"])) + + assert isinstance(out, ScrapeOutput) + assert len(out.items) == 1 + assert out.items[0].id == "123" + + (actor_input, _limit) = scraper.calls[0] + assert [u.url for u in actor_input.startUrls] == [ + "https://www.tiktok.com/@nasa/video/123" + ] + + +async def test_forwards_typed_sources_and_limit(): + scraper = _FakeScraper([]) + execute = build_scrape_executor(scrape_fn=scraper) + + await execute( + ScrapeInput( + profiles=["nasa"], + hashtags=["food"], + search_queries=["cats"], + results_per_page=7, + max_items=25, + ) + ) + + (actor_input, limit) = scraper.calls[0] + assert actor_input.profiles == ["nasa"] + assert actor_input.hashtags == ["food"] + assert actor_input.searchQueries == ["cats"] + assert actor_input.resultsPerPage == 7 + # The outer collection limit is the caller's total-item cap. + assert limit == 25 + + +async def test_access_blocked_maps_to_forbidden(): + async def _blocked(actor_input: TikTokScrapeInput, *, limit: int | None = None): + raise TikTokAccessBlockedError("all IPs refused") + + execute = build_scrape_executor(scrape_fn=_blocked) + + with pytest.raises(ForbiddenError): + await execute(ScrapeInput(hashtags=["food"])) diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py new file mode 100644 index 000000000..f9cec8c3d --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py @@ -0,0 +1,58 @@ +"""``tiktok.scrape`` input guards: a source is required and the batch is bounded.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.scrape.schemas import ( + MAX_TIKTOK_ITEMS, + MAX_TIKTOK_SOURCES, + ScrapeInput, + ScrapeOutput, +) + +pytestmark = pytest.mark.unit + + +def test_rejects_input_with_no_source(): + with pytest.raises(ValidationError): + ScrapeInput() + + +def test_accepts_urls_only(): + payload = ScrapeInput(urls=["https://www.tiktok.com/@nasa"]) + assert payload.profiles == [] + + +def test_accepts_hashtags_only(): + payload = ScrapeInput(hashtags=["food"]) + assert payload.hashtags == ["food"] + + +def test_defaults_and_bounds(): + payload = ScrapeInput(hashtags=["food"]) + assert payload.max_items == 10 + assert payload.results_per_page == 10 + with pytest.raises(ValidationError): + ScrapeInput(hashtags=["food"], max_items=0) + with pytest.raises(ValidationError): + ScrapeInput(hashtags=["food"], max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_rejects_more_sources_than_the_cap(): + too_many = [f"tag{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] + with pytest.raises(ValidationError): + ScrapeInput(hashtags=too_many) + + +def test_error_items_are_not_billed(): + # Real videos count; ErrorItems (blocked/empty targets) are surfaced free. + out = ScrapeOutput( + items=[ + {"id": "1", "webVideoUrl": "https://tiktok.com/@a/video/1"}, + {"errorCode": "no_items", "input": "nasa", "error": "blocked"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py new file mode 100644 index 000000000..c6314ec10 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py @@ -0,0 +1,57 @@ +"""The tiktok namespace registers its verb as one Capability the doors/agent read.""" + +from __future__ import annotations + +import pytest + +from app.capabilities import ( + tiktok, # noqa: F401 — importing the namespace registers its verbs +) +from app.capabilities.core import BillingUnit +from app.capabilities.core.store import get_capability +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) + +pytestmark = pytest.mark.unit + + +def test_tiktok_scrape_is_registered_and_billed_per_video(): + cap = get_capability("tiktok.scrape") + + assert cap.name == "tiktok.scrape" + assert cap.input_schema is ScrapeInput + assert cap.output_schema is ScrapeOutput + assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO + + +def test_tiktok_user_search_is_registered_and_billed_per_profile(): + cap = get_capability("tiktok.user_search") + + assert cap.name == "tiktok.user_search" + assert cap.input_schema is UserSearchInput + assert cap.output_schema is UserSearchOutput + assert cap.billing_unit is BillingUnit.TIKTOK_USER + + +def test_tiktok_comments_is_registered_and_billed_per_comment(): + cap = get_capability("tiktok.comments") + + assert cap.name == "tiktok.comments" + assert cap.input_schema is CommentsInput + assert cap.output_schema is CommentsOutput + assert cap.billing_unit is BillingUnit.TIKTOK_COMMENT + + +def test_tiktok_trending_is_registered_and_billed_per_video(): + cap = get_capability("tiktok.trending") + + assert cap.name == "tiktok.trending" + assert cap.input_schema is TrendingInput + assert cap.output_schema is TrendingOutput + # Trending returns videos, so it shares the per-video meter. + assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/trending/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/trending/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_executor.py new file mode 100644 index 000000000..3539ca595 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_executor.py @@ -0,0 +1,38 @@ +"""``tiktok.trending`` executor: verb input → scraper count → typed video items. + +Boundary mocked: the proprietary trending actor (injected fake). NOT mocked: the +verb's own count forwarding and the dict→TikTokVideoItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.trending.executor import build_trending_executor +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput + +pytestmark = pytest.mark.unit + + +class _FakeTrending: + """Records the count it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[int] = [] + + async def __call__(self, *, count: int) -> list[dict]: + self.calls.append(count) + return self._items + + +async def test_forwards_count_and_wraps_items(): + trending = _FakeTrending([{"id": "1", "text": "viral"}]) + execute = build_trending_executor(trending_fn=trending) + + out = await execute(TrendingInput(max_items=30)) + + assert isinstance(out, TrendingOutput) + assert len(out.items) == 1 + assert out.items[0].id == "1" + assert trending.calls == [30] diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_schemas.py new file mode 100644 index 000000000..f27fec71a --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_schemas.py @@ -0,0 +1,33 @@ +"""``tiktok.trending`` input guards and billing: bounded count, ErrorItems free.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput + +pytestmark = pytest.mark.unit + + +def test_defaults_and_bounds(): + payload = TrendingInput() + assert payload.max_items == 20 + assert payload.estimated_units == 20 + with pytest.raises(ValidationError): + TrendingInput(max_items=0) + with pytest.raises(ValidationError): + TrendingInput(max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_error_items_are_not_billed(): + # Real videos count; an ErrorItem (empty/withheld feed) is surfaced free. + out = TrendingOutput( + items=[ + {"id": "1", "webVideoUrl": "https://tiktok.com/@a/video/1"}, + {"errorCode": "no_items", "input": "explore", "error": "empty"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/user_search/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py new file mode 100644 index 000000000..866968808 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py @@ -0,0 +1,49 @@ +"""``tiktok.user_search`` executor: verb input → search args → typed profile items. + +Boundary mocked: the proprietary search actor (injected fake). NOT mocked: the +verb's own payload→args forwarding and the dict→TikTokProfileItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.user_search.executor import build_user_search_executor +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) + +pytestmark = pytest.mark.unit + + +class _FakeSearch: + """Records the queries + kwargs it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[tuple[list[str], int, int | None]] = [] + + async def __call__( + self, queries: list[str], *, per_query: int, limit: int | None = None + ) -> list[dict]: + self.calls.append((queries, per_query, limit)) + return self._items + + +async def test_forwards_queries_and_limits_and_wraps_items(): + search = _FakeSearch([{"id": "1", "name": "nasa"}]) + execute = build_user_search_executor(search_fn=search) + + out = await execute( + UserSearchInput(queries=["nasa"], results_per_query=7, max_items=25) + ) + + assert isinstance(out, UserSearchOutput) + assert len(out.items) == 1 + assert out.items[0].name == "nasa" + + (queries, per_query, limit) = search.calls[0] + assert queries == ["nasa"] + assert per_query == 7 + assert limit == 25 diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py new file mode 100644 index 000000000..575db3fb5 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py @@ -0,0 +1,49 @@ +"""``tiktok.user_search`` input guards and billing: a query is required, bounded, +and ErrorItems are surfaced free.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) + +pytestmark = pytest.mark.unit + + +def test_rejects_input_with_no_query(): + with pytest.raises(ValidationError): + UserSearchInput(queries=[]) + + +def test_defaults_and_bounds(): + payload = UserSearchInput(queries=["nasa"]) + assert payload.max_items == 10 + assert payload.results_per_query == 10 + assert payload.estimated_units == 10 + with pytest.raises(ValidationError): + UserSearchInput(queries=["nasa"], max_items=0) + with pytest.raises(ValidationError): + UserSearchInput(queries=["nasa"], max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_rejects_more_queries_than_the_cap(): + too_many = [f"q{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] + with pytest.raises(ValidationError): + UserSearchInput(queries=too_many) + + +def test_error_items_are_not_billed(): + # Real accounts count; ErrorItems (empty/withheld queries) are surfaced free. + out = UserSearchOutput( + items=[ + {"id": "1", "name": "nasa"}, + {"errorCode": "no_users", "input": "ghost", "error": "empty"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/platforms/tiktok/__init__.py b/surfsense_backend/tests/unit/platforms/tiktok/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_comments.py b/surfsense_backend/tests/unit/platforms/tiktok/test_comments.py new file mode 100644 index 000000000..e118f3c01 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_comments.py @@ -0,0 +1,89 @@ +"""Comments orchestration over a fake fetch (no network). + +Drives ``scrape_tiktok_comments``: video URLs -> captured raw comments -> items. +""" + +from __future__ import annotations + +from typing import Any + +from app.proprietary.platforms.tiktok import scrape_tiktok_comments + +_VIDEO = "https://www.tiktok.com/@bob/video/123" + + +def _comment(cid: str, reply_id: str = "0") -> dict[str, Any]: + return { + "cid": cid, + "text": f"comment {cid}", + "digg_count": 7, + "reply_comment_total": 2, + "create_time": 1700000000, + "reply_id": reply_id, + "user": { + "uid": "u1", + "unique_id": "alice", + "nickname": "Alice", + "avatar_thumb": {"url_list": ["https://cdn/a.webp"]}, + }, + } + + +async def test_comments_parse_dedupe_and_cap(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_comment("1"), _comment("1"), _comment("2", reply_id="1")] + + items = await scrape_tiktok_comments( + [_VIDEO], per_video=2, fetch_comments_fn=fake_fetch + ) + + assert [i["id"] for i in items] == ["1", "2"] + first = items[0] + assert first["text"] == "comment 1" + assert first["videoWebUrl"] == _VIDEO + assert first["diggCount"] == 7 + assert first["uniqueId"] == "alice" + assert first["avatar"] == "https://cdn/a.webp" + assert first["createTimeISO"] is not None + assert first["repliesToId"] is None # reply_id "0" == top-level + assert first["scrapedAt"] is not None + assert items[1]["repliesToId"] == "1" # a reply carries its parent id + + +async def test_empty_video_emits_error_item(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [] + + items = await scrape_tiktok_comments( + [_VIDEO], per_video=5, fetch_comments_fn=fake_fetch + ) + + assert len(items) == 1 + assert items[0]["errorCode"] == "no_comments" + assert items[0]["input"] == "123" + + +async def test_non_video_url_emits_bad_url_error(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + raise AssertionError("should not fetch for a non-video URL") + + items = await scrape_tiktok_comments( + ["https://www.tiktok.com/@bob"], per_video=5, fetch_comments_fn=fake_fetch + ) + + assert len(items) == 1 + assert items[0]["errorCode"] == "bad_url" + + +async def test_comments_honor_limit_across_videos(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_comment("1"), _comment("2")] + + items = await scrape_tiktok_comments( + [_VIDEO, "https://www.tiktok.com/@bob/video/456"], + per_video=5, + limit=3, + fetch_comments_fn=fake_fetch, + ) + + assert len(items) == 3 diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py new file mode 100644 index 000000000..484157964 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py @@ -0,0 +1,135 @@ +"""Fetch-seam resilience for the TikTok scraper (no network, fake sessions). + +Fake sessions drive the cookie warm-up + rotate-on-block + backoff branches +deterministically; a live first IP normally warms and returns 200s. +""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.session import ( + TikTokAccessBlockedError, + client, +) +from app.proprietary.platforms.tiktok.session.proxy import _current_session + +_HTML = "ok" + + +class _FakePage: + def __init__(self, status: int, *, cookies: dict | None = None, body: str = _HTML): + self.status = status + self.cookies = cookies or {} + self.body = body + + @property + def text(self) -> str: + return self.body + + +class _FakeSession: + """One 'IP': homepage warm mints ``ttwid`` per flag; page GETs return ``status``.""" + + def __init__(self, status: int = 200, *, warms: bool = True, body: str = _HTML): + self.status = status + self.warms = warms + self.body = body + self.page_calls = 0 + self.warm_calls = 0 + + async def get(self, url, headers=None, cookies=None): + if url.rstrip("/") == "https://www.tiktok.com": + self.warm_calls += 1 + return _FakePage(200, cookies={"ttwid": "x"} if self.warms else {}) + self.page_calls += 1 + return _FakePage(self.status, body=self.body) + + +class _FakeHolder: + def __init__(self, sessions: list[_FakeSession]) -> None: + self._sessions = sessions + self.session = sessions[0] + self.rotations = 0 + self.warmed = False + + async def rotate(self): + self.rotations += 1 + self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)] + self.warmed = False + return self.session + + async def pace(self) -> None: + return None + + async def close(self) -> None: + return None + + +def _no_sleep(monkeypatch) -> None: + async def _noop(_seconds): + return None + + monkeypatch.setattr(client.asyncio, "sleep", _noop) + + +async def test_warms_then_returns_html(): + holder = _FakeHolder([_FakeSession(200, warms=True)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@scout2015") + finally: + _current_session.reset(token) + assert result == _HTML + assert holder.rotations == 0 + assert holder.session.warm_calls == 1 + + +async def test_rotates_when_warm_fails_then_succeeds(): + holder = _FakeHolder([_FakeSession(200, warms=False), _FakeSession(200, warms=True)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@scout2015") + finally: + _current_session.reset(token) + assert result == _HTML + assert holder.rotations == 1 + + +async def test_404_returns_none_without_rotating(): + holder = _FakeHolder([_FakeSession(404), _FakeSession(200)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@missing") + finally: + _current_session.reset(token) + assert result is None + assert holder.rotations == 0 + + +async def test_rotates_and_rewarms_on_403(): + holder = _FakeHolder([_FakeSession(403), _FakeSession(200, warms=True)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@scout2015") + finally: + _current_session.reset(token) + assert result == _HTML + assert holder.rotations == 1 + assert holder.session.warm_calls == 1 + + +async def test_persistent_403_raises_blocked(monkeypatch): + _no_sleep(monkeypatch) + holder = _FakeHolder( + [_FakeSession(403) for _ in range(client._MAX_ROTATIONS + 1)] + ) + token = _current_session.set(holder) + try: + raised = False + try: + await client.fetch_html("https://www.tiktok.com/@scout2015") + except TikTokAccessBlockedError: + raised = True + finally: + _current_session.reset(token) + assert raised + assert holder.rotations == client._MAX_ROTATIONS diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py b/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py new file mode 100644 index 000000000..cdca568f7 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py @@ -0,0 +1,32 @@ +"""Rehydration-blob extraction from TikTok page HTML (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import extract_rehydration_data + +_BLOB = ( + '" +) + + +def test_extracts_default_scope_from_blob(): + data = extract_rehydration_data(_BLOB) + assert data is not None + item = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"] + assert item["id"] == "123" + + +def test_returns_none_when_blob_absent(): + assert extract_rehydration_data("no blob here") is None + + +def test_returns_none_when_blob_json_malformed(): + broken = ( + '' + ) + assert extract_rehydration_data(broken) is None diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_input.py b/surfsense_backend/tests/unit/platforms/tiktok/test_input.py new file mode 100644 index 000000000..c080ffec3 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_input.py @@ -0,0 +1,26 @@ +"""Input surface for the TikTok scraper (anonymous, Apify-shaped).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.schemas import TikTokScrapeInput + + +def test_input_has_no_auth_fields(): + forbidden = {"username", "password", "token", "login", "auth", "credentials"} + assert forbidden.isdisjoint(TikTokScrapeInput.model_fields) + + +def test_input_defaults(): + model = TikTokScrapeInput() + assert model.resultsPerPage == 1 + assert model.profileSorting == "latest" + assert model.proxyCountryCode == "None" + assert model.hashtags == [] + assert model.profiles == [] + assert model.searchQueries == [] + assert model.postURLs == [] + + +def test_input_allows_extra_inert_fields(): + model = TikTokScrapeInput(shouldDownloadVideos=True, videoKvStoreIdOrName="x") + assert model.model_dump().get("shouldDownloadVideos") is True diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py b/surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py new file mode 100644 index 000000000..00ab89b02 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py @@ -0,0 +1,25 @@ +"""Pulling item structs out of captured item_list / search API responses.""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import items_from_response + + +def test_reads_item_list_shape(): + body = {"itemList": [{"id": "1"}, {"id": "2"}], "hasMore": True} + assert items_from_response(body) == [{"id": "1"}, {"id": "2"}] + + +def test_reads_search_data_shape(): + body = {"data": [{"type": 1, "item": {"id": "9"}}, {"type": 4, "item": {}}]} + assert items_from_response(body) == [{"id": "9"}, {}] + + +def test_skips_malformed_entries(): + body = {"data": [{"type": 1}, "junk", {"item": {"id": "7"}}]} + assert items_from_response(body) == [{"id": "7"}] + + +def test_returns_empty_for_unrelated_json(): + assert items_from_response({"statusCode": 0}) == [] + assert items_from_response("nope") == [] diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_listing_retry.py b/surfsense_backend/tests/unit/platforms/tiktok/test_listing_retry.py new file mode 100644 index 000000000..2676a2267 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_listing_retry.py @@ -0,0 +1,60 @@ +"""Retry-on-empty for the browser ``item_list`` seam (no browser, fake fetch). + +``fetch_item_list`` re-fetches an empty capture up to ``TIKTOK_LISTING_MAX_ATTEMPTS`` +so a flagged rotating exit IP on the first draw doesn't collapse straight to an +``ErrorItem``. These drive that loop deterministically by faking ``_fetch_sync``. +""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.session import listing + + +class _Fake: + """Returns each queued result once, repeating the last; counts calls.""" + + def __init__(self, results: list[list[dict]]): + self.results = results + self.calls = 0 + + def __call__(self, *args, **kwargs): + out = self.results[min(self.calls, len(self.results) - 1)] + self.calls += 1 + return out + + +def _patch(monkeypatch, fake: _Fake, attempts: int) -> None: + monkeypatch.setattr(listing, "_fetch_sync", fake) + monkeypatch.setattr(listing.config, "TIKTOK_LISTING_MAX_ATTEMPTS", attempts) + + +async def test_returns_first_nonempty_without_retrying(monkeypatch): + fake = _Fake([[{"id": "1"}]]) + _patch(monkeypatch, fake, 3) + items = await listing.fetch_item_list("https://tt/@x", 5) + assert items == [{"id": "1"}] + assert fake.calls == 1 # a draw with items never retries + + +async def test_retries_past_empty_draws_then_hits(monkeypatch): + fake = _Fake([[], [], [{"id": "9"}]]) + _patch(monkeypatch, fake, 3) + items = await listing.fetch_item_list("https://tt/@x", 5) + assert items == [{"id": "9"}] + assert fake.calls == 3 # two empty (flagged-IP) draws retried, third lands + + +async def test_stops_at_attempt_ceiling_when_always_empty(monkeypatch): + fake = _Fake([[]]) + _patch(monkeypatch, fake, 3) + items = await listing.fetch_item_list("https://tt/@x", 5) + assert items == [] + assert fake.calls == 3 # capped; caller then emits the ErrorItem + + +async def test_single_attempt_config_disables_retry(monkeypatch): + fake = _Fake([[]]) + _patch(monkeypatch, fake, 1) + items = await listing.fetch_item_list("https://tt/@x", 5) + assert items == [] + assert fake.calls == 1 # static-IP setups opt out via attempts=1 diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py new file mode 100644 index 000000000..3a4e42776 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py @@ -0,0 +1,185 @@ +"""End-to-end orchestration over a fake fetch (no network). + +Drives the public collector: input -> target -> blob-first flow -> items. +""" + +from __future__ import annotations + +import json + +from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok + + +async def _no_html(_url: str) -> str: + """Fetch stub that yields no rehydration blob (skips profile metadata).""" + return "" + + +def _profile_page(username: str, followers: int, videos: int) -> str: + blob = { + "__DEFAULT_SCOPE__": { + "webapp.user-detail": { + "userInfo": { + "user": { + "id": "u1", + "uniqueId": username, + "nickname": "Nick", + "verified": True, + }, + "stats": {"followerCount": followers, "videoCount": videos}, + } + } + } + } + return ( + '' + ) + + +def _video_page(video_id: str, username: str) -> str: + blob = { + "__DEFAULT_SCOPE__": { + "webapp.video-detail": { + "itemInfo": { + "itemStruct": { + "id": video_id, + "desc": "hello", + "author": {"uniqueId": username}, + "stats": {"diggCount": 5}, + } + } + } + } + } + return ( + '' + ) + + +async def test_scrape_video_url_returns_parsed_item(): + url = "https://www.tiktok.com/@scout2015/video/123" + + async def fake_fetch(_url: str) -> str: + return _video_page("123", "scout2015") + + items = await scrape_tiktok(TikTokScrapeInput(postURLs=[url]), fetch=fake_fetch) + + assert len(items) == 1 + assert items[0]["id"] == "123" + assert items[0]["diggCount"] == 5 + assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@scout2015/video/123" + assert items[0]["scrapedAt"] is not None + + +async def test_scrape_honors_limit_across_targets(): + urls = [ + "https://www.tiktok.com/@a/video/1", + "https://www.tiktok.com/@b/video/2", + ] + + async def fake_fetch(url: str) -> str: + vid = url.rsplit("/", 1)[1] + user = url.split("@")[1].split("/")[0] + return _video_page(vid, user) + + items = await scrape_tiktok( + TikTokScrapeInput(postURLs=urls), limit=1, fetch=fake_fetch + ) + assert len(items) == 1 + + +async def test_scrape_skips_unrecognized_urls(): + async def fake_fetch(_url: str) -> str: + return "" + + items = await scrape_tiktok( + TikTokScrapeInput(postURLs=["https://example.com/x"]), fetch=fake_fetch + ) + assert items == [] + + +async def test_scrape_profile_returns_listing_items(): + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [ + {"id": "1", "author": {"uniqueId": "a"}}, + {"id": "2", "author": {"uniqueId": "a"}}, + ] + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=["a"], resultsPerPage=5), + fetch=_no_html, + fetch_listing=fake_listing, + ) + assert [i["id"] for i in items] == ["1", "2"] + assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@a/video/1" + + +async def test_profile_emits_metadata_then_videos(): + # The blob metadata item comes first and is billable; videos follow. + async def fake_fetch(_url: str) -> str: + return _profile_page("a", followers=100, videos=2) + + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [{"id": "1", "author": {"uniqueId": "a"}}] + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=["a"], resultsPerPage=5), + fetch=fake_fetch, + fetch_listing=fake_listing, + ) + assert len(items) == 2 + profile, video = items + assert "webVideoUrl" not in profile # metadata item, not a video + assert profile["name"] == "a" + assert profile["fans"] == 100 + assert profile["verified"] is True + assert profile["scrapedAt"] is not None + assert video["id"] == "1" + + +async def test_profile_metadata_survives_blocked_listing(): + # Videos withheld from anonymous access: we still return the profile metadata + # (not just an ErrorItem), so a blocked profile isn't a total loss. + async def fake_fetch(_url: str) -> str: + return _profile_page("a", followers=100, videos=9) + + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [] + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=["a"], resultsPerPage=5), + fetch=fake_fetch, + fetch_listing=fake_listing, + ) + assert len(items) == 2 + assert items[0]["name"] == "a" + assert items[0]["fans"] == 100 + assert items[1]["errorCode"] == "no_items" + + +async def test_listing_dedupes_then_caps_per_target(): + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [{"id": "1"}, {"id": "1"}, {"id": "2"}, {"id": "3"}] + + items = await scrape_tiktok( + TikTokScrapeInput(hashtags=["x"], resultsPerPage=2), fetch_listing=fake_listing + ) + assert [i["id"] for i in items] == ["1", "2"] + + +async def test_empty_listing_emits_error_item(): + # A trust-gated/empty feed (0 videos) must surface one honest ErrorItem, + # tagged with errorCode, rather than vanishing silently. + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [] + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=["nasa"], resultsPerPage=5), + fetch=_no_html, + fetch_listing=fake_listing, + ) + assert len(items) == 1 + assert items[0]["errorCode"] == "no_items" + assert items[0]["input"] == "nasa" diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py b/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py new file mode 100644 index 000000000..03fa63133 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py @@ -0,0 +1,102 @@ +"""Raw TikTok payload -> normalized item mapping (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import parse_author, parse_video + +_ITEM_STRUCT = { + "id": "7534061113365859586", + "desc": "haha #comeramabanana", + "createTime": 1_700_000_000, + "author": { + "id": "6733", + "uniqueId": "bruniela_", + "nickname": "Bruni", + "verified": False, + "signature": "bio here", + "avatarLarger": "https://cdn/avatar.jpg", + }, + "authorStats": { + "followerCount": 51200, + "followingCount": 269, + "heartCount": 3_000_000, + "videoCount": 259, + }, + "stats": { + "diggCount": 5344, + "shareCount": 701, + "playCount": 55700, + "commentCount": 24, + "collectCount": 291, + }, + "music": { + "id": "7529", + "title": "som original", + "authorName": "fox_rus0", + "original": True, + "playUrl": "https://cdn/music.mp3", + }, + "video": { + "height": 1024, + "width": 576, + "duration": 16, + "cover": "https://cdn/cover.jpg", + "format": "mp4", + "definition": "540p", + }, + "challenges": [{"id": "4982299", "title": "comeramabanana"}], +} + + +def test_parse_video_maps_core_and_derived_fields(): + item = parse_video(_ITEM_STRUCT) + + assert item["id"] == "7534061113365859586" + assert item["text"] == "haha #comeramabanana" + assert item["createTimeISO"] == "2023-11-14T22:13:20.000Z" + + assert item["authorMeta"]["name"] == "bruniela_" + assert item["authorMeta"]["nickName"] == "Bruni" + assert item["authorMeta"]["profileUrl"] == "https://www.tiktok.com/@bruniela_" + assert item["authorMeta"]["fans"] == 51200 + + assert item["musicMeta"]["musicName"] == "som original" + assert item["videoMeta"]["duration"] == 16 + + assert item["diggCount"] == 5344 + assert item["playCount"] == 55700 + + assert item["hashtags"] == [{"id": "4982299", "name": "comeramabanana"}] + assert ( + item["webVideoUrl"] + == "https://www.tiktok.com/@bruniela_/video/7534061113365859586" + ) + + +_USER_INFO = { + "user": { + "id": "6733", + "uniqueId": "bruniela_", + "nickname": "Bruni", + "verified": True, + "signature": "bio here", + "avatarLarger": "https://cdn/avatar.jpg", + "privateAccount": False, + }, + "stats": { + "followerCount": 51200, + "followingCount": 269, + "heartCount": 3_000_000, + "videoCount": 259, + }, +} + + +def test_parse_author_maps_user_and_stats(): + author = parse_author(_USER_INFO) + assert author["name"] == "bruniela_" + assert author["nickName"] == "Bruni" + assert author["verified"] is True + assert author["profileUrl"] == "https://www.tiktok.com/@bruniela_" + assert author["fans"] == 51200 + assert author["video"] == 259 diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py b/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py new file mode 100644 index 000000000..7a52dd60d --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py @@ -0,0 +1,30 @@ +"""Navigating the rehydration blob to its useful scopes (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import user_info, video_item_struct + + +def test_video_item_struct_navigates_video_detail_scope(): + data = { + "__DEFAULT_SCOPE__": { + "webapp.video-detail": {"itemInfo": {"itemStruct": {"id": "123"}}} + } + } + item = video_item_struct(data) + assert item == {"id": "123"} + + +def test_user_info_navigates_user_detail_scope(): + data = { + "__DEFAULT_SCOPE__": { + "webapp.user-detail": {"userInfo": {"user": {"uniqueId": "scout2015"}}} + } + } + info = user_info(data) + assert info == {"user": {"uniqueId": "scout2015"}} + + +def test_scopes_return_none_when_absent(): + assert video_item_struct({}) is None + assert user_info({"__DEFAULT_SCOPE__": {}}) is None diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py b/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py new file mode 100644 index 000000000..c118f4f86 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py @@ -0,0 +1,47 @@ +"""URL classification for the TikTok scraper (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.targets import resolve_target + + +def test_resolve_video_carries_username_and_id(): + target = resolve_target("https://www.tiktok.com/@scout2015/video/6718335390845095173") + assert target is not None + assert target.kind == "video" + assert target.value == "6718335390845095173" + assert target.username == "scout2015" + + +def test_resolve_profile(): + target = resolve_target("https://www.tiktok.com/@scout2015") + assert target is not None + assert target.kind == "profile" + assert target.value == "scout2015" + + +def test_resolve_hashtag(): + target = resolve_target("https://www.tiktok.com/tag/funny") + assert target is not None + assert target.kind == "hashtag" + assert target.value == "funny" + + +def test_resolve_search_top_video_and_user_sections(): + top = resolve_target("https://www.tiktok.com/search?q=cats") + assert top is not None + assert top.kind == "search" + assert top.value == "cats" + assert top.section is None + + videos = resolve_target("https://www.tiktok.com/search/video?q=cats") + assert videos is not None and videos.section == "video" + + users = resolve_target("https://www.tiktok.com/search/user?q=cats") + assert users is not None and users.section == "user" + + +def test_resolve_rejects_non_tiktok_and_unknown_paths(): + assert resolve_target("https://example.com/@scout2015") is None + assert resolve_target("https://www.tiktok.com/") is None + assert resolve_target("https://www.tiktok.com/foundation") is None diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_trending.py b/surfsense_backend/tests/unit/platforms/tiktok/test_trending.py new file mode 100644 index 000000000..d00d9c09f --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_trending.py @@ -0,0 +1,35 @@ +"""Trending orchestration over a fake fetch (no network). + +Drives ``scrape_tiktok_trending``: the Explore feed -> captured itemStructs -> +video items, reusing the listing flow (parse/dedupe/cap/empty-ErrorItem). +""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok import scrape_tiktok_trending + + +async def test_trending_parses_dedupes_and_caps(): + async def fake_fetch(url: str, _cap: int) -> list[dict]: + assert url == "https://www.tiktok.com/explore" + return [ + {"id": "1", "author": {"uniqueId": "a"}}, + {"id": "1", "author": {"uniqueId": "a"}}, + {"id": "2", "author": {"uniqueId": "b"}}, + ] + + items = await scrape_tiktok_trending(count=2, fetch_trending_fn=fake_fetch) + + assert [i["id"] for i in items] == ["1", "2"] + assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@a/video/1" + assert items[0]["scrapedAt"] is not None + + +async def test_trending_empty_feed_emits_error_item(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [] + + items = await scrape_tiktok_trending(count=5, fetch_trending_fn=fake_fetch) + + assert len(items) == 1 + assert items[0]["errorCode"] == "no_items" diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py b/surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py new file mode 100644 index 000000000..845b5e22d --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py @@ -0,0 +1,70 @@ +"""User-search orchestration over a fake fetch (no network). + +Drives ``search_tiktok_users``: queries -> captured ``user_info`` -> profile items. +""" + +from __future__ import annotations + +from typing import Any + +from app.proprietary.platforms.tiktok import search_tiktok_users + + +def _user(uid: str, unique_id: str, followers: int = 10) -> dict[str, Any]: + return { + "uid": uid, + "unique_id": unique_id, + "nickname": unique_id.upper(), + "signature": "bio", + "follower_count": followers, + "total_favorited": 999, + "sec_uid": f"sec-{uid}", + "enterprise_verify_reason": "official" if uid == "1" else "", + "avatar_thumb": {"url_list": [f"https://cdn/{uid}.webp"]}, + } + + +async def test_user_search_parses_dedupes_and_caps(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_user("1", "nasa"), _user("1", "nasa"), _user("2", "nasa2")] + + items = await search_tiktok_users( + ["nasa"], per_query=2, fetch_users=fake_fetch + ) + + assert [i["id"] for i in items] == ["1", "2"] + first = items[0] + assert first["name"] == "nasa" + assert first["nickName"] == "NASA" + assert first["profileUrl"] == "https://www.tiktok.com/@nasa" + assert first["verified"] is True + assert first["fans"] == 10 + assert first["avatar"] == "https://cdn/1.webp" + assert first["secUid"] == "sec-1" + assert first["scrapedAt"] is not None + assert items[1]["verified"] is False + + +async def test_user_search_empty_query_emits_error_item(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [] + + items = await search_tiktok_users( + ["ghost"], per_query=5, fetch_users=fake_fetch + ) + + assert len(items) == 1 + assert items[0]["errorCode"] == "no_users" + assert items[0]["input"] == "ghost" + + +async def test_user_search_honors_limit_across_queries(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_user("1", "a"), _user("2", "b")] + + items = await search_tiktok_users( + ["q1", "q2"], per_query=5, limit=3, fetch_users=fake_fetch + ) + + # 2 from q1 + 1 from q2, then the cross-query limit stops it. + assert len(items) == 3 diff --git a/surfsense_mcp/README.md b/surfsense_mcp/README.md index c18a857e1..5bac5f8b2 100644 --- a/surfsense_mcp/README.md +++ b/surfsense_mcp/README.md @@ -21,6 +21,7 @@ Connect it two ways: **Scrapers (all platforms)** - `surfsense_web_crawl`, `surfsense_google_search`, `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, + `surfsense_tiktok_scrape`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews` - `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` — retrieve past results in full (useful when a large result was truncated inline) diff --git a/surfsense_mcp/mcp_server/features/scrapers/__init__.py b/surfsense_mcp/mcp_server/features/scrapers/__init__.py index aade7a99d..eb5f72ba5 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/__init__.py +++ b/surfsense_mcp/mcp_server/features/scrapers/__init__.py @@ -13,15 +13,24 @@ from mcp.server.fastmcp import FastMCP from ...core.client import SurfSenseClient from ...core.workspace_context import WorkspaceContext from . import run_history -from .platforms import google_maps, google_search, instagram, reddit, web, youtube +from .platforms import ( + google_maps, + google_search, + instagram, + reddit, + tiktok, + web, + youtube, +) _REGISTRARS = ( web, google_search, reddit, youtube, - google_maps, instagram, + tiktok, + google_maps, run_history, ) diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py new file mode 100644 index 000000000..c295ec3b6 --- /dev/null +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py @@ -0,0 +1,208 @@ +"""TikTok scraper tools: scrape (videos), comments, user search, and trending.""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the TikTok tools.""" + + @mcp.tool( + name="surfsense_tiktok_scrape", + title="Scrape TikTok videos", + annotations=SCRAPE, + structured_output=False, + ) + async def tiktok_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="TikTok URLs: a video, a profile " + "('https://www.tiktok.com/@nasa'), a hashtag " + "('https://www.tiktok.com/tag/food'), or a search URL. Provide " + "urls OR profiles/hashtags/search_queries." + ), + ] = None, + profiles: Annotated[ + list[str] | None, + Field( + description="Profile usernames to scrape, with or without a " + "leading '@', e.g. ['nasa']." + ), + ] = None, + hashtags: Annotated[ + list[str] | None, + Field( + description="Hashtag names to scrape, without the '#', e.g. " + "['food']." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Keyword search terms. Returns no videos — use " + "hashtags/profiles/urls for videos, or the user-search tool for " + "accounts." + ), + ] = None, + results_per_page: Annotated[ + int, + Field(ge=1, description="Max videos per profile/hashtag/search target."), + ] = 10, + max_items: Annotated[ + int, Field(ge=1, description="Maximum videos to return in total.") + ] = 10, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Scrape public TikTok videos by hashtag, profile, or URL. + + Use for TikTok video research — a creator's videos, a hashtag feed, or a + specific video/profile/hashtag URL — instead of a generic web search. + Returns videos with text, author, stats, music, and the web URL. For + accounts by keyword use the user-search tool; keyword search returns no + videos. Example: hashtags=['food'], max_items=20. + """ + return await run_scraper( + client, + context, + platform="tiktok", + verb="scrape", + payload={ + "urls": urls, + "profiles": profiles, + "hashtags": hashtags, + "search_queries": search_queries, + "results_per_page": results_per_page, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_tiktok_comments", + title="Scrape TikTok comments", + annotations=SCRAPE, + structured_output=False, + ) + async def tiktok_comments( + video_urls: Annotated[ + list[str], + Field( + description="TikTok video URLs " + "('https://www.tiktok.com/@user/video/123') to pull comments from." + ), + ], + comments_per_video: Annotated[ + int, Field(ge=1, description="Max comments to return per video.") + ] = 20, + max_items: Annotated[ + int, Field(ge=1, description="Maximum comments to return in total.") + ] = 20, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Scrape the public comments of TikTok videos. + + Returns each comment's text, author, like count, and reply count (replies + carry the parent comment id). Example: video_urls=['https://www.tiktok.com/ + @nasa/video/123'], max_items=50. + """ + return await run_scraper( + client, + context, + platform="tiktok", + verb="comments", + payload={ + "video_urls": video_urls, + "comments_per_video": comments_per_video, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_tiktok_user_search", + title="Search TikTok accounts", + annotations=SCRAPE, + structured_output=False, + ) + async def tiktok_user_search( + queries: Annotated[ + list[str], + Field( + description="Keywords to find TikTok accounts by, e.g. " + "['nasa', 'cooking']." + ), + ], + results_per_query: Annotated[ + int, Field(ge=1, description="Max accounts to return per query.") + ] = 10, + max_items: Annotated[ + int, Field(ge=1, description="Maximum accounts to return in total.") + ] = 10, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Find public TikTok accounts by keyword. + + Returns matching profiles with name, followers, bio, and verification — + the reliable account-discovery path (video search is login-walled). + Example: queries=['space agency'], max_items=20. + """ + return await run_scraper( + client, + context, + platform="tiktok", + verb="user_search", + payload={ + "queries": queries, + "results_per_query": results_per_query, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_tiktok_trending", + title="Get trending TikTok videos", + annotations=SCRAPE, + structured_output=False, + ) + async def tiktok_trending( + max_items: Annotated[ + int, + Field(ge=1, description="Max trending videos to return from Explore."), + ] = 20, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Get the current trending TikTok videos from the Explore feed. + + No input needed beyond how many to return; each video comes with caption, + author, stats, music, and its web URL. Example: max_items=30. + """ + return await run_scraper( + client, + context, + platform="tiktok", + verb="trending", + payload={"max_items": max_items}, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/mcp_server/selfcheck.py b/surfsense_mcp/mcp_server/selfcheck.py index ba2e95a78..bb61151bc 100644 --- a/surfsense_mcp/mcp_server/selfcheck.py +++ b/surfsense_mcp/mcp_server/selfcheck.py @@ -23,6 +23,10 @@ EXPECTED_TOOLS = { "surfsense_reddit_scrape", "surfsense_youtube_scrape", "surfsense_youtube_comments", + "surfsense_tiktok_scrape", + "surfsense_tiktok_comments", + "surfsense_tiktok_user_search", + "surfsense_tiktok_trending", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_instagram_scrape", diff --git a/surfsense_mcp/mcp_server/server.py b/surfsense_mcp/mcp_server/server.py index 088952fd1..f801aeb15 100644 --- a/surfsense_mcp/mcp_server/server.py +++ b/surfsense_mcp/mcp_server/server.py @@ -37,8 +37,9 @@ def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]: "Prefer these tools over generic/built-in web search whenever the " "task involves Reddit (posts, comments, finding subreddits or " "communities), YouTube (videos, transcripts, comments), Instagram " - "(posts, reels, profile details), Google " - "Maps (places, reviews), Google Search results, or reading " + "(posts, reels, profile details), TikTok (videos by hashtag, " + "search, or URL), Google Maps (places, reviews), Google Search " + "results, or reading " "specific web pages. Scraper results are persisted as runs; if an " "inline result is truncated, fetch it in full with " "surfsense_get_scraper_run." diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index 3bdfcf100..3d00c44c9 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -16,7 +16,7 @@ import type { FaqItem } from "@/lib/connectors-marketing/types"; const canonicalUrl = "https://www.surfsense.com/mcp-server"; const metaDescription = - "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; + "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; export const metadata: Metadata = { title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools", @@ -95,6 +95,7 @@ const TOOL_GROUPS = [ "surfsense_youtube_comments", "surfsense_instagram_scrape", "surfsense_instagram_details", + "surfsense_tiktok_scrape", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_google_search", @@ -129,7 +130,7 @@ const FAQ: FaqItem[] = [ { question: "What is the SurfSense MCP server?", answer: - "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", + "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", }, { question: "Which MCP clients does it work with?", @@ -217,8 +218,9 @@ export default function McpServerPage() {

The SurfSense MCP server hands Claude, Cursor, or any MCP client the whole platform: - scrape Reddit, YouTube, Instagram, Google Maps, Google Search, and the open web, and - search, read, and write your knowledge base. One API key, typed tools, pay as you go. + scrape Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the open + web, and search, read, and write your knowledge base. One API key, typed tools, pay + as you go.

+ diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index ddcddf9cc..fc8356041 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -719,7 +719,7 @@ export function HeroSection() { > SurfSense is an open-source competitive intelligence platform. Your AI agents monitor competitors, track rankings, and listen to your market with live data from platforms - like Reddit, Instagram, YouTube, Google Maps, Google Search, and the open web. + like Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the open web.

diff --git a/surfsense_web/components/homepage/persona-paths.tsx b/surfsense_web/components/homepage/persona-paths.tsx index 9c03c5667..50eadba0f 100644 --- a/surfsense_web/components/homepage/persona-paths.tsx +++ b/surfsense_web/components/homepage/persona-paths.tsx @@ -35,7 +35,7 @@ const PATHS: { eyebrow: "For developers & agents", title: "The whole platform is programmable", description: - "Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.", + "Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.", links: [ { label: "Read the docs", href: "/docs" }, { label: "SurfSense MCP server", href: "/mcp-server" }, diff --git a/surfsense_web/components/pricing/pricing-section.tsx b/surfsense_web/components/pricing/pricing-section.tsx index 74c76ad36..1af296e57 100644 --- a/surfsense_web/components/pricing/pricing-section.tsx +++ b/surfsense_web/components/pricing/pricing-section.tsx @@ -35,7 +35,7 @@ const demoPlans = [ billingText: "Your first $5 of credit is free. No subscription, ever", features: [ "$5 of free credit to start, one balance for everything", - "Platform connectors: Reddit, YouTube, Google Maps, Google Search, and the open web", + "Platform connectors: Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web", "Call every connector as a REST API with your key or through the MCP server", "Pay per item returned and per page crawled. Failed calls are never billed", "Premium models like GPT-5.5, Claude Sonnet 5, Gemini 3.1 Pro billed at provider cost", @@ -95,7 +95,7 @@ const faqData: FAQSection[] = [ { question: "How does Pay As You Go work?", answer: - "There is no monthly subscription. Start with $5 of free credit, and when you need more, add any amount. $1 buys exactly $1 of credit, added to your balance immediately. You can enable automatic refills when your balance runs low, and turn them off any time." + "There is no monthly subscription. Start with $5 of free credit, and when you need more, add any amount. $1 buys exactly $1 of credit, added to your balance immediately. You can enable automatic refills when your balance runs low, and turn them off any time.", }, { question: "What happens if I run out of credit?", diff --git a/surfsense_web/components/seo/json-ld.tsx b/surfsense_web/components/seo/json-ld.tsx index c4b3ec09c..063f6a0c0 100644 --- a/surfsense_web/components/seo/json-ld.tsx +++ b/surfsense_web/components/seo/json-ld.tsx @@ -76,11 +76,11 @@ export function SoftwareApplicationJsonLd() { "Free self-hosted from the open-source repo; cloud starts with $5 of free credit, then pay as you go", }, description: - "SurfSense is an open-source competitive intelligence platform. AI agents monitor competitors, track rankings, and listen to your market with platform-native connectors for Reddit, YouTube, Google Maps, Google Search, and the open web, through one API or MCP server.", + "SurfSense is an open-source competitive intelligence platform. AI agents monitor competitors, track rankings, and listen to your market with platform-native connectors for Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web, through one API or MCP server.", url: "https://www.surfsense.com", downloadUrl: "https://github.com/MODSetter/SurfSense/releases", featureList: [ - "Platform-native connectors: Reddit, YouTube, Google Maps, Google Search, Web Crawl", + "Platform-native connectors: Reddit, YouTube, TikTok, Google Maps, Google Search, Web Crawl", "MCP server that exposes every connector as a native agent tool", "Agent harness with retries, structured output, and credit metering", "Competitor, brand, and rank monitoring with briefs and alerts", diff --git a/surfsense_web/content/docs/connectors/index.mdx b/surfsense_web/content/docs/connectors/index.mdx index 747d278e2..f16cf79b9 100644 --- a/surfsense_web/content/docs/connectors/index.mdx +++ b/surfsense_web/content/docs/connectors/index.mdx @@ -12,7 +12,7 @@ Connectors bring data into SurfSense — either as searchable knowledge or as li } title="Native Connectors" - description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" + description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" href="/docs/connectors/native" /> + `), a hashtag (`/tag/`), or a search URL (max 20 sources per call) | +| `profiles` | — | Profile usernames, with or without a leading `@` | +| `hashtags` | — | Hashtag names, without the `#` | +| `search_queries` | — | Search terms to run on TikTok | +| `results_per_page` | `10` | Max videos per profile/hashtag/search target | +| `max_items` | `10` | Max total videos returned across all sources (hard cap 100) | + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/scrape" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ "hashtags": ["food"], "max_items": 25 }' +``` + + +Video and hashtag targets are the reliable video paths. A `profiles` target returns the account's **metadata** (name, followers, bio, verification) reliably, but TikTok often withholds its **video list** from automated clients — so a profile can return metadata with no videos. Keyword **video** search is login-walled and returns a surfaced error; to find accounts by keyword use **user search** below. + + +## Comments + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/tiktok/comments +``` + +Given TikTok video URLs, returns each video's public comment thread: comment text, author, like count, and reply count (replies carry `repliesToId`, the parent comment id). + +| Field | Default | Description | +|-------|---------|-------------| +| `video_urls` | — | TikTok video URLs (`/@/video/`), max 20 per call | +| `comments_per_video` | `20` | Max comments per video | +| `max_items` | `20` | Max total comments returned (hard cap 100) | + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/comments" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ "video_urls": ["https://www.tiktok.com/@nasa/video/123"], "max_items": 50 }' +``` + +## User search + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/tiktok/user_search +``` + +Finds public TikTok accounts by keyword — the reliable account-discovery path. Returns matching profiles with name, followers, bio, and verification. + +| Field | Default | Description | +|-------|---------|-------------| +| `queries` | — | Keywords to find accounts by, e.g. `["nasa", "cooking"]` (max 20) | +| `results_per_query` | `10` | Max accounts per query | +| `max_items` | `10` | Max total accounts returned (hard cap 100) | + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/user_search" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ "queries": ["space agency"], "max_items": 20 }' +``` + +## Trending + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/tiktok/trending +``` + +Returns the current trending videos from TikTok's Explore feed — no input needed beyond how many to return. Items use the same video shape as **scrape** and bill on the same per-video meter. + +| Field | Default | Description | +|-------|---------|-------------| +| `max_items` | `20` | Max trending videos returned (hard cap 100) | + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/trending" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ "max_items": 30 }' +``` + +For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → TikTok** in your workspace. diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx index 1318197f8..782476d07 100644 --- a/surfsense_web/content/docs/how-to/mcp-server.mdx +++ b/surfsense_web/content/docs/how-to/mcp-server.mdx @@ -7,7 +7,7 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs'; # SurfSense MCP Server -The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 21 native, typed tools: every scraper (Reddit, YouTube, Instagram, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. +The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 21 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted. @@ -264,7 +264,7 @@ For self-host (stdio), all settings are environment variables passed by the clie | Group | Tools | |-------|-------| | Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` | -| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_details`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | +| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_details`, `surfsense_tiktok_scrape`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | | Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` | Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**. diff --git a/surfsense_web/lib/connectors-marketing/index.ts b/surfsense_web/lib/connectors-marketing/index.ts index 429fe9b59..826a1f261 100644 --- a/surfsense_web/lib/connectors-marketing/index.ts +++ b/surfsense_web/lib/connectors-marketing/index.ts @@ -2,6 +2,7 @@ import { googleMaps } from "./google-maps"; import { googleSearch } from "./google-search"; import { instagram } from "./instagram"; import { reddit } from "./reddit"; +import { tiktok } from "./tiktok"; import type { ConnectorPageContent } from "./types"; import { webCrawl } from "./web-crawl"; import { youtube } from "./youtube"; @@ -13,6 +14,7 @@ const CONNECTOR_LIST: ConnectorPageContent[] = [ reddit, youtube, instagram, + tiktok, googleMaps, googleSearch, webCrawl, diff --git a/surfsense_web/lib/connectors-marketing/reddit.tsx b/surfsense_web/lib/connectors-marketing/reddit.tsx index 93a8618ef..2a930afe9 100644 --- a/surfsense_web/lib/connectors-marketing/reddit.tsx +++ b/surfsense_web/lib/connectors-marketing/reddit.tsx @@ -311,10 +311,10 @@ export const reddit: ConnectorPageContent = { related: [ { label: "YouTube API", href: "/youtube" }, { label: "Instagram API", href: "/instagram" }, + { label: "TikTok API", href: "/tiktok" }, { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, - { label: "Read the docs", href: "/docs" }, ], }; diff --git a/surfsense_web/lib/connectors-marketing/tiktok.tsx b/surfsense_web/lib/connectors-marketing/tiktok.tsx new file mode 100644 index 000000000..d77865633 --- /dev/null +++ b/surfsense_web/lib/connectors-marketing/tiktok.tsx @@ -0,0 +1,285 @@ +import { IconBrandTiktok } from "@tabler/icons-react"; +import type { ConnectorPageContent } from "./types"; + +export const tiktok: ConnectorPageContent = { + slug: "tiktok", + name: "TikTok", + icon: IconBrandTiktok, + + metaTitle: "TikTok Scraper API for Trend and Creator Research | SurfSense", + metaDescription: + "Scrape public TikTok videos, comments, accounts, and trending feeds by hashtag, profile, or URL with the SurfSense TikTok Scraper API. No approval process or research-API gatekeeping, plus a free tier. Start now.", + keywords: [ + "tiktok scraper", + "tiktok scraper api", + "tiktok api", + "tiktok api alternative", + "scrape tiktok", + "tiktok data api", + "tiktok hashtag scraper", + "tiktok comments scraper", + "tiktok trending scraper", + "tiktok user search", + "tiktok trend tracking", + "tiktok mcp", + "social listening", + "influencer research tool", + "short-form video analytics", + ], + + h1: "TikTok Scraper API for Trend and Creator Research", + heroLede: + "The SurfSense TikTok API extracts public videos by hashtag, creator profile, or URL without TikTok's approval-gated Research API. Give your AI agents a live feed of what your market watches and shares, so you catch a trend while it is still rising.", + + transcript: { + prompt: "Find trending TikToks about meal prep this week", + toolCall: 'tiktok.scrape({ hashtags: ["mealprep"], max_items: 25 })', + rows: [ + { + primary: "5 high-protein lunches under 400 cals", + secondary: "@fitmeals · 2.1M plays · 184K likes", + tag: "breakout", + }, + { + primary: "The $20 weekly meal prep everyone's copying", + secondary: "@budgetbites · 980K plays · 72K likes", + tag: "rising", + }, + { + primary: "POV: your fridge after Sunday meal prep", + secondary: "@cleaneats · 640K plays · 51K likes", + tag: "high engagement", + }, + ], + resultSummary: "25 videos · 8.4M plays · surfaced in 3.2s", + }, + + extractIntro: + "Every call returns structured items. Scrape videos from a hashtag, creator profile, or video URL — or switch verbs to pull a video's comments, discover accounts by keyword, or fetch the current trending feed.", + extractFields: [ + { + label: "Videos", + description: "Caption text, canonical web URL, duration, and cover image for each video.", + }, + { + label: "Engagement", + description: + "Play, like, comment, share, and save counts — the signal for what is breaking out.", + }, + { + label: "Authors", + description: "Creator handle, nickname, follower and heart counts, and verified status.", + }, + { + label: "Music", + description: "Track name, artist, and whether the sound is original — the seed of a trend.", + }, + { + label: "Hashtags", + description: "Every hashtag on a video, so you can map a topic cluster or campaign.", + }, + { + label: "Timestamps", + description: "Created and scraped times so you can track a video's momentum over runs.", + }, + ], + + useCasesHeading: "What teams do with the TikTok API", + useCases: [ + { + title: "Trend and hashtag monitoring", + description: + "Track a hashtag and feed the stream to an agent that flags breakout videos, rising sounds, and formats before they saturate. Catch the wave while it is still rising, not after.", + }, + { + title: "Creator and influencer discovery", + description: + "Surface the creators driving a topic, ranked by real engagement, so your team shortlists partners from data instead of a manager's pitch deck.", + }, + { + title: "Competitor content analysis", + description: + "Watch what your category posts and what actually lands. Turn a competitor's best-performing formats and hooks into your own content brief.", + }, + { + title: "Campaign and sentiment tracking", + description: + "Measure how a launch or branded hashtag spreads across TikTok — video count, reach, and engagement over time — then pull the comments on top videos to read how the audience actually reacts, not just a vanity view count.", + }, + ], + + comparison: { + heading: "A TikTok API alternative built for agents", + intro: + "TikTok's official Research API is approval-gated and largely limited to academic and nonprofit use. If you cannot get access or need it for commercial research, here is how SurfSense compares.", + columnLabel: "Official TikTok API", + rows: [ + { + feature: "Access", + official: "Application and approval, often restricted to academic/nonprofit use", + surfsense: "One API key, no approval process", + }, + { + feature: "Pricing", + official: "Free but gated, with usage quotas per approved project", + surfsense: "Pay per item returned, with a free tier to start", + }, + { + feature: "Signing", + official: "You manage the client-side signature that TikTok rotates", + surfsense: "Handled for you; a real browser signs each request", + }, + { + feature: "Setup", + official: "Register a developer app and await review", + surfsense: "One API key, or add the MCP server to your agent", + }, + { + feature: "Agent-ready", + official: "No; you build the harness yourself", + surfsense: "MCP server exposes scrape, comments, user search, and trending as native tools", + }, + ], + }, + + api: { + platform: "tiktok", + verb: "scrape", + mcpTool: "tiktok.scrape", + requestBody: { + hashtags: ["mealprep"], + max_items: 25, + }, + }, + + schema: { + requestNote: + "Provide at least one source: urls, profiles, hashtags, or search_queries. Up to 20 sources per call.", + request: [ + { + name: "urls", + type: "string[]", + defaultValue: "[]", + description: + "TikTok URLs to scrape: a video, a profile (/@user), or a hashtag (/tag/name). Max 20.", + }, + { + name: "profiles", + type: "string[]", + defaultValue: "[]", + description: "Profile usernames to scrape, with or without a leading @. Max 20.", + }, + { + name: "hashtags", + type: "string[]", + defaultValue: "[]", + description: "Hashtag names to scrape, without the # prefix. Max 20.", + }, + { + name: "search_queries", + type: "string[]", + defaultValue: "[]", + description: + "Keyword search terms. Keyword video search is login-walled and returns no videos — use hashtags/profiles/urls for videos, or user_search for accounts. Max 20.", + }, + { + name: "results_per_page", + type: "integer", + defaultValue: "10", + description: "Max videos to pull per profile or hashtag target. 1 to 100.", + }, + { + name: "max_items", + type: "integer", + defaultValue: "10", + description: "Max total videos to return across all sources. 1 to 100.", + }, + ], + responseNote: + "The response is { items: [...] } with one video item per result. One returned item is one billable unit.", + response: [ + { + name: "id / webVideoUrl", + type: "string", + description: "The TikTok video ID and its canonical web URL.", + }, + { + name: "text", + type: "string", + description: "The video caption, including inline hashtags and mentions.", + }, + { + name: "authorMeta", + type: "object", + description: "Creator handle, nickname, follower and heart counts, and verified flag.", + }, + { + name: "musicMeta", + type: "object", + description: "Track name, artist, and whether the sound is original.", + }, + { + name: "videoMeta", + type: "object", + description: "Duration, dimensions, and cover image for the video.", + }, + { + name: "playCount / diggCount", + type: "integer", + description: "Play count and like (digg) count, the core reach and engagement signals.", + }, + { + name: "commentCount / shareCount", + type: "integer", + description: "Comment and share counts for measuring conversation and spread.", + }, + { + name: "hashtags", + type: "object[]", + description: "The hashtags attached to the video, for topic and campaign mapping.", + }, + { + name: "createTimeISO / scrapedAt", + type: "string", + description: "ISO timestamps for when the video was posted and when it was scraped.", + }, + ], + }, + + faq: [ + { + question: "Is scraping TikTok legal?", + answer: + "SurfSense reads only public TikTok data, the same videos any logged-out visitor can see. It never logs in and cannot access private or deleted content. As always, review TikTok's terms and your own compliance needs before you run at scale.", + }, + { + question: "Does this need the official TikTok API?", + answer: + "No. It is an independent alternative, not a wrapper on the Research API, so there is no application or approval process. You call the SurfSense API with one key, or add the MCP server to your agent, and get structured videos back.", + }, + { + question: "What are the rate limits?", + answer: + "Each call caps at 100 returned videos across all sources, with up to 20 hashtags, profiles, or URLs per request. SurfSense manages the underlying request budget and request signing for you.", + }, + { + question: "Can I scrape a specific creator's videos?", + answer: + "Pass a profile or profile URL and you always get the account's metadata — name, followers, bio, verification. TikTok often withholds the profile video list from automated clients, so that list can come back empty even for a public account; for reliable video results, scrape by hashtag or by a direct video URL.", + }, + { + question: "What TikTok data can I scrape?", + answer: + "Four verbs: scrape (videos by hashtag, profile, or URL), comments (a video's public comment thread), user search (find accounts by keyword — the reliable discovery path, since keyword video search is login-walled), and trending (the current Explore feed). Each returns structured items and is billed per item returned.", + }, + ], + + related: [ + { label: "Reddit API", href: "/reddit" }, + { label: "YouTube API", href: "/youtube" }, + { label: "Google Maps API", href: "/google-maps" }, + { label: "SERP API", href: "/google-search" }, + { label: "Web Crawl API", href: "/web-crawl" }, + { label: "SurfSense MCP Server", href: "/mcp-server" }, + ], +}; diff --git a/surfsense_web/lib/connectors-marketing/youtube.tsx b/surfsense_web/lib/connectors-marketing/youtube.tsx index f5f3215ef..fceaee940 100644 --- a/surfsense_web/lib/connectors-marketing/youtube.tsx +++ b/surfsense_web/lib/connectors-marketing/youtube.tsx @@ -271,10 +271,10 @@ export const youtube: ConnectorPageContent = { related: [ { label: "Reddit API", href: "/reddit" }, { label: "Instagram API", href: "/instagram" }, + { label: "TikTok API", href: "/tiktok" }, { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, - { label: "Read the docs", href: "/docs" }, ], }; diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index 5f72c07ec..456e52008 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -4,6 +4,7 @@ import { GoogleSearchIcon, InstagramIcon, RedditIcon, + TikTokIcon, WebIcon, YouTubeIcon, } from "./platform-icons"; @@ -58,6 +59,21 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ { name: "instagram.details", verb: "details", label: "Details" }, ], }, + { + id: "tiktok", + label: "TikTok", + icon: TikTokIcon, + verbs: [ + { name: "tiktok.scrape", verb: "scrape", label: "Scrape" }, + { name: "tiktok.comments", verb: "comments", label: "Comments" }, + { + name: "tiktok.user_search", + verb: "user_search", + label: "User Search", + }, + { name: "tiktok.trending", verb: "trending", label: "Trending" }, + ], + }, { id: "google_maps", label: "Google Maps", diff --git a/surfsense_web/lib/playground/platform-icons.tsx b/surfsense_web/lib/playground/platform-icons.tsx index 460b7f7a3..c1f61978b 100644 --- a/surfsense_web/lib/playground/platform-icons.tsx +++ b/surfsense_web/lib/playground/platform-icons.tsx @@ -25,6 +25,7 @@ function brandIcon(src: string, alt: string) { export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit"); export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube"); export const InstagramIcon = brandIcon("/connectors/instagram.svg", "Instagram"); +export const TikTokIcon = brandIcon("/connectors/tiktok.svg", "TikTok"); export const GoogleMapsIcon = brandIcon("/connectors/google-maps.svg", "Google Maps"); export const GoogleSearchIcon = brandIcon("/connectors/google-search.svg", "Google Search"); export const WebIcon = brandIcon("/connectors/web.svg", "Web"); diff --git a/surfsense_web/public/connectors/tiktok.svg b/surfsense_web/public/connectors/tiktok.svg new file mode 100644 index 000000000..c3c3fe06e --- /dev/null +++ b/surfsense_web/public/connectors/tiktok.svg @@ -0,0 +1,6 @@ + + + + + +