feat(tiktok): add tiktok.comments verb for public comment scraping

Comments load over a signed /api/comment/list XHR that TikTok serves to
anonymous sessions once the comments panel is opened (unlike profile-video and
general-search feeds), so this is a reliable verb. Given video URLs it returns
CommentItems (text, author, likes, reply counts; replies carry repliesToId),
deduped per video, capped, and degraded to an ErrorItem for empty/withheld
videos or a bad_url ErrorItem for non-video inputs.

Generalizes the browser capture over a pluggable interaction step so the
comments flow (open panel, scroll the panel to paginate) reuses the same
warm+capture scaffolding as listing/user-search. Billed per comment on a new
TIKTOK_COMMENT meter (TIKTOK_MICROS_PER_COMMENT, matching the per-comment
market), surfaced on the chat subagent alongside tiktok.scrape/user_search.
This commit is contained in:
CREDO23 2026-07-09 18:35:26 +02:00
parent 192b6dc31a
commit 7723b5b8b6
24 changed files with 626 additions and 26 deletions

View file

@ -1,4 +1,4 @@
"""``tiktok`` sub-agent tools: the TikTok scrape and user-search capability verbs."""
"""``tiktok`` sub-agent tools: the TikTok scrape, comments, and user-search verbs."""
from __future__ import annotations
@ -8,6 +8,7 @@ from langchain_core.tools import BaseTool
from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
from app.capabilities.core.access.agent import build_capability_tools
from app.capabilities.tiktok.comments.definition import TIKTOK_COMMENTS
from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE
from app.capabilities.tiktok.user_search.definition import TIKTOK_USER_SEARCH
@ -15,7 +16,7 @@ NAME = "tiktok"
RULESET = Ruleset(origin=NAME, rules=[])
_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_USER_SEARCH]
_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_COMMENTS, TIKTOK_USER_SEARCH]
def load_tools(

View file

@ -37,6 +37,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = {
BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT",
BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO",
BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER",
BillingUnit.TIKTOK_COMMENT: "TIKTOK_MICROS_PER_COMMENT",
}
@ -55,6 +56,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = {
BillingUnit.YOUTUBE_COMMENT: "comment",
BillingUnit.TIKTOK_VIDEO: "video",
BillingUnit.TIKTOK_USER: "profile",
BillingUnit.TIKTOK_COMMENT: "comment",
}

View file

@ -27,6 +27,7 @@ class BillingUnit(StrEnum):
YOUTUBE_COMMENT = "youtube_comment"
TIKTOK_VIDEO = "tiktok_video"
TIKTOK_USER = "tiktok_user"
TIKTOK_COMMENT = "tiktok_comment"
class BillableInput(Protocol):

View file

@ -2,5 +2,6 @@
from __future__ import annotations
from app.capabilities.tiktok.comments import definition as _comments # noqa: F401
from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401
from app.capabilities.tiktok.user_search import definition as _user_search # noqa: F401

View file

@ -0,0 +1,3 @@
"""``tiktok.comments``: scrape the public comment thread of TikTok videos."""
from __future__ import annotations

View file

@ -0,0 +1,23 @@
"""``tiktok.comments`` capability registration (billed per comment; see config
``TIKTOK_MICROS_PER_COMMENT``)."""
from __future__ import annotations
from app.capabilities.core import BillingUnit, Capability, register_capability
from app.capabilities.tiktok.comments.executor import build_comments_executor
from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput
TIKTOK_COMMENTS = Capability(
name="tiktok.comments",
description=(
"Scrape the public comments of TikTok videos. Provide video URLs; "
"returns comment text, author, likes, and reply counts."
),
input_schema=CommentsInput,
output_schema=CommentsOutput,
executor=build_comments_executor(),
billing_unit=BillingUnit.TIKTOK_COMMENT,
docs_url="/docs/connectors/native/tiktok",
)
register_capability(TIKTOK_COMMENTS)

View file

@ -0,0 +1,36 @@
"""``tiktok.comments`` executor: video URLs -> scraper -> TikTok comment items."""
from __future__ import annotations
from collections.abc import Awaitable, Callable
from app.capabilities.core import Executor
from app.capabilities.core.progress import emit_progress
from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput
from app.proprietary.platforms.tiktok import scrape_tiktok_comments
CommentsFn = Callable[..., Awaitable[list[dict]]]
def build_comments_executor(comments_fn: CommentsFn | None = None) -> Executor:
"""Bind the executor to a comments fn (defaults to the proprietary actor)."""
comments_fn = comments_fn or scrape_tiktok_comments
async def execute(payload: CommentsInput) -> CommentsOutput:
emit_progress(
"starting",
"Scraping TikTok comments",
total=payload.max_items,
unit="item",
)
items = await comments_fn(
payload.video_urls,
per_video=payload.comments_per_video,
limit=payload.max_items,
)
emit_progress(
"done", f"Scraped {len(items)} comment(s)", current=len(items), unit="item"
)
return CommentsOutput(items=items)
return execute

View file

@ -0,0 +1,56 @@
"""``tiktok.comments`` I/O contracts.
URL-only: given TikTok video URLs, return each video's public comment thread.
Unlike profile-video/general-search feeds, ``/api/comment/list`` is served to
anonymous sessions once the comments panel opens, so this verb is reliable. Each
result is a :class:`CommentItem` (top-level comments; replies carry ``repliesToId``).
"""
from __future__ import annotations
from pydantic import BaseModel, Field
from app.capabilities.tiktok.scrape.schemas import (
MAX_TIKTOK_ITEMS,
MAX_TIKTOK_SOURCES,
)
from app.proprietary.platforms.tiktok import CommentItem
class CommentsInput(BaseModel):
video_urls: list[str] = Field(
min_length=1,
max_length=MAX_TIKTOK_SOURCES,
description="TikTok video URLs (/@<user>/video/<id>) to pull comments from.",
)
comments_per_video: int = Field(
default=20,
ge=1,
le=MAX_TIKTOK_ITEMS,
description="Max comments to return per video.",
)
max_items: int = Field(
default=20,
ge=1,
le=MAX_TIKTOK_ITEMS,
description="Max total comments to return across all videos.",
)
@property
def estimated_units(self) -> int:
"""Worst-case billable comments for the pre-flight gate: ``max_items`` is a
hard cross-video ceiling (le=100), so no call can exceed it."""
return self.max_items
class CommentsOutput(BaseModel):
items: list[CommentItem] = Field(
default_factory=list,
description="One item per comment returned, in emission order.",
)
@property
def billable_units(self) -> int:
"""One returned comment = one billable unit; ErrorItems (``errorCode`` set,
for bad URLs or empty/withheld videos) are surfaced but never charged."""
return sum(1 for item in self.items if not getattr(item, "errorCode", None))

View file

@ -720,6 +720,9 @@ class Config:
# User search returns lighter account records (name/followers/bio), priced
# below the video meter to mirror the cheaper account-discovery market.
TIKTOK_MICROS_PER_USER = int(os.getenv("TIKTOK_MICROS_PER_USER", "2500"))
# Comments are the cheapest per-item TikTok data, matching the per-comment
# market (and YouTube's comment meter).
TIKTOK_MICROS_PER_COMMENT = int(os.getenv("TIKTOK_MICROS_PER_COMMENT", "1500"))
# Low-balance WARNING threshold (micro-USD). Surfaced by the quota service
# so the UI can nudge the user to top up / enable auto-reload. $0.50.

View file

@ -6,16 +6,28 @@ schema, the collector/generator, the video item shape, and the hard-block error.
from __future__ import annotations
from .orchestrator import iter_tiktok, scrape_tiktok, search_tiktok_users
from .schemas import TikTokProfileItem, TikTokScrapeInput, TikTokVideoItem
from .orchestrator import (
iter_tiktok,
scrape_tiktok,
scrape_tiktok_comments,
search_tiktok_users,
)
from .schemas import (
CommentItem,
TikTokProfileItem,
TikTokScrapeInput,
TikTokVideoItem,
)
from .session import TikTokAccessBlockedError
__all__ = [
"CommentItem",
"TikTokAccessBlockedError",
"TikTokProfileItem",
"TikTokScrapeInput",
"TikTokVideoItem",
"iter_tiktok",
"scrape_tiktok",
"scrape_tiktok_comments",
"search_tiktok_users",
]

View file

@ -3,6 +3,7 @@
from __future__ import annotations
from .author import parse_author, parse_profile
from .comments import comments_from_response, parse_comment
from .hydration import extract_rehydration_data
from .item_list import items_from_response
from .scopes import user_info, video_item_struct
@ -10,9 +11,11 @@ from .user_search import parse_search_user, users_from_response
from .video import parse_video
__all__ = [
"comments_from_response",
"extract_rehydration_data",
"items_from_response",
"parse_author",
"parse_comment",
"parse_profile",
"parse_search_user",
"parse_video",

View file

@ -0,0 +1,55 @@
"""Parse a captured ``/api/comment/list`` response into comment items.
The comment API returns ``{"comments": [...]}`` where each entry uses the
mobile-API snake_case shape (``cid``, ``digg_count``, ``reply_comment_total``,
``create_time``, and a nested ``user`` with ``uid``/``unique_id``/``nickname``/
``avatar_thumb``). ``reply_id != "0"`` marks a reply to a parent comment.
"""
from __future__ import annotations
from typing import Any
from .timestamps import epoch_to_iso
def comments_from_response(body: Any) -> list[dict[str, Any]]:
"""Return the raw comment records carried by one API response, or ``[]``."""
if not isinstance(body, dict):
return []
comments = body.get("comments")
if not isinstance(comments, list):
return []
return [c for c in comments if isinstance(c, dict)]
def _avatar(user: dict[str, Any]) -> str | None:
thumb = user.get("avatar_thumb")
if isinstance(thumb, dict):
urls = thumb.get("url_list")
if isinstance(urls, list) and urls:
return urls[0]
return None
def parse_comment(raw: dict[str, Any], video_url: str) -> dict[str, Any]:
"""Map a raw comment record to a :class:`CommentItem` output dict."""
from ..schemas.items import CommentItem
user = raw.get("user") if isinstance(raw.get("user"), dict) else {}
reply_id = raw.get("reply_id")
create_time = raw.get("create_time")
return CommentItem(
id=raw.get("cid"),
text=raw.get("text"),
videoWebUrl=video_url,
diggCount=raw.get("digg_count"),
replyCommentTotal=raw.get("reply_comment_total"),
createTime=create_time,
createTimeISO=epoch_to_iso(create_time),
uid=user.get("uid"),
uniqueId=user.get("unique_id"),
nickName=user.get("nickname"),
avatar=_avatar(user),
repliesToId=reply_id if reply_id and reply_id != "0" else None,
).to_output()

View file

@ -13,4 +13,7 @@ FetchListingFn = Callable[[str, int], Awaitable[list[dict]]]
FetchUsersFn = Callable[[str, int], Awaitable[list[dict]]]
"""Load a user-search page and return up to ``count`` captured ``user_info`` records."""
FetchCommentsFn = Callable[[str, int], Awaitable[list[dict]]]
"""Load a video page and return up to ``count`` captured raw comment records."""
FlowResult = AsyncIterator[dict]

View file

@ -0,0 +1,52 @@
"""Comments flow: a video URL -> its public comment thread.
Comments load over a signed ``/api/comment/list`` XHR that TikTok *does* serve to
anonymous sessions once the comments panel is opened (unlike profile-video/search
feeds). Records are deduped by comment id, capped, and when a video has none or
withholds them degraded to one ErrorItem, mirroring the listing flow.
"""
from __future__ import annotations
from collections.abc import AsyncIterator
from typing import Any
from ..extraction import parse_comment
from ..extraction.timestamps import now_iso
from ..schemas import ErrorItem
from ..targets.types import TikTokTarget
from . import FetchCommentsFn
_EMPTY_MESSAGE = (
"No comments returned. The video may have none, comments disabled, or TikTok "
"withheld them from anonymous access."
)
async def iter_comments(
target: TikTokTarget, *, cap: int, fetch_comments: FetchCommentsFn
) -> AsyncIterator[dict[str, Any]]:
if cap <= 0:
return
seen: set[str] = set()
emitted = 0
for raw in await fetch_comments(target.url, cap):
out = parse_comment(raw, target.url)
cid = out.get("id")
if cid is not None:
if cid in seen:
continue
seen.add(cid)
out["scrapedAt"] = now_iso()
yield out
emitted += 1
if emitted >= cap:
return
if emitted == 0:
yield ErrorItem(
url=target.url,
input=target.value,
error=_EMPTY_MESSAGE,
errorCode="no_comments",
scrapedAt=now_iso(),
).to_output()

View file

@ -12,13 +12,20 @@ from collections.abc import AsyncIterator
from typing import Any
from urllib.parse import quote
from .flows import FetchFn, FetchListingFn, FetchUsersFn
from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn
from .flows.comments import iter_comments
from .flows.listing import iter_listing
from .flows.profile import iter_profile
from .flows.user_search import iter_user_search
from .flows.video import iter_video
from .schemas import TikTokScrapeInput
from .session import fetch_html, fetch_item_list, fetch_user_search
from .extraction.timestamps import now_iso
from .schemas import ErrorItem, TikTokScrapeInput
from .session import (
fetch_comments,
fetch_html,
fetch_item_list,
fetch_user_search,
)
from .targets import resolve_target
from .targets.types import TikTokTarget
@ -123,3 +130,44 @@ async def search_tiktok_users(
if limit is not None and len(results) >= limit:
return results
return results
async def scrape_tiktok_comments(
video_urls: list[str],
*,
per_video: int,
limit: int | None = None,
fetch_comments_fn: FetchCommentsFn = fetch_comments,
) -> list[dict[str, Any]]:
"""Collect comments across video URLs, honoring ``limit``.
A non-video URL yields one ``bad_url`` ErrorItem (never a silent drop) so the
caller can tell "wrong input" from "video has no comments".
"""
from app.capabilities.core.progress import emit_progress
results: list[dict[str, Any]] = []
for url in video_urls:
target = resolve_target(url)
if target is None or target.kind != "video":
results.append(
ErrorItem(
url=url,
input=url,
error="Not a TikTok video URL.",
errorCode="bad_url",
scrapedAt=now_iso(),
).to_output()
)
emit_progress("scraping", current=len(results), total=limit, unit="item")
if limit is not None and len(results) >= limit:
return results
continue
async for item in iter_comments(
target, cap=per_video, fetch_comments=fetch_comments_fn
):
results.append(item)
emit_progress("scraping", current=len(results), total=limit, unit="item")
if limit is not None and len(results) >= limit:
return results
return results

View file

@ -4,12 +4,13 @@ from __future__ import annotations
from .client import fetch_html
from .errors import TikTokAccessBlockedError
from .listing import fetch_item_list, fetch_user_search
from .listing import fetch_comments, fetch_item_list, fetch_user_search
from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session
__all__ = [
"TikTokAccessBlockedError",
"bind_proxy_holder",
"fetch_comments",
"fetch_html",
"fetch_item_list",
"fetch_user_search",

View file

@ -30,11 +30,18 @@ from app.proprietary.web_crawler.stealth import (
)
from app.utils.proxy import get_proxy_url
from ..extraction import items_from_response, users_from_response
from ..extraction import (
comments_from_response,
items_from_response,
users_from_response,
)
logger = logging.getLogger(__name__)
ExtractFn = Callable[[Any], list[dict[str, Any]]]
# Drives the page after navigation to trigger/paginate the target XHRs, filling
# ``collected`` until it reaches ``target_count`` (or the interaction gives up).
InteractFn = Callable[[Any, list[dict[str, Any]], int], None]
# XHR paths that carry itemStructs for the three listing kinds.
_ITEM_LIST_MARKERS = (
@ -44,6 +51,16 @@ _ITEM_LIST_MARKERS = (
)
# The user-search XHR carries account records (user_list), not itemStructs.
_USER_SEARCH_MARKERS = ("/api/search/user",)
# The comment feed fires only after the comments panel is opened.
_COMMENT_MARKERS = ("/api/comment/list",)
_COMMENT_ICON_SELECTORS = (
'[data-e2e="comment-icon"]',
'[data-e2e="browse-comment"]',
)
# The comment icon hydrates a beat after DOM-ready; wait for it before clicking.
_COMMENT_ICON_WAIT_MS = 8000
# First comment page lands shortly after the click — don't declare "empty" early.
_COMMENT_FIRST_PAGE_MS = 3500
_HOME_URL = "https://www.tiktok.com/"
_MSTOKEN_COOKIE = "msToken"
# Bounded scroll: a dead page can't loop forever, and a live one stops early
@ -62,18 +79,87 @@ def _has_mstoken(page: Any) -> bool:
return False
def _scroll_page(page: Any, collected: list[dict[str, Any]], target_count: int) -> None:
"""Page down a listing feed until enough items are captured or it stops growing."""
last_height = 0
for _ in range(_SCROLL_MAX_ROUNDS):
if len(collected) >= target_count:
break
page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
page.wait_for_timeout(_SCROLL_SETTLE_MS)
height = page.evaluate("document.body.scrollHeight")
if not height or height <= last_height:
break
last_height = height
def _open_comments(page: Any) -> None:
"""Click the comment icon so the first ``/api/comment/list`` XHR fires.
The icon must be present and interactive first (the SPA hydrates it a beat
after DOM-ready), so we wait for it, then fall back to a JS click if the
normal click is intercepted (cookie banner / overlay).
"""
for selector in _COMMENT_ICON_SELECTORS:
try:
page.wait_for_selector(selector, timeout=_COMMENT_ICON_WAIT_MS)
except Exception:
continue
try:
page.click(selector, timeout=_COMMENT_ICON_WAIT_MS)
return
except Exception:
try:
page.eval_on_selector(selector, "el => el.click()")
return
except Exception:
continue
def _scroll_comments(
page: Any, collected: list[dict[str, Any]], target_count: int
) -> None:
"""Open the comments panel, then scroll its last comment into view to paginate.
Comment XHRs fire only after the panel is opened, and paging must scroll the
panel (not the page, which would advance the video feed), so we anchor on the
last ``comment-level-1`` element. ponytail: naive scroll-to-last paging,
bounded by ``_SCROLL_MAX_ROUNDS``; upgrade to container-height tracking if
deep threads under-fetch.
"""
_open_comments(page)
# The panel's first page lands a beat after the click; give it room before
# we decide there are no comments to page through.
page.wait_for_timeout(_COMMENT_FIRST_PAGE_MS)
for _ in range(_SCROLL_MAX_ROUNDS):
if len(collected) >= target_count:
break
moved = page.evaluate(
"""() => {
const items = document.querySelectorAll('[data-e2e="comment-level-1"]');
if (!items.length) return false;
items[items.length - 1].scrollIntoView({block: 'end'});
return true;
}"""
)
page.wait_for_timeout(_SCROLL_SETTLE_MS)
if not moved:
break
def _build_page_action(
collected: list[dict[str, Any]],
url: str,
target_count: int,
markers: tuple[str, ...],
extract: ExtractFn,
interact: InteractFn,
):
"""A sync ``page_action`` that warms the session then captures matching XHRs.
A cold context returns an empty body, so we first mint the anonymous
``msToken`` (homepage hit), then navigate to the target with the listener
already attached so page-one fires into it; scrolling pages the rest.
already attached so page-one fires into it; ``interact`` pages the rest.
``markers``/``extract`` select which XHRs to keep and how to unwrap them.
"""
@ -102,28 +188,23 @@ def _build_page_action(
try:
_warm(page)
# Navigate (back) to the target with the listener attached and a
# token in hand, so the page-one item_list fires into the capture.
# token in hand, so the page-one XHR fires into the capture.
page.goto(url, wait_until="domcontentloaded")
page.wait_for_timeout(_SCROLL_SETTLE_MS)
last_height = 0
for _ in range(_SCROLL_MAX_ROUNDS):
if len(collected) >= target_count:
break
page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
page.wait_for_timeout(_SCROLL_SETTLE_MS)
height = page.evaluate("document.body.scrollHeight")
if not height or height <= last_height:
break
last_height = height
interact(page, collected, target_count)
except Exception as exc:
logger.debug("[tiktok] listing scroll aborted: %s", exc)
logger.debug("[tiktok] capture interaction aborted: %s", exc)
return page
return page_action
def _fetch_sync(
url: str, target_count: int, markers: tuple[str, ...], extract: ExtractFn
url: str,
target_count: int,
markers: tuple[str, ...],
extract: ExtractFn,
interact: InteractFn,
) -> list[dict[str, Any]]:
collected: list[dict[str, Any]] = []
kwargs = build_stealthy_kwargs(get_stealth_config())
@ -133,7 +214,7 @@ def _fetch_sync(
network_idle=False,
proxy=get_proxy_url(),
page_action=_build_page_action(
collected, url, target_count, markers, extract
collected, url, target_count, markers, extract, interact
),
**kwargs,
)
@ -143,12 +224,34 @@ def _fetch_sync(
async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]:
"""Return up to ``target_count`` itemStructs from a listing page's XHRs."""
return await asyncio.to_thread(
_fetch_sync, page_url, target_count, _ITEM_LIST_MARKERS, items_from_response
_fetch_sync,
page_url,
target_count,
_ITEM_LIST_MARKERS,
items_from_response,
_scroll_page,
)
async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]:
"""Return up to ``target_count`` ``user_info`` records from a user-search page."""
return await asyncio.to_thread(
_fetch_sync, page_url, target_count, _USER_SEARCH_MARKERS, users_from_response
_fetch_sync,
page_url,
target_count,
_USER_SEARCH_MARKERS,
users_from_response,
_scroll_page,
)
async def fetch_comments(page_url: str, target_count: int) -> list[dict[str, Any]]:
"""Return up to ``target_count`` raw comment records from a video page's XHRs."""
return await asyncio.to_thread(
_fetch_sync,
page_url,
target_count,
_COMMENT_MARKERS,
comments_from_response,
_scroll_comments,
)