diff --git a/docker/.env.example b/docker/.env.example index 86536e94c..19e1a47fd 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -451,6 +451,7 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # YOUTUBE_MICROS_PER_COMMENT=1500 # TIKTOK_MICROS_PER_VIDEO=3500 # TIKTOK_MICROS_PER_USER=2500 +# TIKTOK_MICROS_PER_COMMENT=1500 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index eeb346a1b..20d811839 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -288,6 +288,7 @@ MICROS_PER_PAGE=1000 # YOUTUBE_MICROS_PER_COMMENT=1500 # TIKTOK_MICROS_PER_VIDEO=3500 # TIKTOK_MICROS_PER_USER=2500 +# TIKTOK_MICROS_PER_COMMENT=1500 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. CREDIT_LOW_BALANCE_WARNING_MICROS=500000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py index d826a8a85..44ebc863d 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py @@ -1,4 +1,4 @@ -"""``tiktok`` sub-agent tools: the TikTok scrape and user-search capability verbs.""" +"""``tiktok`` sub-agent tools: the TikTok scrape, comments, and user-search verbs.""" from __future__ import annotations @@ -8,6 +8,7 @@ from langchain_core.tools import BaseTool from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from app.capabilities.core.access.agent import build_capability_tools +from app.capabilities.tiktok.comments.definition import TIKTOK_COMMENTS from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE from app.capabilities.tiktok.user_search.definition import TIKTOK_USER_SEARCH @@ -15,7 +16,7 @@ NAME = "tiktok" RULESET = Ruleset(origin=NAME, rules=[]) -_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_USER_SEARCH] +_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_COMMENTS, TIKTOK_USER_SEARCH] def load_tools( diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 8bc9fbd14..de2e48af3 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -37,6 +37,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT", BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO", BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER", + BillingUnit.TIKTOK_COMMENT: "TIKTOK_MICROS_PER_COMMENT", } @@ -55,6 +56,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.YOUTUBE_COMMENT: "comment", BillingUnit.TIKTOK_VIDEO: "video", BillingUnit.TIKTOK_USER: "profile", + BillingUnit.TIKTOK_COMMENT: "comment", } diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index cbb01c2a6..d41c11ffb 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -27,6 +27,7 @@ class BillingUnit(StrEnum): YOUTUBE_COMMENT = "youtube_comment" TIKTOK_VIDEO = "tiktok_video" TIKTOK_USER = "tiktok_user" + TIKTOK_COMMENT = "tiktok_comment" class BillableInput(Protocol): diff --git a/surfsense_backend/app/capabilities/tiktok/__init__.py b/surfsense_backend/app/capabilities/tiktok/__init__.py index 067c91c0a..1ff1738fa 100644 --- a/surfsense_backend/app/capabilities/tiktok/__init__.py +++ b/surfsense_backend/app/capabilities/tiktok/__init__.py @@ -2,5 +2,6 @@ from __future__ import annotations +from app.capabilities.tiktok.comments import definition as _comments # noqa: F401 from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401 from app.capabilities.tiktok.user_search import definition as _user_search # noqa: F401 diff --git a/surfsense_backend/app/capabilities/tiktok/comments/__init__.py b/surfsense_backend/app/capabilities/tiktok/comments/__init__.py new file mode 100644 index 000000000..f3bd1db88 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/__init__.py @@ -0,0 +1,3 @@ +"""``tiktok.comments``: scrape the public comment thread of TikTok videos.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/tiktok/comments/definition.py b/surfsense_backend/app/capabilities/tiktok/comments/definition.py new file mode 100644 index 000000000..1e5917fa2 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/definition.py @@ -0,0 +1,23 @@ +"""``tiktok.comments`` capability registration (billed per comment; see config +``TIKTOK_MICROS_PER_COMMENT``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.comments.executor import build_comments_executor +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput + +TIKTOK_COMMENTS = Capability( + name="tiktok.comments", + description=( + "Scrape the public comments of TikTok videos. Provide video URLs; " + "returns comment text, author, likes, and reply counts." + ), + input_schema=CommentsInput, + output_schema=CommentsOutput, + executor=build_comments_executor(), + billing_unit=BillingUnit.TIKTOK_COMMENT, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_COMMENTS) diff --git a/surfsense_backend/app/capabilities/tiktok/comments/executor.py b/surfsense_backend/app/capabilities/tiktok/comments/executor.py new file mode 100644 index 000000000..c8bbc776f --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/executor.py @@ -0,0 +1,36 @@ +"""``tiktok.comments`` executor: video URLs -> scraper -> TikTok comment items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput +from app.proprietary.platforms.tiktok import scrape_tiktok_comments + +CommentsFn = Callable[..., Awaitable[list[dict]]] + + +def build_comments_executor(comments_fn: CommentsFn | None = None) -> Executor: + """Bind the executor to a comments fn (defaults to the proprietary actor).""" + comments_fn = comments_fn or scrape_tiktok_comments + + async def execute(payload: CommentsInput) -> CommentsOutput: + emit_progress( + "starting", + "Scraping TikTok comments", + total=payload.max_items, + unit="item", + ) + items = await comments_fn( + payload.video_urls, + per_video=payload.comments_per_video, + limit=payload.max_items, + ) + emit_progress( + "done", f"Scraped {len(items)} comment(s)", current=len(items), unit="item" + ) + return CommentsOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/comments/schemas.py b/surfsense_backend/app/capabilities/tiktok/comments/schemas.py new file mode 100644 index 000000000..f3c68d445 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/schemas.py @@ -0,0 +1,56 @@ +"""``tiktok.comments`` I/O contracts. + +URL-only: given TikTok video URLs, return each video's public comment thread. +Unlike profile-video/general-search feeds, ``/api/comment/list`` is served to +anonymous sessions once the comments panel opens, so this verb is reliable. Each +result is a :class:`CommentItem` (top-level comments; replies carry ``repliesToId``). +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.capabilities.tiktok.scrape.schemas import ( + MAX_TIKTOK_ITEMS, + MAX_TIKTOK_SOURCES, +) +from app.proprietary.platforms.tiktok import CommentItem + + +class CommentsInput(BaseModel): + video_urls: list[str] = Field( + min_length=1, + max_length=MAX_TIKTOK_SOURCES, + description="TikTok video URLs (/@/video/) to pull comments from.", + ) + comments_per_video: int = Field( + default=20, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max comments to return per video.", + ) + max_items: int = Field( + default=20, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max total comments to return across all videos.", + ) + + @property + def estimated_units(self) -> int: + """Worst-case billable comments for the pre-flight gate: ``max_items`` is a + hard cross-video ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class CommentsOutput(BaseModel): + items: list[CommentItem] = Field( + default_factory=list, + description="One item per comment returned, in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned comment = one billable unit; ErrorItems (``errorCode`` set, + for bad URLs or empty/withheld videos) are surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 97abb5eda..21ba1d1c6 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -720,6 +720,9 @@ class Config: # User search returns lighter account records (name/followers/bio), priced # below the video meter to mirror the cheaper account-discovery market. TIKTOK_MICROS_PER_USER = int(os.getenv("TIKTOK_MICROS_PER_USER", "2500")) + # Comments are the cheapest per-item TikTok data, matching the per-comment + # market (and YouTube's comment meter). + TIKTOK_MICROS_PER_COMMENT = int(os.getenv("TIKTOK_MICROS_PER_COMMENT", "1500")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py index 2947498ec..00d345937 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py @@ -6,16 +6,28 @@ schema, the collector/generator, the video item shape, and the hard-block error. from __future__ import annotations -from .orchestrator import iter_tiktok, scrape_tiktok, search_tiktok_users -from .schemas import TikTokProfileItem, TikTokScrapeInput, TikTokVideoItem +from .orchestrator import ( + iter_tiktok, + scrape_tiktok, + scrape_tiktok_comments, + search_tiktok_users, +) +from .schemas import ( + CommentItem, + TikTokProfileItem, + TikTokScrapeInput, + TikTokVideoItem, +) from .session import TikTokAccessBlockedError __all__ = [ + "CommentItem", "TikTokAccessBlockedError", "TikTokProfileItem", "TikTokScrapeInput", "TikTokVideoItem", "iter_tiktok", "scrape_tiktok", + "scrape_tiktok_comments", "search_tiktok_users", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index f4712e724..3afd536f2 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -3,6 +3,7 @@ from __future__ import annotations from .author import parse_author, parse_profile +from .comments import comments_from_response, parse_comment from .hydration import extract_rehydration_data from .item_list import items_from_response from .scopes import user_info, video_item_struct @@ -10,9 +11,11 @@ from .user_search import parse_search_user, users_from_response from .video import parse_video __all__ = [ + "comments_from_response", "extract_rehydration_data", "items_from_response", "parse_author", + "parse_comment", "parse_profile", "parse_search_user", "parse_video", diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/comments.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/comments.py new file mode 100644 index 000000000..8ca96cad3 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/comments.py @@ -0,0 +1,55 @@ +"""Parse a captured ``/api/comment/list`` response into comment items. + +The comment API returns ``{"comments": [...]}`` where each entry uses the +mobile-API snake_case shape (``cid``, ``digg_count``, ``reply_comment_total``, +``create_time``, and a nested ``user`` with ``uid``/``unique_id``/``nickname``/ +``avatar_thumb``). ``reply_id != "0"`` marks a reply to a parent comment. +""" + +from __future__ import annotations + +from typing import Any + +from .timestamps import epoch_to_iso + + +def comments_from_response(body: Any) -> list[dict[str, Any]]: + """Return the raw comment records carried by one API response, or ``[]``.""" + if not isinstance(body, dict): + return [] + comments = body.get("comments") + if not isinstance(comments, list): + return [] + return [c for c in comments if isinstance(c, dict)] + + +def _avatar(user: dict[str, Any]) -> str | None: + thumb = user.get("avatar_thumb") + if isinstance(thumb, dict): + urls = thumb.get("url_list") + if isinstance(urls, list) and urls: + return urls[0] + return None + + +def parse_comment(raw: dict[str, Any], video_url: str) -> dict[str, Any]: + """Map a raw comment record to a :class:`CommentItem` output dict.""" + from ..schemas.items import CommentItem + + user = raw.get("user") if isinstance(raw.get("user"), dict) else {} + reply_id = raw.get("reply_id") + create_time = raw.get("create_time") + return CommentItem( + id=raw.get("cid"), + text=raw.get("text"), + videoWebUrl=video_url, + diggCount=raw.get("digg_count"), + replyCommentTotal=raw.get("reply_comment_total"), + createTime=create_time, + createTimeISO=epoch_to_iso(create_time), + uid=user.get("uid"), + uniqueId=user.get("unique_id"), + nickName=user.get("nickname"), + avatar=_avatar(user), + repliesToId=reply_id if reply_id and reply_id != "0" else None, + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py index 1a8aeaacc..a7e38eba1 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py @@ -13,4 +13,7 @@ FetchListingFn = Callable[[str, int], Awaitable[list[dict]]] FetchUsersFn = Callable[[str, int], Awaitable[list[dict]]] """Load a user-search page and return up to ``count`` captured ``user_info`` records.""" +FetchCommentsFn = Callable[[str, int], Awaitable[list[dict]]] +"""Load a video page and return up to ``count`` captured raw comment records.""" + FlowResult = AsyncIterator[dict] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/comments.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/comments.py new file mode 100644 index 000000000..28a7772a2 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/comments.py @@ -0,0 +1,52 @@ +"""Comments flow: a video URL -> its public comment thread. + +Comments load over a signed ``/api/comment/list`` XHR that TikTok *does* serve to +anonymous sessions once the comments panel is opened (unlike profile-video/search +feeds). Records are deduped by comment id, capped, and — when a video has none or +withholds them — degraded to one ErrorItem, mirroring the listing flow. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from ..extraction import parse_comment +from ..extraction.timestamps import now_iso +from ..schemas import ErrorItem +from ..targets.types import TikTokTarget +from . import FetchCommentsFn + +_EMPTY_MESSAGE = ( + "No comments returned. The video may have none, comments disabled, or TikTok " + "withheld them from anonymous access." +) + + +async def iter_comments( + target: TikTokTarget, *, cap: int, fetch_comments: FetchCommentsFn +) -> AsyncIterator[dict[str, Any]]: + if cap <= 0: + return + seen: set[str] = set() + emitted = 0 + for raw in await fetch_comments(target.url, cap): + out = parse_comment(raw, target.url) + cid = out.get("id") + if cid is not None: + if cid in seen: + continue + seen.add(cid) + out["scrapedAt"] = now_iso() + yield out + emitted += 1 + if emitted >= cap: + return + if emitted == 0: + yield ErrorItem( + url=target.url, + input=target.value, + error=_EMPTY_MESSAGE, + errorCode="no_comments", + scrapedAt=now_iso(), + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index 485a8dfff..dbe0d5ce9 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -12,13 +12,20 @@ from collections.abc import AsyncIterator from typing import Any from urllib.parse import quote -from .flows import FetchFn, FetchListingFn, FetchUsersFn +from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn +from .flows.comments import iter_comments from .flows.listing import iter_listing from .flows.profile import iter_profile from .flows.user_search import iter_user_search from .flows.video import iter_video -from .schemas import TikTokScrapeInput -from .session import fetch_html, fetch_item_list, fetch_user_search +from .extraction.timestamps import now_iso +from .schemas import ErrorItem, TikTokScrapeInput +from .session import ( + fetch_comments, + fetch_html, + fetch_item_list, + fetch_user_search, +) from .targets import resolve_target from .targets.types import TikTokTarget @@ -123,3 +130,44 @@ async def search_tiktok_users( if limit is not None and len(results) >= limit: return results return results + + +async def scrape_tiktok_comments( + video_urls: list[str], + *, + per_video: int, + limit: int | None = None, + fetch_comments_fn: FetchCommentsFn = fetch_comments, +) -> list[dict[str, Any]]: + """Collect comments across video URLs, honoring ``limit``. + + A non-video URL yields one ``bad_url`` ErrorItem (never a silent drop) so the + caller can tell "wrong input" from "video has no comments". + """ + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + for url in video_urls: + target = resolve_target(url) + if target is None or target.kind != "video": + results.append( + ErrorItem( + url=url, + input=url, + error="Not a TikTok video URL.", + errorCode="bad_url", + scrapedAt=now_iso(), + ).to_output() + ) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + return results + continue + async for item in iter_comments( + target, cap=per_video, fetch_comments=fetch_comments_fn + ): + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + return results + return results diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py index 5e29403a0..437fc9643 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py @@ -4,12 +4,13 @@ from __future__ import annotations from .client import fetch_html from .errors import TikTokAccessBlockedError -from .listing import fetch_item_list, fetch_user_search +from .listing import fetch_comments, fetch_item_list, fetch_user_search from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session __all__ = [ "TikTokAccessBlockedError", "bind_proxy_holder", + "fetch_comments", "fetch_html", "fetch_item_list", "fetch_user_search", diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index e0bdd9255..d00e20711 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -30,11 +30,18 @@ from app.proprietary.web_crawler.stealth import ( ) from app.utils.proxy import get_proxy_url -from ..extraction import items_from_response, users_from_response +from ..extraction import ( + comments_from_response, + items_from_response, + users_from_response, +) logger = logging.getLogger(__name__) ExtractFn = Callable[[Any], list[dict[str, Any]]] +# Drives the page after navigation to trigger/paginate the target XHRs, filling +# ``collected`` until it reaches ``target_count`` (or the interaction gives up). +InteractFn = Callable[[Any, list[dict[str, Any]], int], None] # XHR paths that carry itemStructs for the three listing kinds. _ITEM_LIST_MARKERS = ( @@ -44,6 +51,16 @@ _ITEM_LIST_MARKERS = ( ) # The user-search XHR carries account records (user_list), not itemStructs. _USER_SEARCH_MARKERS = ("/api/search/user",) +# The comment feed fires only after the comments panel is opened. +_COMMENT_MARKERS = ("/api/comment/list",) +_COMMENT_ICON_SELECTORS = ( + '[data-e2e="comment-icon"]', + '[data-e2e="browse-comment"]', +) +# The comment icon hydrates a beat after DOM-ready; wait for it before clicking. +_COMMENT_ICON_WAIT_MS = 8000 +# First comment page lands shortly after the click — don't declare "empty" early. +_COMMENT_FIRST_PAGE_MS = 3500 _HOME_URL = "https://www.tiktok.com/" _MSTOKEN_COOKIE = "msToken" # Bounded scroll: a dead page can't loop forever, and a live one stops early @@ -62,18 +79,87 @@ def _has_mstoken(page: Any) -> bool: return False +def _scroll_page(page: Any, collected: list[dict[str, Any]], target_count: int) -> None: + """Page down a listing feed until enough items are captured or it stops growing.""" + last_height = 0 + for _ in range(_SCROLL_MAX_ROUNDS): + if len(collected) >= target_count: + break + page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + page.wait_for_timeout(_SCROLL_SETTLE_MS) + height = page.evaluate("document.body.scrollHeight") + if not height or height <= last_height: + break + last_height = height + + +def _open_comments(page: Any) -> None: + """Click the comment icon so the first ``/api/comment/list`` XHR fires. + + The icon must be present and interactive first (the SPA hydrates it a beat + after DOM-ready), so we wait for it, then fall back to a JS click if the + normal click is intercepted (cookie banner / overlay). + """ + for selector in _COMMENT_ICON_SELECTORS: + try: + page.wait_for_selector(selector, timeout=_COMMENT_ICON_WAIT_MS) + except Exception: + continue + try: + page.click(selector, timeout=_COMMENT_ICON_WAIT_MS) + return + except Exception: + try: + page.eval_on_selector(selector, "el => el.click()") + return + except Exception: + continue + + +def _scroll_comments( + page: Any, collected: list[dict[str, Any]], target_count: int +) -> None: + """Open the comments panel, then scroll its last comment into view to paginate. + + Comment XHRs fire only after the panel is opened, and paging must scroll the + panel (not the page, which would advance the video feed), so we anchor on the + last ``comment-level-1`` element. ponytail: naive scroll-to-last paging, + bounded by ``_SCROLL_MAX_ROUNDS``; upgrade to container-height tracking if + deep threads under-fetch. + """ + _open_comments(page) + # The panel's first page lands a beat after the click; give it room before + # we decide there are no comments to page through. + page.wait_for_timeout(_COMMENT_FIRST_PAGE_MS) + for _ in range(_SCROLL_MAX_ROUNDS): + if len(collected) >= target_count: + break + moved = page.evaluate( + """() => { + const items = document.querySelectorAll('[data-e2e="comment-level-1"]'); + if (!items.length) return false; + items[items.length - 1].scrollIntoView({block: 'end'}); + return true; + }""" + ) + page.wait_for_timeout(_SCROLL_SETTLE_MS) + if not moved: + break + + def _build_page_action( collected: list[dict[str, Any]], url: str, target_count: int, markers: tuple[str, ...], extract: ExtractFn, + interact: InteractFn, ): """A sync ``page_action`` that warms the session then captures matching XHRs. A cold context returns an empty body, so we first mint the anonymous ``msToken`` (homepage hit), then navigate to the target with the listener - already attached so page-one fires into it; scrolling pages the rest. + already attached so page-one fires into it; ``interact`` pages the rest. ``markers``/``extract`` select which XHRs to keep and how to unwrap them. """ @@ -102,28 +188,23 @@ def _build_page_action( try: _warm(page) # Navigate (back) to the target with the listener attached and a - # token in hand, so the page-one item_list fires into the capture. + # token in hand, so the page-one XHR fires into the capture. page.goto(url, wait_until="domcontentloaded") page.wait_for_timeout(_SCROLL_SETTLE_MS) - last_height = 0 - for _ in range(_SCROLL_MAX_ROUNDS): - if len(collected) >= target_count: - break - page.evaluate("window.scrollTo(0, document.body.scrollHeight)") - page.wait_for_timeout(_SCROLL_SETTLE_MS) - height = page.evaluate("document.body.scrollHeight") - if not height or height <= last_height: - break - last_height = height + interact(page, collected, target_count) except Exception as exc: - logger.debug("[tiktok] listing scroll aborted: %s", exc) + logger.debug("[tiktok] capture interaction aborted: %s", exc) return page return page_action def _fetch_sync( - url: str, target_count: int, markers: tuple[str, ...], extract: ExtractFn + url: str, + target_count: int, + markers: tuple[str, ...], + extract: ExtractFn, + interact: InteractFn, ) -> list[dict[str, Any]]: collected: list[dict[str, Any]] = [] kwargs = build_stealthy_kwargs(get_stealth_config()) @@ -133,7 +214,7 @@ def _fetch_sync( network_idle=False, proxy=get_proxy_url(), page_action=_build_page_action( - collected, url, target_count, markers, extract + collected, url, target_count, markers, extract, interact ), **kwargs, ) @@ -143,12 +224,34 @@ def _fetch_sync( async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` itemStructs from a listing page's XHRs.""" return await asyncio.to_thread( - _fetch_sync, page_url, target_count, _ITEM_LIST_MARKERS, items_from_response + _fetch_sync, + page_url, + target_count, + _ITEM_LIST_MARKERS, + items_from_response, + _scroll_page, ) async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` ``user_info`` records from a user-search page.""" return await asyncio.to_thread( - _fetch_sync, page_url, target_count, _USER_SEARCH_MARKERS, users_from_response + _fetch_sync, + page_url, + target_count, + _USER_SEARCH_MARKERS, + users_from_response, + _scroll_page, + ) + + +async def fetch_comments(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` raw comment records from a video page's XHRs.""" + return await asyncio.to_thread( + _fetch_sync, + page_url, + target_count, + _COMMENT_MARKERS, + comments_from_response, + _scroll_comments, ) diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/comments/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/comments/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py new file mode 100644 index 000000000..b23149420 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py @@ -0,0 +1,48 @@ +"""``tiktok.comments`` executor: verb input → scraper args → typed comment items. + +Boundary mocked: the proprietary comments actor (injected fake). NOT mocked: the +verb's own payload→args forwarding and the dict→CommentItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.comments.executor import build_comments_executor +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput + +pytestmark = pytest.mark.unit + +_VIDEO = "https://www.tiktok.com/@bob/video/123" + + +class _FakeComments: + """Records the urls + kwargs it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[tuple[list[str], int, int | None]] = [] + + async def __call__( + self, video_urls: list[str], *, per_video: int, limit: int | None = None + ) -> list[dict]: + self.calls.append((video_urls, per_video, limit)) + return self._items + + +async def test_forwards_urls_and_limits_and_wraps_items(): + comments = _FakeComments([{"id": "1", "text": "hi"}]) + execute = build_comments_executor(comments_fn=comments) + + out = await execute( + CommentsInput(video_urls=[_VIDEO], comments_per_video=15, max_items=40) + ) + + assert isinstance(out, CommentsOutput) + assert len(out.items) == 1 + assert out.items[0].text == "hi" + + (urls, per_video, limit) = comments.calls[0] + assert urls == [_VIDEO] + assert per_video == 15 + assert limit == 40 diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py new file mode 100644 index 000000000..cdcbc201e --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py @@ -0,0 +1,48 @@ +"""``tiktok.comments`` input guards and billing: a video URL is required, bounded, +and ErrorItems are surfaced free.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput +from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES + +pytestmark = pytest.mark.unit + +_VIDEO = "https://www.tiktok.com/@bob/video/123" + + +def test_rejects_input_with_no_url(): + with pytest.raises(ValidationError): + CommentsInput(video_urls=[]) + + +def test_defaults_and_bounds(): + payload = CommentsInput(video_urls=[_VIDEO]) + assert payload.max_items == 20 + assert payload.comments_per_video == 20 + assert payload.estimated_units == 20 + with pytest.raises(ValidationError): + CommentsInput(video_urls=[_VIDEO], max_items=0) + with pytest.raises(ValidationError): + CommentsInput(video_urls=[_VIDEO], max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_rejects_more_urls_than_the_cap(): + too_many = [f"{_VIDEO}{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] + with pytest.raises(ValidationError): + CommentsInput(video_urls=too_many) + + +def test_error_items_are_not_billed(): + # Real comments count; ErrorItems (bad URL / empty video) are surfaced free. + out = CommentsOutput( + items=[ + {"id": "1", "text": "hi"}, + {"errorCode": "no_comments", "input": "123", "error": "empty"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py index 893f3c2ae..c688cf198 100644 --- a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py @@ -9,6 +9,7 @@ from app.capabilities import ( ) from app.capabilities.core import BillingUnit from app.capabilities.core.store import get_capability +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput from app.capabilities.tiktok.user_search.schemas import ( UserSearchInput, @@ -34,3 +35,12 @@ def test_tiktok_user_search_is_registered_and_billed_per_profile(): assert cap.input_schema is UserSearchInput assert cap.output_schema is UserSearchOutput assert cap.billing_unit is BillingUnit.TIKTOK_USER + + +def test_tiktok_comments_is_registered_and_billed_per_comment(): + cap = get_capability("tiktok.comments") + + assert cap.name == "tiktok.comments" + assert cap.input_schema is CommentsInput + assert cap.output_schema is CommentsOutput + assert cap.billing_unit is BillingUnit.TIKTOK_COMMENT diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_comments.py b/surfsense_backend/tests/unit/platforms/tiktok/test_comments.py new file mode 100644 index 000000000..e118f3c01 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_comments.py @@ -0,0 +1,89 @@ +"""Comments orchestration over a fake fetch (no network). + +Drives ``scrape_tiktok_comments``: video URLs -> captured raw comments -> items. +""" + +from __future__ import annotations + +from typing import Any + +from app.proprietary.platforms.tiktok import scrape_tiktok_comments + +_VIDEO = "https://www.tiktok.com/@bob/video/123" + + +def _comment(cid: str, reply_id: str = "0") -> dict[str, Any]: + return { + "cid": cid, + "text": f"comment {cid}", + "digg_count": 7, + "reply_comment_total": 2, + "create_time": 1700000000, + "reply_id": reply_id, + "user": { + "uid": "u1", + "unique_id": "alice", + "nickname": "Alice", + "avatar_thumb": {"url_list": ["https://cdn/a.webp"]}, + }, + } + + +async def test_comments_parse_dedupe_and_cap(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_comment("1"), _comment("1"), _comment("2", reply_id="1")] + + items = await scrape_tiktok_comments( + [_VIDEO], per_video=2, fetch_comments_fn=fake_fetch + ) + + assert [i["id"] for i in items] == ["1", "2"] + first = items[0] + assert first["text"] == "comment 1" + assert first["videoWebUrl"] == _VIDEO + assert first["diggCount"] == 7 + assert first["uniqueId"] == "alice" + assert first["avatar"] == "https://cdn/a.webp" + assert first["createTimeISO"] is not None + assert first["repliesToId"] is None # reply_id "0" == top-level + assert first["scrapedAt"] is not None + assert items[1]["repliesToId"] == "1" # a reply carries its parent id + + +async def test_empty_video_emits_error_item(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [] + + items = await scrape_tiktok_comments( + [_VIDEO], per_video=5, fetch_comments_fn=fake_fetch + ) + + assert len(items) == 1 + assert items[0]["errorCode"] == "no_comments" + assert items[0]["input"] == "123" + + +async def test_non_video_url_emits_bad_url_error(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + raise AssertionError("should not fetch for a non-video URL") + + items = await scrape_tiktok_comments( + ["https://www.tiktok.com/@bob"], per_video=5, fetch_comments_fn=fake_fetch + ) + + assert len(items) == 1 + assert items[0]["errorCode"] == "bad_url" + + +async def test_comments_honor_limit_across_videos(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_comment("1"), _comment("2")] + + items = await scrape_tiktok_comments( + [_VIDEO, "https://www.tiktok.com/@bob/video/456"], + per_video=5, + limit=3, + fetch_comments_fn=fake_fetch, + ) + + assert len(items) == 3