mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-18 23:11:12 +02:00
feat(tiktok): add tiktok.comments verb for public comment scraping
Comments load over a signed /api/comment/list XHR that TikTok serves to anonymous sessions once the comments panel is opened (unlike profile-video and general-search feeds), so this is a reliable verb. Given video URLs it returns CommentItems (text, author, likes, reply counts; replies carry repliesToId), deduped per video, capped, and degraded to an ErrorItem for empty/withheld videos or a bad_url ErrorItem for non-video inputs. Generalizes the browser capture over a pluggable interaction step so the comments flow (open panel, scroll the panel to paginate) reuses the same warm+capture scaffolding as listing/user-search. Billed per comment on a new TIKTOK_COMMENT meter (TIKTOK_MICROS_PER_COMMENT, matching the per-comment market), surfaced on the chat subagent alongside tiktok.scrape/user_search.
This commit is contained in:
parent
192b6dc31a
commit
7723b5b8b6
24 changed files with 626 additions and 26 deletions
|
|
@ -0,0 +1,48 @@
|
|||
"""``tiktok.comments`` executor: verb input → scraper args → typed comment items.
|
||||
|
||||
Boundary mocked: the proprietary comments actor (injected fake). NOT mocked: the
|
||||
verb's own payload→args forwarding and the dict→CommentItem wrapping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.capabilities.tiktok.comments.executor import build_comments_executor
|
||||
from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
_VIDEO = "https://www.tiktok.com/@bob/video/123"
|
||||
|
||||
|
||||
class _FakeComments:
|
||||
"""Records the urls + kwargs it was called with; returns canned items."""
|
||||
|
||||
def __init__(self, items: list[dict]):
|
||||
self._items = items
|
||||
self.calls: list[tuple[list[str], int, int | None]] = []
|
||||
|
||||
async def __call__(
|
||||
self, video_urls: list[str], *, per_video: int, limit: int | None = None
|
||||
) -> list[dict]:
|
||||
self.calls.append((video_urls, per_video, limit))
|
||||
return self._items
|
||||
|
||||
|
||||
async def test_forwards_urls_and_limits_and_wraps_items():
|
||||
comments = _FakeComments([{"id": "1", "text": "hi"}])
|
||||
execute = build_comments_executor(comments_fn=comments)
|
||||
|
||||
out = await execute(
|
||||
CommentsInput(video_urls=[_VIDEO], comments_per_video=15, max_items=40)
|
||||
)
|
||||
|
||||
assert isinstance(out, CommentsOutput)
|
||||
assert len(out.items) == 1
|
||||
assert out.items[0].text == "hi"
|
||||
|
||||
(urls, per_video, limit) = comments.calls[0]
|
||||
assert urls == [_VIDEO]
|
||||
assert per_video == 15
|
||||
assert limit == 40
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
"""``tiktok.comments`` input guards and billing: a video URL is required, bounded,
|
||||
and ErrorItems are surfaced free."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput
|
||||
from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
_VIDEO = "https://www.tiktok.com/@bob/video/123"
|
||||
|
||||
|
||||
def test_rejects_input_with_no_url():
|
||||
with pytest.raises(ValidationError):
|
||||
CommentsInput(video_urls=[])
|
||||
|
||||
|
||||
def test_defaults_and_bounds():
|
||||
payload = CommentsInput(video_urls=[_VIDEO])
|
||||
assert payload.max_items == 20
|
||||
assert payload.comments_per_video == 20
|
||||
assert payload.estimated_units == 20
|
||||
with pytest.raises(ValidationError):
|
||||
CommentsInput(video_urls=[_VIDEO], max_items=0)
|
||||
with pytest.raises(ValidationError):
|
||||
CommentsInput(video_urls=[_VIDEO], max_items=MAX_TIKTOK_ITEMS + 1)
|
||||
|
||||
|
||||
def test_rejects_more_urls_than_the_cap():
|
||||
too_many = [f"{_VIDEO}{i}" for i in range(MAX_TIKTOK_SOURCES + 1)]
|
||||
with pytest.raises(ValidationError):
|
||||
CommentsInput(video_urls=too_many)
|
||||
|
||||
|
||||
def test_error_items_are_not_billed():
|
||||
# Real comments count; ErrorItems (bad URL / empty video) are surfaced free.
|
||||
out = CommentsOutput(
|
||||
items=[
|
||||
{"id": "1", "text": "hi"},
|
||||
{"errorCode": "no_comments", "input": "123", "error": "empty"},
|
||||
]
|
||||
)
|
||||
assert len(out.items) == 2
|
||||
assert out.billable_units == 1
|
||||
|
|
@ -9,6 +9,7 @@ from app.capabilities import (
|
|||
)
|
||||
from app.capabilities.core import BillingUnit
|
||||
from app.capabilities.core.store import get_capability
|
||||
from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput
|
||||
from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
from app.capabilities.tiktok.user_search.schemas import (
|
||||
UserSearchInput,
|
||||
|
|
@ -34,3 +35,12 @@ def test_tiktok_user_search_is_registered_and_billed_per_profile():
|
|||
assert cap.input_schema is UserSearchInput
|
||||
assert cap.output_schema is UserSearchOutput
|
||||
assert cap.billing_unit is BillingUnit.TIKTOK_USER
|
||||
|
||||
|
||||
def test_tiktok_comments_is_registered_and_billed_per_comment():
|
||||
cap = get_capability("tiktok.comments")
|
||||
|
||||
assert cap.name == "tiktok.comments"
|
||||
assert cap.input_schema is CommentsInput
|
||||
assert cap.output_schema is CommentsOutput
|
||||
assert cap.billing_unit is BillingUnit.TIKTOK_COMMENT
|
||||
|
|
|
|||
|
|
@ -0,0 +1,89 @@
|
|||
"""Comments orchestration over a fake fetch (no network).
|
||||
|
||||
Drives ``scrape_tiktok_comments``: video URLs -> captured raw comments -> items.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.proprietary.platforms.tiktok import scrape_tiktok_comments
|
||||
|
||||
_VIDEO = "https://www.tiktok.com/@bob/video/123"
|
||||
|
||||
|
||||
def _comment(cid: str, reply_id: str = "0") -> dict[str, Any]:
|
||||
return {
|
||||
"cid": cid,
|
||||
"text": f"comment {cid}",
|
||||
"digg_count": 7,
|
||||
"reply_comment_total": 2,
|
||||
"create_time": 1700000000,
|
||||
"reply_id": reply_id,
|
||||
"user": {
|
||||
"uid": "u1",
|
||||
"unique_id": "alice",
|
||||
"nickname": "Alice",
|
||||
"avatar_thumb": {"url_list": ["https://cdn/a.webp"]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def test_comments_parse_dedupe_and_cap():
|
||||
async def fake_fetch(_url: str, _cap: int) -> list[dict]:
|
||||
return [_comment("1"), _comment("1"), _comment("2", reply_id="1")]
|
||||
|
||||
items = await scrape_tiktok_comments(
|
||||
[_VIDEO], per_video=2, fetch_comments_fn=fake_fetch
|
||||
)
|
||||
|
||||
assert [i["id"] for i in items] == ["1", "2"]
|
||||
first = items[0]
|
||||
assert first["text"] == "comment 1"
|
||||
assert first["videoWebUrl"] == _VIDEO
|
||||
assert first["diggCount"] == 7
|
||||
assert first["uniqueId"] == "alice"
|
||||
assert first["avatar"] == "https://cdn/a.webp"
|
||||
assert first["createTimeISO"] is not None
|
||||
assert first["repliesToId"] is None # reply_id "0" == top-level
|
||||
assert first["scrapedAt"] is not None
|
||||
assert items[1]["repliesToId"] == "1" # a reply carries its parent id
|
||||
|
||||
|
||||
async def test_empty_video_emits_error_item():
|
||||
async def fake_fetch(_url: str, _cap: int) -> list[dict]:
|
||||
return []
|
||||
|
||||
items = await scrape_tiktok_comments(
|
||||
[_VIDEO], per_video=5, fetch_comments_fn=fake_fetch
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
assert items[0]["errorCode"] == "no_comments"
|
||||
assert items[0]["input"] == "123"
|
||||
|
||||
|
||||
async def test_non_video_url_emits_bad_url_error():
|
||||
async def fake_fetch(_url: str, _cap: int) -> list[dict]:
|
||||
raise AssertionError("should not fetch for a non-video URL")
|
||||
|
||||
items = await scrape_tiktok_comments(
|
||||
["https://www.tiktok.com/@bob"], per_video=5, fetch_comments_fn=fake_fetch
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
assert items[0]["errorCode"] == "bad_url"
|
||||
|
||||
|
||||
async def test_comments_honor_limit_across_videos():
|
||||
async def fake_fetch(_url: str, _cap: int) -> list[dict]:
|
||||
return [_comment("1"), _comment("2")]
|
||||
|
||||
items = await scrape_tiktok_comments(
|
||||
[_VIDEO, "https://www.tiktok.com/@bob/video/456"],
|
||||
per_video=5,
|
||||
limit=3,
|
||||
fetch_comments_fn=fake_fetch,
|
||||
)
|
||||
|
||||
assert len(items) == 3
|
||||
Loading…
Add table
Add a link
Reference in a new issue