mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-16 23:01:06 +02:00
feat(tiktok): add tiktok.user_search verb for account discovery
Video/general search is login-walled for anonymous sessions, but the Users tab (/api/search/user) returns public account records without a redirect, so this exposes the one reliably-unblocked search path. A keyword yields TikTokProfileItems (name, followers, bio, verification), deduped per query, capped, and degraded to an ErrorItem when a query is empty/withheld. Reuses the browser capture (generalized over XHR markers + extractor) and the shared profile item shape. Billed per account on a new TIKTOK_USER meter (TIKTOK_MICROS_PER_USER), surfaced on the chat subagent alongside tiktok.scrape.
This commit is contained in:
parent
6652efd035
commit
192b6dc31a
24 changed files with 502 additions and 18 deletions
|
|
@ -286,6 +286,8 @@ MICROS_PER_PAGE=1000
|
|||
# GOOGLE_MAPS_MICROS_PER_REVIEW=1500
|
||||
# YOUTUBE_MICROS_PER_VIDEO=2500
|
||||
# YOUTUBE_MICROS_PER_COMMENT=1500
|
||||
# TIKTOK_MICROS_PER_VIDEO=3500
|
||||
# TIKTOK_MICROS_PER_USER=2500
|
||||
|
||||
# Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50.
|
||||
CREDIT_LOW_BALANCE_WARNING_MICROS=500000
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""``tiktok`` sub-agent tools: the TikTok scrape capability verb."""
|
||||
"""``tiktok`` sub-agent tools: the TikTok scrape and user-search capability verbs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
|
@ -9,12 +9,13 @@ from langchain_core.tools import BaseTool
|
|||
from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
|
||||
from app.capabilities.core.access.agent import build_capability_tools
|
||||
from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE
|
||||
from app.capabilities.tiktok.user_search.definition import TIKTOK_USER_SEARCH
|
||||
|
||||
NAME = "tiktok"
|
||||
|
||||
RULESET = Ruleset(origin=NAME, rules=[])
|
||||
|
||||
_CI_VERBS = [TIKTOK_SCRAPE]
|
||||
_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_USER_SEARCH]
|
||||
|
||||
|
||||
def load_tools(
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = {
|
|||
BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO",
|
||||
BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT",
|
||||
BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO",
|
||||
BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER",
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -53,6 +54,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = {
|
|||
BillingUnit.YOUTUBE_VIDEO: "video",
|
||||
BillingUnit.YOUTUBE_COMMENT: "comment",
|
||||
BillingUnit.TIKTOK_VIDEO: "video",
|
||||
BillingUnit.TIKTOK_USER: "profile",
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ class BillingUnit(StrEnum):
|
|||
YOUTUBE_VIDEO = "youtube_video"
|
||||
YOUTUBE_COMMENT = "youtube_comment"
|
||||
TIKTOK_VIDEO = "tiktok_video"
|
||||
TIKTOK_USER = "tiktok_user"
|
||||
|
||||
|
||||
class BillableInput(Protocol):
|
||||
|
|
|
|||
|
|
@ -3,3 +3,4 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401
|
||||
from app.capabilities.tiktok.user_search import definition as _user_search # noqa: F401
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
"""``tiktok.user_search``: find public TikTok accounts by keyword."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
"""``tiktok.user_search`` capability registration (billed per account; see config
|
||||
``TIKTOK_MICROS_PER_USER``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.core import BillingUnit, Capability, register_capability
|
||||
from app.capabilities.tiktok.user_search.executor import build_user_search_executor
|
||||
from app.capabilities.tiktok.user_search.schemas import (
|
||||
UserSearchInput,
|
||||
UserSearchOutput,
|
||||
)
|
||||
|
||||
TIKTOK_USER_SEARCH = Capability(
|
||||
name="tiktok.user_search",
|
||||
description=(
|
||||
"Find public TikTok accounts by keyword. Returns profile metadata "
|
||||
"(name, followers, bio, verification) per matching account."
|
||||
),
|
||||
input_schema=UserSearchInput,
|
||||
output_schema=UserSearchOutput,
|
||||
executor=build_user_search_executor(),
|
||||
billing_unit=BillingUnit.TIKTOK_USER,
|
||||
docs_url="/docs/connectors/native/tiktok",
|
||||
)
|
||||
|
||||
register_capability(TIKTOK_USER_SEARCH)
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
"""``tiktok.user_search`` executor: queries -> scraper -> TikTok profile items."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from app.capabilities.core import Executor
|
||||
from app.capabilities.core.progress import emit_progress
|
||||
from app.capabilities.tiktok.user_search.schemas import (
|
||||
UserSearchInput,
|
||||
UserSearchOutput,
|
||||
)
|
||||
from app.proprietary.platforms.tiktok import search_tiktok_users
|
||||
|
||||
SearchFn = Callable[..., Awaitable[list[dict]]]
|
||||
|
||||
|
||||
def build_user_search_executor(search_fn: SearchFn | None = None) -> Executor:
|
||||
"""Bind the executor to a search fn (defaults to the proprietary actor)."""
|
||||
search_fn = search_fn or search_tiktok_users
|
||||
|
||||
async def execute(payload: UserSearchInput) -> UserSearchOutput:
|
||||
emit_progress(
|
||||
"starting",
|
||||
"Searching TikTok accounts",
|
||||
total=payload.max_items,
|
||||
unit="item",
|
||||
)
|
||||
items = await search_fn(
|
||||
payload.queries,
|
||||
per_query=payload.results_per_query,
|
||||
limit=payload.max_items,
|
||||
)
|
||||
emit_progress(
|
||||
"done", f"Found {len(items)} account(s)", current=len(items), unit="item"
|
||||
)
|
||||
return UserSearchOutput(items=items)
|
||||
|
||||
return execute
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
"""``tiktok.user_search`` I/O contracts.
|
||||
|
||||
Account discovery over ``TikTok``'s Users tab. Where video/general search is
|
||||
login-walled for anonymous sessions, ``/api/search/user`` returns public account
|
||||
records, so this verb exposes the one reliably-unblocked search path. Each result
|
||||
is a :class:`TikTokProfileItem` (the same shape the profile verb emits).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.capabilities.tiktok.scrape.schemas import (
|
||||
MAX_TIKTOK_ITEMS,
|
||||
MAX_TIKTOK_SOURCES,
|
||||
)
|
||||
from app.proprietary.platforms.tiktok import TikTokProfileItem
|
||||
|
||||
|
||||
class UserSearchInput(BaseModel):
|
||||
queries: list[str] = Field(
|
||||
min_length=1,
|
||||
max_length=MAX_TIKTOK_SOURCES,
|
||||
description="Keywords to search for TikTok accounts (e.g. names, brands).",
|
||||
)
|
||||
results_per_query: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=MAX_TIKTOK_ITEMS,
|
||||
description="Max accounts to return per query.",
|
||||
)
|
||||
max_items: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=MAX_TIKTOK_ITEMS,
|
||||
description="Max total accounts to return across all queries.",
|
||||
)
|
||||
|
||||
@property
|
||||
def estimated_units(self) -> int:
|
||||
"""Worst-case billable accounts for the pre-flight gate: ``max_items`` is a
|
||||
hard cross-query ceiling (le=100), so no call can exceed it."""
|
||||
return self.max_items
|
||||
|
||||
|
||||
class UserSearchOutput(BaseModel):
|
||||
items: list[TikTokProfileItem] = Field(
|
||||
default_factory=list,
|
||||
description="One item per account found, in emission order.",
|
||||
)
|
||||
|
||||
@property
|
||||
def billable_units(self) -> int:
|
||||
"""One returned account = one billable unit; ErrorItems (``errorCode`` set,
|
||||
for empty/withheld queries) are surfaced but never charged."""
|
||||
return sum(1 for item in self.items if not getattr(item, "errorCode", None))
|
||||
|
|
@ -717,6 +717,9 @@ class Config:
|
|||
# Browser-driven listings make TikTok heavier per item than the API-backed
|
||||
# video meter, so it sits a touch above YouTube's video rate.
|
||||
TIKTOK_MICROS_PER_VIDEO = int(os.getenv("TIKTOK_MICROS_PER_VIDEO", "3500"))
|
||||
# User search returns lighter account records (name/followers/bio), priced
|
||||
# below the video meter to mirror the cheaper account-discovery market.
|
||||
TIKTOK_MICROS_PER_USER = int(os.getenv("TIKTOK_MICROS_PER_USER", "2500"))
|
||||
|
||||
# Low-balance WARNING threshold (micro-USD). Surfaced by the quota service
|
||||
# so the UI can nudge the user to top up / enable auto-reload. $0.50.
|
||||
|
|
|
|||
|
|
@ -6,14 +6,16 @@ schema, the collector/generator, the video item shape, and the hard-block error.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from .orchestrator import iter_tiktok, scrape_tiktok
|
||||
from .schemas import TikTokScrapeInput, TikTokVideoItem
|
||||
from .orchestrator import iter_tiktok, scrape_tiktok, search_tiktok_users
|
||||
from .schemas import TikTokProfileItem, TikTokScrapeInput, TikTokVideoItem
|
||||
from .session import TikTokAccessBlockedError
|
||||
|
||||
__all__ = [
|
||||
"TikTokAccessBlockedError",
|
||||
"TikTokProfileItem",
|
||||
"TikTokScrapeInput",
|
||||
"TikTokVideoItem",
|
||||
"iter_tiktok",
|
||||
"scrape_tiktok",
|
||||
"search_tiktok_users",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from .author import parse_author, parse_profile
|
|||
from .hydration import extract_rehydration_data
|
||||
from .item_list import items_from_response
|
||||
from .scopes import user_info, video_item_struct
|
||||
from .user_search import parse_search_user, users_from_response
|
||||
from .video import parse_video
|
||||
|
||||
__all__ = [
|
||||
|
|
@ -13,7 +14,9 @@ __all__ = [
|
|||
"items_from_response",
|
||||
"parse_author",
|
||||
"parse_profile",
|
||||
"parse_search_user",
|
||||
"parse_video",
|
||||
"user_info",
|
||||
"users_from_response",
|
||||
"video_item_struct",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
"""Parse the ``/api/search/user`` response into profile items.
|
||||
|
||||
User search returns ``{"user_list": [{"user_info": {...}}, ...]}`` where each
|
||||
``user_info`` uses the mobile-API snake_case shape (``uid``, ``unique_id``,
|
||||
``follower_count``, ``total_favorited``, ``avatar_thumb.url_list``) — distinct
|
||||
from the camelCase ``webapp.user-detail`` blob the profile flow reads, so it gets
|
||||
its own mapping into the shared :class:`TikTokProfileItem` output contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
_PROFILE_URL = "https://www.tiktok.com/@{username}"
|
||||
|
||||
|
||||
def users_from_response(body: Any) -> list[dict[str, Any]]:
|
||||
"""Return the ``user_info`` objects carried by one search response, or ``[]``."""
|
||||
if not isinstance(body, dict):
|
||||
return []
|
||||
user_list = body.get("user_list")
|
||||
if not isinstance(user_list, list):
|
||||
return []
|
||||
return [
|
||||
entry["user_info"]
|
||||
for entry in user_list
|
||||
if isinstance(entry, dict) and isinstance(entry.get("user_info"), dict)
|
||||
]
|
||||
|
||||
|
||||
def _avatar(user_info: dict[str, Any]) -> str | None:
|
||||
thumb = user_info.get("avatar_thumb")
|
||||
if isinstance(thumb, dict):
|
||||
urls = thumb.get("url_list")
|
||||
if isinstance(urls, list) and urls:
|
||||
return urls[0]
|
||||
return None
|
||||
|
||||
|
||||
def parse_search_user(user_info: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Map a search ``user_info`` to a :class:`TikTokProfileItem` output dict."""
|
||||
from ..schemas.items import TikTokProfileItem
|
||||
|
||||
username = user_info.get("unique_id")
|
||||
return TikTokProfileItem(
|
||||
id=user_info.get("uid"),
|
||||
name=username,
|
||||
nickName=user_info.get("nickname"),
|
||||
profileUrl=_PROFILE_URL.format(username=username) if username else None,
|
||||
verified=bool(user_info.get("enterprise_verify_reason")),
|
||||
signature=user_info.get("signature"),
|
||||
avatar=_avatar(user_info),
|
||||
fans=user_info.get("follower_count"),
|
||||
heart=user_info.get("total_favorited"),
|
||||
secUid=user_info.get("sec_uid"),
|
||||
).to_output()
|
||||
|
|
@ -10,4 +10,7 @@ FetchFn = Callable[[str], Awaitable[str | None]]
|
|||
FetchListingFn = Callable[[str, int], Awaitable[list[dict]]]
|
||||
"""Load a listing page and return up to ``count`` captured itemStructs."""
|
||||
|
||||
FetchUsersFn = Callable[[str, int], Awaitable[list[dict]]]
|
||||
"""Load a user-search page and return up to ``count`` captured ``user_info`` records."""
|
||||
|
||||
FlowResult = AsyncIterator[dict]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
"""User-search flow: keyword -> public account records.
|
||||
|
||||
Unlike video/general search (login-walled for anonymous sessions), the Users tab
|
||||
hits ``/api/search/user`` and returns account records without a redirect. Each
|
||||
query's results are deduped by uid, capped, and — when a query returns nothing —
|
||||
degraded to one ErrorItem, mirroring the listing flow.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from ..extraction import parse_search_user
|
||||
from ..extraction.timestamps import now_iso
|
||||
from ..schemas import ErrorItem
|
||||
from . import FetchUsersFn
|
||||
|
||||
_USER_SEARCH_URL = "https://www.tiktok.com/search/user?q={query}"
|
||||
_EMPTY_MESSAGE = (
|
||||
"No accounts returned for this query. It may have no matches, or TikTok "
|
||||
"withheld the results from anonymous access."
|
||||
)
|
||||
|
||||
|
||||
async def iter_user_search(
|
||||
query: str, *, cap: int, fetch_users: FetchUsersFn
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
if cap <= 0:
|
||||
return
|
||||
url = _USER_SEARCH_URL.format(query=quote(query))
|
||||
seen: set[str] = set()
|
||||
emitted = 0
|
||||
for user_info in await fetch_users(url, cap):
|
||||
out = parse_search_user(user_info)
|
||||
uid = out.get("id")
|
||||
if uid is not None:
|
||||
if uid in seen:
|
||||
continue
|
||||
seen.add(uid)
|
||||
out["scrapedAt"] = now_iso()
|
||||
yield out
|
||||
emitted += 1
|
||||
if emitted >= cap:
|
||||
return
|
||||
if emitted == 0:
|
||||
yield ErrorItem(
|
||||
url=url,
|
||||
input=query,
|
||||
error=_EMPTY_MESSAGE,
|
||||
errorCode="no_users",
|
||||
scrapedAt=now_iso(),
|
||||
).to_output()
|
||||
|
|
@ -12,12 +12,13 @@ from collections.abc import AsyncIterator
|
|||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from .flows import FetchFn, FetchListingFn
|
||||
from .flows import FetchFn, FetchListingFn, FetchUsersFn
|
||||
from .flows.listing import iter_listing
|
||||
from .flows.profile import iter_profile
|
||||
from .flows.user_search import iter_user_search
|
||||
from .flows.video import iter_video
|
||||
from .schemas import TikTokScrapeInput
|
||||
from .session import fetch_html, fetch_item_list
|
||||
from .session import fetch_html, fetch_item_list, fetch_user_search
|
||||
from .targets import resolve_target
|
||||
from .targets.types import TikTokTarget
|
||||
|
||||
|
|
@ -100,3 +101,25 @@ async def scrape_tiktok(
|
|||
if limit is not None and len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
async def search_tiktok_users(
|
||||
queries: list[str],
|
||||
*,
|
||||
per_query: int,
|
||||
limit: int | None = None,
|
||||
fetch_users: FetchUsersFn = fetch_user_search,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Collect user-search account records across queries, honoring ``limit``."""
|
||||
from app.capabilities.core.progress import emit_progress
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
for query in queries:
|
||||
async for item in iter_user_search(
|
||||
query, cap=per_query, fetch_users=fetch_users
|
||||
):
|
||||
results.append(item)
|
||||
emit_progress("searching", current=len(results), total=limit, unit="item")
|
||||
if limit is not None and len(results) >= limit:
|
||||
return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from __future__ import annotations
|
|||
|
||||
from .client import fetch_html
|
||||
from .errors import TikTokAccessBlockedError
|
||||
from .listing import fetch_item_list
|
||||
from .listing import fetch_item_list, fetch_user_search
|
||||
from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session
|
||||
|
||||
__all__ = [
|
||||
|
|
@ -12,6 +12,7 @@ __all__ = [
|
|||
"bind_proxy_holder",
|
||||
"fetch_html",
|
||||
"fetch_item_list",
|
||||
"fetch_user_search",
|
||||
"open_proxy_holder",
|
||||
"proxy_session",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from scrapling.fetchers import StealthyFetcher
|
||||
|
|
@ -29,16 +30,20 @@ from app.proprietary.web_crawler.stealth import (
|
|||
)
|
||||
from app.utils.proxy import get_proxy_url
|
||||
|
||||
from ..extraction import items_from_response
|
||||
from ..extraction import items_from_response, users_from_response
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ExtractFn = Callable[[Any], list[dict[str, Any]]]
|
||||
|
||||
# XHR paths that carry itemStructs for the three listing kinds.
|
||||
_ITEM_LIST_MARKERS = (
|
||||
"/api/post/item_list",
|
||||
"/api/challenge/item_list",
|
||||
"/api/search/",
|
||||
)
|
||||
# The user-search XHR carries account records (user_list), not itemStructs.
|
||||
_USER_SEARCH_MARKERS = ("/api/search/user",)
|
||||
_HOME_URL = "https://www.tiktok.com/"
|
||||
_MSTOKEN_COOKIE = "msToken"
|
||||
# Bounded scroll: a dead page can't loop forever, and a live one stops early
|
||||
|
|
@ -57,24 +62,31 @@ def _has_mstoken(page: Any) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def _build_page_action(collected: list[dict[str, Any]], url: str, target_count: int):
|
||||
"""A sync ``page_action`` that warms the session then captures item_list XHRs.
|
||||
def _build_page_action(
|
||||
collected: list[dict[str, Any]],
|
||||
url: str,
|
||||
target_count: int,
|
||||
markers: tuple[str, ...],
|
||||
extract: ExtractFn,
|
||||
):
|
||||
"""A sync ``page_action`` that warms the session then captures matching XHRs.
|
||||
|
||||
A cold context returns an empty ``item_list`` body, so we first mint the
|
||||
anonymous ``msToken`` (homepage hit), then navigate to the target with the
|
||||
listener already attached so page-one fires into it; scrolling pages the rest.
|
||||
A cold context returns an empty body, so we first mint the anonymous
|
||||
``msToken`` (homepage hit), then navigate to the target with the listener
|
||||
already attached so page-one fires into it; scrolling pages the rest.
|
||||
``markers``/``extract`` select which XHRs to keep and how to unwrap them.
|
||||
"""
|
||||
|
||||
def _on_response(response: Any) -> None:
|
||||
response_url = getattr(response, "url", "")
|
||||
if not any(marker in response_url for marker in _ITEM_LIST_MARKERS):
|
||||
if not any(marker in response_url for marker in markers):
|
||||
return
|
||||
try:
|
||||
body = response.json()
|
||||
except Exception:
|
||||
# An empty 200 (TikTok soft-block) or a body evicted before read.
|
||||
return
|
||||
collected.extend(items_from_response(body))
|
||||
collected.extend(extract(body))
|
||||
|
||||
def _warm(page: Any) -> None:
|
||||
if _has_mstoken(page):
|
||||
|
|
@ -110,7 +122,9 @@ def _build_page_action(collected: list[dict[str, Any]], url: str, target_count:
|
|||
return page_action
|
||||
|
||||
|
||||
def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]:
|
||||
def _fetch_sync(
|
||||
url: str, target_count: int, markers: tuple[str, ...], extract: ExtractFn
|
||||
) -> list[dict[str, Any]]:
|
||||
collected: list[dict[str, Any]] = []
|
||||
kwargs = build_stealthy_kwargs(get_stealth_config())
|
||||
StealthyFetcher.fetch(
|
||||
|
|
@ -118,7 +132,9 @@ def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]:
|
|||
headless=True,
|
||||
network_idle=False,
|
||||
proxy=get_proxy_url(),
|
||||
page_action=_build_page_action(collected, url, target_count),
|
||||
page_action=_build_page_action(
|
||||
collected, url, target_count, markers, extract
|
||||
),
|
||||
**kwargs,
|
||||
)
|
||||
return collected[:target_count]
|
||||
|
|
@ -126,4 +142,13 @@ def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]:
|
|||
|
||||
async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]:
|
||||
"""Return up to ``target_count`` itemStructs from a listing page's XHRs."""
|
||||
return await asyncio.to_thread(_fetch_sync, page_url, target_count)
|
||||
return await asyncio.to_thread(
|
||||
_fetch_sync, page_url, target_count, _ITEM_LIST_MARKERS, items_from_response
|
||||
)
|
||||
|
||||
|
||||
async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]:
|
||||
"""Return up to ``target_count`` ``user_info`` records from a user-search page."""
|
||||
return await asyncio.to_thread(
|
||||
_fetch_sync, page_url, target_count, _USER_SEARCH_MARKERS, users_from_response
|
||||
)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@ from app.capabilities import (
|
|||
from app.capabilities.core import BillingUnit
|
||||
from app.capabilities.core.store import get_capability
|
||||
from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
from app.capabilities.tiktok.user_search.schemas import (
|
||||
UserSearchInput,
|
||||
UserSearchOutput,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
|
@ -21,3 +25,12 @@ def test_tiktok_scrape_is_registered_and_billed_per_video():
|
|||
assert cap.input_schema is ScrapeInput
|
||||
assert cap.output_schema is ScrapeOutput
|
||||
assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO
|
||||
|
||||
|
||||
def test_tiktok_user_search_is_registered_and_billed_per_profile():
|
||||
cap = get_capability("tiktok.user_search")
|
||||
|
||||
assert cap.name == "tiktok.user_search"
|
||||
assert cap.input_schema is UserSearchInput
|
||||
assert cap.output_schema is UserSearchOutput
|
||||
assert cap.billing_unit is BillingUnit.TIKTOK_USER
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
"""``tiktok.user_search`` executor: verb input → search args → typed profile items.
|
||||
|
||||
Boundary mocked: the proprietary search actor (injected fake). NOT mocked: the
|
||||
verb's own payload→args forwarding and the dict→TikTokProfileItem wrapping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.capabilities.tiktok.user_search.executor import build_user_search_executor
|
||||
from app.capabilities.tiktok.user_search.schemas import (
|
||||
UserSearchInput,
|
||||
UserSearchOutput,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class _FakeSearch:
|
||||
"""Records the queries + kwargs it was called with; returns canned items."""
|
||||
|
||||
def __init__(self, items: list[dict]):
|
||||
self._items = items
|
||||
self.calls: list[tuple[list[str], int, int | None]] = []
|
||||
|
||||
async def __call__(
|
||||
self, queries: list[str], *, per_query: int, limit: int | None = None
|
||||
) -> list[dict]:
|
||||
self.calls.append((queries, per_query, limit))
|
||||
return self._items
|
||||
|
||||
|
||||
async def test_forwards_queries_and_limits_and_wraps_items():
|
||||
search = _FakeSearch([{"id": "1", "name": "nasa"}])
|
||||
execute = build_user_search_executor(search_fn=search)
|
||||
|
||||
out = await execute(
|
||||
UserSearchInput(queries=["nasa"], results_per_query=7, max_items=25)
|
||||
)
|
||||
|
||||
assert isinstance(out, UserSearchOutput)
|
||||
assert len(out.items) == 1
|
||||
assert out.items[0].name == "nasa"
|
||||
|
||||
(queries, per_query, limit) = search.calls[0]
|
||||
assert queries == ["nasa"]
|
||||
assert per_query == 7
|
||||
assert limit == 25
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
"""``tiktok.user_search`` input guards and billing: a query is required, bounded,
|
||||
and ErrorItems are surfaced free."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES
|
||||
from app.capabilities.tiktok.user_search.schemas import (
|
||||
UserSearchInput,
|
||||
UserSearchOutput,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_rejects_input_with_no_query():
|
||||
with pytest.raises(ValidationError):
|
||||
UserSearchInput(queries=[])
|
||||
|
||||
|
||||
def test_defaults_and_bounds():
|
||||
payload = UserSearchInput(queries=["nasa"])
|
||||
assert payload.max_items == 10
|
||||
assert payload.results_per_query == 10
|
||||
assert payload.estimated_units == 10
|
||||
with pytest.raises(ValidationError):
|
||||
UserSearchInput(queries=["nasa"], max_items=0)
|
||||
with pytest.raises(ValidationError):
|
||||
UserSearchInput(queries=["nasa"], max_items=MAX_TIKTOK_ITEMS + 1)
|
||||
|
||||
|
||||
def test_rejects_more_queries_than_the_cap():
|
||||
too_many = [f"q{i}" for i in range(MAX_TIKTOK_SOURCES + 1)]
|
||||
with pytest.raises(ValidationError):
|
||||
UserSearchInput(queries=too_many)
|
||||
|
||||
|
||||
def test_error_items_are_not_billed():
|
||||
# Real accounts count; ErrorItems (empty/withheld queries) are surfaced free.
|
||||
out = UserSearchOutput(
|
||||
items=[
|
||||
{"id": "1", "name": "nasa"},
|
||||
{"errorCode": "no_users", "input": "ghost", "error": "empty"},
|
||||
]
|
||||
)
|
||||
assert len(out.items) == 2
|
||||
assert out.billable_units == 1
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
"""User-search orchestration over a fake fetch (no network).
|
||||
|
||||
Drives ``search_tiktok_users``: queries -> captured ``user_info`` -> profile items.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.proprietary.platforms.tiktok import search_tiktok_users
|
||||
|
||||
|
||||
def _user(uid: str, unique_id: str, followers: int = 10) -> dict[str, Any]:
|
||||
return {
|
||||
"uid": uid,
|
||||
"unique_id": unique_id,
|
||||
"nickname": unique_id.upper(),
|
||||
"signature": "bio",
|
||||
"follower_count": followers,
|
||||
"total_favorited": 999,
|
||||
"sec_uid": f"sec-{uid}",
|
||||
"enterprise_verify_reason": "official" if uid == "1" else "",
|
||||
"avatar_thumb": {"url_list": [f"https://cdn/{uid}.webp"]},
|
||||
}
|
||||
|
||||
|
||||
async def test_user_search_parses_dedupes_and_caps():
|
||||
async def fake_fetch(_url: str, _cap: int) -> list[dict]:
|
||||
return [_user("1", "nasa"), _user("1", "nasa"), _user("2", "nasa2")]
|
||||
|
||||
items = await search_tiktok_users(
|
||||
["nasa"], per_query=2, fetch_users=fake_fetch
|
||||
)
|
||||
|
||||
assert [i["id"] for i in items] == ["1", "2"]
|
||||
first = items[0]
|
||||
assert first["name"] == "nasa"
|
||||
assert first["nickName"] == "NASA"
|
||||
assert first["profileUrl"] == "https://www.tiktok.com/@nasa"
|
||||
assert first["verified"] is True
|
||||
assert first["fans"] == 10
|
||||
assert first["avatar"] == "https://cdn/1.webp"
|
||||
assert first["secUid"] == "sec-1"
|
||||
assert first["scrapedAt"] is not None
|
||||
assert items[1]["verified"] is False
|
||||
|
||||
|
||||
async def test_user_search_empty_query_emits_error_item():
|
||||
async def fake_fetch(_url: str, _cap: int) -> list[dict]:
|
||||
return []
|
||||
|
||||
items = await search_tiktok_users(
|
||||
["ghost"], per_query=5, fetch_users=fake_fetch
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
assert items[0]["errorCode"] == "no_users"
|
||||
assert items[0]["input"] == "ghost"
|
||||
|
||||
|
||||
async def test_user_search_honors_limit_across_queries():
|
||||
async def fake_fetch(_url: str, _cap: int) -> list[dict]:
|
||||
return [_user("1", "a"), _user("2", "b")]
|
||||
|
||||
items = await search_tiktok_users(
|
||||
["q1", "q2"], per_query=5, limit=3, fetch_users=fake_fetch
|
||||
)
|
||||
|
||||
# 2 from q1 + 1 from q2, then the cross-query limit stops it.
|
||||
assert len(items) == 3
|
||||
Loading…
Add table
Add a link
Reference in a new issue