diff --git a/docker/.env.example b/docker/.env.example index 5daa0f3e8..86536e94c 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -449,6 +449,8 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # GOOGLE_MAPS_MICROS_PER_REVIEW=1500 # YOUTUBE_MICROS_PER_VIDEO=2500 # YOUTUBE_MICROS_PER_COMMENT=1500 +# TIKTOK_MICROS_PER_VIDEO=3500 +# TIKTOK_MICROS_PER_USER=2500 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 7e87c186d..eeb346a1b 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -286,6 +286,8 @@ MICROS_PER_PAGE=1000 # GOOGLE_MAPS_MICROS_PER_REVIEW=1500 # YOUTUBE_MICROS_PER_VIDEO=2500 # YOUTUBE_MICROS_PER_COMMENT=1500 +# TIKTOK_MICROS_PER_VIDEO=3500 +# TIKTOK_MICROS_PER_USER=2500 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. CREDIT_LOW_BALANCE_WARNING_MICROS=500000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py index e790d44f0..d826a8a85 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py @@ -1,4 +1,4 @@ -"""``tiktok`` sub-agent tools: the TikTok scrape capability verb.""" +"""``tiktok`` sub-agent tools: the TikTok scrape and user-search capability verbs.""" from __future__ import annotations @@ -9,12 +9,13 @@ from langchain_core.tools import BaseTool from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from app.capabilities.core.access.agent import build_capability_tools from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE +from app.capabilities.tiktok.user_search.definition import TIKTOK_USER_SEARCH NAME = "tiktok" RULESET = Ruleset(origin=NAME, rules=[]) -_CI_VERBS = [TIKTOK_SCRAPE] +_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_USER_SEARCH] def load_tools( diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 048797473..8bc9fbd14 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -36,6 +36,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO", BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT", BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO", + BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER", } @@ -53,6 +54,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.YOUTUBE_VIDEO: "video", BillingUnit.YOUTUBE_COMMENT: "comment", BillingUnit.TIKTOK_VIDEO: "video", + BillingUnit.TIKTOK_USER: "profile", } diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index b45641a2d..cbb01c2a6 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -26,6 +26,7 @@ class BillingUnit(StrEnum): YOUTUBE_VIDEO = "youtube_video" YOUTUBE_COMMENT = "youtube_comment" TIKTOK_VIDEO = "tiktok_video" + TIKTOK_USER = "tiktok_user" class BillableInput(Protocol): diff --git a/surfsense_backend/app/capabilities/tiktok/__init__.py b/surfsense_backend/app/capabilities/tiktok/__init__.py index ed3f32682..067c91c0a 100644 --- a/surfsense_backend/app/capabilities/tiktok/__init__.py +++ b/surfsense_backend/app/capabilities/tiktok/__init__.py @@ -3,3 +3,4 @@ from __future__ import annotations from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401 +from app.capabilities.tiktok.user_search import definition as _user_search # noqa: F401 diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/__init__.py b/surfsense_backend/app/capabilities/tiktok/user_search/__init__.py new file mode 100644 index 000000000..d4344968c --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/__init__.py @@ -0,0 +1,3 @@ +"""``tiktok.user_search``: find public TikTok accounts by keyword.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/definition.py b/surfsense_backend/app/capabilities/tiktok/user_search/definition.py new file mode 100644 index 000000000..6a37f879f --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/definition.py @@ -0,0 +1,26 @@ +"""``tiktok.user_search`` capability registration (billed per account; see config +``TIKTOK_MICROS_PER_USER``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.user_search.executor import build_user_search_executor +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) + +TIKTOK_USER_SEARCH = Capability( + name="tiktok.user_search", + description=( + "Find public TikTok accounts by keyword. Returns profile metadata " + "(name, followers, bio, verification) per matching account." + ), + input_schema=UserSearchInput, + output_schema=UserSearchOutput, + executor=build_user_search_executor(), + billing_unit=BillingUnit.TIKTOK_USER, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_USER_SEARCH) diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/executor.py b/surfsense_backend/app/capabilities/tiktok/user_search/executor.py new file mode 100644 index 000000000..e83f396bd --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/executor.py @@ -0,0 +1,39 @@ +"""``tiktok.user_search`` executor: queries -> scraper -> TikTok profile items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) +from app.proprietary.platforms.tiktok import search_tiktok_users + +SearchFn = Callable[..., Awaitable[list[dict]]] + + +def build_user_search_executor(search_fn: SearchFn | None = None) -> Executor: + """Bind the executor to a search fn (defaults to the proprietary actor).""" + search_fn = search_fn or search_tiktok_users + + async def execute(payload: UserSearchInput) -> UserSearchOutput: + emit_progress( + "starting", + "Searching TikTok accounts", + total=payload.max_items, + unit="item", + ) + items = await search_fn( + payload.queries, + per_query=payload.results_per_query, + limit=payload.max_items, + ) + emit_progress( + "done", f"Found {len(items)} account(s)", current=len(items), unit="item" + ) + return UserSearchOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/schemas.py b/surfsense_backend/app/capabilities/tiktok/user_search/schemas.py new file mode 100644 index 000000000..7cb2aecbb --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/schemas.py @@ -0,0 +1,56 @@ +"""``tiktok.user_search`` I/O contracts. + +Account discovery over ``TikTok``'s Users tab. Where video/general search is +login-walled for anonymous sessions, ``/api/search/user`` returns public account +records, so this verb exposes the one reliably-unblocked search path. Each result +is a :class:`TikTokProfileItem` (the same shape the profile verb emits). +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.capabilities.tiktok.scrape.schemas import ( + MAX_TIKTOK_ITEMS, + MAX_TIKTOK_SOURCES, +) +from app.proprietary.platforms.tiktok import TikTokProfileItem + + +class UserSearchInput(BaseModel): + queries: list[str] = Field( + min_length=1, + max_length=MAX_TIKTOK_SOURCES, + description="Keywords to search for TikTok accounts (e.g. names, brands).", + ) + results_per_query: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max accounts to return per query.", + ) + max_items: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max total accounts to return across all queries.", + ) + + @property + def estimated_units(self) -> int: + """Worst-case billable accounts for the pre-flight gate: ``max_items`` is a + hard cross-query ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class UserSearchOutput(BaseModel): + items: list[TikTokProfileItem] = Field( + default_factory=list, + description="One item per account found, in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned account = one billable unit; ErrorItems (``errorCode`` set, + for empty/withheld queries) are surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index c03ac3598..97abb5eda 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -717,6 +717,9 @@ class Config: # Browser-driven listings make TikTok heavier per item than the API-backed # video meter, so it sits a touch above YouTube's video rate. TIKTOK_MICROS_PER_VIDEO = int(os.getenv("TIKTOK_MICROS_PER_VIDEO", "3500")) + # User search returns lighter account records (name/followers/bio), priced + # below the video meter to mirror the cheaper account-discovery market. + TIKTOK_MICROS_PER_USER = int(os.getenv("TIKTOK_MICROS_PER_USER", "2500")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py index 27867df73..2947498ec 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py @@ -6,14 +6,16 @@ schema, the collector/generator, the video item shape, and the hard-block error. from __future__ import annotations -from .orchestrator import iter_tiktok, scrape_tiktok -from .schemas import TikTokScrapeInput, TikTokVideoItem +from .orchestrator import iter_tiktok, scrape_tiktok, search_tiktok_users +from .schemas import TikTokProfileItem, TikTokScrapeInput, TikTokVideoItem from .session import TikTokAccessBlockedError __all__ = [ "TikTokAccessBlockedError", + "TikTokProfileItem", "TikTokScrapeInput", "TikTokVideoItem", "iter_tiktok", "scrape_tiktok", + "search_tiktok_users", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index 86aefc2d7..f4712e724 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -6,6 +6,7 @@ from .author import parse_author, parse_profile from .hydration import extract_rehydration_data from .item_list import items_from_response from .scopes import user_info, video_item_struct +from .user_search import parse_search_user, users_from_response from .video import parse_video __all__ = [ @@ -13,7 +14,9 @@ __all__ = [ "items_from_response", "parse_author", "parse_profile", + "parse_search_user", "parse_video", "user_info", + "users_from_response", "video_item_struct", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/user_search.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/user_search.py new file mode 100644 index 000000000..5ffc2cd4d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/user_search.py @@ -0,0 +1,56 @@ +"""Parse the ``/api/search/user`` response into profile items. + +User search returns ``{"user_list": [{"user_info": {...}}, ...]}`` where each +``user_info`` uses the mobile-API snake_case shape (``uid``, ``unique_id``, +``follower_count``, ``total_favorited``, ``avatar_thumb.url_list``) — distinct +from the camelCase ``webapp.user-detail`` blob the profile flow reads, so it gets +its own mapping into the shared :class:`TikTokProfileItem` output contract. +""" + +from __future__ import annotations + +from typing import Any + +_PROFILE_URL = "https://www.tiktok.com/@{username}" + + +def users_from_response(body: Any) -> list[dict[str, Any]]: + """Return the ``user_info`` objects carried by one search response, or ``[]``.""" + if not isinstance(body, dict): + return [] + user_list = body.get("user_list") + if not isinstance(user_list, list): + return [] + return [ + entry["user_info"] + for entry in user_list + if isinstance(entry, dict) and isinstance(entry.get("user_info"), dict) + ] + + +def _avatar(user_info: dict[str, Any]) -> str | None: + thumb = user_info.get("avatar_thumb") + if isinstance(thumb, dict): + urls = thumb.get("url_list") + if isinstance(urls, list) and urls: + return urls[0] + return None + + +def parse_search_user(user_info: dict[str, Any]) -> dict[str, Any]: + """Map a search ``user_info`` to a :class:`TikTokProfileItem` output dict.""" + from ..schemas.items import TikTokProfileItem + + username = user_info.get("unique_id") + return TikTokProfileItem( + id=user_info.get("uid"), + name=username, + nickName=user_info.get("nickname"), + profileUrl=_PROFILE_URL.format(username=username) if username else None, + verified=bool(user_info.get("enterprise_verify_reason")), + signature=user_info.get("signature"), + avatar=_avatar(user_info), + fans=user_info.get("follower_count"), + heart=user_info.get("total_favorited"), + secUid=user_info.get("sec_uid"), + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py index a7809567d..1a8aeaacc 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py @@ -10,4 +10,7 @@ FetchFn = Callable[[str], Awaitable[str | None]] FetchListingFn = Callable[[str, int], Awaitable[list[dict]]] """Load a listing page and return up to ``count`` captured itemStructs.""" +FetchUsersFn = Callable[[str, int], Awaitable[list[dict]]] +"""Load a user-search page and return up to ``count`` captured ``user_info`` records.""" + FlowResult = AsyncIterator[dict] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/user_search.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/user_search.py new file mode 100644 index 000000000..872212e80 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/user_search.py @@ -0,0 +1,54 @@ +"""User-search flow: keyword -> public account records. + +Unlike video/general search (login-walled for anonymous sessions), the Users tab +hits ``/api/search/user`` and returns account records without a redirect. Each +query's results are deduped by uid, capped, and — when a query returns nothing — +degraded to one ErrorItem, mirroring the listing flow. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import quote + +from ..extraction import parse_search_user +from ..extraction.timestamps import now_iso +from ..schemas import ErrorItem +from . import FetchUsersFn + +_USER_SEARCH_URL = "https://www.tiktok.com/search/user?q={query}" +_EMPTY_MESSAGE = ( + "No accounts returned for this query. It may have no matches, or TikTok " + "withheld the results from anonymous access." +) + + +async def iter_user_search( + query: str, *, cap: int, fetch_users: FetchUsersFn +) -> AsyncIterator[dict[str, Any]]: + if cap <= 0: + return + url = _USER_SEARCH_URL.format(query=quote(query)) + seen: set[str] = set() + emitted = 0 + for user_info in await fetch_users(url, cap): + out = parse_search_user(user_info) + uid = out.get("id") + if uid is not None: + if uid in seen: + continue + seen.add(uid) + out["scrapedAt"] = now_iso() + yield out + emitted += 1 + if emitted >= cap: + return + if emitted == 0: + yield ErrorItem( + url=url, + input=query, + error=_EMPTY_MESSAGE, + errorCode="no_users", + scrapedAt=now_iso(), + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index 3903a1088..485a8dfff 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -12,12 +12,13 @@ from collections.abc import AsyncIterator from typing import Any from urllib.parse import quote -from .flows import FetchFn, FetchListingFn +from .flows import FetchFn, FetchListingFn, FetchUsersFn from .flows.listing import iter_listing from .flows.profile import iter_profile +from .flows.user_search import iter_user_search from .flows.video import iter_video from .schemas import TikTokScrapeInput -from .session import fetch_html, fetch_item_list +from .session import fetch_html, fetch_item_list, fetch_user_search from .targets import resolve_target from .targets.types import TikTokTarget @@ -100,3 +101,25 @@ async def scrape_tiktok( if limit is not None and len(results) >= limit: break return results + + +async def search_tiktok_users( + queries: list[str], + *, + per_query: int, + limit: int | None = None, + fetch_users: FetchUsersFn = fetch_user_search, +) -> list[dict[str, Any]]: + """Collect user-search account records across queries, honoring ``limit``.""" + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + for query in queries: + async for item in iter_user_search( + query, cap=per_query, fetch_users=fetch_users + ): + results.append(item) + emit_progress("searching", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + return results + return results diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py index 409aa5987..5e29403a0 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py @@ -4,7 +4,7 @@ from __future__ import annotations from .client import fetch_html from .errors import TikTokAccessBlockedError -from .listing import fetch_item_list +from .listing import fetch_item_list, fetch_user_search from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session __all__ = [ @@ -12,6 +12,7 @@ __all__ = [ "bind_proxy_holder", "fetch_html", "fetch_item_list", + "fetch_user_search", "open_proxy_holder", "proxy_session", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index cc84a48b5..e0bdd9255 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -19,6 +19,7 @@ from __future__ import annotations import asyncio import logging +from collections.abc import Callable from typing import Any from scrapling.fetchers import StealthyFetcher @@ -29,16 +30,20 @@ from app.proprietary.web_crawler.stealth import ( ) from app.utils.proxy import get_proxy_url -from ..extraction import items_from_response +from ..extraction import items_from_response, users_from_response logger = logging.getLogger(__name__) +ExtractFn = Callable[[Any], list[dict[str, Any]]] + # XHR paths that carry itemStructs for the three listing kinds. _ITEM_LIST_MARKERS = ( "/api/post/item_list", "/api/challenge/item_list", "/api/search/", ) +# The user-search XHR carries account records (user_list), not itemStructs. +_USER_SEARCH_MARKERS = ("/api/search/user",) _HOME_URL = "https://www.tiktok.com/" _MSTOKEN_COOKIE = "msToken" # Bounded scroll: a dead page can't loop forever, and a live one stops early @@ -57,24 +62,31 @@ def _has_mstoken(page: Any) -> bool: return False -def _build_page_action(collected: list[dict[str, Any]], url: str, target_count: int): - """A sync ``page_action`` that warms the session then captures item_list XHRs. +def _build_page_action( + collected: list[dict[str, Any]], + url: str, + target_count: int, + markers: tuple[str, ...], + extract: ExtractFn, +): + """A sync ``page_action`` that warms the session then captures matching XHRs. - A cold context returns an empty ``item_list`` body, so we first mint the - anonymous ``msToken`` (homepage hit), then navigate to the target with the - listener already attached so page-one fires into it; scrolling pages the rest. + A cold context returns an empty body, so we first mint the anonymous + ``msToken`` (homepage hit), then navigate to the target with the listener + already attached so page-one fires into it; scrolling pages the rest. + ``markers``/``extract`` select which XHRs to keep and how to unwrap them. """ def _on_response(response: Any) -> None: response_url = getattr(response, "url", "") - if not any(marker in response_url for marker in _ITEM_LIST_MARKERS): + if not any(marker in response_url for marker in markers): return try: body = response.json() except Exception: # An empty 200 (TikTok soft-block) or a body evicted before read. return - collected.extend(items_from_response(body)) + collected.extend(extract(body)) def _warm(page: Any) -> None: if _has_mstoken(page): @@ -110,7 +122,9 @@ def _build_page_action(collected: list[dict[str, Any]], url: str, target_count: return page_action -def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]: +def _fetch_sync( + url: str, target_count: int, markers: tuple[str, ...], extract: ExtractFn +) -> list[dict[str, Any]]: collected: list[dict[str, Any]] = [] kwargs = build_stealthy_kwargs(get_stealth_config()) StealthyFetcher.fetch( @@ -118,7 +132,9 @@ def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]: headless=True, network_idle=False, proxy=get_proxy_url(), - page_action=_build_page_action(collected, url, target_count), + page_action=_build_page_action( + collected, url, target_count, markers, extract + ), **kwargs, ) return collected[:target_count] @@ -126,4 +142,13 @@ def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]: async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` itemStructs from a listing page's XHRs.""" - return await asyncio.to_thread(_fetch_sync, page_url, target_count) + return await asyncio.to_thread( + _fetch_sync, page_url, target_count, _ITEM_LIST_MARKERS, items_from_response + ) + + +async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` ``user_info`` records from a user-search page.""" + return await asyncio.to_thread( + _fetch_sync, page_url, target_count, _USER_SEARCH_MARKERS, users_from_response + ) diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py index 3a85fe87f..893f3c2ae 100644 --- a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py @@ -10,6 +10,10 @@ from app.capabilities import ( from app.capabilities.core import BillingUnit from app.capabilities.core.store import get_capability from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) pytestmark = pytest.mark.unit @@ -21,3 +25,12 @@ def test_tiktok_scrape_is_registered_and_billed_per_video(): assert cap.input_schema is ScrapeInput assert cap.output_schema is ScrapeOutput assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO + + +def test_tiktok_user_search_is_registered_and_billed_per_profile(): + cap = get_capability("tiktok.user_search") + + assert cap.name == "tiktok.user_search" + assert cap.input_schema is UserSearchInput + assert cap.output_schema is UserSearchOutput + assert cap.billing_unit is BillingUnit.TIKTOK_USER diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/user_search/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py new file mode 100644 index 000000000..866968808 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py @@ -0,0 +1,49 @@ +"""``tiktok.user_search`` executor: verb input → search args → typed profile items. + +Boundary mocked: the proprietary search actor (injected fake). NOT mocked: the +verb's own payload→args forwarding and the dict→TikTokProfileItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.user_search.executor import build_user_search_executor +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) + +pytestmark = pytest.mark.unit + + +class _FakeSearch: + """Records the queries + kwargs it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[tuple[list[str], int, int | None]] = [] + + async def __call__( + self, queries: list[str], *, per_query: int, limit: int | None = None + ) -> list[dict]: + self.calls.append((queries, per_query, limit)) + return self._items + + +async def test_forwards_queries_and_limits_and_wraps_items(): + search = _FakeSearch([{"id": "1", "name": "nasa"}]) + execute = build_user_search_executor(search_fn=search) + + out = await execute( + UserSearchInput(queries=["nasa"], results_per_query=7, max_items=25) + ) + + assert isinstance(out, UserSearchOutput) + assert len(out.items) == 1 + assert out.items[0].name == "nasa" + + (queries, per_query, limit) = search.calls[0] + assert queries == ["nasa"] + assert per_query == 7 + assert limit == 25 diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py new file mode 100644 index 000000000..575db3fb5 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py @@ -0,0 +1,49 @@ +"""``tiktok.user_search`` input guards and billing: a query is required, bounded, +and ErrorItems are surfaced free.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) + +pytestmark = pytest.mark.unit + + +def test_rejects_input_with_no_query(): + with pytest.raises(ValidationError): + UserSearchInput(queries=[]) + + +def test_defaults_and_bounds(): + payload = UserSearchInput(queries=["nasa"]) + assert payload.max_items == 10 + assert payload.results_per_query == 10 + assert payload.estimated_units == 10 + with pytest.raises(ValidationError): + UserSearchInput(queries=["nasa"], max_items=0) + with pytest.raises(ValidationError): + UserSearchInput(queries=["nasa"], max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_rejects_more_queries_than_the_cap(): + too_many = [f"q{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] + with pytest.raises(ValidationError): + UserSearchInput(queries=too_many) + + +def test_error_items_are_not_billed(): + # Real accounts count; ErrorItems (empty/withheld queries) are surfaced free. + out = UserSearchOutput( + items=[ + {"id": "1", "name": "nasa"}, + {"errorCode": "no_users", "input": "ghost", "error": "empty"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py b/surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py new file mode 100644 index 000000000..845b5e22d --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py @@ -0,0 +1,70 @@ +"""User-search orchestration over a fake fetch (no network). + +Drives ``search_tiktok_users``: queries -> captured ``user_info`` -> profile items. +""" + +from __future__ import annotations + +from typing import Any + +from app.proprietary.platforms.tiktok import search_tiktok_users + + +def _user(uid: str, unique_id: str, followers: int = 10) -> dict[str, Any]: + return { + "uid": uid, + "unique_id": unique_id, + "nickname": unique_id.upper(), + "signature": "bio", + "follower_count": followers, + "total_favorited": 999, + "sec_uid": f"sec-{uid}", + "enterprise_verify_reason": "official" if uid == "1" else "", + "avatar_thumb": {"url_list": [f"https://cdn/{uid}.webp"]}, + } + + +async def test_user_search_parses_dedupes_and_caps(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_user("1", "nasa"), _user("1", "nasa"), _user("2", "nasa2")] + + items = await search_tiktok_users( + ["nasa"], per_query=2, fetch_users=fake_fetch + ) + + assert [i["id"] for i in items] == ["1", "2"] + first = items[0] + assert first["name"] == "nasa" + assert first["nickName"] == "NASA" + assert first["profileUrl"] == "https://www.tiktok.com/@nasa" + assert first["verified"] is True + assert first["fans"] == 10 + assert first["avatar"] == "https://cdn/1.webp" + assert first["secUid"] == "sec-1" + assert first["scrapedAt"] is not None + assert items[1]["verified"] is False + + +async def test_user_search_empty_query_emits_error_item(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [] + + items = await search_tiktok_users( + ["ghost"], per_query=5, fetch_users=fake_fetch + ) + + assert len(items) == 1 + assert items[0]["errorCode"] == "no_users" + assert items[0]["input"] == "ghost" + + +async def test_user_search_honors_limit_across_queries(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_user("1", "a"), _user("2", "b")] + + items = await search_tiktok_users( + ["q1", "q2"], per_query=5, limit=3, fetch_users=fake_fetch + ) + + # 2 from q1 + 1 from q2, then the cross-query limit stops it. + assert len(items) == 3