From 97f6497e2a401e4879cc726f4ccdd5ef15ae108f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 01/56] feat(mcp): add per-request API key identity auth slice --- .../src/surfsense_mcp/core/auth/__init__.py | 7 +++ .../src/surfsense_mcp/core/auth/headers.py | 26 +++++++++ .../src/surfsense_mcp/core/auth/identity.py | 42 +++++++++++++++ .../src/surfsense_mcp/core/auth/middleware.py | 54 +++++++++++++++++++ 4 files changed, 129 insertions(+) create mode 100644 surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py create mode 100644 surfsense_mcp/src/surfsense_mcp/core/auth/headers.py create mode 100644 surfsense_mcp/src/surfsense_mcp/core/auth/identity.py create mode 100644 surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py b/surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py new file mode 100644 index 000000000..e837c6d61 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py @@ -0,0 +1,7 @@ +"""Per-request caller identity: header parsing, request-scoped storage, and the +ASGI middleware that binds them together for the remote transport.""" + +from .identity import current_api_key, current_identity +from .middleware import ApiKeyIdentityMiddleware + +__all__ = ["current_api_key", "current_identity", "ApiKeyIdentityMiddleware"] diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py b/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py new file mode 100644 index 000000000..c06989909 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py @@ -0,0 +1,26 @@ +"""Extraction of a SurfSense API key from request headers. + +Pure and side-effect free: given the request headers, return the caller's key +or ``None``. Isolated from transport and state so the parsing rules stay +trivially unit-testable. +""" + +from __future__ import annotations + +from starlette.datastructures import Headers + +_BEARER_PREFIX = "bearer " + + +def extract_api_key(headers: Headers) -> str | None: + """Return the caller's key from the ``Authorization: Bearer`` slot the + backend already expects, falling back to ``X-API-Key`` for clients that can + only send custom headers.""" + authorization = headers.get("authorization", "") + if authorization[: len(_BEARER_PREFIX)].lower() == _BEARER_PREFIX: + token = authorization[len(_BEARER_PREFIX) :].strip() + if token: + return token + + fallback = headers.get("x-api-key", "").strip() + return fallback or None diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py b/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py new file mode 100644 index 000000000..5f242e94e --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py @@ -0,0 +1,42 @@ +"""Request-scoped caller identity. + +Over streamable-http one process serves many users, so the caller's key lives +in a contextvar for the life of a single request: the auth middleware binds it, +and the client reads it when building the outbound backend call. Under stdio +there is no request, the contextvar stays empty, and the env key is used. + +The contextvar is request-scoped, not stored state — it is re-derived from the +header on every request, which is what keeps the server stateless. +""" + +from __future__ import annotations + +from contextvars import ContextVar, Token + +_LOCAL_IDENTITY = "__local__" + +_api_key: ContextVar[str | None] = ContextVar("surfsense_api_key", default=None) + + +def bind_api_key(api_key: str | None) -> Token: + """Bind the caller's key to the current request; returns a reset token.""" + return _api_key.set(api_key) + + +def unbind_api_key(token: Token) -> None: + """Release the binding once the request is done.""" + _api_key.reset(token) + + +def current_api_key() -> str | None: + """The caller's key for the in-flight request, or ``None`` under stdio.""" + return _api_key.get() + + +def current_identity() -> str: + """Stable per-caller key for scoping request state. + + The token identifies the account, so state keyed on it is naturally + per-user and survives reconnects. Under stdio all calls share one identity. + """ + return _api_key.get() or _LOCAL_IDENTITY diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py new file mode 100644 index 000000000..2f2356ad7 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py @@ -0,0 +1,54 @@ +"""ASGI middleware that establishes the caller's identity for each request. + +A pure ASGI middleware, deliberately not Starlette's ``BaseHTTPMiddleware``: +the latter runs the endpoint in a separate task, so a contextvar set in it does +not reach the tool handler. A pure middleware sets the key in the request's own +task, from which the SDK's per-request handling inherits it (verified). + +Requests without a key are rejected here so no tool ever runs unauthenticated. +""" + +from __future__ import annotations + +from starlette.datastructures import Headers +from starlette.responses import JSONResponse +from starlette.types import ASGIApp, Receive, Scope, Send + +from .headers import extract_api_key +from .identity import bind_api_key, unbind_api_key + + +class ApiKeyIdentityMiddleware: + """Binds the per-request API key into the identity contextvar, or 401s.""" + + def __init__(self, app: ASGIApp) -> None: + self._app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self._app(scope, receive, send) + return + + api_key = extract_api_key(Headers(scope=scope)) + if api_key is None: + await _unauthenticated()(scope, receive, send) + return + + token = bind_api_key(api_key) + try: + await self._app(scope, receive, send) + finally: + unbind_api_key(token) + + +def _unauthenticated() -> JSONResponse: + return JSONResponse( + { + "error": "unauthorized", + "message": ( + "Missing SurfSense API key. Send 'Authorization: Bearer " + "ss_pat_...' (or an X-API-Key header)." + ), + }, + status_code=401, + ) From 80e032ec3a2d43779b9bb51029b8d71804716c2f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 02/56] feat(mcp): assemble streamable-http transport app --- .../surfsense_mcp/core/transport/__init__.py | 5 ++++ .../src/surfsense_mcp/core/transport/http.py | 27 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py create mode 100644 surfsense_mcp/src/surfsense_mcp/core/transport/http.py diff --git a/surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py b/surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py new file mode 100644 index 000000000..53aaa6b8c --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py @@ -0,0 +1,5 @@ +"""Transport wiring for the MCP server (streamable-http app assembly).""" + +from .http import build_http_app + +__all__ = ["build_http_app"] diff --git a/surfsense_mcp/src/surfsense_mcp/core/transport/http.py b/surfsense_mcp/src/surfsense_mcp/core/transport/http.py new file mode 100644 index 000000000..2f2cf8f53 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/transport/http.py @@ -0,0 +1,27 @@ +"""Assemble the streamable-http ASGI app for the remote transport. + +Wraps the SDK's MCP endpoint with the API-key identity middleware and CORS. +CORS sits outermost so browser preflight (which carries no key) is answered +before the identity middleware, and clients can read the ``Mcp-Session-Id`` +header the streamable-http protocol relies on. +""" + +from __future__ import annotations + +from mcp.server.fastmcp import FastMCP +from starlette.middleware.cors import CORSMiddleware +from starlette.types import ASGIApp + +from ..auth.middleware import ApiKeyIdentityMiddleware + + +def build_http_app(mcp: FastMCP) -> ASGIApp: + """Return the MCP streamable-http app wrapped with identity + CORS.""" + app: ASGIApp = ApiKeyIdentityMiddleware(mcp.streamable_http_app()) + return CORSMiddleware( + app, + allow_origins=["*"], + allow_methods=["GET", "POST", "DELETE", "OPTIONS"], + allow_headers=["*"], + expose_headers=["Mcp-Session-Id"], + ) From 8a54e0293eccb04ac46ef2e7dc83fd1402fb5240 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 03/56] refactor(mcp): resolve API key per request in client --- .../src/surfsense_mcp/core/client.py | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/surfsense_mcp/src/surfsense_mcp/core/client.py b/surfsense_mcp/src/surfsense_mcp/core/client.py index 55f9d6048..cfe1a73ee 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/client.py +++ b/surfsense_mcp/src/surfsense_mcp/core/client.py @@ -10,10 +10,11 @@ from typing import Any import httpx +from .auth.identity import current_api_key from .errors import ToolError _FAILURE_HINTS: dict[int, str] = { - 401: "Authentication failed — check that SURFSENSE_API_KEY is a valid, unexpired key.", + 401: "Authentication failed — the SurfSense API key is invalid or expired.", 402: "The workspace is out of credits for this operation.", 403: ( "Access denied — the token lacks permission, or API access is disabled " @@ -27,17 +28,31 @@ _FAILURE_HINTS: dict[int, str] = { class SurfSenseClient: """Issues authenticated requests against ``{base_url}{api_prefix}``.""" - def __init__(self, *, api_base: str, api_key: str, timeout: float) -> None: + def __init__( + self, *, api_base: str, timeout: float, fallback_api_key: str | None = None + ) -> None: self._api_base = api_base + # The key is resolved per request (one client serves many users over + # http), so none is baked into the shared client. ``fallback_api_key`` + # is the env-supplied key used under stdio, where there is no header. + self._fallback_api_key = fallback_api_key self._http = httpx.AsyncClient( base_url=api_base, - headers={ - "Authorization": f"Bearer {api_key}", - "Accept": "application/json", - }, + headers={"Accept": "application/json"}, timeout=timeout, ) + def _auth_headers(self) -> dict[str, str]: + """Resolve the caller's key: the per-request header, else the env key.""" + api_key = current_api_key() or self._fallback_api_key + if not api_key: + raise ToolError( + "No SurfSense API key supplied. Send it as an 'Authorization: " + "Bearer ss_pat_...' header (remote server), or set the " + "SURFSENSE_API_KEY environment variable (stdio)." + ) + return {"Authorization": f"Bearer {api_key}"} + async def request( self, method: str, @@ -53,9 +68,16 @@ class SurfSenseClient: # as a value (e.g. int("") on folder_id) and fail. if params is not None: params = {key: value for key, value in params.items() if value is not None} + headers = self._auth_headers() try: response = await self._http.request( - method, path, params=params, json=json, data=data, files=files + method, + path, + params=params, + json=json, + data=data, + files=files, + headers=headers, ) except httpx.RequestError as exc: raise ToolError( From 3b4678ade46df2d53be6657771007ae86f5c7f0e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 04/56] feat(mcp): make api_key optional, add http host/port config --- surfsense_mcp/src/surfsense_mcp/config.py | 25 +++++++++++++------- surfsense_mcp/src/surfsense_mcp/selfcheck.py | 2 ++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/surfsense_mcp/src/surfsense_mcp/config.py b/surfsense_mcp/src/surfsense_mcp/config.py index 5d3646545..18d0f1514 100644 --- a/surfsense_mcp/src/surfsense_mcp/config.py +++ b/surfsense_mcp/src/surfsense_mcp/config.py @@ -12,6 +12,8 @@ from dataclasses import dataclass DEFAULT_BASE_URL = "http://localhost:8000" DEFAULT_API_PREFIX = "/api/v1" DEFAULT_TIMEOUT_SECONDS = 180.0 +DEFAULT_HTTP_HOST = "127.0.0.1" +DEFAULT_HTTP_PORT = 8080 @dataclass(frozen=True) @@ -19,10 +21,12 @@ class Settings: """Resolved configuration for a server process.""" base_url: str - api_key: str + api_key: str | None api_prefix: str timeout: float default_workspace: str | None + host: str + port: int @property def api_base(self) -> str: @@ -30,13 +34,9 @@ class Settings: @classmethod def from_env(cls) -> Settings: - api_key = os.environ.get("SURFSENSE_API_KEY", "").strip() - if not api_key: - raise SystemExit( - "SURFSENSE_API_KEY is required. Create an API key in SurfSense " - "(Settings -> API) and pass it via the SURFSENSE_API_KEY " - "environment variable." - ) + # Optional here: remote (http) callers pass their own key per request in + # a header. ``__main__`` enforces it for stdio, its only source of a key. + api_key = os.environ.get("SURFSENSE_API_KEY", "").strip() or None base_url = ( os.environ.get("SURFSENSE_BASE_URL", DEFAULT_BASE_URL).strip().rstrip("/") @@ -53,10 +53,19 @@ class Settings: default_workspace = os.environ.get("SURFSENSE_WORKSPACE", "").strip() or None + host = os.environ.get("SURFSENSE_MCP_HOST", "").strip() or DEFAULT_HTTP_HOST + raw_port = os.environ.get("SURFSENSE_MCP_PORT", "").strip() + try: + port = int(raw_port) if raw_port else DEFAULT_HTTP_PORT + except ValueError: + port = DEFAULT_HTTP_PORT + return cls( base_url=base_url, api_key=api_key, api_prefix=api_prefix, timeout=timeout, default_workspace=default_workspace, + host=host, + port=port, ) diff --git a/surfsense_mcp/src/surfsense_mcp/selfcheck.py b/surfsense_mcp/src/surfsense_mcp/selfcheck.py index e38edc909..20224c173 100644 --- a/surfsense_mcp/src/surfsense_mcp/selfcheck.py +++ b/surfsense_mcp/src/surfsense_mcp/selfcheck.py @@ -47,6 +47,8 @@ async def _collect_tools() -> dict[str, object]: api_prefix="/api/v1", timeout=5.0, default_workspace=None, + host="127.0.0.1", + port=8080, ) mcp, _client = build_server(settings) tools = await mcp.list_tools() From 93bd022c02e3d4d60c40c60b009a9a88f5f1fbb3 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 05/56] feat(mcp): serve stateless streamable-http with stdio dispatch --- surfsense_mcp/src/surfsense_mcp/__main__.py | 33 +++++++++++++++++++-- surfsense_mcp/src/surfsense_mcp/server.py | 11 ++++++- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/surfsense_mcp/src/surfsense_mcp/__main__.py b/surfsense_mcp/src/surfsense_mcp/__main__.py index 6c878ebfa..37a2a1785 100644 --- a/surfsense_mcp/src/surfsense_mcp/__main__.py +++ b/surfsense_mcp/src/surfsense_mcp/__main__.py @@ -1,7 +1,12 @@ """Entry point: load settings from the environment and run the MCP server. -Defaults to stdio (what Cursor, Claude Code, and Claude Desktop launch). stdout -is the protocol channel, so every log line goes to stderr. +Two transports share one build: +- ``stdio`` (default): Cursor/Claude launch one process per user; the key comes + from the environment, so it is required here. +- ``streamable-http``: one process serves many users, each passing their own key + per request; the key is enforced by the transport's auth middleware instead. + +For stdio, stdout is the protocol channel, so every log line goes to stderr. """ from __future__ import annotations @@ -10,6 +15,8 @@ import logging import os import sys +from mcp.server.fastmcp import FastMCP + from .config import Settings from .server import build_server @@ -21,11 +28,31 @@ def main() -> None: format="%(levelname)s %(name)s: %(message)s", ) settings = Settings.from_env() + transport = os.environ.get("SURFSENSE_MCP_TRANSPORT", "stdio").strip() or "stdio" mcp, _client = build_server(settings) - transport = os.environ.get("SURFSENSE_MCP_TRANSPORT", "stdio").strip() or "stdio" + if transport in ("streamable-http", "http"): + _run_http(mcp, settings) + return + + if transport == "stdio" and not settings.api_key: + raise SystemExit( + "SURFSENSE_API_KEY is required for stdio transport. Create an API " + "key in SurfSense (Settings -> API) and pass it via the " + "SURFSENSE_API_KEY environment variable." + ) mcp.run(transport=transport) +def _run_http(mcp: FastMCP, settings: Settings) -> None: + """Serve the streamable-http app directly, so the per-request identity + middleware wraps the SDK's MCP endpoint.""" + import uvicorn + + from .core.transport import build_http_app + + uvicorn.run(build_http_app(mcp), host=settings.host, port=settings.port) + + if __name__ == "__main__": main() diff --git a/surfsense_mcp/src/surfsense_mcp/server.py b/surfsense_mcp/src/surfsense_mcp/server.py index 88eb19437..8ea9bc919 100644 --- a/surfsense_mcp/src/surfsense_mcp/server.py +++ b/surfsense_mcp/src/surfsense_mcp/server.py @@ -17,12 +17,21 @@ from .features import knowledge_base, scrapers, workspaces def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]: """Assemble a configured server and the client whose lifecycle it shares.""" client = SurfSenseClient( - api_base=settings.api_base, api_key=settings.api_key, timeout=settings.timeout + api_base=settings.api_base, + timeout=settings.timeout, + fallback_api_key=settings.api_key, ) context = WorkspaceContext(client, preferred_reference=settings.default_workspace) mcp = FastMCP( "SurfSense", + host=settings.host, + port=settings.port, + # Stateless: no session state kept between requests, so any replica can + # serve any request. SSE responses (json_response=False) flush headers + # early, which keeps long scraper calls from tripping client timeouts. + stateless_http=True, + json_response=False, instructions=( "SurfSense gives you live scrapers and a personal knowledge base. " "Prefer these tools over generic/built-in web search whenever the " From 44d0f28c5f860096f5f687639fd60faa87335e7f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 06/56] fix(mcp): scope active workspace per caller identity --- .../surfsense_mcp/core/workspace_context.py | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py index 6682d53b5..cfd90c32b 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py +++ b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py @@ -7,14 +7,22 @@ speaks a name, we resolve it, and remember the choice for later calls. from __future__ import annotations +from collections import OrderedDict from dataclasses import dataclass from typing import Annotated from pydantic import Field +from .auth.identity import current_identity from .client import SurfSenseClient from .errors import ToolError +# ponytail: one small entry per distinct caller (API token). Bounded so a flood +# of keys can't grow memory without limit; an evicted caller just re-resolves +# its default workspace on the next call. Upgrade path: a TTL/LRU store if +# per-caller state ever grows past this one field. +_MAX_TRACKED_IDENTITIES = 2048 + # Shared parameter type for every workspace-scoped tool. WorkspaceParam = Annotated[ str | None, @@ -44,15 +52,21 @@ class WorkspaceContext: ) -> None: self._client = client self._preferred_reference = preferred_reference - self._active: Workspace | None = None + # Active selection is per caller: one shared slot would leak one user's + # choice to every other user on a shared server. + self._active_by_identity: OrderedDict[str, Workspace] = OrderedDict() @property def active(self) -> Workspace | None: - return self._active + return self._active_by_identity.get(current_identity()) def remember(self, workspace: Workspace) -> Workspace: - """Make ``workspace`` the default for later scoped calls.""" - self._active = workspace + """Make ``workspace`` the default for the current caller's later calls.""" + identity = current_identity() + self._active_by_identity[identity] = workspace + self._active_by_identity.move_to_end(identity) + while len(self._active_by_identity) > _MAX_TRACKED_IDENTITIES: + self._active_by_identity.popitem(last=False) return workspace async def fetch_all(self) -> list[Workspace]: @@ -67,8 +81,9 @@ class WorkspaceContext: return self.remember(await self._match(reference)) async def _resolve_default(self) -> Workspace: - if self._active is not None: - return self._active + active = self.active + if active is not None: + return active if self._preferred_reference: return self.remember(await self._match(self._preferred_reference)) return self.remember(await self._only_workspace_or_prompt()) From fa7d5f83f0ea8db081199d6e2afb99154c25c827 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 07/56] test(mcp): cover per-request auth and workspace isolation --- surfsense_mcp/tests/test_auth_headers.py | 40 +++++++ surfsense_mcp/tests/test_client_errors.py | 2 +- surfsense_mcp/tests/test_client_params.py | 4 +- surfsense_mcp/tests/test_request_auth.py | 100 ++++++++++++++++++ surfsense_mcp/tests/test_workspace_context.py | 28 +++++ 5 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 surfsense_mcp/tests/test_auth_headers.py create mode 100644 surfsense_mcp/tests/test_request_auth.py diff --git a/surfsense_mcp/tests/test_auth_headers.py b/surfsense_mcp/tests/test_auth_headers.py new file mode 100644 index 000000000..efb2d85a9 --- /dev/null +++ b/surfsense_mcp/tests/test_auth_headers.py @@ -0,0 +1,40 @@ +"""API key extraction from request headers: Bearer, fallback, and rejection.""" + +from __future__ import annotations + +from starlette.datastructures import Headers + +from surfsense_mcp.core.auth.headers import extract_api_key + + +def _headers(**pairs: str) -> Headers: + return Headers(pairs) + + +def test_reads_bearer_token(): + assert extract_api_key(_headers(authorization="Bearer ss_pat_abc")) == "ss_pat_abc" + + +def test_bearer_scheme_is_case_insensitive(): + assert extract_api_key(_headers(authorization="bearer ss_pat_abc")) == "ss_pat_abc" + + +def test_falls_back_to_x_api_key(): + assert extract_api_key(Headers({"x-api-key": "ss_pat_xyz"})) == "ss_pat_xyz" + + +def test_bearer_wins_over_fallback(): + headers = Headers({"authorization": "Bearer primary", "x-api-key": "secondary"}) + assert extract_api_key(headers) == "primary" + + +def test_missing_headers_return_none(): + assert extract_api_key(_headers()) is None + + +def test_empty_bearer_is_rejected(): + assert extract_api_key(_headers(authorization="Bearer ")) is None + + +def test_non_bearer_authorization_is_ignored(): + assert extract_api_key(_headers(authorization="Basic abc123")) is None diff --git a/surfsense_mcp/tests/test_client_errors.py b/surfsense_mcp/tests/test_client_errors.py index 648bfde3e..0525fb523 100644 --- a/surfsense_mcp/tests/test_client_errors.py +++ b/surfsense_mcp/tests/test_client_errors.py @@ -15,7 +15,7 @@ def _response(status: int, **kwargs) -> httpx.Response: def test_explains_401_with_token_hint(): message = SurfSenseClient._explain_failure(_response(401, json={"detail": "bad"})) - assert "SURFSENSE_API_KEY" in message + assert "API key" in message assert "bad" in message diff --git a/surfsense_mcp/tests/test_client_params.py b/surfsense_mcp/tests/test_client_params.py index 4840ad488..e9cd33a0c 100644 --- a/surfsense_mcp/tests/test_client_params.py +++ b/surfsense_mcp/tests/test_client_params.py @@ -25,7 +25,9 @@ def _capture(client: SurfSenseClient) -> dict: def test_none_params_are_dropped(): - client = SurfSenseClient(api_base="http://test/api/v1", api_key="ss_pat_x", timeout=5) + client = SurfSenseClient( + api_base="http://test/api/v1", timeout=5, fallback_api_key="ss_pat_x" + ) seen = _capture(client) asyncio.run( client.request( diff --git a/surfsense_mcp/tests/test_request_auth.py b/surfsense_mcp/tests/test_request_auth.py new file mode 100644 index 000000000..c548dff7a --- /dev/null +++ b/surfsense_mcp/tests/test_request_auth.py @@ -0,0 +1,100 @@ +"""Per-request key resolution and the Authorization header the backend receives. + +Covers the security-critical behaviors: the per-request key wins over the env +fallback, the fallback covers stdio, a missing key is refused, and concurrent +callers never see each other's key. +""" + +from __future__ import annotations + +import asyncio + +import httpx +import pytest + +from surfsense_mcp.core.auth import identity +from surfsense_mcp.core.client import SurfSenseClient +from surfsense_mcp.core.errors import ToolError + + +def _client_recording_auth(seen: dict, *, fallback: str | None) -> SurfSenseClient: + async def handler(request: httpx.Request) -> httpx.Response: + seen["authorization"] = request.headers.get("authorization") + return httpx.Response(200, json={"ok": True}) + + client = SurfSenseClient( + api_base="http://test/api/v1", timeout=5, fallback_api_key=fallback + ) + client._http = httpx.AsyncClient( + base_url="http://test/api/v1", transport=httpx.MockTransport(handler) + ) + return client + + +async def _get(client: SurfSenseClient) -> None: + await client.request("GET", "/workspaces") + + +def test_request_key_is_sent_as_bearer(): + seen: dict = {} + client = _client_recording_auth(seen, fallback=None) + + async def run() -> None: + token = identity.bind_api_key("ss_pat_request") + try: + await _get(client) + finally: + identity.unbind_api_key(token) + + asyncio.run(run()) + assert seen["authorization"] == "Bearer ss_pat_request" + + +def test_request_key_overrides_env_fallback(): + seen: dict = {} + client = _client_recording_auth(seen, fallback="ss_pat_env") + + async def run() -> None: + token = identity.bind_api_key("ss_pat_request") + try: + await _get(client) + finally: + identity.unbind_api_key(token) + + asyncio.run(run()) + assert seen["authorization"] == "Bearer ss_pat_request" + + +def test_env_fallback_used_without_request_key(): + seen: dict = {} + client = _client_recording_auth(seen, fallback="ss_pat_env") + asyncio.run(_get(client)) + assert seen["authorization"] == "Bearer ss_pat_env" + + +def test_missing_key_is_refused(): + client = _client_recording_auth({}, fallback=None) + with pytest.raises(ToolError): + asyncio.run(_get(client)) + + +def test_concurrent_callers_do_not_leak_keys(): + seen_by_caller: dict[str, str | None] = {} + + async def call_as(key: str) -> None: + # Each caller runs in its own task, so the contextvar is isolated. + recorded: dict = {} + client = _client_recording_auth(recorded, fallback=None) + token = identity.bind_api_key(key) + try: + await _get(client) + finally: + identity.unbind_api_key(token) + seen_by_caller[key] = recorded["authorization"] + + async def run() -> None: + await asyncio.gather(call_as("ss_pat_A"), call_as("ss_pat_B")) + + asyncio.run(run()) + assert seen_by_caller["ss_pat_A"] == "Bearer ss_pat_A" + assert seen_by_caller["ss_pat_B"] == "Bearer ss_pat_B" diff --git a/surfsense_mcp/tests/test_workspace_context.py b/surfsense_mcp/tests/test_workspace_context.py index 9ba9c2ca3..d2024e5d5 100644 --- a/surfsense_mcp/tests/test_workspace_context.py +++ b/surfsense_mcp/tests/test_workspace_context.py @@ -6,6 +6,7 @@ import asyncio import pytest +from surfsense_mcp.core.auth import identity from surfsense_mcp.core.errors import ToolError from surfsense_mcp.core.workspace_context import WorkspaceContext @@ -96,3 +97,30 @@ def test_resolution_is_remembered_as_active(): assert ctx.active is not None and ctx.active.id == 2 # a later default call reuses the active selection without re-choosing assert asyncio.run(ctx.resolve(None)).id == 2 + + +def test_active_workspace_is_isolated_per_identity(): + ctx = _context(_rows(("A", 1), ("B", 2))) + + async def select_as(key: str, reference: str) -> None: + token = identity.bind_api_key(key) + try: + await ctx.resolve(reference) + finally: + identity.unbind_api_key(token) + + async def active_for(key: str) -> int | None: + token = identity.bind_api_key(key) + try: + return ctx.active.id if ctx.active else None + finally: + identity.unbind_api_key(token) + + asyncio.run(select_as("ss_pat_A", "A")) + asyncio.run(select_as("ss_pat_B", "B")) + + # Each caller keeps its own selection; no bleed across identities. + assert asyncio.run(active_for("ss_pat_A")) == 1 + assert asyncio.run(active_for("ss_pat_B")) == 2 + # An unknown caller has no active selection. + assert asyncio.run(active_for("ss_pat_C")) is None From 0d4033682d6bda104d7a16622715bba4976ae393 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 08/56] build(mcp): declare starlette and uvicorn dependencies --- surfsense_mcp/pyproject.toml | 2 ++ surfsense_mcp/uv.lock | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/surfsense_mcp/pyproject.toml b/surfsense_mcp/pyproject.toml index 87db0c223..489bbf455 100644 --- a/surfsense_mcp/pyproject.toml +++ b/surfsense_mcp/pyproject.toml @@ -8,6 +8,8 @@ license = { text = "Apache-2.0" } dependencies = [ "mcp>=1.26.0", "httpx>=0.27.0", + "starlette>=0.37", + "uvicorn>=0.30", ] [project.scripts] diff --git a/surfsense_mcp/uv.lock b/surfsense_mcp/uv.lock index bc97a9b2a..d1bfab702 100644 --- a/surfsense_mcp/uv.lock +++ b/surfsense_mcp/uv.lock @@ -718,6 +718,8 @@ source = { editable = "." } dependencies = [ { name = "httpx" }, { name = "mcp" }, + { name = "starlette" }, + { name = "uvicorn" }, ] [package.dev-dependencies] @@ -729,6 +731,8 @@ dev = [ requires-dist = [ { name = "httpx", specifier = ">=0.27.0" }, { name = "mcp", specifier = ">=1.26.0" }, + { name = "starlette", specifier = ">=0.37" }, + { name = "uvicorn", specifier = ">=0.30" }, ] [package.metadata.requires-dev] From a7215c09dc44a58d0a8686d74cd577deb3c6e15f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:55:20 +0200 Subject: [PATCH 09/56] docs(mcp): tighten comments to intent-only --- .../src/surfsense_mcp/core/auth/headers.py | 6 ++---- .../src/surfsense_mcp/core/auth/identity.py | 18 +++++------------- .../src/surfsense_mcp/core/auth/middleware.py | 4 ++-- surfsense_mcp/src/surfsense_mcp/core/client.py | 5 ++--- 4 files changed, 11 insertions(+), 22 deletions(-) diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py b/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py index c06989909..3bca45967 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py @@ -1,8 +1,6 @@ -"""Extraction of a SurfSense API key from request headers. +"""Extract a SurfSense API key from request headers. -Pure and side-effect free: given the request headers, return the caller's key -or ``None``. Isolated from transport and state so the parsing rules stay -trivially unit-testable. +Pure header parsing, kept separate from transport and state. """ from __future__ import annotations diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py b/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py index 5f242e94e..0c8f6b315 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py @@ -1,12 +1,9 @@ """Request-scoped caller identity. -Over streamable-http one process serves many users, so the caller's key lives -in a contextvar for the life of a single request: the auth middleware binds it, -and the client reads it when building the outbound backend call. Under stdio -there is no request, the contextvar stays empty, and the env key is used. - -The contextvar is request-scoped, not stored state — it is re-derived from the -header on every request, which is what keeps the server stateless. +Over streamable-http one process serves many users, so the caller's key lives in +a contextvar for the life of a request: the auth middleware binds it and the +client reads it when calling the backend. Under stdio there is no request, so the +contextvar is empty and the env key is used instead. """ from __future__ import annotations @@ -24,7 +21,6 @@ def bind_api_key(api_key: str | None) -> Token: def unbind_api_key(token: Token) -> None: - """Release the binding once the request is done.""" _api_key.reset(token) @@ -34,9 +30,5 @@ def current_api_key() -> str | None: def current_identity() -> str: - """Stable per-caller key for scoping request state. - - The token identifies the account, so state keyed on it is naturally - per-user and survives reconnects. Under stdio all calls share one identity. - """ + """Stable per-caller key for scoping request state; shared under stdio.""" return _api_key.get() or _LOCAL_IDENTITY diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py index 2f2356ad7..354ae0a89 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py @@ -2,8 +2,8 @@ A pure ASGI middleware, deliberately not Starlette's ``BaseHTTPMiddleware``: the latter runs the endpoint in a separate task, so a contextvar set in it does -not reach the tool handler. A pure middleware sets the key in the request's own -task, from which the SDK's per-request handling inherits it (verified). +not reach the tool handler. A pure middleware binds the key in the request's own +task, from which the SDK's per-request handling inherits it. Requests without a key are rejected here so no tool ever runs unauthenticated. """ diff --git a/surfsense_mcp/src/surfsense_mcp/core/client.py b/surfsense_mcp/src/surfsense_mcp/core/client.py index cfe1a73ee..9ae446ae2 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/client.py +++ b/surfsense_mcp/src/surfsense_mcp/core/client.py @@ -32,9 +32,8 @@ class SurfSenseClient: self, *, api_base: str, timeout: float, fallback_api_key: str | None = None ) -> None: self._api_base = api_base - # The key is resolved per request (one client serves many users over - # http), so none is baked into the shared client. ``fallback_api_key`` - # is the env-supplied key used under stdio, where there is no header. + # Resolved per request, so no key is baked into the shared client. The + # fallback is the env key used under stdio, where there is no header. self._fallback_api_key = fallback_api_key self._http = httpx.AsyncClient( base_url=api_base, From 2acc1426bf8294f0fd03c5a6e1e7f97b6aba9310 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 17:22:37 +0200 Subject: [PATCH 10/56] refactor(mcp): split scraper and knowledge-base features into SRP modules --- .../surfsense_mcp/core/workspace_context.py | 43 +- .../surfsense_mcp/core/workspace_matching.py | 51 ++ .../features/knowledge_base/__init__.py | 371 +----------- .../features/knowledge_base/annotations.py | 34 ++ .../features/knowledge_base/document_tools.py | 185 ++++++ .../features/knowledge_base/search_tools.py | 188 ++++++ .../features/scrapers/__init__.py | 562 +----------------- .../features/scrapers/annotations.py | 13 + .../features/scrapers/platforms/__init__.py | 1 + .../scrapers/platforms/google_maps.py | 151 +++++ .../scrapers/platforms/google_search.py | 79 +++ .../features/scrapers/platforms/reddit.py | 98 +++ .../features/scrapers/platforms/web.py | 89 +++ .../features/scrapers/platforms/youtube.py | 131 ++++ .../features/scrapers/run_history.py | 112 ++++ 15 files changed, 1153 insertions(+), 955 deletions(-) create mode 100644 surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py diff --git a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py index cfd90c32b..0c5423e6b 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py +++ b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py @@ -16,6 +16,7 @@ from pydantic import Field from .auth.identity import current_identity from .client import SurfSenseClient from .errors import ToolError +from .workspace_matching import as_int, match_by_name, name_list # ponytail: one small entry per distinct caller (API token). Bounded so a flood # of keys can't grow memory without limit; an evicted caller just re-resolves @@ -99,43 +100,20 @@ class WorkspaceContext: ) raise ToolError( "No workspace selected. Choose one first with surfsense_select_workspace, " - f"or pass 'workspace'. Available: {_name_list(workspaces)}." + f"or pass 'workspace'. Available: {name_list(workspaces)}." ) async def _match(self, reference: str | int) -> Workspace: workspaces = await self.fetch_all() - as_id = _as_int(reference) + as_id = as_int(reference) if as_id is not None: found = next((w for w in workspaces if w.id == as_id), None) if found is None: raise ToolError( - f"No workspace with id {as_id}. Available: {_name_list(workspaces)}." + f"No workspace with id {as_id}. Available: {name_list(workspaces)}." ) return found - return _match_by_name(str(reference), workspaces) - - -def _match_by_name(reference: str, workspaces: list[Workspace]) -> Workspace: - """Match on name: exact, then case-insensitive, then unique substring.""" - needle = reference.strip() - exact = [w for w in workspaces if w.name == needle] - if exact: - return exact[0] - lowered = needle.casefold() - insensitive = [w for w in workspaces if w.name.casefold() == lowered] - if insensitive: - return insensitive[0] - partial = [w for w in workspaces if lowered in w.name.casefold()] - if len(partial) == 1: - return partial[0] - if len(partial) > 1: - raise ToolError( - f"'{reference}' matches several workspaces: {_name_list(partial)}. " - "Use a more specific name or the id." - ) - raise ToolError( - f"No workspace named '{reference}'. Available: {_name_list(workspaces)}." - ) + return match_by_name(str(reference), workspaces) def _to_workspace(row: dict) -> Workspace: @@ -146,14 +124,3 @@ def _to_workspace(row: dict) -> Workspace: is_owner=row.get("is_owner", False), member_count=row.get("member_count", 1), ) - - -def _as_int(reference: str | int) -> int | None: - if isinstance(reference, int): - return reference - text = reference.strip() - return int(text) if text.isdigit() else None - - -def _name_list(workspaces: list[Workspace]) -> str: - return ", ".join(f"{w.name} (id {w.id})" for w in workspaces) diff --git a/surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py b/surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py new file mode 100644 index 000000000..33bfc50a5 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py @@ -0,0 +1,51 @@ +"""Resolve a user-supplied workspace reference to a single workspace. + +Pure matching over an already-fetched list: name (exact, then case-insensitive, +then unique substring) or numeric id. Kept apart from WorkspaceContext so the +resolution rules can be read and tested without the network. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from .errors import ToolError + +if TYPE_CHECKING: + from .workspace_context import Workspace + + +def match_by_name(reference: str, workspaces: list[Workspace]) -> Workspace: + """Match on name: exact, then case-insensitive, then unique substring.""" + needle = reference.strip() + exact = [w for w in workspaces if w.name == needle] + if exact: + return exact[0] + lowered = needle.casefold() + insensitive = [w for w in workspaces if w.name.casefold() == lowered] + if insensitive: + return insensitive[0] + partial = [w for w in workspaces if lowered in w.name.casefold()] + if len(partial) == 1: + return partial[0] + if len(partial) > 1: + raise ToolError( + f"'{reference}' matches several workspaces: {name_list(partial)}. " + "Use a more specific name or the id." + ) + raise ToolError( + f"No workspace named '{reference}'. Available: {name_list(workspaces)}." + ) + + +def as_int(reference: str | int) -> int | None: + """Return the reference as an id, or None when it isn't numeric.""" + if isinstance(reference, int): + return reference + text = reference.strip() + return int(text) if text.isdigit() else None + + +def name_list(workspaces: list[Workspace]) -> str: + """Render workspaces as a human-readable 'name (id N)' list.""" + return ", ".join(f"{w.name} (id {w.id})" for w in workspaces) diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py index 7fb1802ae..1a971bfe4 100644 --- a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py +++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py @@ -1,379 +1,22 @@ """Knowledge-base tools: search the KB and manage its documents. Semantic search plus the document lifecycle — list, read, add text, upload a -file, update, and delete — over a workspace's knowledge base. Search and reads -default to the active workspace; document ids identify a single document across -the whole account, so id-addressed tools need no workspace. +file, update, and delete — over a workspace's knowledge base. Read tools live in +search_tools, mutations in document_tools. """ from __future__ import annotations -import mimetypes -from pathlib import Path -from typing import Annotated - from mcp.server.fastmcp import FastMCP -from mcp.types import ToolAnnotations -from pydantic import Field from ...core.client import SurfSenseClient -from ...core.errors import ToolError -from ...core.rendering import ResponseFormatParam, clip, to_json -from ...core.workspace_context import WorkspaceContext, WorkspaceParam -from .note_ingestion import build_note_document - -_READ = ToolAnnotations( - readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False -) -_WRITE = ToolAnnotations( - readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=False -) -_DELETE = ToolAnnotations( - readOnlyHint=False, destructiveHint=True, idempotentHint=False, openWorldHint=False -) - -_DOCUMENT_ID = Annotated[ - int, - Field( - description="Document id from surfsense_search_knowledge_base or " - "surfsense_list_documents results." - ), -] - -_DOCUMENT_TYPES = Annotated[ - list[str] | None, - Field( - description="Restrict to these document types, e.g. " - "['FILE', 'CRAWLED_URL', 'YOUTUBE_VIDEO']. Omit for all types." - ), -] +from ...core.workspace_context import WorkspaceContext +from . import document_tools, search_tools def register( mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext ) -> None: - """Register the knowledge-base tools on the server.""" - - @mcp.tool( - name="surfsense_search_knowledge_base", - title="Search knowledge base", - annotations=_READ, - structured_output=False, - ) - async def search_knowledge_base( - query: Annotated[ - str, - Field( - min_length=1, - description="Natural-language search, e.g. " - "'notebooklm user complaints'.", - ), - ], - top_k: Annotated[ - int, Field(ge=1, le=20, description="Maximum documents to return.") - ] = 5, - document_types: _DOCUMENT_TYPES = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Search the workspace's knowledge base by meaning and keywords. - - Use this FIRST when a question might be answered by content already - stored in SurfSense — notes, uploaded files, saved pages, past - research. Do NOT use it to fetch new data from the web; use the - scraper tools for that. Returns the most relevant documents with the - passages that matched, ranked by relevance score. - Example: query='pricing feedback', top_k=5. - """ - resolved = await context.resolve(workspace) - hits = await client.request( - "POST", - "/documents/search-semantic", - json={ - "workspace_id": resolved.id, - "query": query, - "top_k": max(1, min(top_k, 20)), - "document_types": document_types, - }, - ) - items = (hits or {}).get("items", []) - if response_format == "json": - return to_json(items) - return _render_search(query, items) - - @mcp.tool( - name="surfsense_list_documents", - title="List documents", - annotations=_READ, - structured_output=False, - ) - async def list_documents( - document_types: _DOCUMENT_TYPES = None, - folder_id: Annotated[ - int | None, - Field(description="Only documents in this folder. Omit for all."), - ] = None, - page: Annotated[ - int, Field(ge=0, description="Zero-based page number.") - ] = 0, - page_size: Annotated[ - int, Field(ge=1, description="Documents per page.") - ] = 20, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """List documents in the workspace's knowledge base, newest first. - - Use this to browse or inventory what is stored; to find documents - about a topic, prefer surfsense_search_knowledge_base. Returns each - document's title, id, type, and update time, plus a has_more flag — - request the next page by increasing page. - Example: document_types=['FILE'], page=0, page_size=20. - """ - resolved = await context.resolve(workspace) - result = await client.request( - "GET", - "/documents", - params={ - "workspace_id": resolved.id, - "page": page, - "page_size": page_size, - "document_types": _join(document_types), - "folder_id": folder_id, - }, - ) - if response_format == "json": - return to_json(result) - return _render_document_list(result) - - @mcp.tool( - name="surfsense_get_document", - title="Read one document", - annotations=_READ, - structured_output=False, - ) - async def get_document( - document_id: _DOCUMENT_ID, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Read one document's full content and metadata by id. - - Use this after surfsense_search_knowledge_base or - surfsense_list_documents to open a specific document — search results - only include the matching passages, this returns the whole text. - """ - document = await client.request("GET", f"/documents/{document_id}") - if response_format == "json": - return clip(to_json(document)) - return _render_document(document) - - @mcp.tool( - name="surfsense_add_document", - title="Add a note", - annotations=_WRITE, - structured_output=False, - ) - async def add_document( - title: Annotated[ - str, - Field(min_length=1, description="Short descriptive title for the note."), - ], - content: Annotated[ - str, - Field( - min_length=1, - description="The note's body; plain text or markdown.", - ), - ], - source_url: Annotated[ - str | None, - Field(description="Where the text came from, if anywhere."), - ] = None, - workspace: WorkspaceParam = None, - ) -> str: - """Save a text or markdown note into the workspace's knowledge base. - - Use this to store notes, summaries, or findings so they become - searchable later — e.g. after finishing a piece of research. For files - on disk use surfsense_upload_file instead. Indexing is asynchronous, - so the note may take a moment to appear in search. - Example: title='NotebookLM subreddits', content='- r/notebooklm ...'. - """ - resolved = await context.resolve(workspace) - await client.request( - "POST", - "/documents", - json=build_note_document( - workspace_id=resolved.id, - title=title, - content=content, - source_url=source_url, - ), - ) - return ( - f"Queued '{title}' for indexing in '{resolved.name}'. " - "It will be searchable once processing completes." - ) - - @mcp.tool( - name="surfsense_upload_file", - title="Upload a file", - annotations=_WRITE, - structured_output=False, - ) - async def upload_file( - file_path: Annotated[ - str, - Field( - description="Path to a local file, e.g. " - "'C:/Users/me/report.pdf' or '~/notes/summary.md'." - ), - ], - use_vision_llm: Annotated[ - bool, - Field( - description="True reads scanned or image-heavy files with a " - "vision model (slower)." - ), - ] = False, - workspace: WorkspaceParam = None, - ) -> str: - """Upload a local file (PDF, docx, markdown, etc.) into the knowledge base. - - Use this to ingest a file from disk so its content becomes searchable; - for text you already have in hand use surfsense_add_document instead. - The file is parsed, chunked, and indexed asynchronously. Duplicate - files are detected and skipped. - Example: file_path='C:/Users/me/report.pdf'. - """ - resolved = await context.resolve(workspace) - payload = _read_upload(file_path) - result = await client.request( - "POST", - "/documents/fileupload", - data={ - "workspace_id": str(resolved.id), - "use_vision_llm": str(use_vision_llm).lower(), - "processing_mode": "basic", - }, - files=[("files", payload)], - ) - pending = (result or {}).get("pending_files", 0) - skipped = (result or {}).get("skipped_duplicates", 0) - note = " (already present, skipped)" if skipped and not pending else "" - return ( - f"Uploaded '{Path(file_path).name}' to '{resolved.name}'{note}. " - "It will be searchable once processing completes." - ) - - @mcp.tool( - name="surfsense_update_document", - title="Replace a document's content", - annotations=_WRITE, - structured_output=False, - ) - async def update_document( - document_id: _DOCUMENT_ID, - content: Annotated[ - str, - Field( - min_length=1, - description="New full text; replaces the existing content " - "entirely.", - ), - ], - ) -> str: - """Replace a document's stored content by id. - - Use this to correct or rewrite a document's text. The new content - REPLACES the old entirely — to append, read the document first with - surfsense_get_document and resend the combined text. Search chunks are - not re-indexed by this call. - """ - existing = await client.request("GET", f"/documents/{document_id}") - await client.request( - "PUT", - f"/documents/{document_id}", - json={ - "document_type": existing["document_type"], - "workspace_id": existing["workspace_id"], - "content": content, - }, - ) - return f"Updated document {document_id} ('{existing.get('title', '')}')." - - @mcp.tool( - name="surfsense_delete_document", - title="Delete a document", - annotations=_DELETE, - structured_output=False, - ) - async def delete_document(document_id: _DOCUMENT_ID) -> str: - """Permanently delete a document from the knowledge base by id. - - Use this only when the user explicitly asks to remove a document — - deletion cannot be undone. The document stops appearing in searches - immediately. - """ - await client.request("DELETE", f"/documents/{document_id}") - return f"Deleted document {document_id}." - - -def _read_upload(file_path: str) -> tuple[str, bytes, str]: - path = Path(file_path).expanduser() - if not path.is_file(): - raise ToolError(f"No file at '{file_path}'.") - mime, _ = mimetypes.guess_type(path.name) - return (path.name, path.read_bytes(), mime or "application/octet-stream") - - -def _join(values: list[str] | None) -> str | None: - return ",".join(values) if values else None - - -def _render_search(query: str, items: list[dict]) -> str: - if not items: - return f'No matches for "{query}".' - lines = [f'# {len(items)} result(s) for "{query}"', ""] - for hit in items: - lines.append( - f"## {hit.get('title', 'Untitled')} " - f"(id {hit.get('document_id')}) — score {hit.get('score', 0):.3f}" - ) - for chunk in hit.get("chunks", []): - excerpt = clip(chunk.get("content", "").strip(), 500) - lines.append(f"> {excerpt}") - lines.append("") - return "\n".join(lines).strip() - - -def _render_document_list(result: dict | None) -> str: - items = (result or {}).get("items", []) - if not items: - return "No documents found." - lines = ["# Documents", ""] - for doc in items: - lines.append( - f"- **{doc.get('title', 'Untitled')}** (id {doc.get('id')}) · " - f"{doc.get('document_type')} · updated {doc.get('updated_at')}" - ) - total = (result or {}).get("total", len(items)) - page = (result or {}).get("page", 0) - has_more = (result or {}).get("has_more", False) - lines.append("") - lines.append( - f"_Page {page} · showing {len(items)} of {total}" - + (" · more available_" if has_more else "_") - ) - return "\n".join(lines) - - -def _render_document(document: dict) -> str: - content = clip(document.get("content", "") or "(empty)") - return ( - f"# {document.get('title', 'Untitled')} (id {document.get('id')})\n" - f"- type: {document.get('document_type')}\n" - f"- workspace: {document.get('workspace_id')}\n" - f"- updated: {document.get('updated_at')}\n\n" - f"{content}" - ) + """Register every knowledge-base tool on the server.""" + search_tools.register(mcp, client, context) + document_tools.register(mcp, client, context) diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py new file mode 100644 index 000000000..16322506c --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py @@ -0,0 +1,34 @@ +"""Tool-call policy hints and shared parameter types for knowledge-base tools.""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.types import ToolAnnotations +from pydantic import Field + +READ = ToolAnnotations( + readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False +) +WRITE = ToolAnnotations( + readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=False +) +DELETE = ToolAnnotations( + readOnlyHint=False, destructiveHint=True, idempotentHint=False, openWorldHint=False +) + +DocumentId = Annotated[ + int, + Field( + description="Document id from surfsense_search_knowledge_base or " + "surfsense_list_documents results." + ), +] + +DocumentTypes = Annotated[ + list[str] | None, + Field( + description="Restrict to these document types, e.g. " + "['FILE', 'CRAWLED_URL', 'YOUTUBE_VIDEO']. Omit for all types." + ), +] diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py new file mode 100644 index 000000000..497a2526c --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py @@ -0,0 +1,185 @@ +"""Knowledge-base write tools: add a note, upload a file, update, and delete. + +Add and upload target the active workspace; update and delete address a document +by its account-unique id, so they need no workspace. +""" + +from __future__ import annotations + +import mimetypes +from pathlib import Path +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ...core.client import SurfSenseClient +from ...core.errors import ToolError +from ...core.workspace_context import WorkspaceContext, WorkspaceParam +from .annotations import DELETE, WRITE, DocumentId +from .note_ingestion import build_note_document + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the knowledge-base write and delete tools.""" + + @mcp.tool( + name="surfsense_add_document", + title="Add a note", + annotations=WRITE, + structured_output=False, + ) + async def add_document( + title: Annotated[ + str, + Field(min_length=1, description="Short descriptive title for the note."), + ], + content: Annotated[ + str, + Field( + min_length=1, + description="The note's body; plain text or markdown.", + ), + ], + source_url: Annotated[ + str | None, + Field(description="Where the text came from, if anywhere."), + ] = None, + workspace: WorkspaceParam = None, + ) -> str: + """Save a text or markdown note into the workspace's knowledge base. + + Use this to store notes, summaries, or findings so they become + searchable later — e.g. after finishing a piece of research. For files + on disk use surfsense_upload_file instead. Indexing is asynchronous, + so the note may take a moment to appear in search. + Example: title='NotebookLM subreddits', content='- r/notebooklm ...'. + """ + resolved = await context.resolve(workspace) + await client.request( + "POST", + "/documents", + json=build_note_document( + workspace_id=resolved.id, + title=title, + content=content, + source_url=source_url, + ), + ) + return ( + f"Queued '{title}' for indexing in '{resolved.name}'. " + "It will be searchable once processing completes." + ) + + @mcp.tool( + name="surfsense_upload_file", + title="Upload a file", + annotations=WRITE, + structured_output=False, + ) + async def upload_file( + file_path: Annotated[ + str, + Field( + description="Path to a local file, e.g. " + "'C:/Users/me/report.pdf' or '~/notes/summary.md'." + ), + ], + use_vision_llm: Annotated[ + bool, + Field( + description="True reads scanned or image-heavy files with a " + "vision model (slower)." + ), + ] = False, + workspace: WorkspaceParam = None, + ) -> str: + """Upload a local file (PDF, docx, markdown, etc.) into the knowledge base. + + Use this to ingest a file from disk so its content becomes searchable; + for text you already have in hand use surfsense_add_document instead. + The file is parsed, chunked, and indexed asynchronously. Duplicate + files are detected and skipped. + Example: file_path='C:/Users/me/report.pdf'. + """ + resolved = await context.resolve(workspace) + payload = _read_upload(file_path) + result = await client.request( + "POST", + "/documents/fileupload", + data={ + "workspace_id": str(resolved.id), + "use_vision_llm": str(use_vision_llm).lower(), + "processing_mode": "basic", + }, + files=[("files", payload)], + ) + pending = (result or {}).get("pending_files", 0) + skipped = (result or {}).get("skipped_duplicates", 0) + note = " (already present, skipped)" if skipped and not pending else "" + return ( + f"Uploaded '{Path(file_path).name}' to '{resolved.name}'{note}. " + "It will be searchable once processing completes." + ) + + @mcp.tool( + name="surfsense_update_document", + title="Replace a document's content", + annotations=WRITE, + structured_output=False, + ) + async def update_document( + document_id: DocumentId, + content: Annotated[ + str, + Field( + min_length=1, + description="New full text; replaces the existing content " + "entirely.", + ), + ], + ) -> str: + """Replace a document's stored content by id. + + Use this to correct or rewrite a document's text. The new content + REPLACES the old entirely — to append, read the document first with + surfsense_get_document and resend the combined text. Search chunks are + not re-indexed by this call. + """ + existing = await client.request("GET", f"/documents/{document_id}") + await client.request( + "PUT", + f"/documents/{document_id}", + json={ + "document_type": existing["document_type"], + "workspace_id": existing["workspace_id"], + "content": content, + }, + ) + return f"Updated document {document_id} ('{existing.get('title', '')}')." + + @mcp.tool( + name="surfsense_delete_document", + title="Delete a document", + annotations=DELETE, + structured_output=False, + ) + async def delete_document(document_id: DocumentId) -> str: + """Permanently delete a document from the knowledge base by id. + + Use this only when the user explicitly asks to remove a document — + deletion cannot be undone. The document stops appearing in searches + immediately. + """ + await client.request("DELETE", f"/documents/{document_id}") + return f"Deleted document {document_id}." + + +def _read_upload(file_path: str) -> tuple[str, bytes, str]: + path = Path(file_path).expanduser() + if not path.is_file(): + raise ToolError(f"No file at '{file_path}'.") + mime, _ = mimetypes.guess_type(path.name) + return (path.name, path.read_bytes(), mime or "application/octet-stream") diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py new file mode 100644 index 000000000..a9e60810d --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py @@ -0,0 +1,188 @@ +"""Knowledge-base read tools: semantic search, list, and read one document. + +Search and list default to the active workspace; a document read is addressed by +id, which is unique across the account, so it needs no workspace. +""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ...core.client import SurfSenseClient +from ...core.rendering import ResponseFormatParam, clip, to_json +from ...core.workspace_context import WorkspaceContext, WorkspaceParam +from .annotations import READ, DocumentId, DocumentTypes + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the knowledge-base read tools.""" + + @mcp.tool( + name="surfsense_search_knowledge_base", + title="Search knowledge base", + annotations=READ, + structured_output=False, + ) + async def search_knowledge_base( + query: Annotated[ + str, + Field( + min_length=1, + description="Natural-language search, e.g. " + "'notebooklm user complaints'.", + ), + ], + top_k: Annotated[ + int, Field(ge=1, le=20, description="Maximum documents to return.") + ] = 5, + document_types: DocumentTypes = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Search the workspace's knowledge base by meaning and keywords. + + Use this FIRST when a question might be answered by content already + stored in SurfSense — notes, uploaded files, saved pages, past + research. Do NOT use it to fetch new data from the web; use the + scraper tools for that. Returns the most relevant documents with the + passages that matched, ranked by relevance score. + Example: query='pricing feedback', top_k=5. + """ + resolved = await context.resolve(workspace) + hits = await client.request( + "POST", + "/documents/search-semantic", + json={ + "workspace_id": resolved.id, + "query": query, + "top_k": max(1, min(top_k, 20)), + "document_types": document_types, + }, + ) + items = (hits or {}).get("items", []) + if response_format == "json": + return to_json(items) + return _render_search(query, items) + + @mcp.tool( + name="surfsense_list_documents", + title="List documents", + annotations=READ, + structured_output=False, + ) + async def list_documents( + document_types: DocumentTypes = None, + folder_id: Annotated[ + int | None, + Field(description="Only documents in this folder. Omit for all."), + ] = None, + page: Annotated[ + int, Field(ge=0, description="Zero-based page number.") + ] = 0, + page_size: Annotated[ + int, Field(ge=1, description="Documents per page.") + ] = 20, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """List documents in the workspace's knowledge base, newest first. + + Use this to browse or inventory what is stored; to find documents + about a topic, prefer surfsense_search_knowledge_base. Returns each + document's title, id, type, and update time, plus a has_more flag — + request the next page by increasing page. + Example: document_types=['FILE'], page=0, page_size=20. + """ + resolved = await context.resolve(workspace) + result = await client.request( + "GET", + "/documents", + params={ + "workspace_id": resolved.id, + "page": page, + "page_size": page_size, + "document_types": _join(document_types), + "folder_id": folder_id, + }, + ) + if response_format == "json": + return to_json(result) + return _render_document_list(result) + + @mcp.tool( + name="surfsense_get_document", + title="Read one document", + annotations=READ, + structured_output=False, + ) + async def get_document( + document_id: DocumentId, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Read one document's full content and metadata by id. + + Use this after surfsense_search_knowledge_base or + surfsense_list_documents to open a specific document — search results + only include the matching passages, this returns the whole text. + """ + document = await client.request("GET", f"/documents/{document_id}") + if response_format == "json": + return clip(to_json(document)) + return _render_document(document) + + +def _join(values: list[str] | None) -> str | None: + return ",".join(values) if values else None + + +def _render_search(query: str, items: list[dict]) -> str: + if not items: + return f'No matches for "{query}".' + lines = [f'# {len(items)} result(s) for "{query}"', ""] + for hit in items: + lines.append( + f"## {hit.get('title', 'Untitled')} " + f"(id {hit.get('document_id')}) — score {hit.get('score', 0):.3f}" + ) + for chunk in hit.get("chunks", []): + excerpt = clip(chunk.get("content", "").strip(), 500) + lines.append(f"> {excerpt}") + lines.append("") + return "\n".join(lines).strip() + + +def _render_document_list(result: dict | None) -> str: + items = (result or {}).get("items", []) + if not items: + return "No documents found." + lines = ["# Documents", ""] + for doc in items: + lines.append( + f"- **{doc.get('title', 'Untitled')}** (id {doc.get('id')}) · " + f"{doc.get('document_type')} · updated {doc.get('updated_at')}" + ) + total = (result or {}).get("total", len(items)) + page = (result or {}).get("page", 0) + has_more = (result or {}).get("has_more", False) + lines.append("") + lines.append( + f"_Page {page} · showing {len(items)} of {total}" + + (" · more available_" if has_more else "_") + ) + return "\n".join(lines) + + +def _render_document(document: dict) -> str: + content = clip(document.get("content", "") or "(empty)") + return ( + f"# {document.get('title', 'Untitled')} (id {document.get('id')})\n" + f"- type: {document.get('document_type')}\n" + f"- workspace: {document.get('workspace_id')}\n" + f"- updated: {document.get('updated_at')}\n\n" + f"{content}" + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py index aa8265148..dfa2f3ab2 100644 --- a/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py @@ -1,570 +1,26 @@ """Scraper tools: one MCP surface per SurfSense platform capability. Web crawl, Google Search, Reddit, YouTube, and Google Maps each get a tool that -maps a natural-language request to the workspace's scraper door. Two more tools +maps a natural-language request to the workspace's scraper. Two run-history tools list and fetch past runs, so a large result truncated inline can be retrieved in -full later. +full later. Each platform lives in its own module under platforms/. """ from __future__ import annotations -from typing import Annotated, Literal - from mcp.server.fastmcp import FastMCP -from mcp.types import ToolAnnotations -from pydantic import Field from ...core.client import SurfSenseClient -from ...core.rendering import ResponseFormatParam, clip, to_json -from ...core.workspace_context import WorkspaceContext, WorkspaceParam -from .capability import run_scraper +from ...core.workspace_context import WorkspaceContext +from . import run_history +from .platforms import google_maps, google_search, reddit, web, youtube -# Scrapers reach the open web and record a billable run; they are neither -# read-only nor idempotent, but they do not mutate the knowledge base. -_SCRAPE = ToolAnnotations( - readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=True -) -_READ_RUNS = ToolAnnotations( - readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False -) - -RedditSort = Literal["relevance", "hot", "top", "new", "rising", "comments"] -RedditTime = Literal["hour", "day", "week", "month", "year", "all"] -CommentSort = Literal["TOP_COMMENTS", "NEWEST_FIRST"] -ReviewSort = Literal["newest", "mostRelevant", "highestRanking", "lowestRanking"] +_REGISTRARS = (web, google_search, reddit, youtube, google_maps, run_history) def register( mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext ) -> None: - """Register the scraper and run-history tools on the server.""" - - @mcp.tool( - name="surfsense_web_crawl", - title="Crawl web pages", - annotations=_SCRAPE, - structured_output=False, - ) - async def web_crawl( - start_urls: Annotated[ - list[str], - Field( - min_length=1, - description="Full URLs to fetch, e.g. " - "['https://example.com/blog/post'].", - ), - ], - max_crawl_depth: Annotated[ - int, - Field( - ge=0, - description="Link-hops to follow from start_urls within the " - "same site. 0 fetches only start_urls.", - ), - ] = 0, - max_crawl_pages: Annotated[ - int, Field(ge=1, description="Stop after this many pages in total.") - ] = 10, - max_length: Annotated[ - int, Field(ge=1, description="Max characters kept per page.") - ] = 50_000, - include_url_patterns: Annotated[ - list[str] | None, - Field( - description="Regexes; only discovered links matching one are " - "followed, e.g. ['/docs/.*']." - ), - ] = None, - exclude_url_patterns: Annotated[ - list[str] | None, - Field(description="Regexes; discovered links matching one are skipped."), - ] = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Fetch specific web pages and return their cleaned content as markdown. - - Use this to read a page the user names, or to spider a site from a - starting URL. Do NOT use it to find pages on a topic — use - surfsense_google_search for discovery. Returns one item per crawled - page: url, title, and the page text as markdown. - Example: start_urls=['https://blog.example.com'], max_crawl_depth=1, - include_url_patterns=['/2026/']. - """ - return await run_scraper( - client, - context, - platform="web", - verb="crawl", - payload={ - "startUrls": start_urls, - "maxCrawlDepth": max_crawl_depth, - "maxCrawlPages": max_crawl_pages, - "maxLength": max_length, - "includeUrlPatterns": include_url_patterns, - "excludeUrlPatterns": exclude_url_patterns, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_google_search", - title="Scrape Google Search", - annotations=_SCRAPE, - structured_output=False, - ) - async def google_search( - queries: Annotated[ - list[str], - Field( - min_length=1, - description="Search terms or full Google Search URLs, e.g. " - "['best rss readers 2026'].", - ), - ], - max_pages_per_query: Annotated[ - int, Field(ge=1, description="Result pages to fetch per query.") - ] = 1, - country_code: Annotated[ - str | None, - Field(description="Two-letter country to search from, e.g. 'us'."), - ] = None, - language_code: Annotated[ - str, Field(description="Results language, e.g. 'en'. Empty for default.") - ] = "", - site: Annotated[ - str | None, - Field( - description="Restrict results to one domain, e.g. 'example.com'." - ), - ] = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Scrape Google Search result pages for one or more queries. - - Use this to discover pages on the open web by topic; follow up with - surfsense_web_crawl to read a result in full. Do NOT use it for - Reddit, YouTube, or Google Maps research — the dedicated tools return - richer data. Returns each query's parsed results: title, url, and - snippet per organic result. - Example: queries=['notebooklm review'], site='news.ycombinator.com'. - """ - return await run_scraper( - client, - context, - platform="google_search", - verb="scrape", - payload={ - "queries": queries, - "max_pages_per_query": max_pages_per_query, - "country_code": country_code, - "language_code": language_code, - "site": site, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_reddit_scrape", - title="Search or scrape Reddit", - annotations=_SCRAPE, - structured_output=False, - ) - async def reddit_scrape( - urls: Annotated[ - list[str] | None, - Field( - description="Reddit URLs: a post, a subreddit like " - "'https://reddit.com/r/LocalLLaMA', a user page, or a search " - "URL. Provide urls OR search_queries." - ), - ] = None, - search_queries: Annotated[ - list[str] | None, - Field( - description="Terms to search Reddit for, e.g. " - "['NotebookLM alternatives']. Provide search_queries OR urls." - ), - ] = None, - community: Annotated[ - str | None, - Field( - description="Restrict a search to one subreddit, name without " - "'r/', e.g. 'ArtificialInteligence'." - ), - ] = None, - sort: Annotated[RedditSort, Field(description="Post ordering.")] = "new", - time_filter: Annotated[ - RedditTime | None, - Field(description="Time window; only valid with sort='top'."), - ] = None, - max_items: Annotated[ - int, Field(ge=1, description="Maximum posts to return.") - ] = 10, - skip_comments: Annotated[ - bool, - Field( - description="True fetches posts only (faster); False also " - "fetches each post's comment thread." - ), - ] = False, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Search or scrape Reddit: posts, comments, subreddits, and users. - - Use this for ANY Reddit research — finding relevant subreddits or - communities for a topic, top posts, or discussions — instead of a - generic web search. Returns posts (title, text, score, subreddit, url) - with comment threads unless skip_comments is set. Every post carries - its subreddit, so to find communities for a topic, search posts and - aggregate their subreddits. - Example: search_queries=['NotebookLM'], sort='top', time_filter='month'. - """ - return await run_scraper( - client, - context, - platform="reddit", - verb="scrape", - payload={ - "urls": urls, - "search_queries": search_queries, - "community": community, - "sort": sort, - "time_filter": time_filter, - "max_items": max_items, - "skip_comments": skip_comments, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_youtube_scrape", - title="Search or scrape YouTube", - annotations=_SCRAPE, - structured_output=False, - ) - async def youtube_scrape( - urls: Annotated[ - list[str] | None, - Field( - description="YouTube URLs: video, channel, playlist, shorts, " - "or hashtag pages. Provide urls OR search_queries." - ), - ] = None, - search_queries: Annotated[ - list[str] | None, - Field( - description="Terms to search YouTube for, e.g. " - "['NotebookLM tutorial']. Provide search_queries OR urls." - ), - ] = None, - max_results: Annotated[ - int, Field(ge=1, description="Maximum videos to return.") - ] = 10, - download_subtitles: Annotated[ - bool, - Field(description="True also fetches each video's transcript."), - ] = False, - subtitles_language: Annotated[ - str, Field(description="Transcript language code, e.g. 'en'.") - ] = "en", - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Search or scrape YouTube videos, optionally with transcripts. - - Use this for YouTube research: finding videos on a topic, or reading a - video's details or transcript. For a video's comment section use - surfsense_youtube_comments instead. Returns per-video metadata (title, - channel, views, description, url) and, if requested, the transcript. - Example: search_queries=['NotebookLM tutorial'], download_subtitles=True. - """ - return await run_scraper( - client, - context, - platform="youtube", - verb="scrape", - payload={ - "urls": urls, - "search_queries": search_queries, - "max_results": max_results, - "download_subtitles": download_subtitles, - "subtitles_language": subtitles_language, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_youtube_comments", - title="Fetch YouTube comments", - annotations=_SCRAPE, - structured_output=False, - ) - async def youtube_comments( - urls: Annotated[ - list[str], - Field( - min_length=1, - description="YouTube video URLs, e.g. " - "['https://www.youtube.com/watch?v=abc123'].", - ), - ], - max_comments: Annotated[ - int, - Field( - ge=1, - description="Maximum comments per video, counting top-level " - "comments and replies together.", - ), - ] = 20, - sort_by: Annotated[ - CommentSort, Field(description="Comment ordering.") - ] = "NEWEST_FIRST", - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Fetch the comments (and replies) on one or more YouTube videos. - - Use this when the user wants a video's discussion or audience reaction - rather than the video itself; get video URLs from - surfsense_youtube_scrape if you only have a topic. Returns comment - text, author, likes, and replies. - Example: urls=['https://www.youtube.com/watch?v=abc123'], max_comments=50. - """ - return await run_scraper( - client, - context, - platform="youtube", - verb="comments", - payload={ - "urls": urls, - "max_comments": max_comments, - "sort_by": sort_by, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_google_maps_scrape", - title="Find places on Google Maps", - annotations=_SCRAPE, - structured_output=False, - ) - async def google_maps_scrape( - search_queries: Annotated[ - list[str] | None, - Field( - description="Place searches, e.g. ['coffee shops']. Provide " - "search_queries OR urls OR place_ids." - ), - ] = None, - urls: Annotated[ - list[str] | None, - Field(description="Google Maps URLs of specific places."), - ] = None, - place_ids: Annotated[ - list[str] | None, - Field(description="Google place ids, e.g. ['ChIJj61dQgK6j4AR...']."), - ] = None, - location: Annotated[ - str | None, - Field( - description="Geographic scope for a search, e.g. " - "'Seattle, USA'." - ), - ] = None, - max_places: Annotated[ - int, Field(ge=1, description="Maximum places to return.") - ] = 10, - include_details: Annotated[ - bool, - Field( - description="True adds opening hours and extra contact info " - "(slower)." - ), - ] = False, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Find places on Google Maps by search, URL, or place id. - - Use this for local-business and location research: names, addresses, - ratings, categories, coordinates, place ids. For a place's customer - reviews use surfsense_google_maps_reviews instead. - Example: search_queries=['ramen'], location='Osaka, Japan', max_places=5. - """ - return await run_scraper( - client, - context, - platform="google_maps", - verb="scrape", - payload={ - "search_queries": search_queries, - "urls": urls, - "place_ids": place_ids, - "location": location, - "max_places": max_places, - "include_details": include_details, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_google_maps_reviews", - title="Fetch Google Maps reviews", - annotations=_SCRAPE, - structured_output=False, - ) - async def google_maps_reviews( - urls: Annotated[ - list[str] | None, - Field( - description="Google Maps URLs of places. Provide urls OR " - "place_ids." - ), - ] = None, - place_ids: Annotated[ - list[str] | None, - Field( - description="Google place ids from surfsense_google_maps_scrape." - ), - ] = None, - max_reviews: Annotated[ - int, Field(ge=1, description="Maximum reviews per place.") - ] = 20, - sort_by: Annotated[ - ReviewSort, Field(description="Review ordering.") - ] = "newest", - language: Annotated[ - str, Field(description="Reviews language code, e.g. 'en'.") - ] = "en", - start_date: Annotated[ - str | None, - Field( - description="ISO date like '2026-01-01'; keeps only reviews on " - "or after that day." - ), - ] = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Fetch customer reviews for Google Maps places by URL or place id. - - Use this to read feedback on specific places; get urls or place_ids - from surfsense_google_maps_scrape first if you only have a name. - Returns review text, rating, author, and date per review. - Example: place_ids=['ChIJj61dQgK6j4AR...'], sort_by='newest'. - """ - return await run_scraper( - client, - context, - platform="google_maps", - verb="reviews", - payload={ - "urls": urls, - "place_ids": place_ids, - "max_reviews": max_reviews, - "sort_by": sort_by, - "language": language, - "start_date": start_date, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_list_scraper_runs", - title="List past scraper runs", - annotations=_READ_RUNS, - structured_output=False, - ) - async def list_scraper_runs( - limit: Annotated[ - int, Field(ge=1, description="Maximum runs to list.") - ] = 20, - capability: Annotated[ - str | None, - Field( - description="Filter by capability slug, e.g. 'web.crawl' or " - "'reddit.scrape'." - ), - ] = None, - status: Annotated[ - str | None, - Field(description="Filter by run status: 'success' or 'error'."), - ] = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """List recent scraper runs in the workspace, newest first. - - Use this to find the run_id of an earlier scrape — for example when an - inline result was truncated — then fetch it in full with - surfsense_get_scraper_run. Returns each run's id, capability, status, - item count, and creation time. - Example: capability='reddit.scrape', status='success'. - """ - resolved = await context.resolve(workspace) - runs = await client.request( - "GET", - f"/workspaces/{resolved.id}/scrapers/runs", - params={ - "limit": limit, - "capability": capability, - "status": status, - }, - ) - if response_format == "json": - return to_json(runs) - return _render_runs(runs) - - @mcp.tool( - name="surfsense_get_scraper_run", - title="Fetch one scraper run in full", - annotations=_READ_RUNS, - structured_output=False, - ) - async def get_scraper_run( - run_id: Annotated[ - str, - Field( - description="Run id from surfsense_list_scraper_runs or a " - "prior scrape's output." - ), - ], - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Fetch a single scraper run in full, including its stored output. - - Use this to retrieve the complete, untruncated result of an earlier - scrape. Do NOT re-run a scraper just to recover a truncated result — - fetch the stored run instead. - """ - resolved = await context.resolve(workspace) - run = await client.request( - "GET", f"/workspaces/{resolved.id}/scrapers/runs/{run_id}" - ) - if response_format == "json": - return clip(to_json(run)) - return f"# Run {run.get('id', run_id)}\n\n```json\n{clip(to_json(run))}\n```" - - -def _render_runs(runs: list[dict] | None) -> str: - if not runs: - return "No scraper runs found." - lines = ["# Scraper runs", ""] - for run in runs: - lines.append( - f"- **{run.get('id')}** — {run.get('capability')} · {run.get('status')} · " - f"{run.get('item_count', 0)} item(s) · {run.get('created_at')}" - ) - return "\n".join(lines) + """Register every scraper and run-history tool on the server.""" + for module in _REGISTRARS: + module.register(mcp, client, context) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py new file mode 100644 index 000000000..cb872cc90 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py @@ -0,0 +1,13 @@ +"""Tool-call policy hints shared across scraper tools.""" + +from __future__ import annotations + +from mcp.types import ToolAnnotations + +SCRAPE = ToolAnnotations( + readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=True +) + +READ_RUNS = ToolAnnotations( + readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False +) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py new file mode 100644 index 000000000..f61704380 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py @@ -0,0 +1 @@ +"""One module per scraper platform; each exposes register(mcp, client, context).""" diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py new file mode 100644 index 000000000..e1613ca4e --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py @@ -0,0 +1,151 @@ +"""Google Maps scraper tools: places and reviews.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +ReviewSort = Literal["newest", "mostRelevant", "highestRanking", "lowestRanking"] + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the Google Maps place and review tools.""" + + @mcp.tool( + name="surfsense_google_maps_scrape", + title="Find places on Google Maps", + annotations=SCRAPE, + structured_output=False, + ) + async def google_maps_scrape( + search_queries: Annotated[ + list[str] | None, + Field( + description="Place searches, e.g. ['coffee shops']. Provide " + "search_queries OR urls OR place_ids." + ), + ] = None, + urls: Annotated[ + list[str] | None, + Field(description="Google Maps URLs of specific places."), + ] = None, + place_ids: Annotated[ + list[str] | None, + Field(description="Google place ids, e.g. ['ChIJj61dQgK6j4AR...']."), + ] = None, + location: Annotated[ + str | None, + Field( + description="Geographic scope for a search, e.g. " + "'Seattle, USA'." + ), + ] = None, + max_places: Annotated[ + int, Field(ge=1, description="Maximum places to return.") + ] = 10, + include_details: Annotated[ + bool, + Field( + description="True adds opening hours and extra contact info " + "(slower)." + ), + ] = False, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Find places on Google Maps by search, URL, or place id. + + Use this for local-business and location research: names, addresses, + ratings, categories, coordinates, place ids. For a place's customer + reviews use surfsense_google_maps_reviews instead. + Example: search_queries=['ramen'], location='Osaka, Japan', max_places=5. + """ + return await run_scraper( + client, + context, + platform="google_maps", + verb="scrape", + payload={ + "search_queries": search_queries, + "urls": urls, + "place_ids": place_ids, + "location": location, + "max_places": max_places, + "include_details": include_details, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_google_maps_reviews", + title="Fetch Google Maps reviews", + annotations=SCRAPE, + structured_output=False, + ) + async def google_maps_reviews( + urls: Annotated[ + list[str] | None, + Field( + description="Google Maps URLs of places. Provide urls OR " + "place_ids." + ), + ] = None, + place_ids: Annotated[ + list[str] | None, + Field( + description="Google place ids from surfsense_google_maps_scrape." + ), + ] = None, + max_reviews: Annotated[ + int, Field(ge=1, description="Maximum reviews per place.") + ] = 20, + sort_by: Annotated[ + ReviewSort, Field(description="Review ordering.") + ] = "newest", + language: Annotated[ + str, Field(description="Reviews language code, e.g. 'en'.") + ] = "en", + start_date: Annotated[ + str | None, + Field( + description="ISO date like '2026-01-01'; keeps only reviews on " + "or after that day." + ), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch customer reviews for Google Maps places by URL or place id. + + Use this to read feedback on specific places; get urls or place_ids + from surfsense_google_maps_scrape first if you only have a name. + Returns review text, rating, author, and date per review. + Example: place_ids=['ChIJj61dQgK6j4AR...'], sort_by='newest'. + """ + return await run_scraper( + client, + context, + platform="google_maps", + verb="reviews", + payload={ + "urls": urls, + "place_ids": place_ids, + "max_reviews": max_reviews, + "sort_by": sort_by, + "language": language, + "start_date": start_date, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py new file mode 100644 index 000000000..cc1a1f8ed --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py @@ -0,0 +1,79 @@ +"""Google Search scraper tool.""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the Google Search tool.""" + + @mcp.tool( + name="surfsense_google_search", + title="Scrape Google Search", + annotations=SCRAPE, + structured_output=False, + ) + async def google_search( + queries: Annotated[ + list[str], + Field( + min_length=1, + description="Search terms or full Google Search URLs, e.g. " + "['best rss readers 2026'].", + ), + ], + max_pages_per_query: Annotated[ + int, Field(ge=1, description="Result pages to fetch per query.") + ] = 1, + country_code: Annotated[ + str | None, + Field(description="Two-letter country to search from, e.g. 'us'."), + ] = None, + language_code: Annotated[ + str, Field(description="Results language, e.g. 'en'. Empty for default.") + ] = "", + site: Annotated[ + str | None, + Field( + description="Restrict results to one domain, e.g. 'example.com'." + ), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Scrape Google Search result pages for one or more queries. + + Use this to discover pages on the open web by topic; follow up with + surfsense_web_crawl to read a result in full. Do NOT use it for + Reddit, YouTube, or Google Maps research — the dedicated tools return + richer data. Returns each query's parsed results: title, url, and + snippet per organic result. + Example: queries=['notebooklm review'], site='news.ycombinator.com'. + """ + return await run_scraper( + client, + context, + platform="google_search", + verb="scrape", + payload={ + "queries": queries, + "max_pages_per_query": max_pages_per_query, + "country_code": country_code, + "language_code": language_code, + "site": site, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py new file mode 100644 index 000000000..035193ebc --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py @@ -0,0 +1,98 @@ +"""Reddit scraper tool.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +RedditSort = Literal["relevance", "hot", "top", "new", "rising", "comments"] +RedditTime = Literal["hour", "day", "week", "month", "year", "all"] + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the Reddit tool.""" + + @mcp.tool( + name="surfsense_reddit_scrape", + title="Search or scrape Reddit", + annotations=SCRAPE, + structured_output=False, + ) + async def reddit_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="Reddit URLs: a post, a subreddit like " + "'https://reddit.com/r/LocalLLaMA', a user page, or a search " + "URL. Provide urls OR search_queries." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Terms to search Reddit for, e.g. " + "['NotebookLM alternatives']. Provide search_queries OR urls." + ), + ] = None, + community: Annotated[ + str | None, + Field( + description="Restrict a search to one subreddit, name without " + "'r/', e.g. 'ArtificialInteligence'." + ), + ] = None, + sort: Annotated[RedditSort, Field(description="Post ordering.")] = "new", + time_filter: Annotated[ + RedditTime | None, + Field(description="Time window; only valid with sort='top'."), + ] = None, + max_items: Annotated[ + int, Field(ge=1, description="Maximum posts to return.") + ] = 10, + skip_comments: Annotated[ + bool, + Field( + description="True fetches posts only (faster); False also " + "fetches each post's comment thread." + ), + ] = False, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Search or scrape Reddit: posts, comments, subreddits, and users. + + Use this for ANY Reddit research — finding relevant subreddits or + communities for a topic, top posts, or discussions — instead of a + generic web search. Returns posts (title, text, score, subreddit, url) + with comment threads unless skip_comments is set. Every post carries + its subreddit, so to find communities for a topic, search posts and + aggregate their subreddits. + Example: search_queries=['NotebookLM'], sort='top', time_filter='month'. + """ + return await run_scraper( + client, + context, + platform="reddit", + verb="scrape", + payload={ + "urls": urls, + "search_queries": search_queries, + "community": community, + "sort": sort, + "time_filter": time_filter, + "max_items": max_items, + "skip_comments": skip_comments, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py new file mode 100644 index 000000000..9c24a4352 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py @@ -0,0 +1,89 @@ +"""Web crawl scraper tool.""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the web crawl tool.""" + + @mcp.tool( + name="surfsense_web_crawl", + title="Crawl web pages", + annotations=SCRAPE, + structured_output=False, + ) + async def web_crawl( + start_urls: Annotated[ + list[str], + Field( + min_length=1, + description="Full URLs to fetch, e.g. " + "['https://example.com/blog/post'].", + ), + ], + max_crawl_depth: Annotated[ + int, + Field( + ge=0, + description="Link-hops to follow from start_urls within the " + "same site. 0 fetches only start_urls.", + ), + ] = 0, + max_crawl_pages: Annotated[ + int, Field(ge=1, description="Stop after this many pages in total.") + ] = 10, + max_length: Annotated[ + int, Field(ge=1, description="Max characters kept per page.") + ] = 50_000, + include_url_patterns: Annotated[ + list[str] | None, + Field( + description="Regexes; only discovered links matching one are " + "followed, e.g. ['/docs/.*']." + ), + ] = None, + exclude_url_patterns: Annotated[ + list[str] | None, + Field(description="Regexes; discovered links matching one are skipped."), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch specific web pages and return their cleaned content as markdown. + + Use this to read a page the user names, or to spider a site from a + starting URL. Do NOT use it to find pages on a topic — use + surfsense_google_search for discovery. Returns one item per crawled + page: url, title, and the page text as markdown. + Example: start_urls=['https://blog.example.com'], max_crawl_depth=1, + include_url_patterns=['/2026/']. + """ + return await run_scraper( + client, + context, + platform="web", + verb="crawl", + payload={ + "startUrls": start_urls, + "maxCrawlDepth": max_crawl_depth, + "maxCrawlPages": max_crawl_pages, + "maxLength": max_length, + "includeUrlPatterns": include_url_patterns, + "excludeUrlPatterns": exclude_url_patterns, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py new file mode 100644 index 000000000..5582c82bb --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py @@ -0,0 +1,131 @@ +"""YouTube scraper tools: videos and comments.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +CommentSort = Literal["TOP_COMMENTS", "NEWEST_FIRST"] + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the YouTube video and comment tools.""" + + @mcp.tool( + name="surfsense_youtube_scrape", + title="Search or scrape YouTube", + annotations=SCRAPE, + structured_output=False, + ) + async def youtube_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="YouTube URLs: video, channel, playlist, shorts, " + "or hashtag pages. Provide urls OR search_queries." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Terms to search YouTube for, e.g. " + "['NotebookLM tutorial']. Provide search_queries OR urls." + ), + ] = None, + max_results: Annotated[ + int, Field(ge=1, description="Maximum videos to return.") + ] = 10, + download_subtitles: Annotated[ + bool, + Field(description="True also fetches each video's transcript."), + ] = False, + subtitles_language: Annotated[ + str, Field(description="Transcript language code, e.g. 'en'.") + ] = "en", + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Search or scrape YouTube videos, optionally with transcripts. + + Use this for YouTube research: finding videos on a topic, or reading a + video's details or transcript. For a video's comment section use + surfsense_youtube_comments instead. Returns per-video metadata (title, + channel, views, description, url) and, if requested, the transcript. + Example: search_queries=['NotebookLM tutorial'], download_subtitles=True. + """ + return await run_scraper( + client, + context, + platform="youtube", + verb="scrape", + payload={ + "urls": urls, + "search_queries": search_queries, + "max_results": max_results, + "download_subtitles": download_subtitles, + "subtitles_language": subtitles_language, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_youtube_comments", + title="Fetch YouTube comments", + annotations=SCRAPE, + structured_output=False, + ) + async def youtube_comments( + urls: Annotated[ + list[str], + Field( + min_length=1, + description="YouTube video URLs, e.g. " + "['https://www.youtube.com/watch?v=abc123'].", + ), + ], + max_comments: Annotated[ + int, + Field( + ge=1, + description="Maximum comments per video, counting top-level " + "comments and replies together.", + ), + ] = 20, + sort_by: Annotated[ + CommentSort, Field(description="Comment ordering.") + ] = "NEWEST_FIRST", + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch the comments (and replies) on one or more YouTube videos. + + Use this when the user wants a video's discussion or audience reaction + rather than the video itself; get video URLs from + surfsense_youtube_scrape if you only have a topic. Returns comment + text, author, likes, and replies. + Example: urls=['https://www.youtube.com/watch?v=abc123'], max_comments=50. + """ + return await run_scraper( + client, + context, + platform="youtube", + verb="comments", + payload={ + "urls": urls, + "max_comments": max_comments, + "sort_by": sort_by, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py new file mode 100644 index 000000000..9274a1a69 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py @@ -0,0 +1,112 @@ +"""Scraper run history: list past runs and fetch one in full. + +A scrape whose inline result was truncated is retrievable here by run id, so the +model never re-runs a scraper just to recover output. +""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ...core.client import SurfSenseClient +from ...core.rendering import ResponseFormatParam, clip, to_json +from ...core.workspace_context import WorkspaceContext, WorkspaceParam +from .annotations import READ_RUNS + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the run-history tools.""" + + @mcp.tool( + name="surfsense_list_scraper_runs", + title="List past scraper runs", + annotations=READ_RUNS, + structured_output=False, + ) + async def list_scraper_runs( + limit: Annotated[ + int, Field(ge=1, description="Maximum runs to list.") + ] = 20, + capability: Annotated[ + str | None, + Field( + description="Filter by capability slug, e.g. 'web.crawl' or " + "'reddit.scrape'." + ), + ] = None, + status: Annotated[ + str | None, + Field(description="Filter by run status: 'success' or 'error'."), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """List recent scraper runs in the workspace, newest first. + + Use this to find the run_id of an earlier scrape — for example when an + inline result was truncated — then fetch it in full with + surfsense_get_scraper_run. Returns each run's id, capability, status, + item count, and creation time. + Example: capability='reddit.scrape', status='success'. + """ + resolved = await context.resolve(workspace) + runs = await client.request( + "GET", + f"/workspaces/{resolved.id}/scrapers/runs", + params={ + "limit": limit, + "capability": capability, + "status": status, + }, + ) + if response_format == "json": + return to_json(runs) + return _render_runs(runs) + + @mcp.tool( + name="surfsense_get_scraper_run", + title="Fetch one scraper run in full", + annotations=READ_RUNS, + structured_output=False, + ) + async def get_scraper_run( + run_id: Annotated[ + str, + Field( + description="Run id from surfsense_list_scraper_runs or a " + "prior scrape's output." + ), + ], + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch a single scraper run in full, including its stored output. + + Use this to retrieve the complete, untruncated result of an earlier + scrape. Do NOT re-run a scraper just to recover a truncated result — + fetch the stored run instead. + """ + resolved = await context.resolve(workspace) + run = await client.request( + "GET", f"/workspaces/{resolved.id}/scrapers/runs/{run_id}" + ) + if response_format == "json": + return clip(to_json(run)) + return f"# Run {run.get('id', run_id)}\n\n```json\n{clip(to_json(run))}\n```" + + +def _render_runs(runs: list[dict] | None) -> str: + if not runs: + return "No scraper runs found." + lines = ["# Scraper runs", ""] + for run in runs: + lines.append( + f"- **{run.get('id')}** — {run.get('capability')} · {run.get('status')} · " + f"{run.get('item_count', 0)} item(s) · {run.get('created_at')}" + ) + return "\n".join(lines) From 2b9daead58dcd4f98108d779edd55c1dc19765b3 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 18:05:56 +0200 Subject: [PATCH 11/56] feat(mcp): add health endpoint with public-path auth exemption --- .../src/surfsense_mcp/core/auth/middleware.py | 8 +++++-- .../src/surfsense_mcp/core/transport/http.py | 21 +++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py index 354ae0a89..8ba6ce939 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py @@ -6,10 +6,13 @@ not reach the tool handler. A pure middleware binds the key in the request's own task, from which the SDK's per-request handling inherits it. Requests without a key are rejected here so no tool ever runs unauthenticated. +Paths in ``public_paths`` (e.g. the health probe) skip the check entirely. """ from __future__ import annotations +from collections.abc import Iterable + from starlette.datastructures import Headers from starlette.responses import JSONResponse from starlette.types import ASGIApp, Receive, Scope, Send @@ -21,11 +24,12 @@ from .identity import bind_api_key, unbind_api_key class ApiKeyIdentityMiddleware: """Binds the per-request API key into the identity contextvar, or 401s.""" - def __init__(self, app: ASGIApp) -> None: + def __init__(self, app: ASGIApp, public_paths: Iterable[str] = ()) -> None: self._app = app + self._public_paths = frozenset(public_paths) async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] != "http": + if scope["type"] != "http" or scope["path"] in self._public_paths: await self._app(scope, receive, send) return diff --git a/surfsense_mcp/src/surfsense_mcp/core/transport/http.py b/surfsense_mcp/src/surfsense_mcp/core/transport/http.py index 2f2cf8f53..e375e8b98 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/transport/http.py +++ b/surfsense_mcp/src/surfsense_mcp/core/transport/http.py @@ -1,23 +1,32 @@ -"""Assemble the streamable-http ASGI app for the remote transport. +"""Wrap the SDK's MCP endpoint with identity + CORS for the remote transport. -Wraps the SDK's MCP endpoint with the API-key identity middleware and CORS. -CORS sits outermost so browser preflight (which carries no key) is answered -before the identity middleware, and clients can read the ``Mcp-Session-Id`` -header the streamable-http protocol relies on. +CORS is outermost so keyless browser preflight is answered before the identity +middleware. ``/health`` is a public path, exempt from the key check. """ from __future__ import annotations from mcp.server.fastmcp import FastMCP +from starlette.requests import Request +from starlette.responses import JSONResponse from starlette.middleware.cors import CORSMiddleware from starlette.types import ASGIApp from ..auth.middleware import ApiKeyIdentityMiddleware +HEALTH_PATH = "/health" + + +async def _health(_request: Request) -> JSONResponse: + return JSONResponse({"status": "ok"}) + def build_http_app(mcp: FastMCP) -> ASGIApp: """Return the MCP streamable-http app wrapped with identity + CORS.""" - app: ASGIApp = ApiKeyIdentityMiddleware(mcp.streamable_http_app()) + mcp.custom_route(HEALTH_PATH, methods=["GET"])(_health) + app: ASGIApp = ApiKeyIdentityMiddleware( + mcp.streamable_http_app(), public_paths={HEALTH_PATH} + ) return CORSMiddleware( app, allow_origins=["*"], From 9aef0c27a28181e34275e8b629f27d7ee94877f2 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 18:06:00 +0200 Subject: [PATCH 12/56] docs(mcp): document hosted /mcp remote endpoint --- README.es.md | 2 +- README.hi.md | 2 +- README.md | 2 +- README.pt-BR.md | 2 +- README.zh-CN.md | 2 +- surfsense_mcp/README.md | 68 +++++++++++++------ .../content/docs/how-to/mcp-server.mdx | 63 +++++++++-------- 7 files changed, 88 insertions(+), 53 deletions(-) diff --git a/README.es.md b/README.es.md index 82cfcae1d..70ccd5eb1 100644 --- a/README.es.md +++ b/README.es.md @@ -138,7 +138,7 @@ Agrega el servidor MCP de SurfSense a Claude, Cursor o tu propio framework de ag { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/README.hi.md b/README.hi.md index 1c9cfa4a7..cbf9ae8fa 100644 --- a/README.hi.md +++ b/README.hi.md @@ -138,7 +138,7 @@ SurfSense MCP सर्वर को Claude, Cursor या अपने एज { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/README.md b/README.md index a05548d66..8f2e65118 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ Add the SurfSense MCP server to Claude, Cursor, or your own agent framework: { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/README.pt-BR.md b/README.pt-BR.md index 4f2b945f4..076736bcf 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -138,7 +138,7 @@ Adicione o servidor MCP do SurfSense ao Claude, ao Cursor ou ao seu próprio fra { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/README.zh-CN.md b/README.zh-CN.md index 22410a727..7d2ae35c3 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -138,7 +138,7 @@ curl -X POST "$SURFSENSE_API_URL/workspaces/$WORKSPACE_ID/scrapers/reddit/scrape { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/surfsense_mcp/README.md b/surfsense_mcp/README.md index 47dabf55c..cc5ef19c5 100644 --- a/surfsense_mcp/README.md +++ b/surfsense_mcp/README.md @@ -2,9 +2,15 @@ A [Model Context Protocol](https://modelcontextprotocol.io/) server that exposes SurfSense to MCP clients like **Claude Code**, **Cursor**, and **Claude Desktop**. -It talks to a running SurfSense backend purely over its REST API using a SurfSense -API key — it imports no backend code and can point at any instance (local or -hosted) by changing two environment variables. +It talks to a SurfSense backend purely over its REST API using a SurfSense API +key — it imports no backend code. + +Connect it two ways: + +- **Hosted** (recommended) — point your client at `https://mcp.surfsense.com/mcp` + and pass your API key in a header. Nothing to install or keep running. +- **Self-host (stdio)** — run the server yourself against any backend (cloud or + your own). Best for self-hosters and clients without remote-server support. ## Tools @@ -29,15 +35,42 @@ Workspace-scoped tools default to the active workspace; pass `workspace` (a name or id) to override for a single call. Ids never need to be typed by hand — the model carries them between calls. -## Prerequisites +## Get an API key -1. A running SurfSense backend (default `http://localhost:8000`). -2. A **SurfSense API key**: SurfSense → Settings → API → create key (`ss_pat_…`). -3. **API access enabled** on the workspace(s) you want to use (workspace settings). +1. SurfSense → **API Playground → API Keys**: create a personal key (`ss_pat_…`). + It is shown only once. +2. Toggle **API key access** on for the workspace(s) you want to use. -## Setup +## Connect (hosted) -Uses [uv](https://github.com/astral-sh/uv): +Point your client at the hosted server and send the key as a Bearer token. For +clients that read an `mcpServers` map (Cursor, Claude Desktop, and others): + +```json +{ + "mcpServers": { + "surfsense": { + "url": "https://mcp.surfsense.com/mcp", + "headers": { "Authorization": "Bearer ss_pat_your_key_here" } + } + } +} +``` + +Claude Code, from a terminal: + +```bash +claude mcp add --transport http surfsense https://mcp.surfsense.com/mcp \ + --header "Authorization: Bearer ss_pat_your_key_here" +``` + +Most MCP clients accept this `url` + `headers` form; check your client's docs for +its exact remote-server field. + +## Self-host (stdio) + +Run the server yourself when you host your own backend or use a client without +remote support. It uses [uv](https://github.com/astral-sh/uv): ```bash cd surfsense_mcp @@ -45,11 +78,8 @@ uv sync uv run python -m surfsense_mcp.selfcheck # verify tools register correctly ``` -## Connect it to a client - -### Cursor - -Add to `~/.cursor/mcp.json` (or a project `.cursor/mcp.json`): +Then add it to your client. Cursor (`~/.cursor/mcp.json` or a project +`.cursor/mcp.json`): ```json { @@ -66,7 +96,7 @@ Add to `~/.cursor/mcp.json` (or a project `.cursor/mcp.json`): } ``` -### Claude Code +Claude Code: ```bash claude mcp add surfsense \ @@ -75,15 +105,13 @@ claude mcp add surfsense \ -- uv run --directory /absolute/path/to/SurfSense/surfsense_mcp python -m surfsense_mcp ``` -### Claude Desktop - -Add the same `mcpServers` block as Cursor to +Claude Desktop: add the same `mcpServers` block as Cursor to `claude_desktop_config.json` (Settings → Developer → Edit Config). ## Configuration -See `.env.example`. Secrets are passed as environment variables by the client; -never commit tokens. +See `.env.example`. For self-host, secrets are passed as environment variables by +the client; never commit tokens. ## Backend dependency diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx index ffdb4fd56..62628a659 100644 --- a/surfsense_web/content/docs/how-to/mcp-server.mdx +++ b/surfsense_web/content/docs/how-to/mcp-server.mdx @@ -4,51 +4,58 @@ description: Connect the SurfSense MCP server to Claude Code, Codex, OpenCode, C --- import { Tab, Tabs } from 'fumadocs-ui/components/tabs'; -import { Step, Steps } from 'fumadocs-ui/components/steps'; # SurfSense MCP Server The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 18 native, typed tools: every scraper (Reddit, YouTube, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. -It talks to SurfSense purely over the REST API — point it at SurfSense Cloud or your own self-hosted instance by changing one environment variable. +Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted. -## Prerequisites +## Create an API key - - +You need a SurfSense API key either way. In SurfSense, open **API Playground → API Keys** in your workspace sidebar: -### Install uv +1. Toggle **API key access** on for the workspace. +2. Create a personal API key (`ss_pat_…`) and copy it — it is shown only once. -The server runs with [uv](https://github.com/astral-sh/uv). Install it once, then from the SurfSense repository run: +## Connect (hosted) + +The hosted server runs at `https://mcp.surfsense.com/mcp`. Point your client at it and send the key as a Bearer token — there is nothing to install and no backend to run. For clients that read an `mcpServers` map (Cursor, and others): + +```json +{ + "mcpServers": { + "surfsense": { + "url": "https://mcp.surfsense.com/mcp", + "headers": { "Authorization": "Bearer ss_pat_your_key_here" } + } + } +} +``` + +Claude Code, from a terminal: + +```bash +claude mcp add --transport http surfsense https://mcp.surfsense.com/mcp \ + --header "Authorization: Bearer ss_pat_your_key_here" +``` + +Most MCP clients accept this `url` + `headers` form; check your client's docs for its exact remote-server field. + +## Self-host (stdio) + +Run the server yourself when you host your own backend or use a client without remote support. It runs with [uv](https://github.com/astral-sh/uv) — install it once, then from the SurfSense repository run: ```bash cd surfsense_mcp uv sync ``` - - - -### Create an API key - -In SurfSense, open **API Playground → API Keys** in your workspace sidebar: - -1. Toggle **API key access** on for the workspace. -2. Create a personal API key (`ss_pat_…`) and copy it — it is shown only once. - - - - -### Know your base URL +Point the server at your backend with `SURFSENSE_BASE_URL`: - **SurfSense Cloud**: `https://api.surfsense.com` - **Self-hosted**: wherever your backend runs, e.g. `http://localhost:8000` - - - -## Connect your agent - Every client below launches the same command — `uv run --directory /surfsense_mcp python -m surfsense_mcp` — and passes `SURFSENSE_BASE_URL` and `SURFSENSE_API_KEY` as environment variables. Replace the placeholder paths and key with yours. @@ -221,7 +228,7 @@ Run `/mcp` inside Gemini CLI to confirm the server and its tools. -The server uses stdio transport: your client launches the process on demand and shuts it down with the session. There is no daemon to manage — only your SurfSense backend needs to be up. +In this mode your client launches the process on demand and shuts it down with the session. There is no daemon to manage — only your SurfSense backend needs to be up. (The hosted server above needs none of this.) ## Test it @@ -236,7 +243,7 @@ That calls `surfsense_list_workspaces` — the simplest end-to-end check of the ## Configuration reference -All settings are environment variables passed by the client: +For self-host (stdio), all settings are environment variables passed by the client. The hosted server needs only your API key in the `Authorization` header: | Variable | Required | Default | Purpose | |----------|----------|---------|---------| From 1c5b08561d4505391f5e7ce624908f14c3fa9da3 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 18:06:04 +0200 Subject: [PATCH 13/56] feat(web): add hosted and self-host MCP client configs --- surfsense_web/app/(home)/mcp-server/page.tsx | 21 +- .../connectors-marketing/api-mcp-tabs.tsx | 2 +- .../components/mcp/agent-setup-tabs.tsx | 102 +++-- surfsense_web/lib/mcp/clients.ts | 379 +++++++++++++----- 4 files changed, 350 insertions(+), 154 deletions(-) diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index 6dfc69622..5e4961b1e 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -50,16 +50,13 @@ export const metadata: Metadata = { }, }; -/* Mirrors surfsense_mcp/README.md: the real Cursor config. */ +/* The hosted Cursor config; mirrors lib/mcp/clients.ts. */ const CURSOR_CONFIG = `{ "mcpServers": { "surfsense": { - "command": "uv", - "args": ["run", "--directory", ".../surfsense_mcp", - "python", "-m", "surfsense_mcp"], - "env": { - "SURFSENSE_BASE_URL": "https://api.surfsense.com", - "SURFSENSE_API_KEY": "ss_pat_..." + "url": "https://mcp.surfsense.com/mcp", + "headers": { + "Authorization": "Bearer ss_pat_..." } } } @@ -76,7 +73,7 @@ const STEPS = [ icon: TerminalSquare, title: "Add the server to your client", description: - "Drop the config into Cursor's mcp.json, run claude mcp add for Claude Code, or paste it into Claude Desktop. Point it at the cloud or your own self-hosted instance.", + "Point your client at https://mcp.surfsense.com/mcp with your key in an Authorization header — the hosted config for Cursor, Claude Code, and others is one paste. Prefer stdio? Switch to Self-host and run it against your own backend.", }, { icon: Server, @@ -135,7 +132,7 @@ const FAQ: FaqItem[] = [ { question: "Which MCP clients does it work with?", answer: - "Any MCP client that supports stdio servers. Claude Code, Codex, OpenCode, Cursor, Claude Desktop, VS Code, Windsurf, and Gemini CLI are documented with copy-paste configs on this page, and the same command works in custom agent harnesses built on the MCP SDK.", + "Any MCP client that speaks remote (streamable HTTP) or stdio. Claude Code, Codex, OpenCode, Cursor, Claude Desktop, VS Code, Windsurf, and Gemini CLI all have copy-paste configs on this page — Hosted for the one-paste https://mcp.surfsense.com/mcp endpoint, or Self-host for stdio against your own backend.", }, { question: "How is usage billed?", @@ -278,9 +275,9 @@ export default function McpServerPage() { Step-by-step setup for every agent

- Pick your client, follow its two steps, and paste the config. Replace the placeholder - path with your surfsense_mcp checkout and the key with one from API Playground → API - Keys — or grab a pre-filled config from the playground itself. + Pick your client, choose Hosted or Self-host, and + paste the config. Replace the key with one from API Playground → API Keys — or grab a + pre-filled config from the playground itself.

diff --git a/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx b/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx index 6ca19c47b..f25faf95b 100644 --- a/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx +++ b/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx @@ -206,7 +206,7 @@ function buildMcp({ mcpTool }: ApiSample): string { const config = { mcpServers: { surfsense: { - url: "https://mcp.surfsense.com", + url: "https://mcp.surfsense.com/mcp", headers: { Authorization: "Bearer ${SURFSENSE_API_KEY}" }, }, }, diff --git a/surfsense_web/components/mcp/agent-setup-tabs.tsx b/surfsense_web/components/mcp/agent-setup-tabs.tsx index 1701b8924..3790441f2 100644 --- a/surfsense_web/components/mcp/agent-setup-tabs.tsx +++ b/surfsense_web/components/mcp/agent-setup-tabs.tsx @@ -9,6 +9,8 @@ import { DEFAULT_SERVER_DIR, MCP_CLIENTS, type McpSnippetOptions, + type McpTransport, + REMOTE_URL, } from "@/lib/mcp/clients"; function CopyButton({ text }: { text: string }) { @@ -38,48 +40,82 @@ function CopyButton({ text }: { text: string }) { ); } +const TRANSPORTS: { id: McpTransport; label: string; hint: string }[] = [ + { id: "remote", label: "Hosted", hint: "mcp.surfsense.com — nothing to install" }, + { id: "stdio", label: "Self-host", hint: "run the server against your own backend" }, +]; + /** - * Per-agent MCP setup instructions as tabs: pick a client, follow its steps, - * copy its exact config. Used on the /mcp-server marketing page and in the - * API playground; `options` fills in real values where the caller has them. + * Per-agent MCP setup instructions as tabs: pick a client, then Hosted or + * Self-host, and copy its exact config. Used on the /mcp-server marketing page + * and in the API playground; `options` fills in real values where the caller + * has them. */ export function AgentSetupTabs({ options }: { options?: Partial }) { + const [transport, setTransport] = useState("remote"); + const resolved: McpSnippetOptions = { - baseUrl: options?.baseUrl || "https://api.surfsense.com", + remoteUrl: options?.remoteUrl || REMOTE_URL, apiKey: options?.apiKey || API_KEY_PLACEHOLDER, + baseUrl: options?.baseUrl || "https://api.surfsense.com", serverDir: options?.serverDir || DEFAULT_SERVER_DIR, }; + const active = TRANSPORTS.find((t) => t.id === transport) ?? TRANSPORTS[0]; + return ( - - - {MCP_CLIENTS.map((client) => ( - - {client.label} - - ))} - - {MCP_CLIENTS.map((client) => { - const config = client.buildConfig(resolved); - return ( - -
    - {client.steps.map((step) => ( -
  1. {step}
    2. - ))} -
-
-

{client.configFile}

-
- - 
-									{config}
-
+
+
+
+ {TRANSPORTS.map((t) => ( + + ))} +
+ {active.hint} +
+ + + + {MCP_CLIENTS.map((client) => ( + + {client.label} + + ))} + + {MCP_CLIENTS.map((client) => { + const snippet = client[transport]; + const config = snippet.build(resolved); + return ( + +
    + {snippet.steps.map((step) => ( +
  1. {step}
    2. + ))} +
+
+

+ {snippet.configFile} +

+
+ + 
+										{config}
+
+
-
- - ); - })} - + + ); + })} + +
); } diff --git a/surfsense_web/lib/mcp/clients.ts b/surfsense_web/lib/mcp/clients.ts index 923e99049..795832ad1 100644 --- a/surfsense_web/lib/mcp/clients.ts +++ b/surfsense_web/lib/mcp/clients.ts @@ -1,42 +1,75 @@ /** - * MCP client setup catalog: one entry per popular agent, with the exact - * config file, steps, and snippet needed to connect the SurfSense MCP server. - * Shared by the marketing /mcp-server page and the API playground so the - * instructions can never drift apart. + * MCP client setup catalog: one entry per popular agent, each with a hosted + * (remote) snippet and a self-host (stdio) snippet, plus the exact config file + * and steps. Shared by the marketing /mcp-server page and the API playground so + * the instructions can never drift apart. + * + * Remote snippets point at the hosted server and pass the key as a Bearer token; + * every client's exact remote field is verified against its own docs (Windsurf + * uses `serverUrl`, Gemini CLI `httpUrl`, VS Code needs `type: "http"`, OpenCode + * `type: "remote"` + `oauth: false`, Codex needs the rmcp flag, and Claude + * Desktop has no config-file remote support so it uses the `mcp-remote` bridge). */ +export type McpTransport = "remote" | "stdio"; + export interface McpSnippetOptions { - /** SurfSense backend URL the server should call. */ - baseUrl: string; + /** Hosted MCP endpoint (Bearer-authenticated). */ + remoteUrl: string; /** API key value or placeholder to show in the snippet. */ apiKey: string; - /** Absolute path to the surfsense_mcp directory. */ + /** SurfSense backend URL a self-hosted server should call. */ + baseUrl: string; + /** Absolute path to the surfsense_mcp directory (self-host). */ serverDir: string; } +export interface McpSnippet { + /** Where the snippet goes: a file path or "Terminal". */ + configFile: string; + language: "json" | "toml" | "bash"; + steps: string[]; + build: (options: McpSnippetOptions) => string; +} + export interface McpClient { id: string; label: string; - /** Where the snippet goes: a file path or "Terminal". */ - configFile: string; - language: "json" | "toml" | "bash"; - steps: string[]; - buildConfig: (options: McpSnippetOptions) => string; + remote: McpSnippet; + stdio: McpSnippet; } +export const REMOTE_URL = "https://mcp.surfsense.com/mcp"; export const DEFAULT_SERVER_DIR = "/path/to/SurfSense/surfsense_mcp"; export const API_KEY_PLACEHOLDER = "ss_pat_your_key_here"; -function serverArgs(serverDir: string): string[] { - return ["run", "--directory", serverDir, "python", "-m", "surfsense_mcp"]; -} - function json(value: unknown): string { return JSON.stringify(value, null, 2); } -/** The `mcpServers` JSON shape shared by Cursor, Claude Desktop, Windsurf, and Gemini CLI. */ -function standardJson({ baseUrl, apiKey, serverDir }: McpSnippetOptions): string { +function bearer(apiKey: string): string { + return `Bearer ${apiKey}`; +} + +function serverArgs(serverDir: string): string[] { + return ["run", "--directory", serverDir, "python", "-m", "surfsense_mcp"]; +} + +/** The `mcpServers` remote shape shared by Cursor, Windsurf, and Gemini CLI. */ +function remoteMcpServers(urlField: "url" | "serverUrl" | "httpUrl") { + return ({ remoteUrl, apiKey }: McpSnippetOptions): string => + json({ + mcpServers: { + surfsense: { + [urlField]: remoteUrl, + headers: { Authorization: bearer(apiKey) }, + }, + }, + }); +} + +/** The `mcpServers` stdio shape shared by Cursor, Claude Desktop, Windsurf, Gemini CLI. */ +function stdioMcpServers({ baseUrl, apiKey, serverDir }: McpSnippetOptions): string { return json({ mcpServers: { surfsense: { @@ -52,125 +85,255 @@ export const MCP_CLIENTS: McpClient[] = [ { id: "claude-code", label: "Claude Code", - configFile: "Terminal", - language: "bash", - steps: [ - "Run this command in a terminal (any directory).", - "Start Claude Code and run /mcp — surfsense should be listed as connected.", - ], - buildConfig: ({ baseUrl, apiKey, serverDir }) => - [ - "claude mcp add surfsense \\", - ` -e SURFSENSE_BASE_URL=${baseUrl} \\`, - ` -e SURFSENSE_API_KEY=${apiKey} \\`, - ` -- uv run --directory ${serverDir} python -m surfsense_mcp`, - ].join("\n"), + remote: { + configFile: "Terminal", + language: "bash", + steps: [ + "Run this command in a terminal (any directory).", + "Start Claude Code and run /mcp — surfsense should be listed as connected.", + ], + build: ({ remoteUrl, apiKey }) => + [ + `claude mcp add --transport http surfsense ${remoteUrl} \\`, + ` --header "Authorization: ${bearer(apiKey)}"`, + ].join("\n"), + }, + stdio: { + configFile: "Terminal", + language: "bash", + steps: [ + "Run this command in a terminal (any directory).", + "Start Claude Code and run /mcp — surfsense should be listed as connected.", + ], + build: ({ baseUrl, apiKey, serverDir }) => + [ + "claude mcp add surfsense \\", + ` -e SURFSENSE_BASE_URL=${baseUrl} \\`, + ` -e SURFSENSE_API_KEY=${apiKey} \\`, + ` -- uv run --directory ${serverDir} python -m surfsense_mcp`, + ].join("\n"), + }, }, { id: "codex", label: "Codex", - configFile: "~/.codex/config.toml", - language: "toml", - steps: [ - "Add this to ~/.codex/config.toml (or run `codex mcp add surfsense -- uv run --directory python -m surfsense_mcp`).", - "Restart Codex; `codex mcp list` should show surfsense.", - ], - buildConfig: ({ baseUrl, apiKey, serverDir }) => - [ - "[mcp_servers.surfsense]", - 'command = "uv"', - `args = ${JSON.stringify(serverArgs(serverDir))}`, - "", - "[mcp_servers.surfsense.env]", - `SURFSENSE_BASE_URL = "${baseUrl}"`, - `SURFSENSE_API_KEY = "${apiKey}"`, - ].join("\n"), + remote: { + configFile: "~/.codex/config.toml", + language: "toml", + steps: [ + "Add this to ~/.codex/config.toml. The rmcp flag must sit above every [mcp_servers.*] table.", + "Restart Codex; `codex mcp list` should show surfsense.", + ], + build: ({ remoteUrl, apiKey }) => + [ + "experimental_use_rmcp_client = true", + "", + "[mcp_servers.surfsense]", + `url = "${remoteUrl}"`, + "", + "[mcp_servers.surfsense.http_headers]", + `Authorization = "${bearer(apiKey)}"`, + ].join("\n"), + }, + stdio: { + configFile: "~/.codex/config.toml", + language: "toml", + steps: [ + "Add this to ~/.codex/config.toml (or run `codex mcp add surfsense -- uv run --directory python -m surfsense_mcp`).", + "Restart Codex; `codex mcp list` should show surfsense.", + ], + build: ({ baseUrl, apiKey, serverDir }) => + [ + "[mcp_servers.surfsense]", + 'command = "uv"', + `args = ${JSON.stringify(serverArgs(serverDir))}`, + "", + "[mcp_servers.surfsense.env]", + `SURFSENSE_BASE_URL = "${baseUrl}"`, + `SURFSENSE_API_KEY = "${apiKey}"`, + ].join("\n"), + }, }, { id: "opencode", label: "OpenCode", - configFile: "opencode.json", - language: "json", - steps: [ - "Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).", - "Note OpenCode's format: the key is `mcp`, the command is one array, and env vars go under `environment`.", - ], - buildConfig: ({ baseUrl, apiKey, serverDir }) => - json({ - $schema: "https://opencode.ai/config.json", - mcp: { - surfsense: { - type: "local", - command: ["uv", ...serverArgs(serverDir)], - enabled: true, - environment: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + remote: { + configFile: "opencode.json", + language: "json", + steps: [ + "Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).", + "`oauth: false` tells OpenCode to use the Bearer key instead of starting an OAuth flow.", + ], + build: ({ remoteUrl, apiKey }) => + json({ + $schema: "https://opencode.ai/config.json", + mcp: { + surfsense: { + type: "remote", + url: remoteUrl, + enabled: true, + oauth: false, + headers: { Authorization: bearer(apiKey) }, + }, }, - }, - }), + }), + }, + stdio: { + configFile: "opencode.json", + language: "json", + steps: [ + "Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).", + "Note OpenCode's format: the key is `mcp`, the command is one array, and env vars go under `environment`.", + ], + build: ({ baseUrl, apiKey, serverDir }) => + json({ + $schema: "https://opencode.ai/config.json", + mcp: { + surfsense: { + type: "local", + command: ["uv", ...serverArgs(serverDir)], + enabled: true, + environment: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + }, + }, + }), + }, }, { id: "cursor", label: "Cursor", - configFile: "~/.cursor/mcp.json", - language: "json", - steps: [ - "Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.", - "Refresh the server in Cursor Settings → MCP; its 18 tools should appear.", - ], - buildConfig: standardJson, + remote: { + configFile: "~/.cursor/mcp.json", + language: "json", + steps: [ + "Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.", + "Refresh the server in Cursor Settings → MCP; its 18 tools should appear.", + ], + build: remoteMcpServers("url"), + }, + stdio: { + configFile: "~/.cursor/mcp.json", + language: "json", + steps: [ + "Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.", + "Refresh the server in Cursor Settings → MCP; its 18 tools should appear.", + ], + build: stdioMcpServers, + }, }, { id: "claude-desktop", label: "Claude Desktop", - configFile: "claude_desktop_config.json", - language: "json", - steps: [ - "Open Settings → Developer → Edit Config to reach claude_desktop_config.json and add this.", - "Restart Claude Desktop; surfsense appears under the tools icon.", - ], - buildConfig: standardJson, + remote: { + configFile: "claude_desktop_config.json", + language: "json", + steps: [ + "Claude Desktop can't take a remote URL directly, so this uses the mcp-remote bridge (needs Node 18+).", + "Open Settings → Developer → Edit Config, add this, and restart Claude Desktop.", + ], + build: ({ remoteUrl, apiKey }) => + json({ + mcpServers: { + surfsense: { + command: "npx", + args: ["-y", "mcp-remote", remoteUrl, "--header", `Authorization: ${bearer(apiKey)}`], + }, + }, + }), + }, + stdio: { + configFile: "claude_desktop_config.json", + language: "json", + steps: [ + "Open Settings → Developer → Edit Config to reach claude_desktop_config.json and add this.", + "Restart Claude Desktop; surfsense appears under the tools icon.", + ], + build: stdioMcpServers, + }, }, { id: "vscode", label: "VS Code", - configFile: ".vscode/mcp.json", - language: "json", - steps: [ - "Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).", - "Open Copilot Chat in agent mode and click the tools icon to confirm surfsense is loaded.", - ], - buildConfig: ({ baseUrl, apiKey, serverDir }) => - json({ - servers: { - surfsense: { - type: "stdio", - command: "uv", - args: serverArgs(serverDir), - env: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + remote: { + configFile: ".vscode/mcp.json", + language: "json", + steps: [ + "Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).", + "VS Code requires an explicit `type` field — `http` for the hosted server.", + ], + build: ({ remoteUrl, apiKey }) => + json({ + servers: { + surfsense: { + type: "http", + url: remoteUrl, + headers: { Authorization: bearer(apiKey) }, + }, }, - }, - }), + }), + }, + stdio: { + configFile: ".vscode/mcp.json", + language: "json", + steps: [ + "Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).", + "Open Copilot Chat in agent mode and click the tools icon to confirm surfsense is loaded.", + ], + build: ({ baseUrl, apiKey, serverDir }) => + json({ + servers: { + surfsense: { + type: "stdio", + command: "uv", + args: serverArgs(serverDir), + env: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + }, + }, + }), + }, }, { id: "windsurf", label: "Windsurf", - configFile: "~/.codeium/windsurf/mcp_config.json", - language: "json", - steps: [ - "Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).", - "Press the refresh button in the MCP panel to pick up the server.", - ], - buildConfig: standardJson, + remote: { + configFile: "~/.codeium/windsurf/mcp_config.json", + language: "json", + steps: [ + "Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).", + "Windsurf uses `serverUrl` (not `url`) for remote servers; press refresh in the MCP panel.", + ], + build: remoteMcpServers("serverUrl"), + }, + stdio: { + configFile: "~/.codeium/windsurf/mcp_config.json", + language: "json", + steps: [ + "Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).", + "Press the refresh button in the MCP panel to pick up the server.", + ], + build: stdioMcpServers, + }, }, { id: "gemini-cli", label: "Gemini CLI", - configFile: "~/.gemini/settings.json", - language: "json", - steps: [ - "Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).", - "Run /mcp inside Gemini CLI to confirm the surfsense server and its tools.", - ], - buildConfig: standardJson, + remote: { + configFile: "~/.gemini/settings.json", + language: "json", + steps: [ + "Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).", + "Gemini CLI uses `httpUrl` for streamable-HTTP servers; run /mcp to confirm surfsense.", + ], + build: remoteMcpServers("httpUrl"), + }, + stdio: { + configFile: "~/.gemini/settings.json", + language: "json", + steps: [ + "Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).", + "Run /mcp inside Gemini CLI to confirm the surfsense server and its tools.", + ], + build: stdioMcpServers, + }, }, ]; From ef3f9f2e25ea9661351adb712e2d4f7588af462a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:39:58 +0200 Subject: [PATCH 14/56] feat(tiktok): item output schemas --- .../proprietary/platforms/tiktok/__init__.py | 0 .../platforms/tiktok/schemas/__init__.py | 21 +++ .../platforms/tiktok/schemas/items.py | 132 ++++++++++++++++++ 3 files changed, 153 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py new file mode 100644 index 000000000..28326a544 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py @@ -0,0 +1,21 @@ +"""Apify-compatible input and output contracts for the TikTok scraper.""" + +from __future__ import annotations + +from .items import ( + AuthorMeta, + CommentItem, + ErrorItem, + MusicMeta, + TikTokVideoItem, + VideoMeta, +) + +__all__ = [ + "AuthorMeta", + "CommentItem", + "ErrorItem", + "MusicMeta", + "TikTokVideoItem", + "VideoMeta", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py new file mode 100644 index 000000000..c4ff87618 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py @@ -0,0 +1,132 @@ +# ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API +"""Output items keyed to the Clockworks TikTok actor's shape. + +Every model is open (``extra="allow"``) and defaults unsourced fields to +``None``/``[]`` so the MVP can populate a reliable subset and expand the +contract additively without breaking consumers. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class AuthorMeta(BaseModel): + model_config = ConfigDict(extra="allow") + + id: str | None = None + name: str | None = None + nickName: str | None = None + profileUrl: str | None = None + verified: bool | None = None + signature: str | None = None + avatar: str | None = None + privateAccount: bool | None = None + fans: int | None = None + following: int | None = None + heart: int | None = None + video: int | None = None + + +class MusicMeta(BaseModel): + model_config = ConfigDict(extra="allow") + + musicId: str | None = None + musicName: str | None = None + musicAuthor: str | None = None + musicOriginal: bool | None = None + playUrl: str | None = None + + +class VideoMeta(BaseModel): + model_config = ConfigDict(extra="allow") + + height: int | None = None + width: int | None = None + duration: int | None = None + coverUrl: str | None = None + format: str | None = None + definition: str | None = None + + +class TikTokVideoItem(BaseModel): + """A single scraped post (video or slideshow).""" + + model_config = ConfigDict(extra="allow") + + id: str | None = None + text: str | None = None + textLanguage: str | None = None + createTime: int | None = None + createTimeISO: str | None = None + isAd: bool | None = None + + authorMeta: AuthorMeta = Field(default_factory=AuthorMeta) + musicMeta: MusicMeta = Field(default_factory=MusicMeta) + videoMeta: VideoMeta = Field(default_factory=VideoMeta) + + webVideoUrl: str | None = None + mediaUrls: list[str] = Field(default_factory=list) + + diggCount: int | None = None + shareCount: int | None = None + playCount: int | None = None + collectCount: int | None = None + commentCount: int | None = None + + hashtags: list[dict[str, Any]] = Field(default_factory=list) + mentions: list[str] = Field(default_factory=list) + + isSlideshow: bool | None = None + isPinned: bool | None = None + isSponsored: bool | None = None + + scrapedAt: str | None = None + + def to_output(self) -> dict[str, Any]: + return self.model_dump(exclude_none=False) + + +class CommentItem(BaseModel): + """A single comment or reply under a post.""" + + model_config = ConfigDict(extra="allow") + + id: str | None = None + text: str | None = None + videoWebUrl: str | None = None + diggCount: int | None = None + replyCommentTotal: int | None = None + createTime: int | None = None + createTimeISO: str | None = None + + uid: str | None = None + uniqueId: str | None = None + nickName: str | None = None + avatar: str | None = None + + repliesToId: str | None = None + scrapedAt: str | None = None + + def to_output(self) -> dict[str, Any]: + return self.model_dump(exclude_none=False) + + +class ErrorItem(BaseModel): + """Per-input failure, distinguished from normal items by ``errorCode``. + + Mirrors the actor's convention so a private/deleted/empty target surfaces + as an item instead of silently vanishing from the results. + """ + + model_config = ConfigDict(extra="allow") + + url: str | None = None + input: str | None = None + error: str | None = None + errorCode: str | None = None + + def to_output(self) -> dict[str, Any]: + return self.model_dump(exclude_none=False) From b2d387f994d415a37383d6d2b1e4ff0755e13eae Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:40:07 +0200 Subject: [PATCH 15/56] feat(tiktok): URL target resolver --- .../platforms/tiktok/targets/__init__.py | 8 +++ .../platforms/tiktok/targets/resolver.py | 50 +++++++++++++++++++ .../platforms/tiktok/targets/types.py | 25 ++++++++++ .../tests/unit/platforms/tiktok/__init__.py | 0 .../platforms/tiktok/test_target_resolver.py | 47 +++++++++++++++++ 5 files changed, 130 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/__init__.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py new file mode 100644 index 000000000..ac7410154 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py @@ -0,0 +1,8 @@ +"""TikTok URL classification into scrape targets.""" + +from __future__ import annotations + +from .resolver import resolve_target +from .types import TargetKind, TikTokTarget + +__all__ = ["TargetKind", "TikTokTarget", "resolve_target"] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py new file mode 100644 index 000000000..ffd15ba2a --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py @@ -0,0 +1,50 @@ +"""Classify a TikTok URL into a :class:`TikTokTarget`, or ``None``.""" + +from __future__ import annotations + +from urllib.parse import parse_qs, unquote, urlparse + +from .types import SearchSection, TikTokTarget + +_TIKTOK_HOSTS = frozenset({"tiktok.com", "www.tiktok.com", "m.tiktok.com"}) +_SEARCH_SECTIONS: frozenset[SearchSection] = frozenset({"video", "user"}) + + +def _is_tiktok_host(hostname: str | None) -> bool: + return bool(hostname) and hostname.lower() in _TIKTOK_HOSTS + + +def resolve_target(url: str) -> TikTokTarget | None: + parsed = urlparse(url) + if not _is_tiktok_host(parsed.hostname): + return None + + segments = [s for s in (parsed.path or "").split("/") if s] + if not segments: + return None + + # Profile / video live under /@username[...]. + if segments[0].startswith("@"): + username = segments[0][1:] + if not username: + return None + if len(segments) >= 3 and segments[1] == "video" and segments[2]: + return TikTokTarget("video", segments[2], url, username=username) + return TikTokTarget("profile", username, url) + + if segments[0] == "tag" and len(segments) >= 2 and segments[1]: + return TikTokTarget("hashtag", unquote(segments[1]), url) + + if segments[0] == "search": + query = parse_qs(parsed.query).get("q", [None])[0] + if not query: + return None + section = segments[1] if len(segments) >= 2 else None + return TikTokTarget( + "search", + unquote(query), + url, + section=section if section in _SEARCH_SECTIONS else None, + ) + + return None diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py new file mode 100644 index 000000000..c53d5239d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py @@ -0,0 +1,25 @@ +"""Scrape-target value object produced by URL classification.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +TargetKind = Literal["video", "profile", "hashtag", "search"] +SearchSection = Literal["video", "user"] + + +@dataclass(frozen=True, slots=True) +class TikTokTarget: + """One classified scrape target. + + ``value`` holds the kind-specific identifier: video id, username, hashtag + name, or search query. ``username`` is set for videos (needed to build the + canonical post URL). ``section`` narrows a search to videos or users. + """ + + kind: TargetKind + value: str + url: str + username: str | None = None + section: SearchSection | None = None diff --git a/surfsense_backend/tests/unit/platforms/tiktok/__init__.py b/surfsense_backend/tests/unit/platforms/tiktok/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py b/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py new file mode 100644 index 000000000..c118f4f86 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py @@ -0,0 +1,47 @@ +"""URL classification for the TikTok scraper (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.targets import resolve_target + + +def test_resolve_video_carries_username_and_id(): + target = resolve_target("https://www.tiktok.com/@scout2015/video/6718335390845095173") + assert target is not None + assert target.kind == "video" + assert target.value == "6718335390845095173" + assert target.username == "scout2015" + + +def test_resolve_profile(): + target = resolve_target("https://www.tiktok.com/@scout2015") + assert target is not None + assert target.kind == "profile" + assert target.value == "scout2015" + + +def test_resolve_hashtag(): + target = resolve_target("https://www.tiktok.com/tag/funny") + assert target is not None + assert target.kind == "hashtag" + assert target.value == "funny" + + +def test_resolve_search_top_video_and_user_sections(): + top = resolve_target("https://www.tiktok.com/search?q=cats") + assert top is not None + assert top.kind == "search" + assert top.value == "cats" + assert top.section is None + + videos = resolve_target("https://www.tiktok.com/search/video?q=cats") + assert videos is not None and videos.section == "video" + + users = resolve_target("https://www.tiktok.com/search/user?q=cats") + assert users is not None and users.section == "user" + + +def test_resolve_rejects_non_tiktok_and_unknown_paths(): + assert resolve_target("https://example.com/@scout2015") is None + assert resolve_target("https://www.tiktok.com/") is None + assert resolve_target("https://www.tiktok.com/foundation") is None From 8840fd2c53f1dfd10985069723f152898dbbbfb6 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:40:21 +0200 Subject: [PATCH 16/56] feat(tiktok): rehydration blob extractor --- .../platforms/tiktok/extraction/__init__.py | 7 ++++ .../platforms/tiktok/extraction/hydration.py | 28 ++++++++++++++++ .../unit/platforms/tiktok/test_hydration.py | 32 +++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py new file mode 100644 index 000000000..3bd6b160d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -0,0 +1,7 @@ +"""Turn raw TikTok page/API payloads into normalized items.""" + +from __future__ import annotations + +from .hydration import extract_rehydration_data + +__all__ = ["extract_rehydration_data"] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py new file mode 100644 index 000000000..4cc466849 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py @@ -0,0 +1,28 @@ +"""Extract the ``__UNIVERSAL_DATA_FOR_REHYDRATION__`` JSON embedded in page HTML. + +TikTok server-renders the first page of data into a single script tag; parsing +it yields page-one items (video/profile/hashtag) without any signed API call. +""" + +from __future__ import annotations + +import json +import re +from typing import Any + +_BLOB_RE = re.compile( + r']*id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>(.*?)', + re.DOTALL, +) + + +def extract_rehydration_data(html: str) -> dict[str, Any] | None: + """Return the parsed rehydration blob, or ``None`` if absent/unparseable.""" + match = _BLOB_RE.search(html) + if not match: + return None + try: + data = json.loads(match.group(1)) + except (json.JSONDecodeError, ValueError): + return None + return data if isinstance(data, dict) else None diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py b/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py new file mode 100644 index 000000000..cdca568f7 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py @@ -0,0 +1,32 @@ +"""Rehydration-blob extraction from TikTok page HTML (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import extract_rehydration_data + +_BLOB = ( + '" +) + + +def test_extracts_default_scope_from_blob(): + data = extract_rehydration_data(_BLOB) + assert data is not None + item = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"] + assert item["id"] == "123" + + +def test_returns_none_when_blob_absent(): + assert extract_rehydration_data("no blob here") is None + + +def test_returns_none_when_blob_json_malformed(): + broken = ( + '' + ) + assert extract_rehydration_data(broken) is None From 0bbcf99852c20cb19c03a621149a304dfd2543c0 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:40:38 +0200 Subject: [PATCH 17/56] feat(tiktok): author normalizer --- .../platforms/tiktok/extraction/__init__.py | 3 +- .../platforms/tiktok/extraction/author.py | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index 3bd6b160d..fa4df8731 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations +from .author import parse_author from .hydration import extract_rehydration_data -__all__ = ["extract_rehydration_data"] +__all__ = ["extract_rehydration_data", "parse_author"] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py new file mode 100644 index 000000000..414de39b6 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py @@ -0,0 +1,31 @@ +"""Normalize TikTok author/profile payloads into an ``authorMeta`` dict.""" + +from __future__ import annotations + +from typing import Any + +_PROFILE_URL = "https://www.tiktok.com/@{username}" + + +def build_author_meta(author: dict[str, Any], stats: dict[str, Any]) -> dict[str, Any]: + """Map an author object + its stats to the ``authorMeta`` output shape.""" + username = author.get("uniqueId") + return { + "id": author.get("id"), + "name": username, + "nickName": author.get("nickname"), + "profileUrl": _PROFILE_URL.format(username=username) if username else None, + "verified": author.get("verified"), + "signature": author.get("signature"), + "avatar": author.get("avatarLarger") or author.get("avatarMedium"), + "privateAccount": author.get("privateAccount"), + "fans": stats.get("followerCount"), + "following": stats.get("followingCount"), + "heart": stats.get("heartCount"), + "video": stats.get("videoCount"), + } + + +def parse_author(user_info: dict[str, Any]) -> dict[str, Any]: + """Map a ``webapp.user-detail`` ``userInfo`` (``{user, stats}``) to authorMeta.""" + return build_author_meta(user_info.get("user") or {}, user_info.get("stats") or {}) From ef66063bfc7e43bd329450fadca7e969df5aaf13 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:40:50 +0200 Subject: [PATCH 18/56] feat(tiktok): video normalizer and parser tests --- .../platforms/tiktok/extraction/__init__.py | 3 +- .../platforms/tiktok/extraction/timestamps.py | 18 ++++ .../platforms/tiktok/extraction/video.py | 73 +++++++++++++ .../unit/platforms/tiktok/test_parsers.py | 102 ++++++++++++++++++ 4 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index fa4df8731..9b4f71c68 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -4,5 +4,6 @@ from __future__ import annotations from .author import parse_author from .hydration import extract_rehydration_data +from .video import parse_video -__all__ = ["extract_rehydration_data", "parse_author"] +__all__ = ["extract_rehydration_data", "parse_author", "parse_video"] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py new file mode 100644 index 000000000..4b155c137 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py @@ -0,0 +1,18 @@ +"""Millisecond-ISO timestamps matching the actor's output shape.""" + +from __future__ import annotations + +from datetime import UTC, datetime + + +def epoch_to_iso(seconds: int | None) -> str | None: + """Convert a Unix-seconds timestamp to ``YYYY-MM-DDTHH:MM:SS.000Z``.""" + if not seconds: + return None + stamp = datetime.fromtimestamp(seconds, tz=UTC) + return stamp.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + +def now_iso() -> str: + """Current UTC time in the millisecond-ISO output shape.""" + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py new file mode 100644 index 000000000..1bfe66452 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py @@ -0,0 +1,73 @@ +"""Normalize a raw TikTok ``itemStruct`` into a :class:`TikTokVideoItem` dict.""" + +from __future__ import annotations + +from typing import Any + +from ..schemas.items import TikTokVideoItem +from .author import build_author_meta +from .timestamps import epoch_to_iso + +_VIDEO_URL = "https://www.tiktok.com/@{username}/video/{video_id}" + + +def _music_meta(music: dict[str, Any]) -> dict[str, Any]: + return { + "musicId": music.get("id"), + "musicName": music.get("title"), + "musicAuthor": music.get("authorName"), + "musicOriginal": music.get("original"), + "playUrl": music.get("playUrl"), + } + + +def _video_meta(video: dict[str, Any]) -> dict[str, Any]: + return { + "height": video.get("height"), + "width": video.get("width"), + "duration": video.get("duration"), + "coverUrl": video.get("cover"), + "format": video.get("format"), + "definition": video.get("definition"), + } + + +def _hashtags(challenges: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [ + {"id": c.get("id"), "name": c.get("title")} + for c in challenges + if isinstance(c, dict) + ] + + +def parse_video(item: dict[str, Any]) -> dict[str, Any]: + """Map an ``itemStruct`` to the flat output contract, filling known fields.""" + author = item.get("author") or {} + author_stats = item.get("authorStats") or {} + stats = item.get("stats") or {} + username = author.get("uniqueId") + video_id = item.get("id") + + web_url = ( + _VIDEO_URL.format(username=username, video_id=video_id) + if username and video_id + else None + ) + create_time = item.get("createTime") + + return TikTokVideoItem( + id=video_id, + text=item.get("desc"), + createTime=create_time, + createTimeISO=epoch_to_iso(create_time), + authorMeta=build_author_meta(author, author_stats), + musicMeta=_music_meta(item.get("music") or {}), + videoMeta=_video_meta(item.get("video") or {}), + webVideoUrl=web_url, + diggCount=stats.get("diggCount"), + shareCount=stats.get("shareCount"), + playCount=stats.get("playCount"), + collectCount=stats.get("collectCount"), + commentCount=stats.get("commentCount"), + hashtags=_hashtags(item.get("challenges") or []), + ).to_output() diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py b/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py new file mode 100644 index 000000000..03fa63133 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py @@ -0,0 +1,102 @@ +"""Raw TikTok payload -> normalized item mapping (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import parse_author, parse_video + +_ITEM_STRUCT = { + "id": "7534061113365859586", + "desc": "haha #comeramabanana", + "createTime": 1_700_000_000, + "author": { + "id": "6733", + "uniqueId": "bruniela_", + "nickname": "Bruni", + "verified": False, + "signature": "bio here", + "avatarLarger": "https://cdn/avatar.jpg", + }, + "authorStats": { + "followerCount": 51200, + "followingCount": 269, + "heartCount": 3_000_000, + "videoCount": 259, + }, + "stats": { + "diggCount": 5344, + "shareCount": 701, + "playCount": 55700, + "commentCount": 24, + "collectCount": 291, + }, + "music": { + "id": "7529", + "title": "som original", + "authorName": "fox_rus0", + "original": True, + "playUrl": "https://cdn/music.mp3", + }, + "video": { + "height": 1024, + "width": 576, + "duration": 16, + "cover": "https://cdn/cover.jpg", + "format": "mp4", + "definition": "540p", + }, + "challenges": [{"id": "4982299", "title": "comeramabanana"}], +} + + +def test_parse_video_maps_core_and_derived_fields(): + item = parse_video(_ITEM_STRUCT) + + assert item["id"] == "7534061113365859586" + assert item["text"] == "haha #comeramabanana" + assert item["createTimeISO"] == "2023-11-14T22:13:20.000Z" + + assert item["authorMeta"]["name"] == "bruniela_" + assert item["authorMeta"]["nickName"] == "Bruni" + assert item["authorMeta"]["profileUrl"] == "https://www.tiktok.com/@bruniela_" + assert item["authorMeta"]["fans"] == 51200 + + assert item["musicMeta"]["musicName"] == "som original" + assert item["videoMeta"]["duration"] == 16 + + assert item["diggCount"] == 5344 + assert item["playCount"] == 55700 + + assert item["hashtags"] == [{"id": "4982299", "name": "comeramabanana"}] + assert ( + item["webVideoUrl"] + == "https://www.tiktok.com/@bruniela_/video/7534061113365859586" + ) + + +_USER_INFO = { + "user": { + "id": "6733", + "uniqueId": "bruniela_", + "nickname": "Bruni", + "verified": True, + "signature": "bio here", + "avatarLarger": "https://cdn/avatar.jpg", + "privateAccount": False, + }, + "stats": { + "followerCount": 51200, + "followingCount": 269, + "heartCount": 3_000_000, + "videoCount": 259, + }, +} + + +def test_parse_author_maps_user_and_stats(): + author = parse_author(_USER_INFO) + assert author["name"] == "bruniela_" + assert author["nickName"] == "Bruni" + assert author["verified"] is True + assert author["profileUrl"] == "https://www.tiktok.com/@bruniela_" + assert author["fans"] == 51200 + assert author["video"] == 259 From 5688ab0678b6571bd57ab98f20470270b2c8d8a2 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:51:17 +0200 Subject: [PATCH 19/56] feat(tiktok): scrape input schema --- .../platforms/tiktok/schemas/__init__.py | 3 + .../platforms/tiktok/schemas/input.py | 60 +++++++++++++++++++ .../tests/unit/platforms/tiktok/test_input.py | 26 ++++++++ 3 files changed, 89 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_input.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py index 28326a544..3593f0e28 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations +from .input import StartUrl, TikTokScrapeInput from .items import ( AuthorMeta, CommentItem, @@ -16,6 +17,8 @@ __all__ = [ "CommentItem", "ErrorItem", "MusicMeta", + "StartUrl", + "TikTokScrapeInput", "TikTokVideoItem", "VideoMeta", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py new file mode 100644 index 000000000..fa87772fe --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py @@ -0,0 +1,60 @@ +# ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API +"""Input surface for the TikTok scraper, shaped to the Clockworks actor. + +Anonymous only: no auth-shaped field exists here. Fields the Phase-1 blob-first +path does not yet act on (media downloads, follower add-ons) are still accepted +via ``extra="allow"`` for contract parity and land inert. + +Caps (``resultsPerPage``) are per-target counts; the cross-target ceiling is +caller policy applied by the collector, never baked into the flows. +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +ProfileSorting = Literal["latest", "popular", "oldest"] +ProfileSection = Literal["videos", "reposts"] +SearchSection = Literal["", "/video", "/user"] + + +class StartUrl(BaseModel): + """A single direct URL entry (``{"url": ...}``; extra keys ignored).""" + + model_config = ConfigDict(extra="allow") + + url: str + + +class TikTokScrapeInput(BaseModel): + model_config = ConfigDict(extra="allow") + + # Discovery + startUrls: list[StartUrl] = Field(default_factory=list) + hashtags: list[str] = Field(default_factory=list) + profiles: list[str] = Field(default_factory=list) + searchQueries: list[str] = Field(default_factory=list) + postURLs: list[str] = Field(default_factory=list) + + # Per-target count + resultsPerPage: int = Field(default=1, ge=1) + + # Profile options + profileScrapeSections: list[ProfileSection] = Field( + default_factory=lambda: ["videos"] + ) + profileSorting: ProfileSorting = "latest" + excludePinnedPosts: bool = False + + # Search options + searchSection: SearchSection = "" + maxProfilesPerQuery: int = Field(default=10, ge=1) + + # Incremental filters (ISO date or relative " days" per the actor) + oldestPostDateUnified: str | None = None + newestPostDate: str | None = None + + # Proxy + proxyCountryCode: str = "None" diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_input.py b/surfsense_backend/tests/unit/platforms/tiktok/test_input.py new file mode 100644 index 000000000..c080ffec3 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_input.py @@ -0,0 +1,26 @@ +"""Input surface for the TikTok scraper (anonymous, Apify-shaped).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.schemas import TikTokScrapeInput + + +def test_input_has_no_auth_fields(): + forbidden = {"username", "password", "token", "login", "auth", "credentials"} + assert forbidden.isdisjoint(TikTokScrapeInput.model_fields) + + +def test_input_defaults(): + model = TikTokScrapeInput() + assert model.resultsPerPage == 1 + assert model.profileSorting == "latest" + assert model.proxyCountryCode == "None" + assert model.hashtags == [] + assert model.profiles == [] + assert model.searchQueries == [] + assert model.postURLs == [] + + +def test_input_allows_extra_inert_fields(): + model = TikTokScrapeInput(shouldDownloadVideos=True, videoKvStoreIdOrName="x") + assert model.model_dump().get("shouldDownloadVideos") is True From 44b3e640d36c72423433881925f77c1abea92569 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:51:17 +0200 Subject: [PATCH 20/56] feat(tiktok): rotate-on-block fetch session --- .../platforms/tiktok/session/__init__.py | 15 ++ .../platforms/tiktok/session/client.py | 130 +++++++++++++++++ .../platforms/tiktok/session/errors.py | 11 ++ .../platforms/tiktok/session/proxy.py | 113 +++++++++++++++ .../platforms/tiktok/test_fetch_resilience.py | 135 ++++++++++++++++++ 5 files changed, 404 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/client.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py new file mode 100644 index 000000000..659305055 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py @@ -0,0 +1,15 @@ +"""Cookie-warmed, rotate-on-block proxy session and page-fetch seam.""" + +from __future__ import annotations + +from .client import fetch_html +from .errors import TikTokAccessBlockedError +from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session + +__all__ = [ + "TikTokAccessBlockedError", + "bind_proxy_holder", + "fetch_html", + "open_proxy_holder", + "proxy_session", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py new file mode 100644 index 000000000..e5be1e419 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py @@ -0,0 +1,130 @@ +"""GET TikTok page HTML through a cookie-warmed, sticky-IP proxy session. + +The warm-up mints TikTok's anonymous device cookie (``ttwid``) on the first +homepage hit; the target page then server-renders its rehydration blob. Rotates +the residential IP and re-warms on 403, backs off on 429, and raises +:class:`TikTokAccessBlockedError` only when every rotated IP refuses access. +""" + +from __future__ import annotations + +import asyncio +import logging +import random +from contextlib import suppress +from typing import Any + +from scrapling.fetchers import AsyncFetcher + +from app.utils.proxy import get_proxy_url + +from .errors import TikTokAccessBlockedError +from .proxy import _REQUEST_TIMEOUT_S, _current_session, proxy_session + +logger = logging.getLogger(__name__) + +# 403 => IP blocked; rotate and re-warm. 429 => rate limited; back off same IP. +_ROTATE_STATUS = 403 +_BACKOFF_STATUS = 429 +_MAX_ROTATIONS = 3 +_MAX_BACKOFFS = 4 +_BACKOFF_BASE_S = 5.0 + +_HOME_URL = "https://www.tiktok.com/" +_TTWID_COOKIE = "ttwid" +_HEADERS = {"Accept-Language": "en-US,en;q=0.9"} + + +def _response_cookie_names(page: Any) -> set[str]: + cookies = getattr(page, "cookies", None) + return set(cookies.keys()) if isinstance(cookies, dict) else set() + + +def _page_html(page: Any) -> str | None: + for attr in ("text", "body"): + val = getattr(page, attr, None) + if isinstance(val, bytes): + val = val.decode("utf-8", "replace") + if isinstance(val, str) and val.strip(): + return val + return None + + +async def warm_session(session: Any) -> bool: + """Mint an anonymous ``ttwid`` cookie; ``True`` if the session can now fetch.""" + with suppress(Exception): + page = await session.get(_HOME_URL, headers=_HEADERS) + if _TTWID_COOKIE in _response_cookie_names(page): + return True + return False + + +async def _get_page(session: Any, url: str) -> Any: + if session is not None: + return await session.get(url, headers=_HEADERS) + return await AsyncFetcher.get( + url, + headers=_HEADERS, + proxy=get_proxy_url(), + stealthy_headers=True, + timeout=_REQUEST_TIMEOUT_S, + ) + + +async def fetch_html(url: str) -> str | None: + """Return page HTML, or ``None`` on 404 / non-block failure.""" + holder = _current_session.get() + if holder is None: + async with proxy_session(): + return await fetch_html(url) + + attempt = 0 + backoffs = 0 + while True: + session = holder.session + try: + if session is not None and not holder.warmed: + warmed_ok = await warm_session(session) + holder.warmed = True + if not warmed_ok: + if attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + raise TikTokAccessBlockedError( + f"could not warm session after {attempt} IP rotations: {url}" + ) + + await holder.pace() + page = await _get_page(session, url) + status = page.status + + if status == 200: + return _page_html(page) + if status == 404: + return None + if status == _BACKOFF_STATUS and backoffs < _MAX_BACKOFFS: + backoffs += 1 + delay = _BACKOFF_BASE_S * (2 ** (backoffs - 1)) + logger.warning("[tiktok] 429 on %s; backing off %.1fs", url, delay) + await asyncio.sleep(delay + random.uniform(0, 1)) + continue + if status == _ROTATE_STATUS and attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + if status == _ROTATE_STATUS: + raise TikTokAccessBlockedError( + f"TikTok refused {url} on {attempt} rotated IPs (403)" + ) + logger.warning("[tiktok] GET %s returned %s", url, status) + return None + except TikTokAccessBlockedError: + raise + except Exception as e: + logger.warning("[tiktok] GET %s failed: %s", url, e) + if attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + return None diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py new file mode 100644 index 000000000..4d9fc712d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py @@ -0,0 +1,11 @@ +"""Fetch-seam errors surfaced to the capability layer.""" + +from __future__ import annotations + + +class TikTokAccessBlockedError(RuntimeError): + """Raised when every rotated IP is refused anonymous access. + + Anonymous-only: we cannot log in, so a hard block is surfaced loudly rather + than returning empty data. The route maps it to a 403. + """ diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py new file mode 100644 index 000000000..dca6c776b --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py @@ -0,0 +1,113 @@ +"""Rotate-on-block sticky proxy session, bound per-flow via a ContextVar. + +Reusing one keep-alive connection pins a single residential exit IP so the +warmed cookie jar (``ttwid``/``msToken``, bound to that IP) stays valid across +the warm-up and every subsequent fetch. Ported from the Reddit sibling; the +TikTok-specific warm-up lives in :mod:`client`. +""" + +from __future__ import annotations + +import asyncio +import logging +import random +import time +from contextlib import asynccontextmanager, suppress +from contextvars import ContextVar +from typing import Any + +from scrapling.fetchers import FetcherSession + +from app.utils.proxy import get_proxy_url + +logger = logging.getLogger(__name__) + +# Pace each sticky IP so a fast exit can't burst past TikTok's per-IP threshold. +_MIN_INTERVAL_S = 0.5 +_PACE_JITTER_S = 0.25 +# A healthy fetch lands in ~1-2s; cap a dead IP at one bounded wait before it +# falls through to a rotation. +_REQUEST_TIMEOUT_S = 15.0 + +_current_session: ContextVar[_RotatingSession | None] = ContextVar( + "tiktok_proxy_session", default=None +) + + +class _RotatingSession: + """Owns one live ``FetcherSession`` (sticky IP); ``rotate()`` swaps the IP. + + Used sequentially within a single flow (never shared across concurrent + tasks), so no locking is needed. ``session`` is ``None`` only when no proxy + is configured. + """ + + def __init__(self) -> None: + self._cm: Any | None = None + self.session: Any | None = None + self.rotations = 0 + self.warmed = False + self._last_at = 0.0 + + async def _open(self) -> None: + proxy = get_proxy_url() + self.warmed = False + if proxy is None: + self._cm = self.session = None + return + self._cm = FetcherSession( + proxy=proxy, + stealthy_headers=True, + impersonate="chrome", + timeout=_REQUEST_TIMEOUT_S, + ) + self.session = await self._cm.__aenter__() + + async def close(self) -> None: + if self._cm is not None: + with suppress(Exception): + await self._cm.__aexit__(None, None, None) + self._cm = self.session = None + + async def rotate(self) -> Any | None: + """Drop the current IP and connect through a fresh one.""" + await self.close() + self.rotations += 1 + await self._open() + logger.info("[tiktok] rotated proxy session (rotation #%d)", self.rotations) + return self.session + + async def pace(self) -> None: + """Sleep to hold this sticky IP under TikTok's per-IP rate threshold.""" + wait = _MIN_INTERVAL_S - (time.monotonic() - self._last_at) + if wait > 0: + await asyncio.sleep(wait + random.uniform(0, _PACE_JITTER_S)) + self._last_at = time.monotonic() + + +async def open_proxy_holder() -> _RotatingSession: + """Open a warm rotate-on-block session holder (caller owns ``close()``).""" + holder = _RotatingSession() + await holder._open() + return holder + + +@asynccontextmanager +async def bind_proxy_holder(holder: _RotatingSession): + """Route this task's fetches through ``holder`` for the enclosed block.""" + token = _current_session.set(holder) + try: + yield holder + finally: + _current_session.reset(token) + + +@asynccontextmanager +async def proxy_session(): + """Open one reused, rotate-on-block proxy session for a continuation chain.""" + holder = await open_proxy_holder() + try: + async with bind_proxy_holder(holder): + yield holder + finally: + await holder.close() diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py new file mode 100644 index 000000000..484157964 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py @@ -0,0 +1,135 @@ +"""Fetch-seam resilience for the TikTok scraper (no network, fake sessions). + +Fake sessions drive the cookie warm-up + rotate-on-block + backoff branches +deterministically; a live first IP normally warms and returns 200s. +""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.session import ( + TikTokAccessBlockedError, + client, +) +from app.proprietary.platforms.tiktok.session.proxy import _current_session + +_HTML = "ok" + + +class _FakePage: + def __init__(self, status: int, *, cookies: dict | None = None, body: str = _HTML): + self.status = status + self.cookies = cookies or {} + self.body = body + + @property + def text(self) -> str: + return self.body + + +class _FakeSession: + """One 'IP': homepage warm mints ``ttwid`` per flag; page GETs return ``status``.""" + + def __init__(self, status: int = 200, *, warms: bool = True, body: str = _HTML): + self.status = status + self.warms = warms + self.body = body + self.page_calls = 0 + self.warm_calls = 0 + + async def get(self, url, headers=None, cookies=None): + if url.rstrip("/") == "https://www.tiktok.com": + self.warm_calls += 1 + return _FakePage(200, cookies={"ttwid": "x"} if self.warms else {}) + self.page_calls += 1 + return _FakePage(self.status, body=self.body) + + +class _FakeHolder: + def __init__(self, sessions: list[_FakeSession]) -> None: + self._sessions = sessions + self.session = sessions[0] + self.rotations = 0 + self.warmed = False + + async def rotate(self): + self.rotations += 1 + self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)] + self.warmed = False + return self.session + + async def pace(self) -> None: + return None + + async def close(self) -> None: + return None + + +def _no_sleep(monkeypatch) -> None: + async def _noop(_seconds): + return None + + monkeypatch.setattr(client.asyncio, "sleep", _noop) + + +async def test_warms_then_returns_html(): + holder = _FakeHolder([_FakeSession(200, warms=True)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@scout2015") + finally: + _current_session.reset(token) + assert result == _HTML + assert holder.rotations == 0 + assert holder.session.warm_calls == 1 + + +async def test_rotates_when_warm_fails_then_succeeds(): + holder = _FakeHolder([_FakeSession(200, warms=False), _FakeSession(200, warms=True)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@scout2015") + finally: + _current_session.reset(token) + assert result == _HTML + assert holder.rotations == 1 + + +async def test_404_returns_none_without_rotating(): + holder = _FakeHolder([_FakeSession(404), _FakeSession(200)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@missing") + finally: + _current_session.reset(token) + assert result is None + assert holder.rotations == 0 + + +async def test_rotates_and_rewarms_on_403(): + holder = _FakeHolder([_FakeSession(403), _FakeSession(200, warms=True)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@scout2015") + finally: + _current_session.reset(token) + assert result == _HTML + assert holder.rotations == 1 + assert holder.session.warm_calls == 1 + + +async def test_persistent_403_raises_blocked(monkeypatch): + _no_sleep(monkeypatch) + holder = _FakeHolder( + [_FakeSession(403) for _ in range(client._MAX_ROTATIONS + 1)] + ) + token = _current_session.set(holder) + try: + raised = False + try: + await client.fetch_html("https://www.tiktok.com/@scout2015") + except TikTokAccessBlockedError: + raised = True + finally: + _current_session.reset(token) + assert raised + assert holder.rotations == client._MAX_ROTATIONS From daa0856c443e547739b25cf82e50b80a42964237 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 16:07:07 +0200 Subject: [PATCH 21/56] feat(tiktok): blob-first orchestrator + video flow --- .../proprietary/platforms/tiktok/__init__.py | 19 ++++ .../platforms/tiktok/extraction/__init__.py | 9 +- .../platforms/tiktok/extraction/scopes.py | 30 +++++++ .../platforms/tiktok/flows/__init__.py | 8 ++ .../platforms/tiktok/flows/video.py | 26 ++++++ .../platforms/tiktok/orchestrator.py | 90 +++++++++++++++++++ .../platforms/tiktok/schemas/input.py | 9 +- .../platforms/tiktok/session/errors.py | 3 +- .../platforms/tiktok/test_orchestrator.py | 73 +++++++++++++++ .../unit/platforms/tiktok/test_scopes.py | 30 +++++++ 10 files changed, 289 insertions(+), 8 deletions(-) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py index e69de29bb..27867df73 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py @@ -0,0 +1,19 @@ +"""Anonymous, blob-first TikTok scraper (public interface). + +The capability layer depends only on the names re-exported here: the input +schema, the collector/generator, the video item shape, and the hard-block error. +""" + +from __future__ import annotations + +from .orchestrator import iter_tiktok, scrape_tiktok +from .schemas import TikTokScrapeInput, TikTokVideoItem +from .session import TikTokAccessBlockedError + +__all__ = [ + "TikTokAccessBlockedError", + "TikTokScrapeInput", + "TikTokVideoItem", + "iter_tiktok", + "scrape_tiktok", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index 9b4f71c68..0e6b1c3b7 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -4,6 +4,13 @@ from __future__ import annotations from .author import parse_author from .hydration import extract_rehydration_data +from .scopes import user_info, video_item_struct from .video import parse_video -__all__ = ["extract_rehydration_data", "parse_author", "parse_video"] +__all__ = [ + "extract_rehydration_data", + "parse_author", + "parse_video", + "user_info", + "video_item_struct", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py new file mode 100644 index 000000000..a20a72b2b --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py @@ -0,0 +1,30 @@ +"""Navigate the rehydration blob to the scopes the flows consume.""" + +from __future__ import annotations + +from typing import Any + +_DEFAULT = "__DEFAULT_SCOPE__" + + +def _scope(data: dict[str, Any], name: str) -> dict[str, Any] | None: + scope = (data.get(_DEFAULT) or {}).get(name) + return scope if isinstance(scope, dict) else None + + +def video_item_struct(data: dict[str, Any]) -> dict[str, Any] | None: + """The ``itemStruct`` of a video-detail page, or ``None``.""" + scope = _scope(data, "webapp.video-detail") + if not scope: + return None + item = (scope.get("itemInfo") or {}).get("itemStruct") + return item if isinstance(item, dict) else None + + +def user_info(data: dict[str, Any]) -> dict[str, Any] | None: + """The ``userInfo`` (``{user, stats}``) of a profile page, or ``None``.""" + scope = _scope(data, "webapp.user-detail") + if not scope: + return None + info = scope.get("userInfo") + return info if isinstance(info, dict) else None diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py new file mode 100644 index 000000000..f2f4b67cc --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py @@ -0,0 +1,8 @@ +"""Per-target scrape flows: resolved target -> normalized items.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator, Awaitable, Callable + +FetchFn = Callable[[str], Awaitable[str | None]] +FlowResult = AsyncIterator[dict] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py new file mode 100644 index 000000000..48e558677 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py @@ -0,0 +1,26 @@ +"""Video-URL flow: fetch a post page, read its rehydration blob, emit one item.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from ..extraction import extract_rehydration_data, parse_video, video_item_struct +from ..extraction.timestamps import now_iso +from ..targets.types import TikTokTarget +from . import FetchFn + + +async def iter_video(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]: + html = await fetch(target.url) + if not html: + return + data = extract_rehydration_data(html) + if not data: + return + item = video_item_struct(data) + if item is None: + return + out = parse_video(item) + out["scrapedAt"] = now_iso() + yield out diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py new file mode 100644 index 000000000..d924e3a9e --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -0,0 +1,90 @@ +"""Resolve a :class:`TikTokScrapeInput` into targets and stream their items. + +Targets run sequentially on one warm sticky IP; ``limit`` is collector policy +applied by :func:`scrape_tiktok`, never baked into a flow. Each kind routes to +its flow via :func:`_dispatch`. +""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import quote + +from .flows import FetchFn +from .flows.video import iter_video +from .schemas import TikTokScrapeInput +from .session import fetch_html, proxy_session +from .targets import resolve_target +from .targets.types import TikTokTarget + +logger = logging.getLogger(__name__) + +_PROFILE_URL = "https://www.tiktok.com/@{name}" +_HASHTAG_URL = "https://www.tiktok.com/tag/{tag}" +_SEARCH_URL = "https://www.tiktok.com/search?q={query}" + + +async def _empty() -> AsyncIterator[dict[str, Any]]: + for _ in (): + yield {} + + +def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]: + """Build the target list from every input source, dropping unresolved URLs.""" + targets: list[TikTokTarget] = [] + for entry in input_model.startUrls: + resolved = resolve_target(entry.url) + if resolved is not None: + targets.append(resolved) + for url in input_model.postURLs: + resolved = resolve_target(url) + if resolved is not None: + targets.append(resolved) + for profile in input_model.profiles: + name = profile.lstrip("@") + targets.append(TikTokTarget("profile", name, _PROFILE_URL.format(name=name))) + for tag in input_model.hashtags: + targets.append(TikTokTarget("hashtag", tag, _HASHTAG_URL.format(tag=tag))) + for query in input_model.searchQueries: + targets.append( + TikTokTarget("search", query, _SEARCH_URL.format(query=quote(query))) + ) + return targets + + +def _dispatch(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]: + if target.kind == "video": + return iter_video(target, fetch=fetch) + # Listings come from the signed item_list API, not the blob. + logger.debug("[tiktok] no blob flow for %s target", target.kind) + return _empty() + + +async def iter_tiktok( + input_model: TikTokScrapeInput, *, fetch: FetchFn = fetch_html +) -> AsyncIterator[dict[str, Any]]: + """Yield normalized items for every resolved target, in order.""" + async with proxy_session(): + for target in _resolve_targets(input_model): + async for item in _dispatch(target, fetch=fetch): + yield item + + +async def scrape_tiktok( + input_model: TikTokScrapeInput, + *, + limit: int | None = None, + fetch: FetchFn = fetch_html, +) -> list[dict[str, Any]]: + """Collect :func:`iter_tiktok` into a list, honoring an optional ``limit``.""" + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + async for item in iter_tiktok(input_model, fetch=fetch): + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + break + return results diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py index fa87772fe..f710eed72 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py @@ -1,12 +1,11 @@ # ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API """Input surface for the TikTok scraper, shaped to the Clockworks actor. -Anonymous only: no auth-shaped field exists here. Fields the Phase-1 blob-first -path does not yet act on (media downloads, follower add-ons) are still accepted -via ``extra="allow"`` for contract parity and land inert. +Anonymous only: no auth-shaped field exists here. ``extra="allow"`` keeps the +contract in parity with the actor for fields the scraper does not read. -Caps (``resultsPerPage``) are per-target counts; the cross-target ceiling is -caller policy applied by the collector, never baked into the flows. +``resultsPerPage`` is a per-target count; the cross-target ceiling is caller +policy applied by the collector, never baked into a flow. """ from __future__ import annotations diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py index 4d9fc712d..708cb0212 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py @@ -6,6 +6,5 @@ from __future__ import annotations class TikTokAccessBlockedError(RuntimeError): """Raised when every rotated IP is refused anonymous access. - Anonymous-only: we cannot log in, so a hard block is surfaced loudly rather - than returning empty data. The route maps it to a 403. + Distinguishes a hard block from an empty result; the route maps it to 403. """ diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py new file mode 100644 index 000000000..85c601047 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py @@ -0,0 +1,73 @@ +"""End-to-end orchestration over a fake fetch (no network). + +Drives the public collector: input -> target -> blob-first flow -> items. +""" + +from __future__ import annotations + +import json + +from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok + + +def _video_page(video_id: str, username: str) -> str: + blob = { + "__DEFAULT_SCOPE__": { + "webapp.video-detail": { + "itemInfo": { + "itemStruct": { + "id": video_id, + "desc": "hello", + "author": {"uniqueId": username}, + "stats": {"diggCount": 5}, + } + } + } + } + } + return ( + '' + ) + + +async def test_scrape_video_url_returns_parsed_item(): + url = "https://www.tiktok.com/@scout2015/video/123" + + async def fake_fetch(_url: str) -> str: + return _video_page("123", "scout2015") + + items = await scrape_tiktok(TikTokScrapeInput(postURLs=[url]), fetch=fake_fetch) + + assert len(items) == 1 + assert items[0]["id"] == "123" + assert items[0]["diggCount"] == 5 + assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@scout2015/video/123" + assert items[0]["scrapedAt"] is not None + + +async def test_scrape_honors_limit_across_targets(): + urls = [ + "https://www.tiktok.com/@a/video/1", + "https://www.tiktok.com/@b/video/2", + ] + + async def fake_fetch(url: str) -> str: + vid = url.rsplit("/", 1)[1] + user = url.split("@")[1].split("/")[0] + return _video_page(vid, user) + + items = await scrape_tiktok( + TikTokScrapeInput(postURLs=urls), limit=1, fetch=fake_fetch + ) + assert len(items) == 1 + + +async def test_scrape_skips_unrecognized_urls(): + async def fake_fetch(_url: str) -> str: + return "" + + items = await scrape_tiktok( + TikTokScrapeInput(postURLs=["https://example.com/x"]), fetch=fake_fetch + ) + assert items == [] diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py b/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py new file mode 100644 index 000000000..7a52dd60d --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py @@ -0,0 +1,30 @@ +"""Navigating the rehydration blob to its useful scopes (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import user_info, video_item_struct + + +def test_video_item_struct_navigates_video_detail_scope(): + data = { + "__DEFAULT_SCOPE__": { + "webapp.video-detail": {"itemInfo": {"itemStruct": {"id": "123"}}} + } + } + item = video_item_struct(data) + assert item == {"id": "123"} + + +def test_user_info_navigates_user_detail_scope(): + data = { + "__DEFAULT_SCOPE__": { + "webapp.user-detail": {"userInfo": {"user": {"uniqueId": "scout2015"}}} + } + } + info = user_info(data) + assert info == {"user": {"uniqueId": "scout2015"}} + + +def test_scopes_return_none_when_absent(): + assert video_item_struct({}) is None + assert user_info({"__DEFAULT_SCOPE__": {}}) is None From ad8b73477a281a175cd579c12105f5ed02245207 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 16:37:36 +0200 Subject: [PATCH 22/56] feat(tiktok): browser-driven signed listings --- .../platforms/tiktok/extraction/__init__.py | 2 + .../platforms/tiktok/extraction/item_list.py | 30 ++++++ .../platforms/tiktok/flows/__init__.py | 5 + .../platforms/tiktok/flows/listing.py | 37 ++++++++ .../platforms/tiktok/orchestrator.py | 41 +++++---- .../platforms/tiktok/session/__init__.py | 2 + .../platforms/tiktok/session/listing.py | 92 +++++++++++++++++++ .../unit/platforms/tiktok/test_item_list.py | 25 +++++ .../platforms/tiktok/test_orchestrator.py | 24 +++++ 9 files changed, 240 insertions(+), 18 deletions(-) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index 0e6b1c3b7..d70a494a5 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -4,11 +4,13 @@ from __future__ import annotations from .author import parse_author from .hydration import extract_rehydration_data +from .item_list import items_from_response from .scopes import user_info, video_item_struct from .video import parse_video __all__ = [ "extract_rehydration_data", + "items_from_response", "parse_author", "parse_video", "user_info", diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py new file mode 100644 index 000000000..7dddf891f --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py @@ -0,0 +1,30 @@ +"""Item structs from a captured ``item_list`` / search API response body. + +Profile and hashtag listings return ``{"itemList": [...]}``; search returns +``{"data": [{"item": {...}}]}``. Both element shapes are the same itemStruct +:func:`parse_video` already consumes. +""" + +from __future__ import annotations + +from typing import Any + + +def items_from_response(body: Any) -> list[dict[str, Any]]: + """Return the itemStructs carried by one API response, or ``[]``.""" + if not isinstance(body, dict): + return [] + + item_list = body.get("itemList") + if isinstance(item_list, list): + return [i for i in item_list if isinstance(i, dict)] + + data = body.get("data") + if isinstance(data, list): + return [ + entry["item"] + for entry in data + if isinstance(entry, dict) and isinstance(entry.get("item"), dict) + ] + + return [] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py index f2f4b67cc..a7809567d 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py @@ -5,4 +5,9 @@ from __future__ import annotations from collections.abc import AsyncIterator, Awaitable, Callable FetchFn = Callable[[str], Awaitable[str | None]] +"""Fetch a page's HTML by URL (blob-first video flow).""" + +FetchListingFn = Callable[[str, int], Awaitable[list[dict]]] +"""Load a listing page and return up to ``count`` captured itemStructs.""" + FlowResult = AsyncIterator[dict] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py new file mode 100644 index 000000000..44fcc33e2 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py @@ -0,0 +1,37 @@ +"""Listing flow shared by profile, hashtag, and search targets. + +The browser seam returns raw itemStructs captured from the signed ``item_list`` +XHRs; this maps each to the output contract, drops duplicate video ids, and +stops at the per-target ``cap``. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from ..extraction import parse_video +from ..extraction.timestamps import now_iso +from ..targets.types import TikTokTarget +from . import FetchListingFn + + +async def iter_listing( + target: TikTokTarget, *, cap: int, fetch_listing: FetchListingFn +) -> AsyncIterator[dict[str, Any]]: + if cap <= 0: + return + seen: set[str] = set() + emitted = 0 + for item in await fetch_listing(target.url, cap): + out = parse_video(item) + video_id = out.get("id") + if video_id is not None: + if video_id in seen: + continue + seen.add(video_id) + out["scrapedAt"] = now_iso() + yield out + emitted += 1 + if emitted >= cap: + return diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index d924e3a9e..c7f2e9008 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -2,35 +2,29 @@ Targets run sequentially on one warm sticky IP; ``limit`` is collector policy applied by :func:`scrape_tiktok`, never baked into a flow. Each kind routes to -its flow via :func:`_dispatch`. +its flow via :func:`_dispatch`: video URLs read the rehydration blob over HTTP, +listings capture signed item_list XHRs through the stealth browser. """ from __future__ import annotations -import logging from collections.abc import AsyncIterator from typing import Any from urllib.parse import quote -from .flows import FetchFn +from .flows import FetchFn, FetchListingFn +from .flows.listing import iter_listing from .flows.video import iter_video from .schemas import TikTokScrapeInput -from .session import fetch_html, proxy_session +from .session import fetch_html, fetch_item_list, proxy_session from .targets import resolve_target from .targets.types import TikTokTarget -logger = logging.getLogger(__name__) - _PROFILE_URL = "https://www.tiktok.com/@{name}" _HASHTAG_URL = "https://www.tiktok.com/tag/{tag}" _SEARCH_URL = "https://www.tiktok.com/search?q={query}" -async def _empty() -> AsyncIterator[dict[str, Any]]: - for _ in (): - yield {} - - def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]: """Build the target list from every input source, dropping unresolved URLs.""" targets: list[TikTokTarget] = [] @@ -54,21 +48,31 @@ def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]: return targets -def _dispatch(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]: +def _dispatch( + target: TikTokTarget, + *, + cap: int, + fetch: FetchFn, + fetch_listing: FetchListingFn, +) -> AsyncIterator[dict[str, Any]]: if target.kind == "video": return iter_video(target, fetch=fetch) - # Listings come from the signed item_list API, not the blob. - logger.debug("[tiktok] no blob flow for %s target", target.kind) - return _empty() + return iter_listing(target, cap=cap, fetch_listing=fetch_listing) async def iter_tiktok( - input_model: TikTokScrapeInput, *, fetch: FetchFn = fetch_html + input_model: TikTokScrapeInput, + *, + fetch: FetchFn = fetch_html, + fetch_listing: FetchListingFn = fetch_item_list, ) -> AsyncIterator[dict[str, Any]]: """Yield normalized items for every resolved target, in order.""" + cap = input_model.resultsPerPage async with proxy_session(): for target in _resolve_targets(input_model): - async for item in _dispatch(target, fetch=fetch): + async for item in _dispatch( + target, cap=cap, fetch=fetch, fetch_listing=fetch_listing + ): yield item @@ -77,12 +81,13 @@ async def scrape_tiktok( *, limit: int | None = None, fetch: FetchFn = fetch_html, + fetch_listing: FetchListingFn = fetch_item_list, ) -> list[dict[str, Any]]: """Collect :func:`iter_tiktok` into a list, honoring an optional ``limit``.""" from app.capabilities.core.progress import emit_progress results: list[dict[str, Any]] = [] - async for item in iter_tiktok(input_model, fetch=fetch): + async for item in iter_tiktok(input_model, fetch=fetch, fetch_listing=fetch_listing): results.append(item) emit_progress("scraping", current=len(results), total=limit, unit="item") if limit is not None and len(results) >= limit: diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py index 659305055..409aa5987 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py @@ -4,12 +4,14 @@ from __future__ import annotations from .client import fetch_html from .errors import TikTokAccessBlockedError +from .listing import fetch_item_list from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session __all__ = [ "TikTokAccessBlockedError", "bind_proxy_holder", "fetch_html", + "fetch_item_list", "open_proxy_holder", "proxy_session", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py new file mode 100644 index 000000000..0d19281a2 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -0,0 +1,92 @@ +"""Browser-driven listing fetch: let TikTok sign its own ``item_list`` XHRs. + +Profile/hashtag/search listings need signed requests (``X-Gnarly``) whose +algorithm rev's monthly and reads a browser canvas fingerprint. Rather than port +and chase that signer, we load the page in the stealth browser we already run +(patchright-Chromium, via the web-crawler tier) and capture the itemStruct JSON +the page's own scripts fetch while scrolling. The browser is the client, so it +signs correctly for whatever version TikTok ships. + +The pure response-shape parsing lives in :func:`items_from_response`; this module +is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +from scrapling.fetchers import StealthyFetcher + +from app.proprietary.web_crawler.stealth import ( + build_stealthy_kwargs, + get_stealth_config, +) +from app.utils.proxy import get_proxy_url + +from ..extraction import items_from_response + +logger = logging.getLogger(__name__) + +# XHR paths that carry itemStructs for the three listing kinds. +_ITEM_LIST_MARKERS = ( + "/api/post/item_list", + "/api/challenge/item_list", + "/api/search/", +) +# Bounded scroll: a dead page can't loop forever, and a live one stops early +# once enough items are captured. +_SCROLL_MAX_ROUNDS = 20 +_SCROLL_SETTLE_MS = 1200 + + +def _build_page_action(collected: list[dict[str, Any]], target_count: int): + """A sync ``page_action`` that captures item_list XHRs while scrolling.""" + + def _on_response(response: Any) -> None: + try: + if not any(marker in response.url for marker in _ITEM_LIST_MARKERS): + return + body = response.json() + except Exception: + return + collected.extend(items_from_response(body)) + + def page_action(page: Any) -> Any: + page.on("response", _on_response) + try: + last_height = 0 + for _ in range(_SCROLL_MAX_ROUNDS): + if len(collected) >= target_count: + break + page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + page.wait_for_timeout(_SCROLL_SETTLE_MS) + height = page.evaluate("document.body.scrollHeight") + if not height or height <= last_height: + break + last_height = height + except Exception as exc: + logger.debug("[tiktok] listing scroll aborted: %s", exc) + return page + + return page_action + + +def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]: + collected: list[dict[str, Any]] = [] + kwargs = build_stealthy_kwargs(get_stealth_config()) + StealthyFetcher.fetch( + url, + headless=True, + network_idle=False, + proxy=get_proxy_url(), + page_action=_build_page_action(collected, target_count), + **kwargs, + ) + return collected[:target_count] + + +async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` itemStructs from a listing page's XHRs.""" + return await asyncio.to_thread(_fetch_sync, page_url, target_count) diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py b/surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py new file mode 100644 index 000000000..00ab89b02 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py @@ -0,0 +1,25 @@ +"""Pulling item structs out of captured item_list / search API responses.""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import items_from_response + + +def test_reads_item_list_shape(): + body = {"itemList": [{"id": "1"}, {"id": "2"}], "hasMore": True} + assert items_from_response(body) == [{"id": "1"}, {"id": "2"}] + + +def test_reads_search_data_shape(): + body = {"data": [{"type": 1, "item": {"id": "9"}}, {"type": 4, "item": {}}]} + assert items_from_response(body) == [{"id": "9"}, {}] + + +def test_skips_malformed_entries(): + body = {"data": [{"type": 1}, "junk", {"item": {"id": "7"}}]} + assert items_from_response(body) == [{"id": "7"}] + + +def test_returns_empty_for_unrelated_json(): + assert items_from_response({"statusCode": 0}) == [] + assert items_from_response("nope") == [] diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py index 85c601047..8520a1d85 100644 --- a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py @@ -71,3 +71,27 @@ async def test_scrape_skips_unrecognized_urls(): TikTokScrapeInput(postURLs=["https://example.com/x"]), fetch=fake_fetch ) assert items == [] + + +async def test_scrape_profile_returns_listing_items(): + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [ + {"id": "1", "author": {"uniqueId": "a"}}, + {"id": "2", "author": {"uniqueId": "a"}}, + ] + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=["a"], resultsPerPage=5), fetch_listing=fake_listing + ) + assert [i["id"] for i in items] == ["1", "2"] + assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@a/video/1" + + +async def test_listing_dedupes_then_caps_per_target(): + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [{"id": "1"}, {"id": "1"}, {"id": "2"}, {"id": "3"}] + + items = await scrape_tiktok( + TikTokScrapeInput(hashtags=["x"], resultsPerPage=2), fetch_listing=fake_listing + ) + assert [i["id"] for i in items] == ["1", "2"] From 5a326f3818158e40dea0fc8847b4f563ba1d4486 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 17:14:13 +0200 Subject: [PATCH 23/56] fix(tiktok): warm browser listing + drop ctx-var-in-generator --- .../platforms/tiktok/orchestrator.py | 20 ++++--- .../platforms/tiktok/session/listing.py | 60 ++++++++++++++++--- 2 files changed, 65 insertions(+), 15 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index c7f2e9008..95bc10fb9 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -16,7 +16,7 @@ from .flows import FetchFn, FetchListingFn from .flows.listing import iter_listing from .flows.video import iter_video from .schemas import TikTokScrapeInput -from .session import fetch_html, fetch_item_list, proxy_session +from .session import fetch_html, fetch_item_list from .targets import resolve_target from .targets.types import TikTokTarget @@ -66,14 +66,18 @@ async def iter_tiktok( fetch: FetchFn = fetch_html, fetch_listing: FetchListingFn = fetch_item_list, ) -> AsyncIterator[dict[str, Any]]: - """Yield normalized items for every resolved target, in order.""" + """Yield normalized items for every resolved target, in order. + + The video flow's ``fetch_html`` opens its own warmed proxy session per call + when none is bound; the listing flow drives its own browser. Neither binds a + ContextVar across these ``yield``s, so the generator stays context-safe. + """ cap = input_model.resultsPerPage - async with proxy_session(): - for target in _resolve_targets(input_model): - async for item in _dispatch( - target, cap=cap, fetch=fetch, fetch_listing=fetch_listing - ): - yield item + for target in _resolve_targets(input_model): + async for item in _dispatch( + target, cap=cap, fetch=fetch, fetch_listing=fetch_listing + ): + yield item async def scrape_tiktok( diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index 0d19281a2..98b2e6e8b 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -9,12 +9,17 @@ signs correctly for whatever version TikTok ships. The pure response-shape parsing lives in :func:`items_from_response`; this module is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). + +Requires a residential proxy: TikTok throttles bare/datacenter IPs, returning +empty ``item_list`` bodies (and 429s) after a few hits. Set +``TIKTOK_LISTING_DEBUG=1`` to print captured XHR URLs while diagnosing. """ from __future__ import annotations import asyncio import logging +import os from typing import Any from scrapling.fetchers import StealthyFetcher @@ -35,27 +40,68 @@ _ITEM_LIST_MARKERS = ( "/api/challenge/item_list", "/api/search/", ) +_HOME_URL = "https://www.tiktok.com/" +_MSTOKEN_COOKIE = "msToken" # Bounded scroll: a dead page can't loop forever, and a live one stops early # once enough items are captured. _SCROLL_MAX_ROUNDS = 20 -_SCROLL_SETTLE_MS = 1200 +_SCROLL_SETTLE_MS = 1500 +# Warm-up poll for the anonymous msToken cookie the item_list API requires. +_WARM_POLLS = 8 +_WARM_POLL_MS = 500 -def _build_page_action(collected: list[dict[str, Any]], target_count: int): - """A sync ``page_action`` that captures item_list XHRs while scrolling.""" +def _has_mstoken(page: Any) -> bool: + try: + return any(c.get("name") == _MSTOKEN_COOKIE for c in page.context.cookies()) + except Exception: + return False + + +def _build_page_action(collected: list[dict[str, Any]], url: str, target_count: int): + """A sync ``page_action`` that warms the session then captures item_list XHRs. + + A cold context returns an empty ``item_list`` body, so we first mint the + anonymous ``msToken`` (homepage hit), then navigate to the target with the + listener already attached so page-one fires into it; scrolling pages the rest. + """ + + debug = bool(os.getenv("TIKTOK_LISTING_DEBUG")) def _on_response(response: Any) -> None: + response_url = getattr(response, "url", "") + if debug and "/api/" in response_url: + print(f" [xhr] {getattr(response, 'status', '?')} {response_url[:120]}") try: - if not any(marker in response.url for marker in _ITEM_LIST_MARKERS): + if not any(marker in response_url for marker in _ITEM_LIST_MARKERS): return body = response.json() - except Exception: + except Exception as exc: + if debug: + print(f" [xhr-parse-fail] {exc} :: {response_url[:120]}") return - collected.extend(items_from_response(body)) + items = items_from_response(body) + if debug: + print(f" [match] +{len(items)} items from {response_url[:100]}") + collected.extend(items) + + def _warm(page: Any) -> None: + if _has_mstoken(page): + return + page.goto(_HOME_URL, wait_until="domcontentloaded") + for _ in range(_WARM_POLLS): + page.wait_for_timeout(_WARM_POLL_MS) + if _has_mstoken(page): + break def page_action(page: Any) -> Any: page.on("response", _on_response) try: + _warm(page) + # Navigate (back) to the target with the listener attached and a + # token in hand, so the page-one item_list fires into the capture. + page.goto(url, wait_until="domcontentloaded") + page.wait_for_timeout(_SCROLL_SETTLE_MS) last_height = 0 for _ in range(_SCROLL_MAX_ROUNDS): if len(collected) >= target_count: @@ -81,7 +127,7 @@ def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]: headless=True, network_idle=False, proxy=get_proxy_url(), - page_action=_build_page_action(collected, target_count), + page_action=_build_page_action(collected, url, target_count), **kwargs, ) return collected[:target_count] From ea95a0529fbfba5864606bdef353df32224d8e7e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 17:14:13 +0200 Subject: [PATCH 24/56] test(tiktok): live e2e smoke script --- .../scripts/e2e_tiktok_scrape.py | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 surfsense_backend/scripts/e2e_tiktok_scrape.py diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py new file mode 100644 index 000000000..29cf1edc7 --- /dev/null +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -0,0 +1,201 @@ +"""Manual functional e2e for the TikTok scraper (blob + browser-listing seams). + +Run from the backend directory: + cd surfsense_backend + uv run python scripts/e2e_tiktok_scrape.py + +What it exercises (everything REAL — live network, live proxy, live browser): + + Stage 1 — proxy egress proof (informational). + Stage 2 — profile listing via the stealth browser (captures item_list XHRs). + Stage 3 — blob video path over HTTP (a video URL derived from Stage 2). + Stage 4 — hashtag listing via the stealth browser. + Stage 5 — full scrape_tiktok() pipeline on a profile. + +On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser +suites can pin against real-shaped data without network. + +This is NOT a pytest test (it needs a live stack + proxy + a real browser). It is +the manual counterpart to the unit suites and the truth check for +``session/listing.py``, which is otherwise unverified. +""" + +import asyncio +import json +import sys +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +from dotenv import load_dotenv + +# --- bootstrap: load .env and put the backend root on sys.path before app.* --- +_BACKEND_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_BACKEND_ROOT)) +for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): + if _candidate.exists(): + load_dotenv(_candidate) + break + +_FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok" + +# Evergreen public targets: the official account and a broad hashtag. +_PROFILE = "tiktok" +_HASHTAG = "food" +_COUNT = 5 + + +def _mask(url: str | None) -> str: + if not url: + return "" + p = urlsplit(url) + creds = "***@" if p.username else "" + port = f":{p.port}" if p.port else "" + return f"{p.scheme}://{creds}{p.hostname or '?'}{port}" + + +def _hr(title: str) -> None: + print(f"\n{'=' * 70}\n{title}\n{'=' * 70}") + + +def _check(label: str, ok: bool, detail: str = "") -> bool: + print(f" [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}") + return ok + + +def _dump_fixture(name: str, data: Any) -> None: + _FIXTURES.mkdir(parents=True, exist_ok=True) + path = _FIXTURES / name + path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + print(f" [fixture] wrote {path.relative_to(_BACKEND_ROOT)}") + + +async def _discover_video_url() -> str | None: + """Find a live video URL over plain HTTP (independent of the browser path).""" + import re + + from app.proprietary.platforms.tiktok.session import fetch_html + + html = await fetch_html(f"https://www.tiktok.com/@{_PROFILE}") + if not html: + return None + match = re.search(r"/@[\w.]+/video/\d+", html) + return f"https://www.tiktok.com{match.group(0)}" if match else None + + +async def stage_proxy() -> bool: + _hr("STAGE 1 — proxy egress (informational)") + from app.utils.proxy import get_active_provider, get_proxy_url, is_pool_backed + + provider = get_active_provider() + proxy_url = get_proxy_url() + print(f" active proxy provider : {provider.name}") + print(f" proxy url : {_mask(proxy_url)}") + print(f" pool-backed (rotates) : {is_pool_backed()}") + if not proxy_url: + print(" [INFO] no proxy configured — TikTok may block anonymous access") + return True + + +async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]: + _hr(f"STAGE 2 — profile listing (browser): @{_PROFILE}") + from app.proprietary.platforms.tiktok.session import fetch_item_list + + url = f"https://www.tiktok.com/@{_PROFILE}" + raw = await fetch_item_list(url, _COUNT) + ok = _check( + "captured itemStructs from item_list XHRs", + len(raw) > 0 and isinstance(raw[0], dict) and bool(raw[0].get("id")), + f"{len(raw)} struct(s)", + ) + if ok: + _dump_fixture("listing_item.json", raw[0]) + return ok, raw + + +async def stage_blob_video(video_url: str) -> tuple[bool, dict[str, Any] | None]: + _hr("STAGE 3 — blob video path (HTTP)") + print(f" target: {video_url}") + from app.proprietary.platforms.tiktok.extraction import ( + extract_rehydration_data, + video_item_struct, + ) + from app.proprietary.platforms.tiktok.session import fetch_html + + html = await fetch_html(video_url) + if not _check("fetched page HTML", bool(html), f"{len(html or '')} chars"): + return False, None + data = extract_rehydration_data(html or "") + if not _check("extracted rehydration blob", data is not None): + return False, None + raw = video_item_struct(data or {}) + ok = _check( + "blob carries the video itemStruct", + raw is not None and bool(raw.get("id")), + f"id={None if raw is None else raw.get('id')}", + ) + if ok and raw is not None: + _dump_fixture("video_item_struct.json", raw) + return ok, raw + + +async def stage_hashtag_listing() -> bool: + _hr(f"STAGE 4 — hashtag listing (browser): #{_HASHTAG}") + from app.proprietary.platforms.tiktok.session import fetch_item_list + + url = f"https://www.tiktok.com/tag/{_HASHTAG}" + raw = await fetch_item_list(url, _COUNT) + return _check( + "captured itemStructs for hashtag", + len(raw) > 0 and bool(raw[0].get("id")), + f"{len(raw)} struct(s)", + ) + + +async def stage_pipeline() -> bool: + _hr("STAGE 5 — full scrape_tiktok() pipeline") + from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=3), limit=3 + ) + ok = _check( + "pipeline returns normalized video items", + len(items) > 0 + and bool(items[0].get("id")) + and bool(items[0].get("webVideoUrl")), + f"{len(items)} item(s)", + ) + if items: + print(f" sample: {items[0].get('webVideoUrl')} — {items[0].get('text', '')[:60]!r}") + return ok + + +async def main() -> int: + print("TikTok scraper functional e2e — live network + proxy + browser") + results: dict[str, bool] = {} + + results["Stage 1 proxy"] = await stage_proxy() + + # Blob path first: HTTP-only, so it's validated independently of the + # proxy-sensitive browser listing path. + video_url = await _discover_video_url() + if video_url: + ok_video, _ = await stage_blob_video(video_url) + results["Stage 3 blob video"] = ok_video + else: + print("\n [SKIP] Stage 3 — could not discover a video URL over HTTP") + + ok_profile, _ = await stage_profile_listing() + results["Stage 2 profile listing"] = ok_profile + results["Stage 4 hashtag listing"] = await stage_hashtag_listing() + results["Stage 5 pipeline"] = await stage_pipeline() + + _hr("SUMMARY") + for name, ok in results.items(): + print(f" {'PASS' if ok else 'FAIL/SKIP'} — {name}") + return 0 if all(results.values()) else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) From 9dd39faa50331c6df0e07790520a395d51177d93 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 17:53:29 +0200 Subject: [PATCH 25/56] test(tiktok): pin live fixtures, trim listing debug scaffolding --- .../platforms/tiktok/session/listing.py | 29 +- .../scripts/e2e_tiktok_scrape.py | 52 +- .../tests/fixtures/tiktok/listing_item.json | 559 +++++++++++++++ .../fixtures/tiktok/video_item_struct.json | 660 ++++++++++++++++++ 4 files changed, 1254 insertions(+), 46 deletions(-) create mode 100644 surfsense_backend/tests/fixtures/tiktok/listing_item.json create mode 100644 surfsense_backend/tests/fixtures/tiktok/video_item_struct.json diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index 98b2e6e8b..cc84a48b5 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -10,16 +10,15 @@ signs correctly for whatever version TikTok ships. The pure response-shape parsing lives in :func:`items_from_response`; this module is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). -Requires a residential proxy: TikTok throttles bare/datacenter IPs, returning -empty ``item_list`` bodies (and 429s) after a few hits. Set -``TIKTOK_LISTING_DEBUG=1`` to print captured XHR URLs while diagnosing. +Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile +feed (``/api/post/item_list``) is additionally soft-blocked: TikTok returns an +empty 200 even to its own signed request, so profile targets yield nothing. """ from __future__ import annotations import asyncio import logging -import os from typing import Any from scrapling.fetchers import StealthyFetcher @@ -66,24 +65,16 @@ def _build_page_action(collected: list[dict[str, Any]], url: str, target_count: listener already attached so page-one fires into it; scrolling pages the rest. """ - debug = bool(os.getenv("TIKTOK_LISTING_DEBUG")) - def _on_response(response: Any) -> None: response_url = getattr(response, "url", "") - if debug and "/api/" in response_url: - print(f" [xhr] {getattr(response, 'status', '?')} {response_url[:120]}") - try: - if not any(marker in response_url for marker in _ITEM_LIST_MARKERS): - return - body = response.json() - except Exception as exc: - if debug: - print(f" [xhr-parse-fail] {exc} :: {response_url[:120]}") + if not any(marker in response_url for marker in _ITEM_LIST_MARKERS): return - items = items_from_response(body) - if debug: - print(f" [match] +{len(items)} items from {response_url[:100]}") - collected.extend(items) + try: + body = response.json() + except Exception: + # An empty 200 (TikTok soft-block) or a body evicted before read. + return + collected.extend(items_from_response(body)) def _warm(page: Any) -> None: if _has_mstoken(page): diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py index 29cf1edc7..22bebff96 100644 --- a/surfsense_backend/scripts/e2e_tiktok_scrape.py +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -7,10 +7,11 @@ Run from the backend directory: What it exercises (everything REAL — live network, live proxy, live browser): Stage 1 — proxy egress proof (informational). - Stage 2 — profile listing via the stealth browser (captures item_list XHRs). - Stage 3 — blob video path over HTTP (a video URL derived from Stage 2). - Stage 4 — hashtag listing via the stealth browser. - Stage 5 — full scrape_tiktok() pipeline on a profile. + Stage 2 — profile listing via the stealth browser (soft-blocked by TikTok; + expected empty until a stronger anti-detection path exists). + Stage 3 — blob video path over HTTP (URL taken from a captured hashtag struct). + Stage 4 — hashtag listing via the stealth browser (captures item_list XHRs). + Stage 5 — full scrape_tiktok() pipeline on a hashtag. On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser suites can pin against real-shaped data without network. @@ -39,8 +40,8 @@ for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): _FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok" -# Evergreen public targets: the official account and a broad hashtag. -_PROFILE = "tiktok" +# Evergreen public targets: a regular high-volume creator and a broad hashtag. +_PROFILE = "nasa" _HASHTAG = "food" _COUNT = 5 @@ -70,17 +71,11 @@ def _dump_fixture(name: str, data: Any) -> None: print(f" [fixture] wrote {path.relative_to(_BACKEND_ROOT)}") -async def _discover_video_url() -> str | None: - """Find a live video URL over plain HTTP (independent of the browser path).""" - import re - - from app.proprietary.platforms.tiktok.session import fetch_html - - html = await fetch_html(f"https://www.tiktok.com/@{_PROFILE}") - if not html: - return None - match = re.search(r"/@[\w.]+/video/\d+", html) - return f"https://www.tiktok.com{match.group(0)}" if match else None +def _url_from_struct(struct: dict[str, Any]) -> str | None: + """Build a canonical video URL from a captured itemStruct.""" + author = (struct.get("author") or {}).get("uniqueId") + vid = struct.get("id") + return f"https://www.tiktok.com/@{author}/video/{vid}" if author and vid else None async def stage_proxy() -> bool: @@ -108,8 +103,6 @@ async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]: len(raw) > 0 and isinstance(raw[0], dict) and bool(raw[0].get("id")), f"{len(raw)} struct(s)", ) - if ok: - _dump_fixture("listing_item.json", raw[0]) return ok, raw @@ -139,17 +132,20 @@ async def stage_blob_video(video_url: str) -> tuple[bool, dict[str, Any] | None] return ok, raw -async def stage_hashtag_listing() -> bool: +async def stage_hashtag_listing() -> tuple[bool, list[dict[str, Any]]]: _hr(f"STAGE 4 — hashtag listing (browser): #{_HASHTAG}") from app.proprietary.platforms.tiktok.session import fetch_item_list url = f"https://www.tiktok.com/tag/{_HASHTAG}" raw = await fetch_item_list(url, _COUNT) - return _check( + ok = _check( "captured itemStructs for hashtag", len(raw) > 0 and bool(raw[0].get("id")), f"{len(raw)} struct(s)", ) + if ok: + _dump_fixture("listing_item.json", raw[0]) + return ok, raw async def stage_pipeline() -> bool: @@ -157,7 +153,7 @@ async def stage_pipeline() -> bool: from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok items = await scrape_tiktok( - TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=3), limit=3 + TikTokScrapeInput(hashtags=[_HASHTAG], resultsPerPage=3), limit=3 ) ok = _check( "pipeline returns normalized video items", @@ -177,18 +173,20 @@ async def main() -> int: results["Stage 1 proxy"] = await stage_proxy() - # Blob path first: HTTP-only, so it's validated independently of the - # proxy-sensitive browser listing path. - video_url = await _discover_video_url() + # Hashtag listing is the reliable browser path; use one of its captured + # structs to build a real video URL for the HTTP blob path. + ok_tag, tag_structs = await stage_hashtag_listing() + results["Stage 4 hashtag listing"] = ok_tag + + video_url = _url_from_struct(tag_structs[0]) if tag_structs else None if video_url: ok_video, _ = await stage_blob_video(video_url) results["Stage 3 blob video"] = ok_video else: - print("\n [SKIP] Stage 3 — could not discover a video URL over HTTP") + print("\n [SKIP] Stage 3 — no captured struct to build a video URL") ok_profile, _ = await stage_profile_listing() results["Stage 2 profile listing"] = ok_profile - results["Stage 4 hashtag listing"] = await stage_hashtag_listing() results["Stage 5 pipeline"] = await stage_pipeline() _hr("SUMMARY") diff --git a/surfsense_backend/tests/fixtures/tiktok/listing_item.json b/surfsense_backend/tests/fixtures/tiktok/listing_item.json new file mode 100644 index 000000000..374a9bf67 --- /dev/null +++ b/surfsense_backend/tests/fixtures/tiktok/listing_item.json @@ -0,0 +1,559 @@ +{ + "AIGCDescription": "", + "CategoryType": 111, + "IsHDBitrate": false, + "anchors": [ + { + "description": "CapCut · Video Editor", + "extraInfo": { + "subtype": "" + }, + "icon": { + "urlList": [ + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=44xwWHkyYJdE%2FIwqS3zAAjU4XQ4%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=lMwgk8e0wLaM3WkpnsMqYcxZN6w%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=1EMaRlc0Snbs9Ltwr0s95%2BDvpFk%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5" + ] + }, + "id": "0", + "keyword": "CapCut · Editing made easy", + "logExtra": "{\"anchor_id\":0,\"anchor_name\":\"CapCut · Editing made easy\",\"anchor_title_detail\":\"None\",\"anchor_title_id\":21502485763,\"anchor_type\":\"TT_CAPCUT\",\"capability_key\":\"default\",\"ccep_vip\":0,\"if_race_trigger\":0,\"is_two_line\":0,\"landing_page_style_id\":21502486275,\"maker_source\":\"\",\"publish_key\":\"default\",\"template_id\":\"none\",\"tutorial_id\":\"none\",\"viamaker_anchor_capability_key_weight\":1,\"viamaker_anchor_style_idx\":-1,\"viamaker_anchor_style_source\":3,\"video_source\":0,\"video_type_id\":1}", + "schema": "https://www.capcut.com/editor?scenario=tiktok&utm_source=tiktok_anchor_tool&utm_medium=tiktok_anchor&utm_campaign=70361230", + "thumbnail": { + "height": 64, + "urlList": [ + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=GOZUldk%2BrxINygKT0jwxeJz9VLQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=SJ6q86bjtCoRArlVNS1DjDDFWoc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=c7mUcXdyEOFc0SMcLAkkib3v9RU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5" + ], + "width": 64 + }, + "type": 54 + } + ], + "author": { + "avatarLarger": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "avatarMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "avatarThumb": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=8c557171&x-expires=1783695600&x-signature=VaAysM3zFkbl%2FOT74d0h2kdkLqo%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "commentSetting": 0, + "downloadSetting": 3, + "duetSetting": 0, + "ftc": false, + "id": "7501138183701103662", + "isADVirtual": false, + "isEmbedBanned": false, + "nickname": "Cookwithhali", + "openFavorite": false, + "privateAccount": false, + "relation": 0, + "secUid": "MS4wLjABAAAAywZ6lCXK3u4zKz8eQxxcjVblLiugz6zMysU98G7dyKVxu6IvMOJ97mpIfpDxUL4q", + "secret": false, + "shortDramaCreator": {}, + "signature": "From my kitchen to your screen! 🍞\nFollow for more ✨\nIG: halikit25\n\n📩 Halikitchen25@gmail.com", + "stitchSetting": 0, + "uniqueId": "cookwithhali", + "verified": false + }, + "authorStats": { + "diggCount": 223, + "followerCount": 372600, + "followingCount": 0, + "friendCount": 0, + "heart": 8000000, + "heartCount": 8000000, + "videoCount": 279 + }, + "authorStatsV2": { + "diggCount": "223", + "followerCount": "372600", + "followingCount": "0", + "friendCount": "0", + "heart": "8000000", + "heartCount": "8000000", + "videoCount": "279" + }, + "backendSourceEventTracking": "", + "challenges": [ + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "", + "id": "6983", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "title": "bread" + }, + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "", + "id": "285169", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "title": "garlicbread" + }, + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "", + "id": "1643317566954502", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "title": "breadtok" + }, + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "Whether you're baking the perfect pie or just searching for a recipe, you've come to the right place for all things #Baking.", + "id": "7250", + "profileLarger": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "profileMedium": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "profileThumb": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "title": "baking" + }, + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "", + "id": "10488587", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "title": "homemadebread" + } + ], + "collected": false, + "contents": [ + { + "desc": "Pull Apart Garlic Bread " + }, + { + "desc": "" + }, + { + "desc": "Ingredients" + }, + { + "desc": "" + }, + { + "desc": "* 1 tsp yeast" + }, + { + "desc": "* 1 tbsp sugar" + }, + { + "desc": "* 270g milk (about 1 cup + 2 tbsp)" + }, + { + "desc": "* 380g flour (about 3 cups)" + }, + { + "desc": "* 1 tsp salt" + }, + { + "desc": "* 43g softened butter (about 3 tbsp)" + }, + { + "desc": "" + }, + { + "desc": "Butter Mixture" + }, + { + "desc": "" + }, + { + "desc": "* 100g butter, softened" + }, + { + "desc": "* 1 tbsp chopped cilantro" + }, + { + "desc": "* 1 garlic clove, minced" + }, + { + "desc": "* Shredded cheese" + }, + { + "desc": "" + }, + { + "desc": "Instructions" + }, + { + "desc": "" + }, + { + "desc": "1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined." + }, + { + "desc": "2. Add the salt and softened butter, then coil fold until the butter is fully incorporated." + }, + { + "desc": "3. Cover and let the dough rest for 2 hours or until doubled." + }, + { + "desc": "4. Mix the softened butter with the cilantro and minced garlic." + }, + { + "desc": "5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese." + }, + { + "desc": "6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan." + }, + { + "desc": "7. Cover and let rest for 20 minutes." + }, + { + "desc": "8. Bake at 370°F for 30 minutes, or until golden brown." + }, + { + "desc": "9. Brush with the remaining garlic butter mixture while still warm." + }, + { + "desc": "" + }, + { + "desc": "#bread #garlicbread #breadtok #baking #homemadebread ", + "textExtra": [ + { + "awemeId": "", + "end": 6, + "hashtagName": "bread", + "isCommerce": false, + "start": 0, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 19, + "hashtagName": "garlicbread", + "isCommerce": false, + "start": 7, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 29, + "hashtagName": "breadtok", + "isCommerce": false, + "start": 20, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 37, + "hashtagName": "baking", + "isCommerce": false, + "start": 30, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 52, + "hashtagName": "homemadebread", + "isCommerce": false, + "start": 38, + "subType": 0, + "type": 1 + } + ] + } + ], + "createTime": 1782509863, + "desc": "Pull Apart Garlic Bread Ingredients * 1 tsp yeast * 1 tbsp sugar * 270g milk (about 1 cup + 2 tbsp) * 380g flour (about 3 cups) * 1 tsp salt * 43g softened butter (about 3 tbsp) Butter Mixture * 100g butter, softened * 1 tbsp chopped cilantro * 1 garlic clove, minced * Shredded cheese Instructions 1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined. 2. Add the salt and softened butter, then coil fold until the butter is fully incorporated. 3. Cover and let the dough rest for 2 hours or until doubled. 4. Mix the softened butter with the cilantro and minced garlic. 5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese. 6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan. 7. Cover and let rest for 20 minutes. 8. Bake at 370°F for 30 minutes, or until golden brown. 9. Brush with the remaining garlic butter mixture while still warm. #bread #garlicbread #breadtok #baking #homemadebread ", + "digged": false, + "diversificationId": 10040, + "duetDisplay": 0, + "duetEnabled": true, + "forFriend": false, + "id": "7655821461772406047", + "isAd": false, + "isReviewing": false, + "itemCommentStatus": 0, + "item_control": { + "can_repost": true + }, + "music": { + "authorName": "Cookwithhali", + "coverLarge": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "coverMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "coverThumb": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=8c557171&x-expires=1783695600&x-signature=VaAysM3zFkbl%2FOT74d0h2kdkLqo%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "duration": 62, + "id": "7655821492566919966", + "isCopyrighted": false, + "is_commerce_music": true, + "is_unlimited_music": false, + "original": true, + "playUrl": "https://v16m.tiktokcdn-us.com/ecd83422a3e7bf5450bd92ebf07e6fce/6a4ec449/video/tos/useast8/tos-useast8-v-27dcd7-tx2/ocBIhANWILSemeAAINEGoTTPe58AJaRegg6eKS/?a=1233&bti=ODszNWYuMDE6&&bt=125&ft=GSDrKInz7Th2sbSGXq8Zmo&mime_type=audio_mpeg&rc=anY5cXA5cmRpOzMzaTU8NEBpanY5cXA5cmRpOzMzaTU8NEBpYzZrMmRjL3NhLS1kMTJzYSNpYzZrMmRjL3NhLS1kMTJzcw%3D%3D&vvpl=1&l=2026070815413036F7D0D0506FB569C78C&btag=e00050000", + "private": false, + "shoot_duration": 62, + "title": "original sound", + "tt2dsp": {} + }, + "officalItem": false, + "originalItem": false, + "privateItem": false, + "secret": false, + "shareEnabled": true, + "stats": { + "collectCount": 396400, + "commentCount": 3720, + "diggCount": 754100, + "playCount": 8300000, + "shareCount": 144500 + }, + "statsV2": { + "collectCount": "396397", + "commentCount": "3720", + "diggCount": "754100", + "playCount": "8300000", + "repostCount": "0", + "shareCount": "144500" + }, + "stickersOnItem": [ + { + "stickerText": [ + "Cheesy Pull Apart Garlic Bread! 🧄🧀\n", + "(Bakery-Style & No-Knead)" + ], + "stickerType": 4 + } + ], + "stitchDisplay": 0, + "stitchEnabled": true, + "textExtra": [ + { + "awemeId": "", + "end": 1030, + "hashtagName": "bread", + "isCommerce": false, + "start": 1024, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 1043, + "hashtagName": "garlicbread", + "isCommerce": false, + "start": 1031, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 1053, + "hashtagName": "breadtok", + "isCommerce": false, + "start": 1044, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 1061, + "hashtagName": "baking", + "isCommerce": false, + "start": 1054, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 1076, + "hashtagName": "homemadebread", + "isCommerce": false, + "start": 1062, + "subType": 0, + "type": 1 + } + ], + "textLanguage": "en", + "textTranslatable": true, + "video": { + "PlayAddrStruct": { + "DataSize": 13761060, + "FileCs": "c:0-51456-f8d1", + "FileHash": "485174c8492db027adbbf6dca7c7c201", + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 720 + }, + "VQScore": "69.69", + "bitrate": 1773703, + "bitrateInfo": [ + { + "Bitrate": 1836180, + "BitrateFPS": 29, + "CodecType": "h265_hvc1", + "Format": "mp4", + "GearName": "adapt_lowest_1080_1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 96.83, \"v960\": 97.64, \"v864\": 98.256, \"v720\": 98.838}, \"ori\": {\"v1080\": 91.834, \"v960\": 93.423, \"v864\": 94.54, \"v720\": 95.872}}}", + "PlayAddr": { + "DataSize": 14245777, + "FileCs": "c:0-52754-ed06", + "FileHash": "38996a98cfeb2714fca1b94768090c37", + "Height": 1920, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_1080p_1836180", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=db26af0a98ff05e504d7121b25048527&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=db26af0a98ff05e504d7121b25048527&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=5de0e21d367c4e2bb42b386ed1aca6fe&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjAzMmU1YzJlMTAwMDJiYjIwMmFiN2RhNTIzOWVhYWRk&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 1080 + }, + "QualityType": 2, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 385342}, {\\\"time\\\": 2, \\\"offset\\\": 561014}, {\\\"time\\\": 3, \\\"offset\\\": 703814}, {\\\"time\\\": 4, \\\"offset\\\": 855867}, {\\\"time\\\": 5, \\\"offset\\\": 1019333}, {\\\"time\\\": 10, \\\"offset\\\": 2028412}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 96.83, \\\"v960\\\": 97.64, \\\"v864\\\": 98.256, \\\"v720\\\": 98.838}, \\\"ori\\\": {\\\"v1080\\\": 91.834, \\\"v960\\\": 93.423, \\\"v864\\\": 94.54, \\\"v720\\\": 95.872}}}\",\"ufq\":\"{\\\"enh\\\":89.317,\\\"playback\\\":{\\\"ori\\\":85.151,\\\"srv1\\\":90.147},\\\"src\\\":85.957,\\\"trans\\\":85.151,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"11b0345f3bf185ddbe7c48bd7301980f\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwCWAACdzkgS\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwCWAACgAhyAHgWWdzkySUQs0RJJJJify4d/1+bPl/1+M4gQ5qAgICCAAAADAIAAAA8E\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 1773703, + "BitrateFPS": 29, + "CodecType": "h264", + "Format": "mp4", + "GearName": "normal_720_0", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}, \"ori\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}}}", + "PlayAddr": { + "DataSize": 13761060, + "FileCs": "c:0-51456-f8d1", + "FileHash": "485174c8492db027adbbf6dca7c7c201", + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 720 + }, + "QualityType": 10, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 400712}, {\\\"time\\\": 2, \\\"offset\\\": 659243}, {\\\"time\\\": 3, \\\"offset\\\": 886968}, {\\\"time\\\": 4, \\\"offset\\\": 1101663}, {\\\"time\\\": 5, \\\"offset\\\": 1368565}, {\\\"time\\\": 10, \\\"offset\\\": 2375704}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}, \\\"ori\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"65b88c41c0963253ad31b6f68981a242\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":64193}" + }, + { + "Bitrate": 966380, + "BitrateFPS": 29, + "CodecType": "h265_hvc1", + "Format": "mp4", + "GearName": "adapt_lower_720_1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 90.05, \"v960\": 91.958, \"v864\": 93.608, \"v720\": 95.869}, \"ori\": {\"v1080\": 82.259, \"v960\": 85.339, \"v864\": 87.288, \"v720\": 90.851}}}", + "PlayAddr": { + "DataSize": 7497540, + "FileCs": "c:0-52786-7f65", + "FileHash": "1376df24c917669727b27273697fb826", + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_720p_966380", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=e3690d67b2ecae289320f5db6c338d05&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=e3690d67b2ecae289320f5db6c338d05&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=87b87e594a2b40ec86b4c3d6e1ca50a6&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRjNWNmZGVmMmFkNTIyNWQ1ZTA1NzExM2NjOTVlNzMy&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 720 + }, + "QualityType": 14, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 284307}, {\\\"time\\\": 2, \\\"offset\\\": 357012}, {\\\"time\\\": 3, \\\"offset\\\": 438159}, {\\\"time\\\": 4, \\\"offset\\\": 532720}, {\\\"time\\\": 5, \\\"offset\\\": 636824}, {\\\"time\\\": 10, \\\"offset\\\": 1142427}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 90.05, \\\"v960\\\": 91.958, \\\"v864\\\": 93.608, \\\"v720\\\": 95.869}, \\\"ori\\\": {\\\"v1080\\\": 82.259, \\\"v960\\\": 85.339, \\\"v864\\\": 87.288, \\\"v720\\\": 90.851}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":76.711,\\\"srv1\\\":84.502},\\\"src\\\":85.957,\\\"trans\\\":76.711,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"97eb57dffd9e7a0333979f6ca6c9418d\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 91.625, \\\"sr20\\\": 97.355}}\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBaIAUBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 799620, + "BitrateFPS": 29, + "CodecType": "h265_hvc1", + "Format": "mp4", + "GearName": "adapt_540_1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 87.061, \"v960\": 89.844, \"v864\": 91.813, \"v720\": 94.499}, \"ori\": {\"v1080\": 77.717, \"v960\": 80.972, \"v864\": 83.632, \"v720\": 87.945}}}", + "PlayAddr": { + "DataSize": 6203756, + "FileCs": "c:0-52786-7f6d", + "FileHash": "41f62f740ad13045000450f28b9ff2cb", + "Height": 1024, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_540p_799620", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=05aaeb04a6cf8d012bba46bb131f00af&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=05aaeb04a6cf8d012bba46bb131f00af&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=ccd4b0e63fe840d587420c7a116a7552&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRlMWFkM2IxY2JkODUxODVlZDJjMDFjYTJmZWE0NTEx&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 576 + }, + "QualityType": 28, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 254747}, {\\\"time\\\": 2, \\\"offset\\\": 310409}, {\\\"time\\\": 3, \\\"offset\\\": 378126}, {\\\"time\\\": 4, \\\"offset\\\": 458158}, {\\\"time\\\": 5, \\\"offset\\\": 546764}, {\\\"time\\\": 10, \\\"offset\\\": 946117}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 87.061, \\\"v960\\\": 89.844, \\\"v864\\\": 91.813, \\\"v720\\\": 94.499}, \\\"ori\\\": {\\\"v1080\\\": 77.717, \\\"v960\\\": 80.972, \\\"v864\\\": 83.632, \\\"v720\\\": 87.945}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":73.304,\\\"srv1\\\":82.648},\\\"src\\\":85.957,\\\"trans\\\":73.304,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"86fda13c754bde67159dea14b7d55d6a\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 87.963, \\\"sr20\\\": 95.722}}\",\"audio_bit_rate\":64143,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBIIAQBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 622546, + "BitrateFPS": 29, + "CodecType": "h264", + "Format": "mp4", + "GearName": "lowest_540_0", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 72.483, \"v960\": 75.424, \"v864\": 79.843, \"v720\": 84.897}, \"ori\": {\"v1080\": 64.527, \"v960\": 67.93, \"v864\": 72.279, \"v720\": 74.495}}}", + "PlayAddr": { + "DataSize": 4829948, + "FileCs": "c:0-51513-3d83", + "FileHash": "a74fd8fabed8b36131a8e31bf78e9d1d", + "Height": 1024, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_540p_622546", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=fa8ddc2bcbad8ebb4394c4a8b55b28df&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=fa8ddc2bcbad8ebb4394c4a8b55b28df&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=1ce99a90ac5e412b94fb5981ce7e9e8e&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjU0ZWZiNDU4NWVmYzFlY2IyNmEyODAyMGY0ZWJjMGZj&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 576 + }, + "QualityType": 25, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 176720}, {\\\"time\\\": 2, \\\"offset\\\": 268125}, {\\\"time\\\": 3, \\\"offset\\\": 342814}, {\\\"time\\\": 4, \\\"offset\\\": 414132}, {\\\"time\\\": 5, \\\"offset\\\": 511717}, {\\\"time\\\": 10, \\\"offset\\\": 841405}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 72.483, \\\"v960\\\": 75.424, \\\"v864\\\": 79.843, \\\"v720\\\": 84.897}, \\\"ori\\\": {\\\"v1080\\\": 64.527, \\\"v960\\\": 67.93, \\\"v864\\\": 72.279, \\\"v720\\\": 74.495}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"25b3faebdda74a9e438e851988a6efcc\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":32099}" + } + ], + "claInfo": { + "enableAutoCaption": true, + "hasOriginalAudio": true, + "noCaptionReason": 3 + }, + "codecType": "h264", + "cover": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=ZIp3Rdhs6GNzMatSMsNWHFZX%2F7s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "definition": "720p", + "duration": 62, + "dynamicCover": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoPG0RxQTGE89EEqvAuADpFvfGABVMxQBeza~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=JWF%2F5g2txpFXhNhJzOwleSJgKik%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "encodeUserTag": "", + "encodedType": "normal", + "format": "mp4", + "height": 1280, + "id": "7655821461772406047", + "originCover": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoQGFsU9VvLxieEDDAQAPpRTqA8B5EzvEGfF~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=yHDGczHGo3joyDeerZP6Dp4S1Jc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "playAddr": "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "ratio": "720p", + "size": 13761060, + "videoID": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "videoQuality": "normal", + "volumeInfo": { + "Loudness": -48.5, + "Peak": 0.18408 + }, + "width": 720, + "zoomCover": { + "240": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:240:240.avif?dr=9616&x-expires=1783695600&x-signature=ZONw1V50ZmupszDX0NiC%2Bf8BimU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "480": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:480:480.avif?dr=9616&x-expires=1783695600&x-signature=LztWYGkvUC4%2BvhtQb0TO0l1ky0s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "720": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:720:720.avif?dr=9616&x-expires=1783695600&x-signature=X%2F3HQ%2BkiKqEAxnuk%2BdFfBfeOsko%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "960": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:960:960.avif?dr=9616&x-expires=1783695600&x-signature=aJaYG%2BDAo%2BhO7nZ8t8XP7g0DArU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1" + } + } +} \ No newline at end of file diff --git a/surfsense_backend/tests/fixtures/tiktok/video_item_struct.json b/surfsense_backend/tests/fixtures/tiktok/video_item_struct.json new file mode 100644 index 000000000..af3bd651b --- /dev/null +++ b/surfsense_backend/tests/fixtures/tiktok/video_item_struct.json @@ -0,0 +1,660 @@ +{ + "id": "7655821461772406047", + "desc": "Pull Apart Garlic Bread Ingredients * 1 tsp yeast * 1 tbsp sugar * 270g milk (about 1 cup + 2 tbsp) * 380g flour (about 3 cups) * 1 tsp salt * 43g softened butter (about 3 tbsp) Butter Mixture * 100g butter, softened * 1 tbsp chopped cilantro * 1 garlic clove, minced * Shredded cheese Instructions 1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined. 2. Add the salt and softened butter, then coil fold until the butter is fully incorporated. 3. Cover and let the dough rest for 2 hours or until doubled. 4. Mix the softened butter with the cilantro and minced garlic. 5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese. 6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan. 7. Cover and let rest for 20 minutes. 8. Bake at 370°F for 30 minutes, or until golden brown. 9. Brush with the remaining garlic butter mixture while still warm. #bread #garlicbread #breadtok #baking #homemadebread ", + "createTime": "1782509863", + "scheduleTime": 0, + "video": { + "id": "7655821461772406047", + "height": 1280, + "width": 720, + "duration": 62, + "ratio": "720p", + "cover": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=ZIp3Rdhs6GNzMatSMsNWHFZX%2F7s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "originCover": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoQGFsU9VvLxieEDDAQAPpRTqA8B5EzvEGfF~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=yHDGczHGo3joyDeerZP6Dp4S1Jc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "dynamicCover": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoPG0RxQTGE89EEqvAuADpFvfGABVMxQBeza~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=Jz%2BPtpSXdbjTV4qFbG4H7JFD0f8%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "playAddr": "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "downloadAddr": "", + "shareCover": [ + "" + ], + "reflowCover": "", + "bitrate": 1773703, + "encodedType": "normal", + "format": "mp4", + "videoQuality": "normal", + "encodeUserTag": "", + "codecType": "h264", + "definition": "720p", + "subtitleInfos": [], + "zoomCover": { + "240": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:240:240.avif?dr=9616&x-expires=1783695600&x-signature=cBXoX16WOKJyUth9fnaohYv%2B%2BBs%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "480": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:480:480.avif?dr=9616&x-expires=1783695600&x-signature=Ff5B1aV4teSPQc6JXMoidMXqJWk%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "720": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:720:720.avif?dr=9616&x-expires=1783695600&x-signature=X%2F3HQ%2BkiKqEAxnuk%2BdFfBfeOsko%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "960": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:960:960.avif?dr=9616&x-expires=1783695600&x-signature=aJaYG%2BDAo%2BhO7nZ8t8XP7g0DArU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1" + }, + "volumeInfo": { + "Loudness": -48.5, + "Peak": 0.18408 + }, + "bitrateInfo": [ + { + "Bitrate": 1836180, + "QualityType": 2, + "BitrateFPS": 29, + "GearName": "adapt_lowest_1080_1", + "PlayAddr": { + "DataSize": "14245777", + "Width": 1080, + "Height": 1920, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=0648319cea95ee08fd7b4f93a95cb14c&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=0648319cea95ee08fd7b4f93a95cb14c&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=5de0e21d367c4e2bb42b386ed1aca6fe&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjAzMmU1YzJlMTAwMDJiYjIwMmFiN2RhNTIzOWVhYWRk&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_1080p_1836180", + "FileHash": "38996a98cfeb2714fca1b94768090c37", + "FileCs": "c:0-52754-ed06" + }, + "CodecType": "h265_hvc1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 96.83, \"v960\": 97.64, \"v864\": 98.256, \"v720\": 98.838}, \"ori\": {\"v1080\": 91.834, \"v960\": 93.423, \"v864\": 94.54, \"v720\": 95.872}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 385342}, {\\\"time\\\": 2, \\\"offset\\\": 561014}, {\\\"time\\\": 3, \\\"offset\\\": 703814}, {\\\"time\\\": 4, \\\"offset\\\": 855867}, {\\\"time\\\": 5, \\\"offset\\\": 1019333}, {\\\"time\\\": 10, \\\"offset\\\": 2028412}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 96.83, \\\"v960\\\": 97.64, \\\"v864\\\": 98.256, \\\"v720\\\": 98.838}, \\\"ori\\\": {\\\"v1080\\\": 91.834, \\\"v960\\\": 93.423, \\\"v864\\\": 94.54, \\\"v720\\\": 95.872}}}\",\"ufq\":\"{\\\"enh\\\":89.317,\\\"playback\\\":{\\\"ori\\\":85.151,\\\"srv1\\\":90.147},\\\"src\\\":85.957,\\\"trans\\\":85.151,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"11b0345f3bf185ddbe7c48bd7301980f\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwCWAACdzkgS\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwCWAACgAhyAHgWWdzkySUQs0RJJJJify4d/1+bPl/1+M4gQ5qAgICCAAAADAIAAAA8E\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 1773703, + "QualityType": 10, + "BitrateFPS": 29, + "GearName": "normal_720_0", + "PlayAddr": { + "DataSize": "13761060", + "Width": 720, + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703", + "FileHash": "485174c8492db027adbbf6dca7c7c201", + "FileCs": "c:0-51456-f8d1" + }, + "CodecType": "h264", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}, \"ori\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 400712}, {\\\"time\\\": 2, \\\"offset\\\": 659243}, {\\\"time\\\": 3, \\\"offset\\\": 886968}, {\\\"time\\\": 4, \\\"offset\\\": 1101663}, {\\\"time\\\": 5, \\\"offset\\\": 1368565}, {\\\"time\\\": 10, \\\"offset\\\": 2375704}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}, \\\"ori\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"65b88c41c0963253ad31b6f68981a242\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":64193}" + }, + { + "Bitrate": 966380, + "QualityType": 14, + "BitrateFPS": 29, + "GearName": "adapt_lower_720_1", + "PlayAddr": { + "DataSize": "7497540", + "Width": 720, + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=3a0c7313f1a5d9908cc2f5b0dd24d048&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=3a0c7313f1a5d9908cc2f5b0dd24d048&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=87b87e594a2b40ec86b4c3d6e1ca50a6&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRjNWNmZGVmMmFkNTIyNWQ1ZTA1NzExM2NjOTVlNzMy&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_720p_966380", + "FileHash": "1376df24c917669727b27273697fb826", + "FileCs": "c:0-52786-7f65" + }, + "CodecType": "h265_hvc1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 90.05, \"v960\": 91.958, \"v864\": 93.608, \"v720\": 95.869}, \"ori\": {\"v1080\": 82.259, \"v960\": 85.339, \"v864\": 87.288, \"v720\": 90.851}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 284307}, {\\\"time\\\": 2, \\\"offset\\\": 357012}, {\\\"time\\\": 3, \\\"offset\\\": 438159}, {\\\"time\\\": 4, \\\"offset\\\": 532720}, {\\\"time\\\": 5, \\\"offset\\\": 636824}, {\\\"time\\\": 10, \\\"offset\\\": 1142427}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 90.05, \\\"v960\\\": 91.958, \\\"v864\\\": 93.608, \\\"v720\\\": 95.869}, \\\"ori\\\": {\\\"v1080\\\": 82.259, \\\"v960\\\": 85.339, \\\"v864\\\": 87.288, \\\"v720\\\": 90.851}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":76.711,\\\"srv1\\\":84.502},\\\"src\\\":85.957,\\\"trans\\\":76.711,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"97eb57dffd9e7a0333979f6ca6c9418d\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 91.625, \\\"sr20\\\": 97.355}}\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBaIAUBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 799620, + "QualityType": 28, + "BitrateFPS": 29, + "GearName": "adapt_540_1", + "PlayAddr": { + "DataSize": "6203756", + "Width": 576, + "Height": 1024, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=a8516d4d8e79814753dc2cd0fde9f7bd&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=a8516d4d8e79814753dc2cd0fde9f7bd&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=ccd4b0e63fe840d587420c7a116a7552&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRlMWFkM2IxY2JkODUxODVlZDJjMDFjYTJmZWE0NTEx&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_540p_799620", + "FileHash": "41f62f740ad13045000450f28b9ff2cb", + "FileCs": "c:0-52786-7f6d" + }, + "CodecType": "h265_hvc1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 87.061, \"v960\": 89.844, \"v864\": 91.813, \"v720\": 94.499}, \"ori\": {\"v1080\": 77.717, \"v960\": 80.972, \"v864\": 83.632, \"v720\": 87.945}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 254747}, {\\\"time\\\": 2, \\\"offset\\\": 310409}, {\\\"time\\\": 3, \\\"offset\\\": 378126}, {\\\"time\\\": 4, \\\"offset\\\": 458158}, {\\\"time\\\": 5, \\\"offset\\\": 546764}, {\\\"time\\\": 10, \\\"offset\\\": 946117}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 87.061, \\\"v960\\\": 89.844, \\\"v864\\\": 91.813, \\\"v720\\\": 94.499}, \\\"ori\\\": {\\\"v1080\\\": 77.717, \\\"v960\\\": 80.972, \\\"v864\\\": 83.632, \\\"v720\\\": 87.945}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":73.304,\\\"srv1\\\":82.648},\\\"src\\\":85.957,\\\"trans\\\":73.304,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"86fda13c754bde67159dea14b7d55d6a\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 87.963, \\\"sr20\\\": 95.722}}\",\"audio_bit_rate\":64143,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBIIAQBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 622546, + "QualityType": 25, + "BitrateFPS": 29, + "GearName": "lowest_540_0", + "PlayAddr": { + "DataSize": "4829948", + "Width": 576, + "Height": 1024, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=f44ac110e4573a7bc11c73155b9141d0&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=f44ac110e4573a7bc11c73155b9141d0&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=1ce99a90ac5e412b94fb5981ce7e9e8e&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjU0ZWZiNDU4NWVmYzFlY2IyNmEyODAyMGY0ZWJjMGZj&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_540p_622546", + "FileHash": "a74fd8fabed8b36131a8e31bf78e9d1d", + "FileCs": "c:0-51513-3d83" + }, + "CodecType": "h264", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 72.483, \"v960\": 75.424, \"v864\": 79.843, \"v720\": 84.897}, \"ori\": {\"v1080\": 64.527, \"v960\": 67.93, \"v864\": 72.279, \"v720\": 74.495}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 176720}, {\\\"time\\\": 2, \\\"offset\\\": 268125}, {\\\"time\\\": 3, \\\"offset\\\": 342814}, {\\\"time\\\": 4, \\\"offset\\\": 414132}, {\\\"time\\\": 5, \\\"offset\\\": 511717}, {\\\"time\\\": 10, \\\"offset\\\": 841405}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 72.483, \\\"v960\\\": 75.424, \\\"v864\\\": 79.843, \\\"v720\\\": 84.897}, \\\"ori\\\": {\\\"v1080\\\": 64.527, \\\"v960\\\": 67.93, \\\"v864\\\": 72.279, \\\"v720\\\": 74.495}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"25b3faebdda74a9e438e851988a6efcc\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":32099}" + } + ], + "size": "13761060", + "VQScore": "69.69", + "claInfo": { + "hasOriginalAudio": true, + "enableAutoCaption": true, + "captionInfos": [], + "noCaptionReason": 3 + }, + "videoID": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "PlayAddrStruct": { + "DataSize": "13761060", + "Width": 720, + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703", + "FileHash": "485174c8492db027adbbf6dca7c7c201", + "FileCs": "c:0-51456-f8d1" + } + }, + "author": { + "id": "7501138183701103662", + "shortId": "", + "uniqueId": "cookwithhali", + "nickname": "Cookwithhali", + "avatarLarger": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "avatarMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "avatarThumb": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=84bba9f5&x-expires=1783695600&x-signature=Et06On6iQeXFbtycXoVodzl0fdM%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "signature": "From my kitchen to your screen! 🍞\nFollow for more ✨\nIG: halikit25\n\n📩 Halikitchen25@gmail.com", + "createTime": 1746494976, + "verified": false, + "secUid": "MS4wLjABAAAAywZ6lCXK3u4zKz8eQxxcjVblLiugz6zMysU98G7dyKVxu6IvMOJ97mpIfpDxUL4q", + "ftc": false, + "relation": 0, + "openFavorite": false, + "commentSetting": 0, + "duetSetting": 0, + "stitchSetting": 0, + "privateAccount": false, + "secret": false, + "isADVirtual": false, + "roomId": "", + "uniqueIdModifyTime": 0, + "ttSeller": false, + "downloadSetting": 3, + "recommendReason": "", + "nowInvitationCardUrl": "", + "nickNameModifyTime": 0, + "isEmbedBanned": false, + "canExpPlaylist": false, + "suggestAccountBind": false, + "UserStoryStatus": 0, + "shortDramaCreator": {} + }, + "music": { + "id": "7655821492566919966", + "title": "original sound", + "playUrl": "https://v16m.tiktokcdn-us.com/84ccc1629d7199ad68f976d419148401/6a4ec44e/video/tos/useast8/tos-useast8-v-27dcd7-tx2/ocBIhANWILSemeAAINEGoTTPe58AJaRegg6eKS/?a=1233&bti=ODszNWYuMDE6&&bt=125&ft=GSDrKInz7ThasbSGXq8Zmo&mime_type=audio_mpeg&rc=anY5cXA5cmRpOzMzaTU8NEBpanY5cXA5cmRpOzMzaTU8NEBpYzZrMmRjL3NhLS1kMTJzYSNpYzZrMmRjL3NhLS1kMTJzcw%3D%3D&vvpl=1&l=20260708154135271AF032BAC01F42EF1E&btag=e00050000", + "coverLarge": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "coverMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "coverThumb": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=84bba9f5&x-expires=1783695600&x-signature=Et06On6iQeXFbtycXoVodzl0fdM%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "authorName": "Cookwithhali", + "original": true, + "private": false, + "duration": 62, + "scheduleSearchTime": 0, + "collected": false, + "preciseDuration": { + "preciseDuration": 62.093063, + "preciseShootDuration": 62.093063, + "preciseAuditionDuration": 62.093063, + "preciseVideoDuration": 62.093063 + }, + "isCopyrighted": false, + "tt2dsp": { + "tt_to_dsp_song_infos": [] + }, + "is_unlimited_music": false, + "is_commerce_music": true, + "shoot_duration": 62 + }, + "challenges": [ + { + "id": "6983", + "title": "bread", + "desc": "", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + }, + { + "id": "285169", + "title": "garlicbread", + "desc": "", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + }, + { + "id": "1643317566954502", + "title": "breadtok", + "desc": "", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + }, + { + "id": "7250", + "title": "baking", + "desc": "Whether you're baking the perfect pie or just searching for a recipe, you've come to the right place for all things #Baking.", + "profileLarger": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "profileMedium": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "profileThumb": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + }, + { + "id": "10488587", + "title": "homemadebread", + "desc": "", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + } + ], + "stats": { + "diggCount": 754100, + "shareCount": 144500, + "commentCount": 3720, + "playCount": 8300000, + "collectCount": "396399" + }, + "statsV2": { + "diggCount": "754100", + "shareCount": "144500", + "commentCount": "3720", + "playCount": "8300000", + "collectCount": "396399", + "repostCount": "0" + }, + "warnInfo": [], + "originalItem": false, + "officalItem": false, + "textExtra": [ + { + "awemeId": "", + "start": 1024, + "end": 1030, + "hashtagId": "6983", + "hashtagName": "bread", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 1031, + "end": 1043, + "hashtagId": "285169", + "hashtagName": "garlicbread", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 1044, + "end": 1053, + "hashtagId": "1643317566954502", + "hashtagName": "breadtok", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 1054, + "end": 1061, + "hashtagId": "7250", + "hashtagName": "baking", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 1062, + "end": 1076, + "hashtagId": "10488587", + "hashtagName": "homemadebread", + "type": 1, + "subType": 0, + "isCommerce": false + } + ], + "secret": false, + "forFriend": false, + "digged": false, + "itemCommentStatus": 0, + "takeDown": 0, + "effectStickers": [], + "authorStats": { + "followerCount": 372600, + "followingCount": 0, + "heart": 8000000, + "heartCount": 8000000, + "videoCount": 279, + "diggCount": 223, + "friendCount": 0 + }, + "privateItem": false, + "duetEnabled": true, + "stitchEnabled": true, + "stickersOnItem": [ + { + "stickerText": [ + "Cheesy Pull Apart Garlic Bread! 🧄🧀\n", + "(Bakery-Style & No-Knead)" + ], + "stickerType": 4 + } + ], + "isAd": false, + "shareEnabled": true, + "comments": [], + "duetDisplay": 0, + "stitchDisplay": 0, + "indexEnabled": true, + "diversificationLabels": [ + "Cooking", + "Food & Drink", + "Lifestyle" + ], + "locationCreated": "US", + "suggestedWords": [ + "Garlic Bread", + "garlic bread recipe", + "pull apart bread", + "Sourdough Bread", + "How To Make Garlic Bread", + "Baking", + "food", + "bread", + "Baking Recipes", + "Garlic Pull Apart Bread Recipe" + ], + "contents": [ + { + "desc": "Pull Apart Garlic Bread ", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "Ingredients", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "* 1 tsp yeast", + "textExtra": [] + }, + { + "desc": "* 1 tbsp sugar", + "textExtra": [] + }, + { + "desc": "* 270g milk (about 1 cup + 2 tbsp)", + "textExtra": [] + }, + { + "desc": "* 380g flour (about 3 cups)", + "textExtra": [] + }, + { + "desc": "* 1 tsp salt", + "textExtra": [] + }, + { + "desc": "* 43g softened butter (about 3 tbsp)", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "Butter Mixture", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "* 100g butter, softened", + "textExtra": [] + }, + { + "desc": "* 1 tbsp chopped cilantro", + "textExtra": [] + }, + { + "desc": "* 1 garlic clove, minced", + "textExtra": [] + }, + { + "desc": "* Shredded cheese", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "Instructions", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined.", + "textExtra": [] + }, + { + "desc": "2. Add the salt and softened butter, then coil fold until the butter is fully incorporated.", + "textExtra": [] + }, + { + "desc": "3. Cover and let the dough rest for 2 hours or until doubled.", + "textExtra": [] + }, + { + "desc": "4. Mix the softened butter with the cilantro and minced garlic.", + "textExtra": [] + }, + { + "desc": "5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese.", + "textExtra": [] + }, + { + "desc": "6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan.", + "textExtra": [] + }, + { + "desc": "7. Cover and let rest for 20 minutes.", + "textExtra": [] + }, + { + "desc": "8. Bake at 370°F for 30 minutes, or until golden brown.", + "textExtra": [] + }, + { + "desc": "9. Brush with the remaining garlic butter mixture while still warm.", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "#bread #garlicbread #breadtok #baking #homemadebread ", + "textExtra": [ + { + "awemeId": "", + "start": 0, + "end": 6, + "hashtagId": "6983", + "hashtagName": "bread", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 7, + "end": 19, + "hashtagId": "285169", + "hashtagName": "garlicbread", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 20, + "end": 29, + "hashtagId": "1643317566954502", + "hashtagName": "breadtok", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 30, + "end": 37, + "hashtagId": "7250", + "hashtagName": "baking", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 38, + "end": 52, + "hashtagId": "10488587", + "hashtagName": "homemadebread", + "type": 1, + "subType": 0, + "isCommerce": false + } + ] + } + ], + "diversificationId": 10040, + "anchors": [ + { + "id": "0", + "type": 54, + "keyword": "CapCut · Editing made easy", + "icon": { + "uri": "tiktok-obj/capcut_logo_64px_bk.png", + "urlList": [ + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=lMwgk8e0wLaM3WkpnsMqYcxZN6w%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=44xwWHkyYJdE%2FIwqS3zAAjU4XQ4%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=lzVxquDTAbut%2BvshGEhsFsYQc2s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5" + ] + }, + "schema": "https://www.capcut.com/tools/desktop-video-editor?channel=capcutpc_ttweb_anchor_edit1&download_channel=capcutpc_ttweb_anchor_edit1&enter_from=capcutpc_ttweb_anchor_edit1&force_dp=1&from_page=capcutpc_ttweb_anchor_edit1&type=tools", + "logExtra": "{\"anchor_id\":0,\"anchor_name\":\"CapCut · Editing made easy\",\"anchor_title_detail\":\"None\",\"anchor_title_id\":21502485763,\"anchor_type\":\"TT_CAPCUT\",\"capability_key\":\"default\",\"ccep_vip\":0,\"if_race_trigger\":0,\"is_two_line\":0,\"landing_page_style_id\":21502486275,\"maker_source\":\"\",\"publish_key\":\"default\",\"template_id\":\"none\",\"tutorial_id\":\"none\",\"viamaker_anchor_capability_key_weight\":1,\"viamaker_anchor_style_idx\":-1,\"viamaker_anchor_style_source\":3,\"video_source\":0,\"video_type_id\":1}", + "description": "CapCut · Video Editor", + "thumbnail": { + "uri": "tiktok-obj/64x_Capcut3x.png", + "urlList": [ + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=SJ6q86bjtCoRArlVNS1DjDDFWoc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=GOZUldk%2BrxINygKT0jwxeJz9VLQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=NBemEand1jjAoV81cQ1ANQzVqqc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5" + ], + "width": 64, + "height": 64 + }, + "extraInfo": { + "subtype": "" + } + } + ], + "collected": false, + "channelTags": [], + "item_control": { + "can_repost": true + }, + "IsAigc": false, + "AIGCDescription": "", + "backendSourceEventTracking": "", + "CategoryType": 111, + "textLanguage": "en", + "textTranslatable": true, + "authorStatsV2": { + "followerCount": "372600", + "followingCount": "0", + "heart": "8000000", + "heartCount": "8000000", + "videoCount": "279", + "diggCount": "223", + "friendCount": "0" + }, + "isReviewing": false, + "creatorAIComment": { + "hasAITopic": false, + "categoryList": [], + "eligibleVideo": false, + "notEligibleReason": 101 + } +} \ No newline at end of file From 2943d8b23cb1b03f5b97a07eecf9298b2fafe2d1 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 18:21:44 +0200 Subject: [PATCH 26/56] feat(tiktok): tiktok.scrape capability + billing wire-up --- .../app/capabilities/core/billing.py | 2 + .../app/capabilities/core/types.py | 1 + .../app/capabilities/tiktok/__init__.py | 5 ++ .../capabilities/tiktok/scrape/__init__.py | 1 + .../capabilities/tiktok/scrape/definition.py | 23 +++++ .../capabilities/tiktok/scrape/executor.py | 48 +++++++++++ .../app/capabilities/tiktok/scrape/schemas.py | 86 +++++++++++++++++++ surfsense_backend/app/config/__init__.py | 3 + surfsense_backend/app/routes/__init__.py | 1 + .../unit/capabilities/tiktok/__init__.py | 0 .../capabilities/tiktok/scrape/__init__.py | 0 .../tiktok/scrape/test_executor.py | 79 +++++++++++++++++ .../tiktok/scrape/test_schemas.py | 45 ++++++++++ .../unit/capabilities/tiktok/test_registry.py | 23 +++++ 14 files changed, 317 insertions(+) create mode 100644 surfsense_backend/app/capabilities/tiktok/__init__.py create mode 100644 surfsense_backend/app/capabilities/tiktok/scrape/__init__.py create mode 100644 surfsense_backend/app/capabilities/tiktok/scrape/definition.py create mode 100644 surfsense_backend/app/capabilities/tiktok/scrape/executor.py create mode 100644 surfsense_backend/app/capabilities/tiktok/scrape/schemas.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/scrape/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 71aa45c58..048797473 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -35,6 +35,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.GOOGLE_MAPS_REVIEW: "GOOGLE_MAPS_MICROS_PER_REVIEW", BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO", BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT", + BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO", } @@ -51,6 +52,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.GOOGLE_MAPS_REVIEW: "review", BillingUnit.YOUTUBE_VIDEO: "video", BillingUnit.YOUTUBE_COMMENT: "comment", + BillingUnit.TIKTOK_VIDEO: "video", } diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index c87601832..b45641a2d 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -25,6 +25,7 @@ class BillingUnit(StrEnum): GOOGLE_MAPS_REVIEW = "google_maps_review" YOUTUBE_VIDEO = "youtube_video" YOUTUBE_COMMENT = "youtube_comment" + TIKTOK_VIDEO = "tiktok_video" class BillableInput(Protocol): diff --git a/surfsense_backend/app/capabilities/tiktok/__init__.py b/surfsense_backend/app/capabilities/tiktok/__init__.py new file mode 100644 index 000000000..ed3f32682 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/__init__.py @@ -0,0 +1,5 @@ +"""``tiktok.*`` namespace: platform-native TikTok data verbs.""" + +from __future__ import annotations + +from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401 diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py b/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py new file mode 100644 index 000000000..9e5acc165 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py @@ -0,0 +1 @@ +"""``tiktok.scrape``: public TikTok videos over the browser-driven scraper.""" diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/definition.py b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py new file mode 100644 index 000000000..9f8632f3b --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py @@ -0,0 +1,23 @@ +"""``tiktok.scrape`` capability registration (billed per video; see config +``TIKTOK_MICROS_PER_VIDEO``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.scrape.executor import build_scrape_executor +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput + +TIKTOK_SCRAPE = Capability( + name="tiktok.scrape", + description=( + "Scrape public TikTok videos. Use urls, profiles, hashtags, or " + "search_queries." + ), + input_schema=ScrapeInput, + output_schema=ScrapeOutput, + executor=build_scrape_executor(), + billing_unit=BillingUnit.TIKTOK_VIDEO, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_SCRAPE) diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/executor.py b/surfsense_backend/app/capabilities/tiktok/scrape/executor.py new file mode 100644 index 000000000..7ef71486d --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/executor.py @@ -0,0 +1,48 @@ +"""``tiktok.scrape`` executor: verb input → scraper → TikTok video items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.tiktok import ( + TikTokAccessBlockedError, + TikTokScrapeInput, + scrape_tiktok, +) + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the executor to a scraper fn (defaults to the proprietary actor).""" + scrape_fn = scrape_fn or scrape_tiktok + + async def execute(payload: ScrapeInput) -> ScrapeOutput: + actor_input = TikTokScrapeInput( + startUrls=[{"url": url} for url in payload.urls], + profiles=payload.profiles, + hashtags=payload.hashtags, + searchQueries=payload.search_queries, + resultsPerPage=payload.results_per_page, + ) + emit_progress( + "starting", "Resolving TikTok targets", total=payload.max_items, unit="item" + ) + try: + items = await scrape_fn(actor_input, limit=payload.max_items) + except TikTokAccessBlockedError as exc: + # Anonymous-only scraper; a hard block can't be retried with creds. + raise ForbiddenError( + f"TikTok refused anonymous access: {exc}", + code="TIKTOK_ACCESS_BLOCKED", + ) from exc + emit_progress( + "done", f"Scraped {len(items)} item(s)", current=len(items), unit="item" + ) + return ScrapeOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py new file mode 100644 index 000000000..697a94043 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py @@ -0,0 +1,86 @@ +"""``tiktok.scrape`` I/O contracts. + +A lean, agent-friendly surface over ``TikTokScrapeInput`` +(``app/proprietary/platforms/tiktok``). The executor maps this to the full +scraper input; the scraper's ``TikTokVideoItem`` is reused verbatim as the +output element. Any TikTok URL kind (video, profile, hashtag, search) goes in +``urls``; ``profiles``/``hashtags``/``search_queries`` are typed shortcuts. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field, model_validator + +from app.proprietary.platforms.tiktok import TikTokVideoItem + +MAX_TIKTOK_SOURCES = 20 +"""Per-call cap on each source list: bounds a synchronous request's fan-out.""" + +MAX_TIKTOK_ITEMS = 100 +"""Hard ceiling on items returned per call, regardless of the per-target count.""" + + +class ScrapeInput(BaseModel): + urls: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description=( + "TikTok URLs to scrape: a video, a profile (/@), a hashtag " + "(/tag/), or a search URL. Provide these OR profiles/hashtags/" + "search_queries (at least one source is required)." + ), + ) + profiles: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description="Profile usernames (with or without a leading '@').", + ) + hashtags: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description="Hashtag names to scrape, without the leading '#'.", + ) + search_queries: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description="Search terms to run on TikTok.", + ) + results_per_page: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max videos to pull per profile/hashtag/search target.", + ) + max_items: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max total items to return across all sources.", + ) + + @model_validator(mode="after") + def _require_a_source(self) -> ScrapeInput: + if not any((self.urls, self.profiles, self.hashtags, self.search_queries)): + raise ValueError( + "Provide at least one of 'urls', 'profiles', 'hashtags', or " + "'search_queries'." + ) + return self + + @property + def estimated_units(self) -> int: + """Worst-case billable items for the pre-flight gate: ``max_items`` is a + hard cross-source ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class ScrapeOutput(BaseModel): + items: list[TikTokVideoItem] = Field( + default_factory=list, + description="One item per video returned, in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned item = one billable unit.""" + return len(self.items) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 5d8e96881..c03ac3598 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -714,6 +714,9 @@ class Config: # Kept separate from the video rate so comments can be re-tuned toward the # cheaper per-comment market ($0.40-2.00/1k) without touching video pricing. YOUTUBE_MICROS_PER_COMMENT = int(os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500")) + # Browser-driven listings make TikTok heavier per item than the API-backed + # video meter, so it sits a touch above YouTube's video rate. + TIKTOK_MICROS_PER_VIDEO = int(os.getenv("TIKTOK_MICROS_PER_VIDEO", "3500")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 1a5b182b8..3c28fd65c 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -4,6 +4,7 @@ from fastapi import APIRouter, Depends import app.capabilities.google_maps import app.capabilities.google_search import app.capabilities.reddit +import app.capabilities.tiktok import app.capabilities.web import app.capabilities.youtube # noqa: F401 from app.automations.api import router as automations_router diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py new file mode 100644 index 000000000..1bf07b0c3 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py @@ -0,0 +1,79 @@ +"""``tiktok.scrape`` executor: verb input → actor input mapping → typed items. + +Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's +own payload→TikTokScrapeInput mapping and the dict→TikTokVideoItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.scrape.executor import build_scrape_executor +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.tiktok import TikTokAccessBlockedError, TikTokScrapeInput + +pytestmark = pytest.mark.unit + + +class _FakeScraper: + """Records the actor input + limit it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[tuple[TikTokScrapeInput, int | None]] = [] + + async def __call__( + self, actor_input: TikTokScrapeInput, *, limit: int | None = None + ) -> list[dict]: + self.calls.append((actor_input, limit)) + return self._items + + +async def test_maps_urls_to_start_urls_and_wraps_items(): + scraper = _FakeScraper([{"id": "123", "text": "hello"}]) + execute = build_scrape_executor(scrape_fn=scraper) + + out = await execute(ScrapeInput(urls=["https://www.tiktok.com/@nasa/video/123"])) + + assert isinstance(out, ScrapeOutput) + assert len(out.items) == 1 + assert out.items[0].id == "123" + + (actor_input, _limit) = scraper.calls[0] + assert [u.url for u in actor_input.startUrls] == [ + "https://www.tiktok.com/@nasa/video/123" + ] + + +async def test_forwards_typed_sources_and_limit(): + scraper = _FakeScraper([]) + execute = build_scrape_executor(scrape_fn=scraper) + + await execute( + ScrapeInput( + profiles=["nasa"], + hashtags=["food"], + search_queries=["cats"], + results_per_page=7, + max_items=25, + ) + ) + + (actor_input, limit) = scraper.calls[0] + assert actor_input.profiles == ["nasa"] + assert actor_input.hashtags == ["food"] + assert actor_input.searchQueries == ["cats"] + assert actor_input.resultsPerPage == 7 + # The outer collection limit is the caller's total-item cap. + assert limit == 25 + + +async def test_access_blocked_maps_to_forbidden(): + async def _blocked(actor_input: TikTokScrapeInput, *, limit: int | None = None): + raise TikTokAccessBlockedError("all IPs refused") + + execute = build_scrape_executor(scrape_fn=_blocked) + + with pytest.raises(ForbiddenError): + await execute(ScrapeInput(hashtags=["food"])) diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py new file mode 100644 index 000000000..e7c978bcc --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py @@ -0,0 +1,45 @@ +"""``tiktok.scrape`` input guards: a source is required and the batch is bounded.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.scrape.schemas import ( + MAX_TIKTOK_ITEMS, + MAX_TIKTOK_SOURCES, + ScrapeInput, +) + +pytestmark = pytest.mark.unit + + +def test_rejects_input_with_no_source(): + with pytest.raises(ValidationError): + ScrapeInput() + + +def test_accepts_urls_only(): + payload = ScrapeInput(urls=["https://www.tiktok.com/@nasa"]) + assert payload.profiles == [] + + +def test_accepts_hashtags_only(): + payload = ScrapeInput(hashtags=["food"]) + assert payload.hashtags == ["food"] + + +def test_defaults_and_bounds(): + payload = ScrapeInput(hashtags=["food"]) + assert payload.max_items == 10 + assert payload.results_per_page == 10 + with pytest.raises(ValidationError): + ScrapeInput(hashtags=["food"], max_items=0) + with pytest.raises(ValidationError): + ScrapeInput(hashtags=["food"], max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_rejects_more_sources_than_the_cap(): + too_many = [f"tag{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] + with pytest.raises(ValidationError): + ScrapeInput(hashtags=too_many) diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py new file mode 100644 index 000000000..3a85fe87f --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py @@ -0,0 +1,23 @@ +"""The tiktok namespace registers its verb as one Capability the doors/agent read.""" + +from __future__ import annotations + +import pytest + +from app.capabilities import ( + tiktok, # noqa: F401 — importing the namespace registers its verbs +) +from app.capabilities.core import BillingUnit +from app.capabilities.core.store import get_capability +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput + +pytestmark = pytest.mark.unit + + +def test_tiktok_scrape_is_registered_and_billed_per_video(): + cap = get_capability("tiktok.scrape") + + assert cap.name == "tiktok.scrape" + assert cap.input_schema is ScrapeInput + assert cap.output_schema is ScrapeOutput + assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO From ed1c3a1f3d43cb77e214c88d87e1a793080ac9d8 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 20:20:20 +0200 Subject: [PATCH 27/56] feat(tiktok): expose tiktok.scrape on MCP, chat subagent, playground --- .../agents/chat/multi_agent_chat/constants.py | 1 + .../subagents/builtins/tiktok/__init__.py | 1 + .../subagents/builtins/tiktok/agent.py | 43 +++++++++ .../subagents/builtins/tiktok/description.md | 2 + .../builtins/tiktok/system_prompt.md | 64 ++++++++++++++ .../builtins/tiktok/tools/__init__.py | 0 .../subagents/builtins/tiktok/tools/index.py | 27 ++++++ .../multi_agent_chat/subagents/registry.py | 4 + .../test_subagent_composition.py | 1 + .../mcp_server/features/scrapers/__init__.py | 4 +- .../features/scrapers/platforms/tiktok.py | 88 +++++++++++++++++++ surfsense_mcp/mcp_server/selfcheck.py | 1 + surfsense_web/lib/playground/catalog.ts | 7 ++ .../lib/playground/platform-icons.tsx | 1 + surfsense_web/public/connectors/tiktok.svg | 6 ++ 15 files changed, 248 insertions(+), 2 deletions(-) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/__init__.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py create mode 100644 surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py create mode 100644 surfsense_web/public/connectors/tiktok.svg diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py index b132618d7..c396fceaf 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py @@ -36,6 +36,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = { "google_maps": frozenset(), "google_search": frozenset(), "reddit": frozenset(), + "tiktok": frozenset(), "mcp_discovery": frozenset( { "SLACK_CONNECTOR", diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py new file mode 100644 index 000000000..ec7d5955f --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py @@ -0,0 +1 @@ +"""``tiktok`` builtin subagent: structured public TikTok videos and listings.""" diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py new file mode 100644 index 000000000..2c7dd014b --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py @@ -0,0 +1,43 @@ +"""``tiktok`` route: ``SurfSenseSubagentSpec`` builder for deepagents.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.language_models import BaseChatModel +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) + +from .tools.index import NAME, RULESET, load_tools + + +def build_subagent( + *, + dependencies: dict[str, Any], + model: BaseChatModel | None = None, + middleware_stack: dict[str, Any] | None = None, + mcp_tools: list[BaseTool] | None = None, +) -> SurfSenseSubagentSpec: + tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] + description = ( + read_md_file(__package__, "description").strip() + or "Pulls structured data from public TikTok videos, hashtags, and searches." + ) + system_prompt = read_md_file(__package__, "system_prompt").strip() + return pack_subagent( + name=NAME, + description=description, + system_prompt=system_prompt, + tools=tools, + ruleset=RULESET, + dependencies=dependencies, + model=model, + middleware_stack=middleware_stack, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md new file mode 100644 index 000000000..fc13d864d --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md @@ -0,0 +1,2 @@ +TikTok specialist: pulls structured public TikTok data — videos (caption/text, author, play/like/comment/share counts, music, hashtags, timestamps, web URL) from a hashtag feed, a search query, a creator profile, or a known video URL. Also compares fresh TikTok results against earlier findings in this chat. +Use whenever the task is to find what is trending or being said on TikTok about a topic, gather a creator's or hashtag's videos, or scrape a specific video URL. Triggers include "search TikTok for X", "trending TikTok videos about X", "videos with #X", and "scrape this TikTok video". Not for general web pages (use the web crawling specialist), Google results (use the Google Search specialist), Reddit (use the Reddit specialist), or YouTube (use the YouTube specialist). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md new file mode 100644 index 000000000..49c20164e --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md @@ -0,0 +1,64 @@ +You are the SurfSense TikTok sub-agent. +You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis. + + +Answer the delegated question from live TikTok data gathered with your verb, comparing against earlier results already in this conversation when the task calls for it. + + + +- `tiktok_scrape` +- `read_run` / `search_run` (free readers for stored scrape output) + + + +- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#') and/or `search_queries`. +- Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`. +- Profiles: a creator's `profiles` feed can come back empty — TikTok restricts the profile video endpoint. Prefer `hashtags`, `search_queries`, or a direct video URL, and treat an empty profile result as a known limit, not a failure to retry endlessly. +- Controlling volume: use `max_items` for the total cap and `results_per_page` per target. +- Requested counts: `max_items` defaults to only 10 — when the task asks for N videos, set `max_items` and `results_per_page` above N. A call that caps below the target can never satisfy it. +- Batch multiple hashtags or search terms into one call rather than many single-term calls. + +- Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, count changes). + + + +- Use only tools in ``. +- Report only results present in the tool output. Never invent captions, URLs, authors, or counts. + + + +- Do not read arbitrary web pages — that belongs to the web crawling specialist. +- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on. +- Reddit belongs to the Reddit specialist; YouTube belongs to the YouTube specialist; Google results belong to the Google Search specialist. + + + +- Report uncertainty explicitly when evidence is incomplete or conflicting. +- Never present unverified claims as facts. + + + +- Underspecified request — no usable hashtag, query, or URL — return `status=blocked` with the missing fields. +- Tool failure: return `status=error` with a concise recovery `next_step`. +- No useful evidence: return `status=blocked` with a narrower query or the scope you still need. + + + +Return **only** one JSON object (no markdown/prose): +{ + "status": "success" | "partial" | "blocked" | "error", + "action_summary": string, + "evidence": { + "findings": string[], + "sources": string[], + "confidence": "high" | "medium" | "low" + }, + "next_step": string | null, + "missing_fields": string[] | null, + "assumptions": string[] | null +} + +Route-specific rules: +- `evidence.findings`: one entry per distinct video or delta — a single sentence each; do not paste raw payloads. Max 10 entries, unless the delegated task asks for N items: then return up to N (each backed by a real scraped result, never padded). +- `evidence.sources`: one TikTok URL per finding when applicable, same cap as findings. List each URL once. + diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py new file mode 100644 index 000000000..e790d44f0 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py @@ -0,0 +1,27 @@ +"""``tiktok`` sub-agent tools: the TikTok scrape capability verb.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset +from app.capabilities.core.access.agent import build_capability_tools +from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE + +NAME = "tiktok" + +RULESET = Ruleset(origin=NAME, rules=[]) + +_CI_VERBS = [TIKTOK_SCRAPE] + + +def load_tools( + *, dependencies: dict[str, Any] | None = None, **kwargs: Any +) -> list[BaseTool]: + d = {**(dependencies or {}), **kwargs} + return build_capability_tools( + workspace_id=d.get("workspace_id"), + capabilities=_CI_VERBS, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index 34895a514..71c2e6f3d 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -33,6 +33,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.memory.agent import ( from app.agents.chat.multi_agent_chat.subagents.builtins.reddit.agent import ( build_subagent as build_reddit_subagent, ) +from app.agents.chat.multi_agent_chat.subagents.builtins.tiktok.agent import ( + build_subagent as build_tiktok_subagent, +) from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import ( build_subagent as build_web_crawler_subagent, ) @@ -84,6 +87,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { "memory": build_memory_subagent, "onedrive": build_onedrive_subagent, "reddit": build_reddit_subagent, + "tiktok": build_tiktok_subagent, "web_crawler": build_web_crawler_subagent, "youtube": build_youtube_subagent, } diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py index c3ad04250..66e8f28db 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py @@ -37,6 +37,7 @@ _EXPECTED_SUBAGENTS = frozenset( "memory", "onedrive", "reddit", + "tiktok", "web_crawler", "youtube", } diff --git a/surfsense_mcp/mcp_server/features/scrapers/__init__.py b/surfsense_mcp/mcp_server/features/scrapers/__init__.py index dfa2f3ab2..c0e7dea07 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/__init__.py +++ b/surfsense_mcp/mcp_server/features/scrapers/__init__.py @@ -13,9 +13,9 @@ from mcp.server.fastmcp import FastMCP from ...core.client import SurfSenseClient from ...core.workspace_context import WorkspaceContext from . import run_history -from .platforms import google_maps, google_search, reddit, web, youtube +from .platforms import google_maps, google_search, reddit, tiktok, web, youtube -_REGISTRARS = (web, google_search, reddit, youtube, google_maps, run_history) +_REGISTRARS = (web, google_search, reddit, youtube, tiktok, google_maps, run_history) def register( diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py new file mode 100644 index 000000000..1e5f472ee --- /dev/null +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py @@ -0,0 +1,88 @@ +"""TikTok scraper tool.""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the TikTok tool.""" + + @mcp.tool( + name="surfsense_tiktok_scrape", + title="Search or scrape TikTok", + annotations=SCRAPE, + structured_output=False, + ) + async def tiktok_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="TikTok URLs: a video, a profile " + "('https://www.tiktok.com/@nasa'), a hashtag " + "('https://www.tiktok.com/tag/food'), or a search URL. Provide " + "urls OR profiles/hashtags/search_queries." + ), + ] = None, + profiles: Annotated[ + list[str] | None, + Field( + description="Profile usernames to scrape, with or without a " + "leading '@', e.g. ['nasa']." + ), + ] = None, + hashtags: Annotated[ + list[str] | None, + Field( + description="Hashtag names to scrape, without the '#', e.g. " + "['food']." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field(description="Terms to search TikTok for, e.g. ['cooking']."), + ] = None, + results_per_page: Annotated[ + int, + Field(ge=1, description="Max videos per profile/hashtag/search target."), + ] = 10, + max_items: Annotated[ + int, Field(ge=1, description="Maximum videos to return in total.") + ] = 10, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Search or scrape public TikTok videos. + + Use this for ANY TikTok research — a creator's videos, a hashtag feed, + a search, or a specific video URL — instead of a generic web search. + Returns videos with text, author, stats, music, and the web URL. + Example: hashtags=['food'], max_items=20. + """ + return await run_scraper( + client, + context, + platform="tiktok", + verb="scrape", + payload={ + "urls": urls, + "profiles": profiles, + "hashtags": hashtags, + "search_queries": search_queries, + "results_per_page": results_per_page, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/mcp_server/selfcheck.py b/surfsense_mcp/mcp_server/selfcheck.py index 20224c173..e5ac6bd2b 100644 --- a/surfsense_mcp/mcp_server/selfcheck.py +++ b/surfsense_mcp/mcp_server/selfcheck.py @@ -23,6 +23,7 @@ EXPECTED_TOOLS = { "surfsense_reddit_scrape", "surfsense_youtube_scrape", "surfsense_youtube_comments", + "surfsense_tiktok_scrape", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_list_scraper_runs", diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index 9b508228a..4503662f8 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -3,6 +3,7 @@ import { GoogleMapsIcon, GoogleSearchIcon, RedditIcon, + TikTokIcon, WebIcon, YouTubeIcon, } from "./platform-icons"; @@ -48,6 +49,12 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ { name: "youtube.comments", verb: "comments", label: "Comments" }, ], }, + { + id: "tiktok", + label: "TikTok", + icon: TikTokIcon, + verbs: [{ name: "tiktok.scrape", verb: "scrape", label: "Scrape" }], + }, { id: "google_maps", label: "Google Maps", diff --git a/surfsense_web/lib/playground/platform-icons.tsx b/surfsense_web/lib/playground/platform-icons.tsx index e7a443c2f..474bff317 100644 --- a/surfsense_web/lib/playground/platform-icons.tsx +++ b/surfsense_web/lib/playground/platform-icons.tsx @@ -24,6 +24,7 @@ function brandIcon(src: string, alt: string) { export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit"); export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube"); +export const TikTokIcon = brandIcon("/connectors/tiktok.svg", "TikTok"); export const GoogleMapsIcon = brandIcon("/connectors/google-maps.svg", "Google Maps"); export const GoogleSearchIcon = brandIcon("/connectors/google-search.svg", "Google Search"); export const WebIcon = brandIcon("/connectors/web.svg", "Web"); diff --git a/surfsense_web/public/connectors/tiktok.svg b/surfsense_web/public/connectors/tiktok.svg new file mode 100644 index 000000000..c3c3fe06e --- /dev/null +++ b/surfsense_web/public/connectors/tiktok.svg @@ -0,0 +1,6 @@ + + + + + + From ea6befdf595e91613b40d3c5f26f2f4dad116df5 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 20:21:29 +0200 Subject: [PATCH 28/56] docs(tiktok): native connector page + nav card --- .../content/docs/connectors/native/index.mdx | 7 ++- .../content/docs/connectors/native/meta.json | 2 +- .../content/docs/connectors/native/tiktok.mdx | 45 +++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 surfsense_web/content/docs/connectors/native/tiktok.mdx diff --git a/surfsense_web/content/docs/connectors/native/index.mdx b/surfsense_web/content/docs/connectors/native/index.mdx index 5c5d24444..f915b5533 100644 --- a/surfsense_web/content/docs/connectors/native/index.mdx +++ b/surfsense_web/content/docs/connectors/native/index.mdx @@ -1,6 +1,6 @@ --- title: Native Connectors -description: SurfSense's built-in scraper APIs for Reddit, YouTube, Google Maps, Google Search, and the web +description: SurfSense's built-in scraper APIs for Reddit, YouTube, TikTok, Google Maps, Google Search, and the web --- import { Card, Cards } from 'fumadocs-ui/components/card'; @@ -18,6 +18,11 @@ Native connectors are SurfSense's own scraper APIs — built into the platform, description="Videos, channels, playlists, subtitles, and comments" href="/docs/connectors/native/youtube" /> + `), a hashtag (`/tag/`), or a search URL (max 20 sources per call) | +| `profiles` | — | Profile usernames, with or without a leading `@` | +| `hashtags` | — | Hashtag names, without the `#` | +| `search_queries` | — | Search terms to run on TikTok | +| `results_per_page` | `10` | Max videos per profile/hashtag/search target | +| `max_items` | `10` | Max total videos returned across all sources (hard cap 100) | + +## Example + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/scrape" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "hashtags": ["food"], + "max_items": 25 + }' +``` + +The response is `{ "items": [...] }` — one item per video. Billing is per returned item. + + +Video, hashtag, and search targets are the reliable paths. TikTok restricts the profile video endpoint for automated clients, so a `profiles` target can return no items even when the account is public — prefer hashtags, search, or a direct video URL. + + +For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → TikTok → Scrape** in your workspace. From 5f9472b122157c3b9ab8e590ae26bcdd2f0cfea9 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 20:30:38 +0200 Subject: [PATCH 29/56] feat(tiktok): advertise tiktok across agent prompts, MCP instructions, docs --- .../main_agent/system_prompt/prompts/identity/private.md | 6 +++--- .../main_agent/system_prompt/prompts/identity/team.md | 6 +++--- .../main_agent/system_prompt/prompts/kb_first.md | 5 +++-- .../main_agent/system_prompt/prompts/routing.md | 1 + surfsense_mcp/README.md | 1 + surfsense_mcp/mcp_server/server.py | 5 +++-- surfsense_web/content/docs/how-to/mcp-server.mdx | 4 ++-- 7 files changed, 16 insertions(+), 12 deletions(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md index 378c052d4..0c7e4071c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md @@ -6,9 +6,9 @@ reviews are moving, and what is being said across the open web — and to put that intelligence to work alongside their own knowledge base. You do this by dispatching **specialist subagents** via the `task` tool: -- **Live market data** — Reddit, YouTube, Google Maps, Google Search, and the - web crawler return structured, current platform data (posts, comments, - transcripts, reviews, SERPs, full page content). +- **Live market data** — Reddit, YouTube, TikTok, Google Maps, Google Search, + and the web crawler return structured, current platform data (posts, + comments, transcripts, videos, reviews, SERPs, full page content). - **The user's own context** — their knowledge base, connected apps, and persistent memory. - **Deliverables** — reports, podcasts, and presentations built from what the diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md index 2765c06b7..6c1076a89 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md @@ -6,9 +6,9 @@ reviews are moving, and what is being said across the open web — and to put that intelligence to work alongside the team's shared knowledge base. You do this by dispatching **specialist subagents** via the `task` tool: -- **Live market data** — Reddit, YouTube, Google Maps, Google Search, and the - web crawler return structured, current platform data (posts, comments, - transcripts, reviews, SERPs, full page content). +- **Live market data** — Reddit, YouTube, TikTok, Google Maps, Google Search, + and the web crawler return structured, current platform data (posts, + comments, transcripts, videos, reviews, SERPs, full page content). - **The team's own context** — its shared knowledge base, connected apps, and persistent team memory. - **Deliverables** — reports, podcasts, and presentations built from what the diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md index c69ed400c..ef753b93c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md @@ -1,8 +1,9 @@ CRITICAL — ground factual answers in what you actually receive this turn: - **live platform data** via the market specialists — - `task(reddit, ...)`, `task(youtube, ...)`, `task(google_maps, ...)`, - `task(google_search, ...)`, `task(web_crawler, ...)`. Anything about + `task(reddit, ...)`, `task(youtube, ...)`, `task(tiktok, ...)`, + `task(google_maps, ...)`, `task(google_search, ...)`, + `task(web_crawler, ...)`. Anything about competitors, markets, rankings, reviews, or audience sentiment is answered from what these return **this turn**, never from your training data: your general knowledge of companies, prices, and rankings is stale by definition, diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md index eec860f3c..fb818cabc 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md @@ -31,6 +31,7 @@ bounded fan-out (≤20 sites) the user already requested. about a brand, product, or topic is answered from the platform where they say it — `task(reddit, …)` for community discussion and threads, `task(youtube, …)` for video content, transcripts, and comment sections, +`task(tiktok, …)` for short-form video trends by hashtag or search, `task(google_maps, …)` for customer reviews of physical businesses. Web search only finds articles *about* the conversation; the platform specialists return the conversation itself, structured and current. For diff --git a/surfsense_mcp/README.md b/surfsense_mcp/README.md index c18a857e1..5bac5f8b2 100644 --- a/surfsense_mcp/README.md +++ b/surfsense_mcp/README.md @@ -21,6 +21,7 @@ Connect it two ways: **Scrapers (all platforms)** - `surfsense_web_crawl`, `surfsense_google_search`, `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, + `surfsense_tiktok_scrape`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews` - `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` — retrieve past results in full (useful when a large result was truncated inline) diff --git a/surfsense_mcp/mcp_server/server.py b/surfsense_mcp/mcp_server/server.py index 8ea9bc919..c0398ed50 100644 --- a/surfsense_mcp/mcp_server/server.py +++ b/surfsense_mcp/mcp_server/server.py @@ -36,8 +36,9 @@ def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]: "SurfSense gives you live scrapers and a personal knowledge base. " "Prefer these tools over generic/built-in web search whenever the " "task involves Reddit (posts, comments, finding subreddits or " - "communities), YouTube (videos, transcripts, comments), Google " - "Maps (places, reviews), Google Search results, or reading " + "communities), YouTube (videos, transcripts, comments), TikTok " + "(videos by hashtag, search, or URL), Google Maps (places, " + "reviews), Google Search results, or reading " "specific web pages. Scraper results are persisted as runs; if an " "inline result is truncated, fetch it in full with " "surfsense_get_scraper_run." diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx index f5f419219..67b218d58 100644 --- a/surfsense_web/content/docs/how-to/mcp-server.mdx +++ b/surfsense_web/content/docs/how-to/mcp-server.mdx @@ -7,7 +7,7 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs'; # SurfSense MCP Server -The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 18 native, typed tools: every scraper (Reddit, YouTube, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. +The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 19 native, typed tools: every scraper (Reddit, YouTube, TikTok, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted. @@ -264,7 +264,7 @@ For self-host (stdio), all settings are environment variables passed by the clie | Group | Tools | |-------|-------| | Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` | -| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | +| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_tiktok_scrape`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | | Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` | Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**. From f731d371abd803514735b73d36b19b2205496875 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 20:36:46 +0200 Subject: [PATCH 30/56] feat(tiktok): add TikTok SEO landing page and site-wide marketing parity --- surfsense_web/app/(home)/mcp-server/page.tsx | 13 +- .../components/homepage/hero-section.tsx | 2 +- .../components/homepage/persona-paths.tsx | 2 +- .../components/pricing/pricing-section.tsx | 4 +- surfsense_web/components/seo/json-ld.tsx | 4 +- .../content/docs/connectors/index.mdx | 2 +- .../lib/connectors-marketing/index.ts | 2 + .../lib/connectors-marketing/reddit.tsx | 2 +- .../lib/connectors-marketing/tiktok.tsx | 277 ++++++++++++++++++ .../lib/connectors-marketing/youtube.tsx | 2 +- 10 files changed, 297 insertions(+), 13 deletions(-) create mode 100644 surfsense_web/lib/connectors-marketing/tiktok.tsx diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index 5e4961b1e..22c867c73 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -16,7 +16,7 @@ import type { FaqItem } from "@/lib/connectors-marketing/types"; const canonicalUrl = "https://www.surfsense.com/mcp-server"; const metaDescription = - "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; + "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, TikTok, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; export const metadata: Metadata = { title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools", @@ -93,6 +93,7 @@ const TOOL_GROUPS = [ "surfsense_reddit_scrape", "surfsense_youtube_scrape", "surfsense_youtube_comments", + "surfsense_tiktok_scrape", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_google_search", @@ -127,7 +128,7 @@ const FAQ: FaqItem[] = [ { question: "What is the SurfSense MCP server?", answer: - "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", + "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, TikTok, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", }, { question: "Which MCP clients does it work with?", @@ -215,8 +216,9 @@ export default function McpServerPage() {

The SurfSense MCP server hands Claude, Cursor, or any MCP client the whole platform: - scrape Reddit, YouTube, Google Maps, Google Search, and the open web, and search, - read, and write your knowledge base. One API key, typed tools, pay as you go. + scrape Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web, and + search, read, and write your knowledge base. One API key, typed tools, pay as you + go.

+ diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index 89d1e46f5..febe8a62b 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -719,7 +719,7 @@ export function HeroSection() { > SurfSense is an open-source competitive intelligence platform. Your AI agents monitor competitors, track rankings, and listen to your market with live data from platforms - like Reddit, YouTube, Google Maps, Google Search, and the open web. + like Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web.

diff --git a/surfsense_web/components/homepage/persona-paths.tsx b/surfsense_web/components/homepage/persona-paths.tsx index 9c03c5667..50eadba0f 100644 --- a/surfsense_web/components/homepage/persona-paths.tsx +++ b/surfsense_web/components/homepage/persona-paths.tsx @@ -35,7 +35,7 @@ const PATHS: { eyebrow: "For developers & agents", title: "The whole platform is programmable", description: - "Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.", + "Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.", links: [ { label: "Read the docs", href: "/docs" }, { label: "SurfSense MCP server", href: "/mcp-server" }, diff --git a/surfsense_web/components/pricing/pricing-section.tsx b/surfsense_web/components/pricing/pricing-section.tsx index 74c76ad36..1af296e57 100644 --- a/surfsense_web/components/pricing/pricing-section.tsx +++ b/surfsense_web/components/pricing/pricing-section.tsx @@ -35,7 +35,7 @@ const demoPlans = [ billingText: "Your first $5 of credit is free. No subscription, ever", features: [ "$5 of free credit to start, one balance for everything", - "Platform connectors: Reddit, YouTube, Google Maps, Google Search, and the open web", + "Platform connectors: Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web", "Call every connector as a REST API with your key or through the MCP server", "Pay per item returned and per page crawled. Failed calls are never billed", "Premium models like GPT-5.5, Claude Sonnet 5, Gemini 3.1 Pro billed at provider cost", @@ -95,7 +95,7 @@ const faqData: FAQSection[] = [ { question: "How does Pay As You Go work?", answer: - "There is no monthly subscription. Start with $5 of free credit, and when you need more, add any amount. $1 buys exactly $1 of credit, added to your balance immediately. You can enable automatic refills when your balance runs low, and turn them off any time." + "There is no monthly subscription. Start with $5 of free credit, and when you need more, add any amount. $1 buys exactly $1 of credit, added to your balance immediately. You can enable automatic refills when your balance runs low, and turn them off any time.", }, { question: "What happens if I run out of credit?", diff --git a/surfsense_web/components/seo/json-ld.tsx b/surfsense_web/components/seo/json-ld.tsx index c4b3ec09c..063f6a0c0 100644 --- a/surfsense_web/components/seo/json-ld.tsx +++ b/surfsense_web/components/seo/json-ld.tsx @@ -76,11 +76,11 @@ export function SoftwareApplicationJsonLd() { "Free self-hosted from the open-source repo; cloud starts with $5 of free credit, then pay as you go", }, description: - "SurfSense is an open-source competitive intelligence platform. AI agents monitor competitors, track rankings, and listen to your market with platform-native connectors for Reddit, YouTube, Google Maps, Google Search, and the open web, through one API or MCP server.", + "SurfSense is an open-source competitive intelligence platform. AI agents monitor competitors, track rankings, and listen to your market with platform-native connectors for Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web, through one API or MCP server.", url: "https://www.surfsense.com", downloadUrl: "https://github.com/MODSetter/SurfSense/releases", featureList: [ - "Platform-native connectors: Reddit, YouTube, Google Maps, Google Search, Web Crawl", + "Platform-native connectors: Reddit, YouTube, TikTok, Google Maps, Google Search, Web Crawl", "MCP server that exposes every connector as a native agent tool", "Agent harness with retries, structured output, and credit metering", "Competitor, brand, and rank monitoring with briefs and alerts", diff --git a/surfsense_web/content/docs/connectors/index.mdx b/surfsense_web/content/docs/connectors/index.mdx index 1ee9209e6..ab136c372 100644 --- a/surfsense_web/content/docs/connectors/index.mdx +++ b/surfsense_web/content/docs/connectors/index.mdx @@ -12,7 +12,7 @@ Connectors bring data into SurfSense — either as searchable knowledge or as li } title="Native Connectors" - description="SurfSense's built-in scraper APIs: Reddit, YouTube, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" + description="SurfSense's built-in scraper APIs: Reddit, YouTube, TikTok, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" href="/docs/connectors/native" /> Date: Wed, 8 Jul 2026 23:14:50 +0200 Subject: [PATCH 31/56] feat(tiktok): emit graceful ErrorItem for blocked/empty listings Profile and search feeds are trust-gated: an anonymous headless session gets an empty item_list (profile) or no results XHR (search), while hashtag feeds load. A zero-item listing now yields one honest ErrorItem (errorCode="no_items") instead of vanishing silently, and ErrorItems are excluded from billing so a blocked target is surfaced but never charged. --- .../app/capabilities/tiktok/scrape/schemas.py | 5 +- .../platforms/tiktok/flows/listing.py | 18 ++++++ .../scripts/e2e_tiktok_scrape.py | 59 +++++++++++++++---- .../tiktok/scrape/test_schemas.py | 13 ++++ .../platforms/tiktok/test_orchestrator.py | 15 +++++ 5 files changed, 96 insertions(+), 14 deletions(-) diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py index 697a94043..2f5005353 100644 --- a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py @@ -82,5 +82,6 @@ class ScrapeOutput(BaseModel): @property def billable_units(self) -> int: - """One returned item = one billable unit.""" - return len(self.items) + """One returned video = one billable unit; ErrorItems (``errorCode`` set, + for blocked/empty targets) are surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py index 44fcc33e2..3e930271f 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py @@ -12,9 +12,19 @@ from typing import Any from ..extraction import parse_video from ..extraction.timestamps import now_iso +from ..schemas import ErrorItem from ..targets.types import TikTokTarget from . import FetchListingFn +# Profile and search feeds are trust-gated: an anonymous headless session gets an +# empty body (profile) or no results XHR (search), while hashtag feeds load. We +# can't tell "genuinely empty" from "blocked" here, so a zero-item listing emits +# one honest ErrorItem instead of vanishing silently. +_EMPTY_LISTING_MESSAGE = ( + "No videos returned. The target may be empty/private/removed, or TikTok " + "withheld this feed from anonymous access (common for profiles and search)." +) + async def iter_listing( target: TikTokTarget, *, cap: int, fetch_listing: FetchListingFn @@ -35,3 +45,11 @@ async def iter_listing( emitted += 1 if emitted >= cap: return + if emitted == 0: + yield ErrorItem( + url=target.url, + input=target.value, + error=_EMPTY_LISTING_MESSAGE, + errorCode="no_items", + scrapedAt=now_iso(), + ).to_output() diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py index 22bebff96..eedeaacac 100644 --- a/surfsense_backend/scripts/e2e_tiktok_scrape.py +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -7,11 +7,14 @@ Run from the backend directory: What it exercises (everything REAL — live network, live proxy, live browser): Stage 1 — proxy egress proof (informational). - Stage 2 — profile listing via the stealth browser (soft-blocked by TikTok; - expected empty until a stronger anti-detection path exists). + Stage 2 — profile via the full pipeline: TikTok soft-blocks the anonymous + profile feed, so this asserts graceful degradation — real videos OR + a single honest ErrorItem, never a silent empty. Stage 3 — blob video path over HTTP (URL taken from a captured hashtag struct). Stage 4 — hashtag listing via the stealth browser (captures item_list XHRs). Stage 5 — full scrape_tiktok() pipeline on a hashtag. + Stage 6 — search via the full pipeline: same graceful-degrade contract as + profile (results feed doesn't load for anonymous sessions). On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser suites can pin against real-shaped data without network. @@ -40,9 +43,11 @@ for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): _FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok" -# Evergreen public targets: a regular high-volume creator and a broad hashtag. +# Evergreen public targets: a regular high-volume creator, a broad hashtag, and +# a common search term. _PROFILE = "nasa" _HASHTAG = "food" +_SEARCH = "meal prep" _COUNT = 5 @@ -93,17 +98,24 @@ async def stage_proxy() -> bool: async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]: - _hr(f"STAGE 2 — profile listing (browser): @{_PROFILE}") - from app.proprietary.platforms.tiktok.session import fetch_item_list + _hr(f"STAGE 2 — profile listing graceful-degrade: @{_PROFILE}") + from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok - url = f"https://www.tiktok.com/@{_PROFILE}" - raw = await fetch_item_list(url, _COUNT) - ok = _check( - "captured itemStructs from item_list XHRs", - len(raw) > 0 and isinstance(raw[0], dict) and bool(raw[0].get("id")), - f"{len(raw)} struct(s)", + # TikTok soft-blocks the profile feed for anonymous headless sessions + # (empty 200). The shipped contract is a single honest ErrorItem, not a + # silent empty. This stage verifies that graceful degradation, and passes if + # EITHER real videos come back (session got trusted) OR an ErrorItem does. + items = await scrape_tiktok( + TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=_COUNT), limit=_COUNT ) - return ok, raw + has_video = any(it.get("id") and not it.get("errorCode") for it in items) + has_error = any(it.get("errorCode") == "no_items" for it in items) + ok = _check( + "profile yields videos or a graceful ErrorItem (never silent empty)", + has_video or has_error, + f"{len(items)} item(s); video={has_video} error={has_error}", + ) + return ok, items async def stage_blob_video(video_url: str) -> tuple[bool, dict[str, Any] | None]: @@ -167,6 +179,26 @@ async def stage_pipeline() -> bool: return ok +async def stage_search_listing() -> tuple[bool, list[dict[str, Any]]]: + _hr(f"STAGE 6 — search listing graceful-degrade: {_SEARCH!r}") + from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok + + # The search results feed doesn't load for anonymous headless sessions + # (results XHR never fires on a cold deep-link). Same contract as profile: + # verify a graceful ErrorItem instead of a silent empty. + items = await scrape_tiktok( + TikTokScrapeInput(searchQueries=[_SEARCH], resultsPerPage=_COUNT), limit=_COUNT + ) + has_video = any(it.get("id") and not it.get("errorCode") for it in items) + has_error = any(it.get("errorCode") == "no_items" for it in items) + ok = _check( + "search yields videos or a graceful ErrorItem (never silent empty)", + has_video or has_error, + f"{len(items)} item(s); video={has_video} error={has_error}", + ) + return ok, items + + async def main() -> int: print("TikTok scraper functional e2e — live network + proxy + browser") results: dict[str, bool] = {} @@ -185,6 +217,9 @@ async def main() -> int: else: print("\n [SKIP] Stage 3 — no captured struct to build a video URL") + ok_search, _ = await stage_search_listing() + results["Stage 6 search listing"] = ok_search + ok_profile, _ = await stage_profile_listing() results["Stage 2 profile listing"] = ok_profile results["Stage 5 pipeline"] = await stage_pipeline() diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py index e7c978bcc..f9cec8c3d 100644 --- a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py +++ b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py @@ -9,6 +9,7 @@ from app.capabilities.tiktok.scrape.schemas import ( MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES, ScrapeInput, + ScrapeOutput, ) pytestmark = pytest.mark.unit @@ -43,3 +44,15 @@ def test_rejects_more_sources_than_the_cap(): too_many = [f"tag{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] with pytest.raises(ValidationError): ScrapeInput(hashtags=too_many) + + +def test_error_items_are_not_billed(): + # Real videos count; ErrorItems (blocked/empty targets) are surfaced free. + out = ScrapeOutput( + items=[ + {"id": "1", "webVideoUrl": "https://tiktok.com/@a/video/1"}, + {"errorCode": "no_items", "input": "nasa", "error": "blocked"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py index 8520a1d85..777ed49de 100644 --- a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py @@ -95,3 +95,18 @@ async def test_listing_dedupes_then_caps_per_target(): TikTokScrapeInput(hashtags=["x"], resultsPerPage=2), fetch_listing=fake_listing ) assert [i["id"] for i in items] == ["1", "2"] + + +async def test_empty_listing_emits_error_item(): + # A trust-gated/empty feed (0 videos) must surface one honest ErrorItem, + # tagged with errorCode, rather than vanishing silently. + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [] + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=["nasa"], resultsPerPage=5), + fetch_listing=fake_listing, + ) + assert len(items) == 1 + assert items[0]["errorCode"] == "no_items" + assert items[0]["input"] == "nasa" From 6652efd0358e131bb8ca1c3696ef687718a8ff3e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 9 Jul 2026 17:48:19 +0200 Subject: [PATCH 32/56] feat(tiktok): return profile metadata even when video listing is gated A profile's account data (name, followers, bio, verification) lives in the page's rehydration blob and loads over plain HTTP without a signed request, so emit it first and always. The video listing needs a signed item_list XHR that TikTok withholds from anonymous sessions, so it stays best-effort and degrades to an ErrorItem. A blocked profile now yields its metadata instead of only an ErrorItem. --- .../platforms/tiktok/extraction/__init__.py | 3 +- .../platforms/tiktok/extraction/author.py | 7 ++ .../platforms/tiktok/flows/profile.py | 37 +++++++++ .../platforms/tiktok/orchestrator.py | 3 + .../platforms/tiktok/schemas/__init__.py | 2 + .../platforms/tiktok/schemas/items.py | 15 ++++ .../platforms/tiktok/test_orchestrator.py | 75 ++++++++++++++++++- 7 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/profile.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index d70a494a5..86aefc2d7 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -2,7 +2,7 @@ from __future__ import annotations -from .author import parse_author +from .author import parse_author, parse_profile from .hydration import extract_rehydration_data from .item_list import items_from_response from .scopes import user_info, video_item_struct @@ -12,6 +12,7 @@ __all__ = [ "extract_rehydration_data", "items_from_response", "parse_author", + "parse_profile", "parse_video", "user_info", "video_item_struct", diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py index 414de39b6..7dc1783e4 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py @@ -29,3 +29,10 @@ def build_author_meta(author: dict[str, Any], stats: dict[str, Any]) -> dict[str def parse_author(user_info: dict[str, Any]) -> dict[str, Any]: """Map a ``webapp.user-detail`` ``userInfo`` (``{user, stats}``) to authorMeta.""" return build_author_meta(user_info.get("user") or {}, user_info.get("stats") or {}) + + +def parse_profile(user_info: dict[str, Any]) -> dict[str, Any]: + """Map a ``userInfo`` to a standalone :class:`TikTokProfileItem` output dict.""" + from ..schemas.items import TikTokProfileItem + + return TikTokProfileItem(**parse_author(user_info)).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/profile.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/profile.py new file mode 100644 index 000000000..feebd3c0d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/profile.py @@ -0,0 +1,37 @@ +"""Profile flow: reliable blob metadata first, then the (gated) video listing. + +A profile's account data (name, followers, bio, verification) lives in the page's +rehydration blob and loads over plain HTTP without a signed request, so we emit it +first and always. The video listing needs a signed ``item_list`` XHR that TikTok +withholds from anonymous sessions, so it is best-effort: it streams videos when it +loads and degrades to an ErrorItem (via :func:`iter_listing`) when withheld. The +metadata item therefore survives even when the videos are blocked. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from ..extraction import extract_rehydration_data, parse_profile, user_info +from ..extraction.timestamps import now_iso +from ..targets.types import TikTokTarget +from . import FetchFn, FetchListingFn +from .listing import iter_listing + + +async def iter_profile( + target: TikTokTarget, + *, + cap: int, + fetch: FetchFn, + fetch_listing: FetchListingFn, +) -> AsyncIterator[dict[str, Any]]: + html = await fetch(target.url) + info = user_info(extract_rehydration_data(html) or {}) if html else None + if info: + item = parse_profile(info) + item["scrapedAt"] = now_iso() + yield item + async for out in iter_listing(target, cap=cap, fetch_listing=fetch_listing): + yield out diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index 95bc10fb9..3903a1088 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -14,6 +14,7 @@ from urllib.parse import quote from .flows import FetchFn, FetchListingFn from .flows.listing import iter_listing +from .flows.profile import iter_profile from .flows.video import iter_video from .schemas import TikTokScrapeInput from .session import fetch_html, fetch_item_list @@ -57,6 +58,8 @@ def _dispatch( ) -> AsyncIterator[dict[str, Any]]: if target.kind == "video": return iter_video(target, fetch=fetch) + if target.kind == "profile": + return iter_profile(target, cap=cap, fetch=fetch, fetch_listing=fetch_listing) return iter_listing(target, cap=cap, fetch_listing=fetch_listing) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py index 3593f0e28..a52400a58 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py @@ -8,6 +8,7 @@ from .items import ( CommentItem, ErrorItem, MusicMeta, + TikTokProfileItem, TikTokVideoItem, VideoMeta, ) @@ -18,6 +19,7 @@ __all__ = [ "ErrorItem", "MusicMeta", "StartUrl", + "TikTokProfileItem", "TikTokScrapeInput", "TikTokVideoItem", "VideoMeta", diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py index c4ff87618..50a5c4293 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py @@ -30,6 +30,21 @@ class AuthorMeta(BaseModel): video: int | None = None +class TikTokProfileItem(AuthorMeta): + """A profile's public metadata, read from the page's rehydration blob. + + Emitted even when the video listing is withheld from anonymous access, so a + blocked profile still yields its account data (name, followers, bio, + verification) instead of only an ErrorItem. Distinguishable from a video item + by the absence of ``webVideoUrl``/``text``. + """ + + scrapedAt: str | None = None + + def to_output(self) -> dict[str, Any]: + return self.model_dump(exclude_none=False) + + class MusicMeta(BaseModel): model_config = ConfigDict(extra="allow") diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py index 777ed49de..3a4e42776 100644 --- a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py @@ -10,6 +10,33 @@ import json from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok +async def _no_html(_url: str) -> str: + """Fetch stub that yields no rehydration blob (skips profile metadata).""" + return "" + + +def _profile_page(username: str, followers: int, videos: int) -> str: + blob = { + "__DEFAULT_SCOPE__": { + "webapp.user-detail": { + "userInfo": { + "user": { + "id": "u1", + "uniqueId": username, + "nickname": "Nick", + "verified": True, + }, + "stats": {"followerCount": followers, "videoCount": videos}, + } + } + } + } + return ( + '' + ) + + def _video_page(video_id: str, username: str) -> str: blob = { "__DEFAULT_SCOPE__": { @@ -81,12 +108,57 @@ async def test_scrape_profile_returns_listing_items(): ] items = await scrape_tiktok( - TikTokScrapeInput(profiles=["a"], resultsPerPage=5), fetch_listing=fake_listing + TikTokScrapeInput(profiles=["a"], resultsPerPage=5), + fetch=_no_html, + fetch_listing=fake_listing, ) assert [i["id"] for i in items] == ["1", "2"] assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@a/video/1" +async def test_profile_emits_metadata_then_videos(): + # The blob metadata item comes first and is billable; videos follow. + async def fake_fetch(_url: str) -> str: + return _profile_page("a", followers=100, videos=2) + + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [{"id": "1", "author": {"uniqueId": "a"}}] + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=["a"], resultsPerPage=5), + fetch=fake_fetch, + fetch_listing=fake_listing, + ) + assert len(items) == 2 + profile, video = items + assert "webVideoUrl" not in profile # metadata item, not a video + assert profile["name"] == "a" + assert profile["fans"] == 100 + assert profile["verified"] is True + assert profile["scrapedAt"] is not None + assert video["id"] == "1" + + +async def test_profile_metadata_survives_blocked_listing(): + # Videos withheld from anonymous access: we still return the profile metadata + # (not just an ErrorItem), so a blocked profile isn't a total loss. + async def fake_fetch(_url: str) -> str: + return _profile_page("a", followers=100, videos=9) + + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [] + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=["a"], resultsPerPage=5), + fetch=fake_fetch, + fetch_listing=fake_listing, + ) + assert len(items) == 2 + assert items[0]["name"] == "a" + assert items[0]["fans"] == 100 + assert items[1]["errorCode"] == "no_items" + + async def test_listing_dedupes_then_caps_per_target(): async def fake_listing(_url: str, _count: int) -> list[dict]: return [{"id": "1"}, {"id": "1"}, {"id": "2"}, {"id": "3"}] @@ -105,6 +177,7 @@ async def test_empty_listing_emits_error_item(): items = await scrape_tiktok( TikTokScrapeInput(profiles=["nasa"], resultsPerPage=5), + fetch=_no_html, fetch_listing=fake_listing, ) assert len(items) == 1 From 192b6dc31a39ead462f0bf4cd1ebac1da01f18e6 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 9 Jul 2026 18:00:40 +0200 Subject: [PATCH 33/56] feat(tiktok): add tiktok.user_search verb for account discovery Video/general search is login-walled for anonymous sessions, but the Users tab (/api/search/user) returns public account records without a redirect, so this exposes the one reliably-unblocked search path. A keyword yields TikTokProfileItems (name, followers, bio, verification), deduped per query, capped, and degraded to an ErrorItem when a query is empty/withheld. Reuses the browser capture (generalized over XHR markers + extractor) and the shared profile item shape. Billed per account on a new TIKTOK_USER meter (TIKTOK_MICROS_PER_USER), surfaced on the chat subagent alongside tiktok.scrape. --- docker/.env.example | 2 + surfsense_backend/.env.example | 2 + .../subagents/builtins/tiktok/tools/index.py | 5 +- .../app/capabilities/core/billing.py | 2 + .../app/capabilities/core/types.py | 1 + .../app/capabilities/tiktok/__init__.py | 1 + .../tiktok/user_search/__init__.py | 3 + .../tiktok/user_search/definition.py | 26 +++++++ .../tiktok/user_search/executor.py | 39 +++++++++++ .../tiktok/user_search/schemas.py | 56 +++++++++++++++ surfsense_backend/app/config/__init__.py | 3 + .../proprietary/platforms/tiktok/__init__.py | 6 +- .../platforms/tiktok/extraction/__init__.py | 3 + .../tiktok/extraction/user_search.py | 56 +++++++++++++++ .../platforms/tiktok/flows/__init__.py | 3 + .../platforms/tiktok/flows/user_search.py | 54 ++++++++++++++ .../platforms/tiktok/orchestrator.py | 27 ++++++- .../platforms/tiktok/session/__init__.py | 3 +- .../platforms/tiktok/session/listing.py | 47 ++++++++++--- .../unit/capabilities/tiktok/test_registry.py | 13 ++++ .../tiktok/user_search/__init__.py | 0 .../tiktok/user_search/test_executor.py | 49 +++++++++++++ .../tiktok/user_search/test_schemas.py | 49 +++++++++++++ .../unit/platforms/tiktok/test_user_search.py | 70 +++++++++++++++++++ 24 files changed, 502 insertions(+), 18 deletions(-) create mode 100644 surfsense_backend/app/capabilities/tiktok/user_search/__init__.py create mode 100644 surfsense_backend/app/capabilities/tiktok/user_search/definition.py create mode 100644 surfsense_backend/app/capabilities/tiktok/user_search/executor.py create mode 100644 surfsense_backend/app/capabilities/tiktok/user_search/schemas.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/user_search.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/user_search.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/user_search/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py diff --git a/docker/.env.example b/docker/.env.example index 5daa0f3e8..86536e94c 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -449,6 +449,8 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # GOOGLE_MAPS_MICROS_PER_REVIEW=1500 # YOUTUBE_MICROS_PER_VIDEO=2500 # YOUTUBE_MICROS_PER_COMMENT=1500 +# TIKTOK_MICROS_PER_VIDEO=3500 +# TIKTOK_MICROS_PER_USER=2500 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 7e87c186d..eeb346a1b 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -286,6 +286,8 @@ MICROS_PER_PAGE=1000 # GOOGLE_MAPS_MICROS_PER_REVIEW=1500 # YOUTUBE_MICROS_PER_VIDEO=2500 # YOUTUBE_MICROS_PER_COMMENT=1500 +# TIKTOK_MICROS_PER_VIDEO=3500 +# TIKTOK_MICROS_PER_USER=2500 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. CREDIT_LOW_BALANCE_WARNING_MICROS=500000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py index e790d44f0..d826a8a85 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py @@ -1,4 +1,4 @@ -"""``tiktok`` sub-agent tools: the TikTok scrape capability verb.""" +"""``tiktok`` sub-agent tools: the TikTok scrape and user-search capability verbs.""" from __future__ import annotations @@ -9,12 +9,13 @@ from langchain_core.tools import BaseTool from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from app.capabilities.core.access.agent import build_capability_tools from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE +from app.capabilities.tiktok.user_search.definition import TIKTOK_USER_SEARCH NAME = "tiktok" RULESET = Ruleset(origin=NAME, rules=[]) -_CI_VERBS = [TIKTOK_SCRAPE] +_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_USER_SEARCH] def load_tools( diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 048797473..8bc9fbd14 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -36,6 +36,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO", BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT", BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO", + BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER", } @@ -53,6 +54,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.YOUTUBE_VIDEO: "video", BillingUnit.YOUTUBE_COMMENT: "comment", BillingUnit.TIKTOK_VIDEO: "video", + BillingUnit.TIKTOK_USER: "profile", } diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index b45641a2d..cbb01c2a6 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -26,6 +26,7 @@ class BillingUnit(StrEnum): YOUTUBE_VIDEO = "youtube_video" YOUTUBE_COMMENT = "youtube_comment" TIKTOK_VIDEO = "tiktok_video" + TIKTOK_USER = "tiktok_user" class BillableInput(Protocol): diff --git a/surfsense_backend/app/capabilities/tiktok/__init__.py b/surfsense_backend/app/capabilities/tiktok/__init__.py index ed3f32682..067c91c0a 100644 --- a/surfsense_backend/app/capabilities/tiktok/__init__.py +++ b/surfsense_backend/app/capabilities/tiktok/__init__.py @@ -3,3 +3,4 @@ from __future__ import annotations from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401 +from app.capabilities.tiktok.user_search import definition as _user_search # noqa: F401 diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/__init__.py b/surfsense_backend/app/capabilities/tiktok/user_search/__init__.py new file mode 100644 index 000000000..d4344968c --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/__init__.py @@ -0,0 +1,3 @@ +"""``tiktok.user_search``: find public TikTok accounts by keyword.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/definition.py b/surfsense_backend/app/capabilities/tiktok/user_search/definition.py new file mode 100644 index 000000000..6a37f879f --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/definition.py @@ -0,0 +1,26 @@ +"""``tiktok.user_search`` capability registration (billed per account; see config +``TIKTOK_MICROS_PER_USER``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.user_search.executor import build_user_search_executor +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) + +TIKTOK_USER_SEARCH = Capability( + name="tiktok.user_search", + description=( + "Find public TikTok accounts by keyword. Returns profile metadata " + "(name, followers, bio, verification) per matching account." + ), + input_schema=UserSearchInput, + output_schema=UserSearchOutput, + executor=build_user_search_executor(), + billing_unit=BillingUnit.TIKTOK_USER, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_USER_SEARCH) diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/executor.py b/surfsense_backend/app/capabilities/tiktok/user_search/executor.py new file mode 100644 index 000000000..e83f396bd --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/executor.py @@ -0,0 +1,39 @@ +"""``tiktok.user_search`` executor: queries -> scraper -> TikTok profile items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) +from app.proprietary.platforms.tiktok import search_tiktok_users + +SearchFn = Callable[..., Awaitable[list[dict]]] + + +def build_user_search_executor(search_fn: SearchFn | None = None) -> Executor: + """Bind the executor to a search fn (defaults to the proprietary actor).""" + search_fn = search_fn or search_tiktok_users + + async def execute(payload: UserSearchInput) -> UserSearchOutput: + emit_progress( + "starting", + "Searching TikTok accounts", + total=payload.max_items, + unit="item", + ) + items = await search_fn( + payload.queries, + per_query=payload.results_per_query, + limit=payload.max_items, + ) + emit_progress( + "done", f"Found {len(items)} account(s)", current=len(items), unit="item" + ) + return UserSearchOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/schemas.py b/surfsense_backend/app/capabilities/tiktok/user_search/schemas.py new file mode 100644 index 000000000..7cb2aecbb --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/schemas.py @@ -0,0 +1,56 @@ +"""``tiktok.user_search`` I/O contracts. + +Account discovery over ``TikTok``'s Users tab. Where video/general search is +login-walled for anonymous sessions, ``/api/search/user`` returns public account +records, so this verb exposes the one reliably-unblocked search path. Each result +is a :class:`TikTokProfileItem` (the same shape the profile verb emits). +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.capabilities.tiktok.scrape.schemas import ( + MAX_TIKTOK_ITEMS, + MAX_TIKTOK_SOURCES, +) +from app.proprietary.platforms.tiktok import TikTokProfileItem + + +class UserSearchInput(BaseModel): + queries: list[str] = Field( + min_length=1, + max_length=MAX_TIKTOK_SOURCES, + description="Keywords to search for TikTok accounts (e.g. names, brands).", + ) + results_per_query: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max accounts to return per query.", + ) + max_items: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max total accounts to return across all queries.", + ) + + @property + def estimated_units(self) -> int: + """Worst-case billable accounts for the pre-flight gate: ``max_items`` is a + hard cross-query ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class UserSearchOutput(BaseModel): + items: list[TikTokProfileItem] = Field( + default_factory=list, + description="One item per account found, in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned account = one billable unit; ErrorItems (``errorCode`` set, + for empty/withheld queries) are surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index c03ac3598..97abb5eda 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -717,6 +717,9 @@ class Config: # Browser-driven listings make TikTok heavier per item than the API-backed # video meter, so it sits a touch above YouTube's video rate. TIKTOK_MICROS_PER_VIDEO = int(os.getenv("TIKTOK_MICROS_PER_VIDEO", "3500")) + # User search returns lighter account records (name/followers/bio), priced + # below the video meter to mirror the cheaper account-discovery market. + TIKTOK_MICROS_PER_USER = int(os.getenv("TIKTOK_MICROS_PER_USER", "2500")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py index 27867df73..2947498ec 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py @@ -6,14 +6,16 @@ schema, the collector/generator, the video item shape, and the hard-block error. from __future__ import annotations -from .orchestrator import iter_tiktok, scrape_tiktok -from .schemas import TikTokScrapeInput, TikTokVideoItem +from .orchestrator import iter_tiktok, scrape_tiktok, search_tiktok_users +from .schemas import TikTokProfileItem, TikTokScrapeInput, TikTokVideoItem from .session import TikTokAccessBlockedError __all__ = [ "TikTokAccessBlockedError", + "TikTokProfileItem", "TikTokScrapeInput", "TikTokVideoItem", "iter_tiktok", "scrape_tiktok", + "search_tiktok_users", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index 86aefc2d7..f4712e724 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -6,6 +6,7 @@ from .author import parse_author, parse_profile from .hydration import extract_rehydration_data from .item_list import items_from_response from .scopes import user_info, video_item_struct +from .user_search import parse_search_user, users_from_response from .video import parse_video __all__ = [ @@ -13,7 +14,9 @@ __all__ = [ "items_from_response", "parse_author", "parse_profile", + "parse_search_user", "parse_video", "user_info", + "users_from_response", "video_item_struct", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/user_search.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/user_search.py new file mode 100644 index 000000000..5ffc2cd4d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/user_search.py @@ -0,0 +1,56 @@ +"""Parse the ``/api/search/user`` response into profile items. + +User search returns ``{"user_list": [{"user_info": {...}}, ...]}`` where each +``user_info`` uses the mobile-API snake_case shape (``uid``, ``unique_id``, +``follower_count``, ``total_favorited``, ``avatar_thumb.url_list``) — distinct +from the camelCase ``webapp.user-detail`` blob the profile flow reads, so it gets +its own mapping into the shared :class:`TikTokProfileItem` output contract. +""" + +from __future__ import annotations + +from typing import Any + +_PROFILE_URL = "https://www.tiktok.com/@{username}" + + +def users_from_response(body: Any) -> list[dict[str, Any]]: + """Return the ``user_info`` objects carried by one search response, or ``[]``.""" + if not isinstance(body, dict): + return [] + user_list = body.get("user_list") + if not isinstance(user_list, list): + return [] + return [ + entry["user_info"] + for entry in user_list + if isinstance(entry, dict) and isinstance(entry.get("user_info"), dict) + ] + + +def _avatar(user_info: dict[str, Any]) -> str | None: + thumb = user_info.get("avatar_thumb") + if isinstance(thumb, dict): + urls = thumb.get("url_list") + if isinstance(urls, list) and urls: + return urls[0] + return None + + +def parse_search_user(user_info: dict[str, Any]) -> dict[str, Any]: + """Map a search ``user_info`` to a :class:`TikTokProfileItem` output dict.""" + from ..schemas.items import TikTokProfileItem + + username = user_info.get("unique_id") + return TikTokProfileItem( + id=user_info.get("uid"), + name=username, + nickName=user_info.get("nickname"), + profileUrl=_PROFILE_URL.format(username=username) if username else None, + verified=bool(user_info.get("enterprise_verify_reason")), + signature=user_info.get("signature"), + avatar=_avatar(user_info), + fans=user_info.get("follower_count"), + heart=user_info.get("total_favorited"), + secUid=user_info.get("sec_uid"), + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py index a7809567d..1a8aeaacc 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py @@ -10,4 +10,7 @@ FetchFn = Callable[[str], Awaitable[str | None]] FetchListingFn = Callable[[str, int], Awaitable[list[dict]]] """Load a listing page and return up to ``count`` captured itemStructs.""" +FetchUsersFn = Callable[[str, int], Awaitable[list[dict]]] +"""Load a user-search page and return up to ``count`` captured ``user_info`` records.""" + FlowResult = AsyncIterator[dict] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/user_search.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/user_search.py new file mode 100644 index 000000000..872212e80 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/user_search.py @@ -0,0 +1,54 @@ +"""User-search flow: keyword -> public account records. + +Unlike video/general search (login-walled for anonymous sessions), the Users tab +hits ``/api/search/user`` and returns account records without a redirect. Each +query's results are deduped by uid, capped, and — when a query returns nothing — +degraded to one ErrorItem, mirroring the listing flow. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import quote + +from ..extraction import parse_search_user +from ..extraction.timestamps import now_iso +from ..schemas import ErrorItem +from . import FetchUsersFn + +_USER_SEARCH_URL = "https://www.tiktok.com/search/user?q={query}" +_EMPTY_MESSAGE = ( + "No accounts returned for this query. It may have no matches, or TikTok " + "withheld the results from anonymous access." +) + + +async def iter_user_search( + query: str, *, cap: int, fetch_users: FetchUsersFn +) -> AsyncIterator[dict[str, Any]]: + if cap <= 0: + return + url = _USER_SEARCH_URL.format(query=quote(query)) + seen: set[str] = set() + emitted = 0 + for user_info in await fetch_users(url, cap): + out = parse_search_user(user_info) + uid = out.get("id") + if uid is not None: + if uid in seen: + continue + seen.add(uid) + out["scrapedAt"] = now_iso() + yield out + emitted += 1 + if emitted >= cap: + return + if emitted == 0: + yield ErrorItem( + url=url, + input=query, + error=_EMPTY_MESSAGE, + errorCode="no_users", + scrapedAt=now_iso(), + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index 3903a1088..485a8dfff 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -12,12 +12,13 @@ from collections.abc import AsyncIterator from typing import Any from urllib.parse import quote -from .flows import FetchFn, FetchListingFn +from .flows import FetchFn, FetchListingFn, FetchUsersFn from .flows.listing import iter_listing from .flows.profile import iter_profile +from .flows.user_search import iter_user_search from .flows.video import iter_video from .schemas import TikTokScrapeInput -from .session import fetch_html, fetch_item_list +from .session import fetch_html, fetch_item_list, fetch_user_search from .targets import resolve_target from .targets.types import TikTokTarget @@ -100,3 +101,25 @@ async def scrape_tiktok( if limit is not None and len(results) >= limit: break return results + + +async def search_tiktok_users( + queries: list[str], + *, + per_query: int, + limit: int | None = None, + fetch_users: FetchUsersFn = fetch_user_search, +) -> list[dict[str, Any]]: + """Collect user-search account records across queries, honoring ``limit``.""" + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + for query in queries: + async for item in iter_user_search( + query, cap=per_query, fetch_users=fetch_users + ): + results.append(item) + emit_progress("searching", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + return results + return results diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py index 409aa5987..5e29403a0 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py @@ -4,7 +4,7 @@ from __future__ import annotations from .client import fetch_html from .errors import TikTokAccessBlockedError -from .listing import fetch_item_list +from .listing import fetch_item_list, fetch_user_search from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session __all__ = [ @@ -12,6 +12,7 @@ __all__ = [ "bind_proxy_holder", "fetch_html", "fetch_item_list", + "fetch_user_search", "open_proxy_holder", "proxy_session", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index cc84a48b5..e0bdd9255 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -19,6 +19,7 @@ from __future__ import annotations import asyncio import logging +from collections.abc import Callable from typing import Any from scrapling.fetchers import StealthyFetcher @@ -29,16 +30,20 @@ from app.proprietary.web_crawler.stealth import ( ) from app.utils.proxy import get_proxy_url -from ..extraction import items_from_response +from ..extraction import items_from_response, users_from_response logger = logging.getLogger(__name__) +ExtractFn = Callable[[Any], list[dict[str, Any]]] + # XHR paths that carry itemStructs for the three listing kinds. _ITEM_LIST_MARKERS = ( "/api/post/item_list", "/api/challenge/item_list", "/api/search/", ) +# The user-search XHR carries account records (user_list), not itemStructs. +_USER_SEARCH_MARKERS = ("/api/search/user",) _HOME_URL = "https://www.tiktok.com/" _MSTOKEN_COOKIE = "msToken" # Bounded scroll: a dead page can't loop forever, and a live one stops early @@ -57,24 +62,31 @@ def _has_mstoken(page: Any) -> bool: return False -def _build_page_action(collected: list[dict[str, Any]], url: str, target_count: int): - """A sync ``page_action`` that warms the session then captures item_list XHRs. +def _build_page_action( + collected: list[dict[str, Any]], + url: str, + target_count: int, + markers: tuple[str, ...], + extract: ExtractFn, +): + """A sync ``page_action`` that warms the session then captures matching XHRs. - A cold context returns an empty ``item_list`` body, so we first mint the - anonymous ``msToken`` (homepage hit), then navigate to the target with the - listener already attached so page-one fires into it; scrolling pages the rest. + A cold context returns an empty body, so we first mint the anonymous + ``msToken`` (homepage hit), then navigate to the target with the listener + already attached so page-one fires into it; scrolling pages the rest. + ``markers``/``extract`` select which XHRs to keep and how to unwrap them. """ def _on_response(response: Any) -> None: response_url = getattr(response, "url", "") - if not any(marker in response_url for marker in _ITEM_LIST_MARKERS): + if not any(marker in response_url for marker in markers): return try: body = response.json() except Exception: # An empty 200 (TikTok soft-block) or a body evicted before read. return - collected.extend(items_from_response(body)) + collected.extend(extract(body)) def _warm(page: Any) -> None: if _has_mstoken(page): @@ -110,7 +122,9 @@ def _build_page_action(collected: list[dict[str, Any]], url: str, target_count: return page_action -def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]: +def _fetch_sync( + url: str, target_count: int, markers: tuple[str, ...], extract: ExtractFn +) -> list[dict[str, Any]]: collected: list[dict[str, Any]] = [] kwargs = build_stealthy_kwargs(get_stealth_config()) StealthyFetcher.fetch( @@ -118,7 +132,9 @@ def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]: headless=True, network_idle=False, proxy=get_proxy_url(), - page_action=_build_page_action(collected, url, target_count), + page_action=_build_page_action( + collected, url, target_count, markers, extract + ), **kwargs, ) return collected[:target_count] @@ -126,4 +142,13 @@ def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]: async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` itemStructs from a listing page's XHRs.""" - return await asyncio.to_thread(_fetch_sync, page_url, target_count) + return await asyncio.to_thread( + _fetch_sync, page_url, target_count, _ITEM_LIST_MARKERS, items_from_response + ) + + +async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` ``user_info`` records from a user-search page.""" + return await asyncio.to_thread( + _fetch_sync, page_url, target_count, _USER_SEARCH_MARKERS, users_from_response + ) diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py index 3a85fe87f..893f3c2ae 100644 --- a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py @@ -10,6 +10,10 @@ from app.capabilities import ( from app.capabilities.core import BillingUnit from app.capabilities.core.store import get_capability from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) pytestmark = pytest.mark.unit @@ -21,3 +25,12 @@ def test_tiktok_scrape_is_registered_and_billed_per_video(): assert cap.input_schema is ScrapeInput assert cap.output_schema is ScrapeOutput assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO + + +def test_tiktok_user_search_is_registered_and_billed_per_profile(): + cap = get_capability("tiktok.user_search") + + assert cap.name == "tiktok.user_search" + assert cap.input_schema is UserSearchInput + assert cap.output_schema is UserSearchOutput + assert cap.billing_unit is BillingUnit.TIKTOK_USER diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/user_search/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py new file mode 100644 index 000000000..866968808 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py @@ -0,0 +1,49 @@ +"""``tiktok.user_search`` executor: verb input → search args → typed profile items. + +Boundary mocked: the proprietary search actor (injected fake). NOT mocked: the +verb's own payload→args forwarding and the dict→TikTokProfileItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.user_search.executor import build_user_search_executor +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) + +pytestmark = pytest.mark.unit + + +class _FakeSearch: + """Records the queries + kwargs it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[tuple[list[str], int, int | None]] = [] + + async def __call__( + self, queries: list[str], *, per_query: int, limit: int | None = None + ) -> list[dict]: + self.calls.append((queries, per_query, limit)) + return self._items + + +async def test_forwards_queries_and_limits_and_wraps_items(): + search = _FakeSearch([{"id": "1", "name": "nasa"}]) + execute = build_user_search_executor(search_fn=search) + + out = await execute( + UserSearchInput(queries=["nasa"], results_per_query=7, max_items=25) + ) + + assert isinstance(out, UserSearchOutput) + assert len(out.items) == 1 + assert out.items[0].name == "nasa" + + (queries, per_query, limit) = search.calls[0] + assert queries == ["nasa"] + assert per_query == 7 + assert limit == 25 diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py new file mode 100644 index 000000000..575db3fb5 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py @@ -0,0 +1,49 @@ +"""``tiktok.user_search`` input guards and billing: a query is required, bounded, +and ErrorItems are surfaced free.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) + +pytestmark = pytest.mark.unit + + +def test_rejects_input_with_no_query(): + with pytest.raises(ValidationError): + UserSearchInput(queries=[]) + + +def test_defaults_and_bounds(): + payload = UserSearchInput(queries=["nasa"]) + assert payload.max_items == 10 + assert payload.results_per_query == 10 + assert payload.estimated_units == 10 + with pytest.raises(ValidationError): + UserSearchInput(queries=["nasa"], max_items=0) + with pytest.raises(ValidationError): + UserSearchInput(queries=["nasa"], max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_rejects_more_queries_than_the_cap(): + too_many = [f"q{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] + with pytest.raises(ValidationError): + UserSearchInput(queries=too_many) + + +def test_error_items_are_not_billed(): + # Real accounts count; ErrorItems (empty/withheld queries) are surfaced free. + out = UserSearchOutput( + items=[ + {"id": "1", "name": "nasa"}, + {"errorCode": "no_users", "input": "ghost", "error": "empty"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py b/surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py new file mode 100644 index 000000000..845b5e22d --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py @@ -0,0 +1,70 @@ +"""User-search orchestration over a fake fetch (no network). + +Drives ``search_tiktok_users``: queries -> captured ``user_info`` -> profile items. +""" + +from __future__ import annotations + +from typing import Any + +from app.proprietary.platforms.tiktok import search_tiktok_users + + +def _user(uid: str, unique_id: str, followers: int = 10) -> dict[str, Any]: + return { + "uid": uid, + "unique_id": unique_id, + "nickname": unique_id.upper(), + "signature": "bio", + "follower_count": followers, + "total_favorited": 999, + "sec_uid": f"sec-{uid}", + "enterprise_verify_reason": "official" if uid == "1" else "", + "avatar_thumb": {"url_list": [f"https://cdn/{uid}.webp"]}, + } + + +async def test_user_search_parses_dedupes_and_caps(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_user("1", "nasa"), _user("1", "nasa"), _user("2", "nasa2")] + + items = await search_tiktok_users( + ["nasa"], per_query=2, fetch_users=fake_fetch + ) + + assert [i["id"] for i in items] == ["1", "2"] + first = items[0] + assert first["name"] == "nasa" + assert first["nickName"] == "NASA" + assert first["profileUrl"] == "https://www.tiktok.com/@nasa" + assert first["verified"] is True + assert first["fans"] == 10 + assert first["avatar"] == "https://cdn/1.webp" + assert first["secUid"] == "sec-1" + assert first["scrapedAt"] is not None + assert items[1]["verified"] is False + + +async def test_user_search_empty_query_emits_error_item(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [] + + items = await search_tiktok_users( + ["ghost"], per_query=5, fetch_users=fake_fetch + ) + + assert len(items) == 1 + assert items[0]["errorCode"] == "no_users" + assert items[0]["input"] == "ghost" + + +async def test_user_search_honors_limit_across_queries(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_user("1", "a"), _user("2", "b")] + + items = await search_tiktok_users( + ["q1", "q2"], per_query=5, limit=3, fetch_users=fake_fetch + ) + + # 2 from q1 + 1 from q2, then the cross-query limit stops it. + assert len(items) == 3 From 7723b5b8b6c0a4fc516735b6f4a59ecf67faf9fd Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 9 Jul 2026 18:35:26 +0200 Subject: [PATCH 34/56] feat(tiktok): add tiktok.comments verb for public comment scraping Comments load over a signed /api/comment/list XHR that TikTok serves to anonymous sessions once the comments panel is opened (unlike profile-video and general-search feeds), so this is a reliable verb. Given video URLs it returns CommentItems (text, author, likes, reply counts; replies carry repliesToId), deduped per video, capped, and degraded to an ErrorItem for empty/withheld videos or a bad_url ErrorItem for non-video inputs. Generalizes the browser capture over a pluggable interaction step so the comments flow (open panel, scroll the panel to paginate) reuses the same warm+capture scaffolding as listing/user-search. Billed per comment on a new TIKTOK_COMMENT meter (TIKTOK_MICROS_PER_COMMENT, matching the per-comment market), surfaced on the chat subagent alongside tiktok.scrape/user_search. --- docker/.env.example | 1 + surfsense_backend/.env.example | 1 + .../subagents/builtins/tiktok/tools/index.py | 5 +- .../app/capabilities/core/billing.py | 2 + .../app/capabilities/core/types.py | 1 + .../app/capabilities/tiktok/__init__.py | 1 + .../capabilities/tiktok/comments/__init__.py | 3 + .../tiktok/comments/definition.py | 23 +++ .../capabilities/tiktok/comments/executor.py | 36 +++++ .../capabilities/tiktok/comments/schemas.py | 56 +++++++ surfsense_backend/app/config/__init__.py | 3 + .../proprietary/platforms/tiktok/__init__.py | 16 +- .../platforms/tiktok/extraction/__init__.py | 3 + .../platforms/tiktok/extraction/comments.py | 55 +++++++ .../platforms/tiktok/flows/__init__.py | 3 + .../platforms/tiktok/flows/comments.py | 52 +++++++ .../platforms/tiktok/orchestrator.py | 54 ++++++- .../platforms/tiktok/session/__init__.py | 3 +- .../platforms/tiktok/session/listing.py | 139 +++++++++++++++--- .../capabilities/tiktok/comments/__init__.py | 0 .../tiktok/comments/test_executor.py | 48 ++++++ .../tiktok/comments/test_schemas.py | 48 ++++++ .../unit/capabilities/tiktok/test_registry.py | 10 ++ .../unit/platforms/tiktok/test_comments.py | 89 +++++++++++ 24 files changed, 626 insertions(+), 26 deletions(-) create mode 100644 surfsense_backend/app/capabilities/tiktok/comments/__init__.py create mode 100644 surfsense_backend/app/capabilities/tiktok/comments/definition.py create mode 100644 surfsense_backend/app/capabilities/tiktok/comments/executor.py create mode 100644 surfsense_backend/app/capabilities/tiktok/comments/schemas.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/comments.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/comments.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/comments/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_comments.py diff --git a/docker/.env.example b/docker/.env.example index 86536e94c..19e1a47fd 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -451,6 +451,7 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # YOUTUBE_MICROS_PER_COMMENT=1500 # TIKTOK_MICROS_PER_VIDEO=3500 # TIKTOK_MICROS_PER_USER=2500 +# TIKTOK_MICROS_PER_COMMENT=1500 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index eeb346a1b..20d811839 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -288,6 +288,7 @@ MICROS_PER_PAGE=1000 # YOUTUBE_MICROS_PER_COMMENT=1500 # TIKTOK_MICROS_PER_VIDEO=3500 # TIKTOK_MICROS_PER_USER=2500 +# TIKTOK_MICROS_PER_COMMENT=1500 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. CREDIT_LOW_BALANCE_WARNING_MICROS=500000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py index d826a8a85..44ebc863d 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py @@ -1,4 +1,4 @@ -"""``tiktok`` sub-agent tools: the TikTok scrape and user-search capability verbs.""" +"""``tiktok`` sub-agent tools: the TikTok scrape, comments, and user-search verbs.""" from __future__ import annotations @@ -8,6 +8,7 @@ from langchain_core.tools import BaseTool from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from app.capabilities.core.access.agent import build_capability_tools +from app.capabilities.tiktok.comments.definition import TIKTOK_COMMENTS from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE from app.capabilities.tiktok.user_search.definition import TIKTOK_USER_SEARCH @@ -15,7 +16,7 @@ NAME = "tiktok" RULESET = Ruleset(origin=NAME, rules=[]) -_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_USER_SEARCH] +_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_COMMENTS, TIKTOK_USER_SEARCH] def load_tools( diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 8bc9fbd14..de2e48af3 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -37,6 +37,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT", BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO", BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER", + BillingUnit.TIKTOK_COMMENT: "TIKTOK_MICROS_PER_COMMENT", } @@ -55,6 +56,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.YOUTUBE_COMMENT: "comment", BillingUnit.TIKTOK_VIDEO: "video", BillingUnit.TIKTOK_USER: "profile", + BillingUnit.TIKTOK_COMMENT: "comment", } diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index cbb01c2a6..d41c11ffb 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -27,6 +27,7 @@ class BillingUnit(StrEnum): YOUTUBE_COMMENT = "youtube_comment" TIKTOK_VIDEO = "tiktok_video" TIKTOK_USER = "tiktok_user" + TIKTOK_COMMENT = "tiktok_comment" class BillableInput(Protocol): diff --git a/surfsense_backend/app/capabilities/tiktok/__init__.py b/surfsense_backend/app/capabilities/tiktok/__init__.py index 067c91c0a..1ff1738fa 100644 --- a/surfsense_backend/app/capabilities/tiktok/__init__.py +++ b/surfsense_backend/app/capabilities/tiktok/__init__.py @@ -2,5 +2,6 @@ from __future__ import annotations +from app.capabilities.tiktok.comments import definition as _comments # noqa: F401 from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401 from app.capabilities.tiktok.user_search import definition as _user_search # noqa: F401 diff --git a/surfsense_backend/app/capabilities/tiktok/comments/__init__.py b/surfsense_backend/app/capabilities/tiktok/comments/__init__.py new file mode 100644 index 000000000..f3bd1db88 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/__init__.py @@ -0,0 +1,3 @@ +"""``tiktok.comments``: scrape the public comment thread of TikTok videos.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/tiktok/comments/definition.py b/surfsense_backend/app/capabilities/tiktok/comments/definition.py new file mode 100644 index 000000000..1e5917fa2 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/definition.py @@ -0,0 +1,23 @@ +"""``tiktok.comments`` capability registration (billed per comment; see config +``TIKTOK_MICROS_PER_COMMENT``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.comments.executor import build_comments_executor +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput + +TIKTOK_COMMENTS = Capability( + name="tiktok.comments", + description=( + "Scrape the public comments of TikTok videos. Provide video URLs; " + "returns comment text, author, likes, and reply counts." + ), + input_schema=CommentsInput, + output_schema=CommentsOutput, + executor=build_comments_executor(), + billing_unit=BillingUnit.TIKTOK_COMMENT, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_COMMENTS) diff --git a/surfsense_backend/app/capabilities/tiktok/comments/executor.py b/surfsense_backend/app/capabilities/tiktok/comments/executor.py new file mode 100644 index 000000000..c8bbc776f --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/executor.py @@ -0,0 +1,36 @@ +"""``tiktok.comments`` executor: video URLs -> scraper -> TikTok comment items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput +from app.proprietary.platforms.tiktok import scrape_tiktok_comments + +CommentsFn = Callable[..., Awaitable[list[dict]]] + + +def build_comments_executor(comments_fn: CommentsFn | None = None) -> Executor: + """Bind the executor to a comments fn (defaults to the proprietary actor).""" + comments_fn = comments_fn or scrape_tiktok_comments + + async def execute(payload: CommentsInput) -> CommentsOutput: + emit_progress( + "starting", + "Scraping TikTok comments", + total=payload.max_items, + unit="item", + ) + items = await comments_fn( + payload.video_urls, + per_video=payload.comments_per_video, + limit=payload.max_items, + ) + emit_progress( + "done", f"Scraped {len(items)} comment(s)", current=len(items), unit="item" + ) + return CommentsOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/comments/schemas.py b/surfsense_backend/app/capabilities/tiktok/comments/schemas.py new file mode 100644 index 000000000..f3c68d445 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/schemas.py @@ -0,0 +1,56 @@ +"""``tiktok.comments`` I/O contracts. + +URL-only: given TikTok video URLs, return each video's public comment thread. +Unlike profile-video/general-search feeds, ``/api/comment/list`` is served to +anonymous sessions once the comments panel opens, so this verb is reliable. Each +result is a :class:`CommentItem` (top-level comments; replies carry ``repliesToId``). +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.capabilities.tiktok.scrape.schemas import ( + MAX_TIKTOK_ITEMS, + MAX_TIKTOK_SOURCES, +) +from app.proprietary.platforms.tiktok import CommentItem + + +class CommentsInput(BaseModel): + video_urls: list[str] = Field( + min_length=1, + max_length=MAX_TIKTOK_SOURCES, + description="TikTok video URLs (/@/video/) to pull comments from.", + ) + comments_per_video: int = Field( + default=20, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max comments to return per video.", + ) + max_items: int = Field( + default=20, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max total comments to return across all videos.", + ) + + @property + def estimated_units(self) -> int: + """Worst-case billable comments for the pre-flight gate: ``max_items`` is a + hard cross-video ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class CommentsOutput(BaseModel): + items: list[CommentItem] = Field( + default_factory=list, + description="One item per comment returned, in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned comment = one billable unit; ErrorItems (``errorCode`` set, + for bad URLs or empty/withheld videos) are surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 97abb5eda..21ba1d1c6 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -720,6 +720,9 @@ class Config: # User search returns lighter account records (name/followers/bio), priced # below the video meter to mirror the cheaper account-discovery market. TIKTOK_MICROS_PER_USER = int(os.getenv("TIKTOK_MICROS_PER_USER", "2500")) + # Comments are the cheapest per-item TikTok data, matching the per-comment + # market (and YouTube's comment meter). + TIKTOK_MICROS_PER_COMMENT = int(os.getenv("TIKTOK_MICROS_PER_COMMENT", "1500")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py index 2947498ec..00d345937 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py @@ -6,16 +6,28 @@ schema, the collector/generator, the video item shape, and the hard-block error. from __future__ import annotations -from .orchestrator import iter_tiktok, scrape_tiktok, search_tiktok_users -from .schemas import TikTokProfileItem, TikTokScrapeInput, TikTokVideoItem +from .orchestrator import ( + iter_tiktok, + scrape_tiktok, + scrape_tiktok_comments, + search_tiktok_users, +) +from .schemas import ( + CommentItem, + TikTokProfileItem, + TikTokScrapeInput, + TikTokVideoItem, +) from .session import TikTokAccessBlockedError __all__ = [ + "CommentItem", "TikTokAccessBlockedError", "TikTokProfileItem", "TikTokScrapeInput", "TikTokVideoItem", "iter_tiktok", "scrape_tiktok", + "scrape_tiktok_comments", "search_tiktok_users", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index f4712e724..3afd536f2 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -3,6 +3,7 @@ from __future__ import annotations from .author import parse_author, parse_profile +from .comments import comments_from_response, parse_comment from .hydration import extract_rehydration_data from .item_list import items_from_response from .scopes import user_info, video_item_struct @@ -10,9 +11,11 @@ from .user_search import parse_search_user, users_from_response from .video import parse_video __all__ = [ + "comments_from_response", "extract_rehydration_data", "items_from_response", "parse_author", + "parse_comment", "parse_profile", "parse_search_user", "parse_video", diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/comments.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/comments.py new file mode 100644 index 000000000..8ca96cad3 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/comments.py @@ -0,0 +1,55 @@ +"""Parse a captured ``/api/comment/list`` response into comment items. + +The comment API returns ``{"comments": [...]}`` where each entry uses the +mobile-API snake_case shape (``cid``, ``digg_count``, ``reply_comment_total``, +``create_time``, and a nested ``user`` with ``uid``/``unique_id``/``nickname``/ +``avatar_thumb``). ``reply_id != "0"`` marks a reply to a parent comment. +""" + +from __future__ import annotations + +from typing import Any + +from .timestamps import epoch_to_iso + + +def comments_from_response(body: Any) -> list[dict[str, Any]]: + """Return the raw comment records carried by one API response, or ``[]``.""" + if not isinstance(body, dict): + return [] + comments = body.get("comments") + if not isinstance(comments, list): + return [] + return [c for c in comments if isinstance(c, dict)] + + +def _avatar(user: dict[str, Any]) -> str | None: + thumb = user.get("avatar_thumb") + if isinstance(thumb, dict): + urls = thumb.get("url_list") + if isinstance(urls, list) and urls: + return urls[0] + return None + + +def parse_comment(raw: dict[str, Any], video_url: str) -> dict[str, Any]: + """Map a raw comment record to a :class:`CommentItem` output dict.""" + from ..schemas.items import CommentItem + + user = raw.get("user") if isinstance(raw.get("user"), dict) else {} + reply_id = raw.get("reply_id") + create_time = raw.get("create_time") + return CommentItem( + id=raw.get("cid"), + text=raw.get("text"), + videoWebUrl=video_url, + diggCount=raw.get("digg_count"), + replyCommentTotal=raw.get("reply_comment_total"), + createTime=create_time, + createTimeISO=epoch_to_iso(create_time), + uid=user.get("uid"), + uniqueId=user.get("unique_id"), + nickName=user.get("nickname"), + avatar=_avatar(user), + repliesToId=reply_id if reply_id and reply_id != "0" else None, + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py index 1a8aeaacc..a7e38eba1 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py @@ -13,4 +13,7 @@ FetchListingFn = Callable[[str, int], Awaitable[list[dict]]] FetchUsersFn = Callable[[str, int], Awaitable[list[dict]]] """Load a user-search page and return up to ``count`` captured ``user_info`` records.""" +FetchCommentsFn = Callable[[str, int], Awaitable[list[dict]]] +"""Load a video page and return up to ``count`` captured raw comment records.""" + FlowResult = AsyncIterator[dict] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/comments.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/comments.py new file mode 100644 index 000000000..28a7772a2 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/comments.py @@ -0,0 +1,52 @@ +"""Comments flow: a video URL -> its public comment thread. + +Comments load over a signed ``/api/comment/list`` XHR that TikTok *does* serve to +anonymous sessions once the comments panel is opened (unlike profile-video/search +feeds). Records are deduped by comment id, capped, and — when a video has none or +withholds them — degraded to one ErrorItem, mirroring the listing flow. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from ..extraction import parse_comment +from ..extraction.timestamps import now_iso +from ..schemas import ErrorItem +from ..targets.types import TikTokTarget +from . import FetchCommentsFn + +_EMPTY_MESSAGE = ( + "No comments returned. The video may have none, comments disabled, or TikTok " + "withheld them from anonymous access." +) + + +async def iter_comments( + target: TikTokTarget, *, cap: int, fetch_comments: FetchCommentsFn +) -> AsyncIterator[dict[str, Any]]: + if cap <= 0: + return + seen: set[str] = set() + emitted = 0 + for raw in await fetch_comments(target.url, cap): + out = parse_comment(raw, target.url) + cid = out.get("id") + if cid is not None: + if cid in seen: + continue + seen.add(cid) + out["scrapedAt"] = now_iso() + yield out + emitted += 1 + if emitted >= cap: + return + if emitted == 0: + yield ErrorItem( + url=target.url, + input=target.value, + error=_EMPTY_MESSAGE, + errorCode="no_comments", + scrapedAt=now_iso(), + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index 485a8dfff..dbe0d5ce9 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -12,13 +12,20 @@ from collections.abc import AsyncIterator from typing import Any from urllib.parse import quote -from .flows import FetchFn, FetchListingFn, FetchUsersFn +from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn +from .flows.comments import iter_comments from .flows.listing import iter_listing from .flows.profile import iter_profile from .flows.user_search import iter_user_search from .flows.video import iter_video -from .schemas import TikTokScrapeInput -from .session import fetch_html, fetch_item_list, fetch_user_search +from .extraction.timestamps import now_iso +from .schemas import ErrorItem, TikTokScrapeInput +from .session import ( + fetch_comments, + fetch_html, + fetch_item_list, + fetch_user_search, +) from .targets import resolve_target from .targets.types import TikTokTarget @@ -123,3 +130,44 @@ async def search_tiktok_users( if limit is not None and len(results) >= limit: return results return results + + +async def scrape_tiktok_comments( + video_urls: list[str], + *, + per_video: int, + limit: int | None = None, + fetch_comments_fn: FetchCommentsFn = fetch_comments, +) -> list[dict[str, Any]]: + """Collect comments across video URLs, honoring ``limit``. + + A non-video URL yields one ``bad_url`` ErrorItem (never a silent drop) so the + caller can tell "wrong input" from "video has no comments". + """ + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + for url in video_urls: + target = resolve_target(url) + if target is None or target.kind != "video": + results.append( + ErrorItem( + url=url, + input=url, + error="Not a TikTok video URL.", + errorCode="bad_url", + scrapedAt=now_iso(), + ).to_output() + ) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + return results + continue + async for item in iter_comments( + target, cap=per_video, fetch_comments=fetch_comments_fn + ): + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + return results + return results diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py index 5e29403a0..437fc9643 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py @@ -4,12 +4,13 @@ from __future__ import annotations from .client import fetch_html from .errors import TikTokAccessBlockedError -from .listing import fetch_item_list, fetch_user_search +from .listing import fetch_comments, fetch_item_list, fetch_user_search from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session __all__ = [ "TikTokAccessBlockedError", "bind_proxy_holder", + "fetch_comments", "fetch_html", "fetch_item_list", "fetch_user_search", diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index e0bdd9255..d00e20711 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -30,11 +30,18 @@ from app.proprietary.web_crawler.stealth import ( ) from app.utils.proxy import get_proxy_url -from ..extraction import items_from_response, users_from_response +from ..extraction import ( + comments_from_response, + items_from_response, + users_from_response, +) logger = logging.getLogger(__name__) ExtractFn = Callable[[Any], list[dict[str, Any]]] +# Drives the page after navigation to trigger/paginate the target XHRs, filling +# ``collected`` until it reaches ``target_count`` (or the interaction gives up). +InteractFn = Callable[[Any, list[dict[str, Any]], int], None] # XHR paths that carry itemStructs for the three listing kinds. _ITEM_LIST_MARKERS = ( @@ -44,6 +51,16 @@ _ITEM_LIST_MARKERS = ( ) # The user-search XHR carries account records (user_list), not itemStructs. _USER_SEARCH_MARKERS = ("/api/search/user",) +# The comment feed fires only after the comments panel is opened. +_COMMENT_MARKERS = ("/api/comment/list",) +_COMMENT_ICON_SELECTORS = ( + '[data-e2e="comment-icon"]', + '[data-e2e="browse-comment"]', +) +# The comment icon hydrates a beat after DOM-ready; wait for it before clicking. +_COMMENT_ICON_WAIT_MS = 8000 +# First comment page lands shortly after the click — don't declare "empty" early. +_COMMENT_FIRST_PAGE_MS = 3500 _HOME_URL = "https://www.tiktok.com/" _MSTOKEN_COOKIE = "msToken" # Bounded scroll: a dead page can't loop forever, and a live one stops early @@ -62,18 +79,87 @@ def _has_mstoken(page: Any) -> bool: return False +def _scroll_page(page: Any, collected: list[dict[str, Any]], target_count: int) -> None: + """Page down a listing feed until enough items are captured or it stops growing.""" + last_height = 0 + for _ in range(_SCROLL_MAX_ROUNDS): + if len(collected) >= target_count: + break + page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + page.wait_for_timeout(_SCROLL_SETTLE_MS) + height = page.evaluate("document.body.scrollHeight") + if not height or height <= last_height: + break + last_height = height + + +def _open_comments(page: Any) -> None: + """Click the comment icon so the first ``/api/comment/list`` XHR fires. + + The icon must be present and interactive first (the SPA hydrates it a beat + after DOM-ready), so we wait for it, then fall back to a JS click if the + normal click is intercepted (cookie banner / overlay). + """ + for selector in _COMMENT_ICON_SELECTORS: + try: + page.wait_for_selector(selector, timeout=_COMMENT_ICON_WAIT_MS) + except Exception: + continue + try: + page.click(selector, timeout=_COMMENT_ICON_WAIT_MS) + return + except Exception: + try: + page.eval_on_selector(selector, "el => el.click()") + return + except Exception: + continue + + +def _scroll_comments( + page: Any, collected: list[dict[str, Any]], target_count: int +) -> None: + """Open the comments panel, then scroll its last comment into view to paginate. + + Comment XHRs fire only after the panel is opened, and paging must scroll the + panel (not the page, which would advance the video feed), so we anchor on the + last ``comment-level-1`` element. ponytail: naive scroll-to-last paging, + bounded by ``_SCROLL_MAX_ROUNDS``; upgrade to container-height tracking if + deep threads under-fetch. + """ + _open_comments(page) + # The panel's first page lands a beat after the click; give it room before + # we decide there are no comments to page through. + page.wait_for_timeout(_COMMENT_FIRST_PAGE_MS) + for _ in range(_SCROLL_MAX_ROUNDS): + if len(collected) >= target_count: + break + moved = page.evaluate( + """() => { + const items = document.querySelectorAll('[data-e2e="comment-level-1"]'); + if (!items.length) return false; + items[items.length - 1].scrollIntoView({block: 'end'}); + return true; + }""" + ) + page.wait_for_timeout(_SCROLL_SETTLE_MS) + if not moved: + break + + def _build_page_action( collected: list[dict[str, Any]], url: str, target_count: int, markers: tuple[str, ...], extract: ExtractFn, + interact: InteractFn, ): """A sync ``page_action`` that warms the session then captures matching XHRs. A cold context returns an empty body, so we first mint the anonymous ``msToken`` (homepage hit), then navigate to the target with the listener - already attached so page-one fires into it; scrolling pages the rest. + already attached so page-one fires into it; ``interact`` pages the rest. ``markers``/``extract`` select which XHRs to keep and how to unwrap them. """ @@ -102,28 +188,23 @@ def _build_page_action( try: _warm(page) # Navigate (back) to the target with the listener attached and a - # token in hand, so the page-one item_list fires into the capture. + # token in hand, so the page-one XHR fires into the capture. page.goto(url, wait_until="domcontentloaded") page.wait_for_timeout(_SCROLL_SETTLE_MS) - last_height = 0 - for _ in range(_SCROLL_MAX_ROUNDS): - if len(collected) >= target_count: - break - page.evaluate("window.scrollTo(0, document.body.scrollHeight)") - page.wait_for_timeout(_SCROLL_SETTLE_MS) - height = page.evaluate("document.body.scrollHeight") - if not height or height <= last_height: - break - last_height = height + interact(page, collected, target_count) except Exception as exc: - logger.debug("[tiktok] listing scroll aborted: %s", exc) + logger.debug("[tiktok] capture interaction aborted: %s", exc) return page return page_action def _fetch_sync( - url: str, target_count: int, markers: tuple[str, ...], extract: ExtractFn + url: str, + target_count: int, + markers: tuple[str, ...], + extract: ExtractFn, + interact: InteractFn, ) -> list[dict[str, Any]]: collected: list[dict[str, Any]] = [] kwargs = build_stealthy_kwargs(get_stealth_config()) @@ -133,7 +214,7 @@ def _fetch_sync( network_idle=False, proxy=get_proxy_url(), page_action=_build_page_action( - collected, url, target_count, markers, extract + collected, url, target_count, markers, extract, interact ), **kwargs, ) @@ -143,12 +224,34 @@ def _fetch_sync( async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` itemStructs from a listing page's XHRs.""" return await asyncio.to_thread( - _fetch_sync, page_url, target_count, _ITEM_LIST_MARKERS, items_from_response + _fetch_sync, + page_url, + target_count, + _ITEM_LIST_MARKERS, + items_from_response, + _scroll_page, ) async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` ``user_info`` records from a user-search page.""" return await asyncio.to_thread( - _fetch_sync, page_url, target_count, _USER_SEARCH_MARKERS, users_from_response + _fetch_sync, + page_url, + target_count, + _USER_SEARCH_MARKERS, + users_from_response, + _scroll_page, + ) + + +async def fetch_comments(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` raw comment records from a video page's XHRs.""" + return await asyncio.to_thread( + _fetch_sync, + page_url, + target_count, + _COMMENT_MARKERS, + comments_from_response, + _scroll_comments, ) diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/comments/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/comments/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py new file mode 100644 index 000000000..b23149420 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py @@ -0,0 +1,48 @@ +"""``tiktok.comments`` executor: verb input → scraper args → typed comment items. + +Boundary mocked: the proprietary comments actor (injected fake). NOT mocked: the +verb's own payload→args forwarding and the dict→CommentItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.comments.executor import build_comments_executor +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput + +pytestmark = pytest.mark.unit + +_VIDEO = "https://www.tiktok.com/@bob/video/123" + + +class _FakeComments: + """Records the urls + kwargs it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[tuple[list[str], int, int | None]] = [] + + async def __call__( + self, video_urls: list[str], *, per_video: int, limit: int | None = None + ) -> list[dict]: + self.calls.append((video_urls, per_video, limit)) + return self._items + + +async def test_forwards_urls_and_limits_and_wraps_items(): + comments = _FakeComments([{"id": "1", "text": "hi"}]) + execute = build_comments_executor(comments_fn=comments) + + out = await execute( + CommentsInput(video_urls=[_VIDEO], comments_per_video=15, max_items=40) + ) + + assert isinstance(out, CommentsOutput) + assert len(out.items) == 1 + assert out.items[0].text == "hi" + + (urls, per_video, limit) = comments.calls[0] + assert urls == [_VIDEO] + assert per_video == 15 + assert limit == 40 diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py new file mode 100644 index 000000000..cdcbc201e --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py @@ -0,0 +1,48 @@ +"""``tiktok.comments`` input guards and billing: a video URL is required, bounded, +and ErrorItems are surfaced free.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput +from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES + +pytestmark = pytest.mark.unit + +_VIDEO = "https://www.tiktok.com/@bob/video/123" + + +def test_rejects_input_with_no_url(): + with pytest.raises(ValidationError): + CommentsInput(video_urls=[]) + + +def test_defaults_and_bounds(): + payload = CommentsInput(video_urls=[_VIDEO]) + assert payload.max_items == 20 + assert payload.comments_per_video == 20 + assert payload.estimated_units == 20 + with pytest.raises(ValidationError): + CommentsInput(video_urls=[_VIDEO], max_items=0) + with pytest.raises(ValidationError): + CommentsInput(video_urls=[_VIDEO], max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_rejects_more_urls_than_the_cap(): + too_many = [f"{_VIDEO}{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] + with pytest.raises(ValidationError): + CommentsInput(video_urls=too_many) + + +def test_error_items_are_not_billed(): + # Real comments count; ErrorItems (bad URL / empty video) are surfaced free. + out = CommentsOutput( + items=[ + {"id": "1", "text": "hi"}, + {"errorCode": "no_comments", "input": "123", "error": "empty"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py index 893f3c2ae..c688cf198 100644 --- a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py @@ -9,6 +9,7 @@ from app.capabilities import ( ) from app.capabilities.core import BillingUnit from app.capabilities.core.store import get_capability +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput from app.capabilities.tiktok.user_search.schemas import ( UserSearchInput, @@ -34,3 +35,12 @@ def test_tiktok_user_search_is_registered_and_billed_per_profile(): assert cap.input_schema is UserSearchInput assert cap.output_schema is UserSearchOutput assert cap.billing_unit is BillingUnit.TIKTOK_USER + + +def test_tiktok_comments_is_registered_and_billed_per_comment(): + cap = get_capability("tiktok.comments") + + assert cap.name == "tiktok.comments" + assert cap.input_schema is CommentsInput + assert cap.output_schema is CommentsOutput + assert cap.billing_unit is BillingUnit.TIKTOK_COMMENT diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_comments.py b/surfsense_backend/tests/unit/platforms/tiktok/test_comments.py new file mode 100644 index 000000000..e118f3c01 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_comments.py @@ -0,0 +1,89 @@ +"""Comments orchestration over a fake fetch (no network). + +Drives ``scrape_tiktok_comments``: video URLs -> captured raw comments -> items. +""" + +from __future__ import annotations + +from typing import Any + +from app.proprietary.platforms.tiktok import scrape_tiktok_comments + +_VIDEO = "https://www.tiktok.com/@bob/video/123" + + +def _comment(cid: str, reply_id: str = "0") -> dict[str, Any]: + return { + "cid": cid, + "text": f"comment {cid}", + "digg_count": 7, + "reply_comment_total": 2, + "create_time": 1700000000, + "reply_id": reply_id, + "user": { + "uid": "u1", + "unique_id": "alice", + "nickname": "Alice", + "avatar_thumb": {"url_list": ["https://cdn/a.webp"]}, + }, + } + + +async def test_comments_parse_dedupe_and_cap(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_comment("1"), _comment("1"), _comment("2", reply_id="1")] + + items = await scrape_tiktok_comments( + [_VIDEO], per_video=2, fetch_comments_fn=fake_fetch + ) + + assert [i["id"] for i in items] == ["1", "2"] + first = items[0] + assert first["text"] == "comment 1" + assert first["videoWebUrl"] == _VIDEO + assert first["diggCount"] == 7 + assert first["uniqueId"] == "alice" + assert first["avatar"] == "https://cdn/a.webp" + assert first["createTimeISO"] is not None + assert first["repliesToId"] is None # reply_id "0" == top-level + assert first["scrapedAt"] is not None + assert items[1]["repliesToId"] == "1" # a reply carries its parent id + + +async def test_empty_video_emits_error_item(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [] + + items = await scrape_tiktok_comments( + [_VIDEO], per_video=5, fetch_comments_fn=fake_fetch + ) + + assert len(items) == 1 + assert items[0]["errorCode"] == "no_comments" + assert items[0]["input"] == "123" + + +async def test_non_video_url_emits_bad_url_error(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + raise AssertionError("should not fetch for a non-video URL") + + items = await scrape_tiktok_comments( + ["https://www.tiktok.com/@bob"], per_video=5, fetch_comments_fn=fake_fetch + ) + + assert len(items) == 1 + assert items[0]["errorCode"] == "bad_url" + + +async def test_comments_honor_limit_across_videos(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_comment("1"), _comment("2")] + + items = await scrape_tiktok_comments( + [_VIDEO, "https://www.tiktok.com/@bob/video/456"], + per_video=5, + limit=3, + fetch_comments_fn=fake_fetch, + ) + + assert len(items) == 3 From 67b5472b9fc2775727af94273aab88816e303c75 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 9 Jul 2026 18:52:56 +0200 Subject: [PATCH 35/56] feat(tiktok): add tiktok.trending verb for the Explore feed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Explore feed (/api/explore/item_list) is a global trending-video feed served to anonymous sessions, and it returns the same itemStruct shape as the other listings — so the verb reuses parse_video, the listing flow, the TikTokVideoItem output, and the per-video billing meter wholesale. Adds a browser-capture marker + fetch_trending, a synthetic-target orchestrator entry, and the tiktok.trending capability, surfaced on the chat subagent. --- .../subagents/builtins/tiktok/tools/index.py | 5 ++- .../app/capabilities/tiktok/__init__.py | 1 + .../capabilities/tiktok/trending/__init__.py | 3 ++ .../tiktok/trending/definition.py | 23 +++++++++++ .../capabilities/tiktok/trending/executor.py | 32 +++++++++++++++ .../capabilities/tiktok/trending/schemas.py | 41 +++++++++++++++++++ .../proprietary/platforms/tiktok/__init__.py | 2 + .../platforms/tiktok/orchestrator.py | 24 ++++++++++- .../platforms/tiktok/session/__init__.py | 8 +++- .../platforms/tiktok/session/listing.py | 14 +++++++ .../platforms/tiktok/targets/types.py | 2 +- .../unit/capabilities/tiktok/test_registry.py | 11 +++++ .../capabilities/tiktok/trending/__init__.py | 0 .../tiktok/trending/test_executor.py | 38 +++++++++++++++++ .../tiktok/trending/test_schemas.py | 33 +++++++++++++++ .../unit/platforms/tiktok/test_trending.py | 35 ++++++++++++++++ 16 files changed, 267 insertions(+), 5 deletions(-) create mode 100644 surfsense_backend/app/capabilities/tiktok/trending/__init__.py create mode 100644 surfsense_backend/app/capabilities/tiktok/trending/definition.py create mode 100644 surfsense_backend/app/capabilities/tiktok/trending/executor.py create mode 100644 surfsense_backend/app/capabilities/tiktok/trending/schemas.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/trending/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/trending/test_executor.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/trending/test_schemas.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_trending.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py index 44ebc863d..f1bec7fc2 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py @@ -1,4 +1,4 @@ -"""``tiktok`` sub-agent tools: the TikTok scrape, comments, and user-search verbs.""" +"""``tiktok`` sub-agent tools: scrape, comments, user-search, and trending verbs.""" from __future__ import annotations @@ -10,13 +10,14 @@ from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from app.capabilities.core.access.agent import build_capability_tools from app.capabilities.tiktok.comments.definition import TIKTOK_COMMENTS from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE +from app.capabilities.tiktok.trending.definition import TIKTOK_TRENDING from app.capabilities.tiktok.user_search.definition import TIKTOK_USER_SEARCH NAME = "tiktok" RULESET = Ruleset(origin=NAME, rules=[]) -_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_COMMENTS, TIKTOK_USER_SEARCH] +_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_COMMENTS, TIKTOK_USER_SEARCH, TIKTOK_TRENDING] def load_tools( diff --git a/surfsense_backend/app/capabilities/tiktok/__init__.py b/surfsense_backend/app/capabilities/tiktok/__init__.py index 1ff1738fa..4962d3162 100644 --- a/surfsense_backend/app/capabilities/tiktok/__init__.py +++ b/surfsense_backend/app/capabilities/tiktok/__init__.py @@ -4,4 +4,5 @@ from __future__ import annotations from app.capabilities.tiktok.comments import definition as _comments # noqa: F401 from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401 +from app.capabilities.tiktok.trending import definition as _trending # noqa: F401 from app.capabilities.tiktok.user_search import definition as _user_search # noqa: F401 diff --git a/surfsense_backend/app/capabilities/tiktok/trending/__init__.py b/surfsense_backend/app/capabilities/tiktok/trending/__init__.py new file mode 100644 index 000000000..c335258b3 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/trending/__init__.py @@ -0,0 +1,3 @@ +"""``tiktok.trending``: pull the current trending videos from the Explore feed.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/tiktok/trending/definition.py b/surfsense_backend/app/capabilities/tiktok/trending/definition.py new file mode 100644 index 000000000..7f1b006ce --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/trending/definition.py @@ -0,0 +1,23 @@ +"""``tiktok.trending`` capability registration (billed per video on the shared +``TIKTOK_MICROS_PER_VIDEO`` meter).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.trending.executor import build_trending_executor +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput + +TIKTOK_TRENDING = Capability( + name="tiktok.trending", + description=( + "Get the current trending TikTok videos from the Explore feed. No input " + "needed beyond how many to return." + ), + input_schema=TrendingInput, + output_schema=TrendingOutput, + executor=build_trending_executor(), + billing_unit=BillingUnit.TIKTOK_VIDEO, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_TRENDING) diff --git a/surfsense_backend/app/capabilities/tiktok/trending/executor.py b/surfsense_backend/app/capabilities/tiktok/trending/executor.py new file mode 100644 index 000000000..9551f4b6a --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/trending/executor.py @@ -0,0 +1,32 @@ +"""``tiktok.trending`` executor: Explore feed -> scraper -> TikTok video items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput +from app.proprietary.platforms.tiktok import scrape_tiktok_trending + +TrendingFn = Callable[..., Awaitable[list[dict]]] + + +def build_trending_executor(trending_fn: TrendingFn | None = None) -> Executor: + """Bind the executor to a trending fn (defaults to the proprietary actor).""" + trending_fn = trending_fn or scrape_tiktok_trending + + async def execute(payload: TrendingInput) -> TrendingOutput: + emit_progress( + "starting", + "Fetching TikTok trending videos", + total=payload.max_items, + unit="item", + ) + items = await trending_fn(count=payload.max_items) + emit_progress( + "done", f"Fetched {len(items)} video(s)", current=len(items), unit="item" + ) + return TrendingOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/trending/schemas.py b/surfsense_backend/app/capabilities/tiktok/trending/schemas.py new file mode 100644 index 000000000..879206907 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/trending/schemas.py @@ -0,0 +1,41 @@ +"""``tiktok.trending`` I/O contracts. + +The Explore feed (``/api/explore/item_list``) is a single global feed of trending +videos, served to anonymous sessions. No source input is needed — just how many +to return. Each result reuses :class:`TikTokVideoItem`, so trending videos bill on +the same per-video meter as ``tiktok.scrape``. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS +from app.proprietary.platforms.tiktok import TikTokVideoItem + + +class TrendingInput(BaseModel): + max_items: int = Field( + default=20, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max trending videos to return from the Explore feed.", + ) + + @property + def estimated_units(self) -> int: + """Worst-case billable videos for the pre-flight gate (le=100 ceiling).""" + return self.max_items + + +class TrendingOutput(BaseModel): + items: list[TikTokVideoItem] = Field( + default_factory=list, + description="One item per trending video returned, in feed order.", + ) + + @property + def billable_units(self) -> int: + """One returned video = one billable unit; an ErrorItem (``errorCode`` set, + for an empty/withheld feed) is surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py index 00d345937..91f716d1f 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py @@ -10,6 +10,7 @@ from .orchestrator import ( iter_tiktok, scrape_tiktok, scrape_tiktok_comments, + scrape_tiktok_trending, search_tiktok_users, ) from .schemas import ( @@ -29,5 +30,6 @@ __all__ = [ "iter_tiktok", "scrape_tiktok", "scrape_tiktok_comments", + "scrape_tiktok_trending", "search_tiktok_users", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index dbe0d5ce9..5d6a24545 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -12,18 +12,19 @@ from collections.abc import AsyncIterator from typing import Any from urllib.parse import quote +from .extraction.timestamps import now_iso from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn from .flows.comments import iter_comments from .flows.listing import iter_listing from .flows.profile import iter_profile from .flows.user_search import iter_user_search from .flows.video import iter_video -from .extraction.timestamps import now_iso from .schemas import ErrorItem, TikTokScrapeInput from .session import ( fetch_comments, fetch_html, fetch_item_list, + fetch_trending, fetch_user_search, ) from .targets import resolve_target @@ -32,6 +33,7 @@ from .targets.types import TikTokTarget _PROFILE_URL = "https://www.tiktok.com/@{name}" _HASHTAG_URL = "https://www.tiktok.com/tag/{tag}" _SEARCH_URL = "https://www.tiktok.com/search?q={query}" +_EXPLORE_URL = "https://www.tiktok.com/explore" def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]: @@ -132,6 +134,26 @@ async def search_tiktok_users( return results +async def scrape_tiktok_trending( + *, + count: int, + fetch_trending_fn: FetchListingFn = fetch_trending, +) -> list[dict[str, Any]]: + """Collect up to ``count`` trending videos from the Explore feed. + + A single global feed, so it reuses the listing flow (parse/dedupe/cap/empty- + ErrorItem) over a synthetic target — no user input to resolve. + """ + from app.capabilities.core.progress import emit_progress + + target = TikTokTarget(kind="trending", value="explore", url=_EXPLORE_URL) + results: list[dict[str, Any]] = [] + async for item in iter_listing(target, cap=count, fetch_listing=fetch_trending_fn): + results.append(item) + emit_progress("scraping", current=len(results), total=count, unit="item") + return results + + async def scrape_tiktok_comments( video_urls: list[str], *, diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py index 437fc9643..6fa1c1b45 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py @@ -4,7 +4,12 @@ from __future__ import annotations from .client import fetch_html from .errors import TikTokAccessBlockedError -from .listing import fetch_comments, fetch_item_list, fetch_user_search +from .listing import ( + fetch_comments, + fetch_item_list, + fetch_trending, + fetch_user_search, +) from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session __all__ = [ @@ -13,6 +18,7 @@ __all__ = [ "fetch_comments", "fetch_html", "fetch_item_list", + "fetch_trending", "fetch_user_search", "open_proxy_holder", "proxy_session", diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index d00e20711..e94984440 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -51,6 +51,8 @@ _ITEM_LIST_MARKERS = ( ) # The user-search XHR carries account records (user_list), not itemStructs. _USER_SEARCH_MARKERS = ("/api/search/user",) +# The Explore feed's trending videos arrive as ordinary itemStructs. +_EXPLORE_MARKERS = ("/api/explore/item_list",) # The comment feed fires only after the comments panel is opened. _COMMENT_MARKERS = ("/api/comment/list",) _COMMENT_ICON_SELECTORS = ( @@ -255,3 +257,15 @@ async def fetch_comments(page_url: str, target_count: int) -> list[dict[str, Any comments_from_response, _scroll_comments, ) + + +async def fetch_trending(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` trending itemStructs from the Explore feed.""" + return await asyncio.to_thread( + _fetch_sync, + page_url, + target_count, + _EXPLORE_MARKERS, + items_from_response, + _scroll_page, + ) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py index c53d5239d..417d27741 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py @@ -5,7 +5,7 @@ from __future__ import annotations from dataclasses import dataclass from typing import Literal -TargetKind = Literal["video", "profile", "hashtag", "search"] +TargetKind = Literal["video", "profile", "hashtag", "search", "trending"] SearchSection = Literal["video", "user"] diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py index c688cf198..c6314ec10 100644 --- a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py @@ -11,6 +11,7 @@ from app.capabilities.core import BillingUnit from app.capabilities.core.store import get_capability from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput from app.capabilities.tiktok.user_search.schemas import ( UserSearchInput, UserSearchOutput, @@ -44,3 +45,13 @@ def test_tiktok_comments_is_registered_and_billed_per_comment(): assert cap.input_schema is CommentsInput assert cap.output_schema is CommentsOutput assert cap.billing_unit is BillingUnit.TIKTOK_COMMENT + + +def test_tiktok_trending_is_registered_and_billed_per_video(): + cap = get_capability("tiktok.trending") + + assert cap.name == "tiktok.trending" + assert cap.input_schema is TrendingInput + assert cap.output_schema is TrendingOutput + # Trending returns videos, so it shares the per-video meter. + assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/trending/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/trending/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_executor.py new file mode 100644 index 000000000..3539ca595 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_executor.py @@ -0,0 +1,38 @@ +"""``tiktok.trending`` executor: verb input → scraper count → typed video items. + +Boundary mocked: the proprietary trending actor (injected fake). NOT mocked: the +verb's own count forwarding and the dict→TikTokVideoItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.trending.executor import build_trending_executor +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput + +pytestmark = pytest.mark.unit + + +class _FakeTrending: + """Records the count it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[int] = [] + + async def __call__(self, *, count: int) -> list[dict]: + self.calls.append(count) + return self._items + + +async def test_forwards_count_and_wraps_items(): + trending = _FakeTrending([{"id": "1", "text": "viral"}]) + execute = build_trending_executor(trending_fn=trending) + + out = await execute(TrendingInput(max_items=30)) + + assert isinstance(out, TrendingOutput) + assert len(out.items) == 1 + assert out.items[0].id == "1" + assert trending.calls == [30] diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_schemas.py new file mode 100644 index 000000000..f27fec71a --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_schemas.py @@ -0,0 +1,33 @@ +"""``tiktok.trending`` input guards and billing: bounded count, ErrorItems free.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput + +pytestmark = pytest.mark.unit + + +def test_defaults_and_bounds(): + payload = TrendingInput() + assert payload.max_items == 20 + assert payload.estimated_units == 20 + with pytest.raises(ValidationError): + TrendingInput(max_items=0) + with pytest.raises(ValidationError): + TrendingInput(max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_error_items_are_not_billed(): + # Real videos count; an ErrorItem (empty/withheld feed) is surfaced free. + out = TrendingOutput( + items=[ + {"id": "1", "webVideoUrl": "https://tiktok.com/@a/video/1"}, + {"errorCode": "no_items", "input": "explore", "error": "empty"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_trending.py b/surfsense_backend/tests/unit/platforms/tiktok/test_trending.py new file mode 100644 index 000000000..d00d9c09f --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_trending.py @@ -0,0 +1,35 @@ +"""Trending orchestration over a fake fetch (no network). + +Drives ``scrape_tiktok_trending``: the Explore feed -> captured itemStructs -> +video items, reusing the listing flow (parse/dedupe/cap/empty-ErrorItem). +""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok import scrape_tiktok_trending + + +async def test_trending_parses_dedupes_and_caps(): + async def fake_fetch(url: str, _cap: int) -> list[dict]: + assert url == "https://www.tiktok.com/explore" + return [ + {"id": "1", "author": {"uniqueId": "a"}}, + {"id": "1", "author": {"uniqueId": "a"}}, + {"id": "2", "author": {"uniqueId": "b"}}, + ] + + items = await scrape_tiktok_trending(count=2, fetch_trending_fn=fake_fetch) + + assert [i["id"] for i in items] == ["1", "2"] + assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@a/video/1" + assert items[0]["scrapedAt"] is not None + + +async def test_trending_empty_feed_emits_error_item(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [] + + items = await scrape_tiktok_trending(count=5, fetch_trending_fn=fake_fetch) + + assert len(items) == 1 + assert items[0]["errorCode"] == "no_items" From 6b15d610d92fdcb51cfd5d89b54a512eab4daa3c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 9 Jul 2026 19:09:00 +0200 Subject: [PATCH 36/56] feat(tiktok): surface comments, user_search, trending across all product surfaces Brings the three newer verbs to parity with tiktok.scrape everywhere it was wired: MCP tools (+ selfcheck manifest), the chat subagent prompt/description, the playground catalog, the native docs page, and the SEO marketing page. Also adds live e2e stages for comments, user search, and trending. Docs/FAQ now state the real contract: profile metadata is reliable while its video list can be withheld, and keyword video search is walled (use user search for accounts). --- .../subagents/builtins/tiktok/description.md | 4 +- .../builtins/tiktok/system_prompt.md | 18 ++- .../scripts/e2e_tiktok_scrape.py | 70 ++++++++++- .../features/scrapers/platforms/tiktok.py | 119 +++++++++++++++++- surfsense_mcp/mcp_server/selfcheck.py | 3 + .../content/docs/connectors/native/tiktok.mdx | 84 ++++++++++--- .../lib/connectors-marketing/tiktok.tsx | 18 ++- surfsense_web/lib/playground/catalog.ts | 11 +- 8 files changed, 294 insertions(+), 33 deletions(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md index fc13d864d..d57c551d4 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md @@ -1,2 +1,2 @@ -TikTok specialist: pulls structured public TikTok data — videos (caption/text, author, play/like/comment/share counts, music, hashtags, timestamps, web URL) from a hashtag feed, a search query, a creator profile, or a known video URL. Also compares fresh TikTok results against earlier findings in this chat. -Use whenever the task is to find what is trending or being said on TikTok about a topic, gather a creator's or hashtag's videos, or scrape a specific video URL. Triggers include "search TikTok for X", "trending TikTok videos about X", "videos with #X", and "scrape this TikTok video". Not for general web pages (use the web crawling specialist), Google results (use the Google Search specialist), Reddit (use the Reddit specialist), or YouTube (use the YouTube specialist). +TikTok specialist: pulls structured public TikTok data — videos (caption/text, author, play/like/comment/share counts, music, hashtags, timestamps, web URL) from a hashtag feed, a search query, a creator profile, or a known video URL; a video's public comment thread; accounts found by keyword; and the current Explore trending-video feed. Also compares fresh TikTok results against earlier findings in this chat. +Use whenever the task is to find what is trending or being said on TikTok about a topic, gather a creator's or hashtag's videos, read a video's comments, discover accounts by keyword, or scrape a specific video URL. Triggers include "search TikTok for X", "what's trending on TikTok", "videos with #X", "comments on this TikTok", "find TikTok accounts about X", and "scrape this TikTok video". Not for general web pages (use the web crawling specialist), Google results (use the Google Search specialist), Reddit (use the Reddit specialist), or YouTube (use the YouTube specialist). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md index 49c20164e..5b6de993c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md @@ -6,17 +6,23 @@ Answer the delegated question from live TikTok data gathered with your verb, com -- `tiktok_scrape` +- `tiktok_scrape` — videos from urls / profiles / hashtags / search_queries +- `tiktok_comments` — a video's comment thread, from `video_urls` +- `tiktok_user_search` — find accounts by keyword, from `queries` +- `tiktok_trending` — the current Explore trending-video feed - `read_run` / `search_run` (free readers for stored scrape output) - Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#') and/or `search_queries`. - Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`. -- Profiles: a creator's `profiles` feed can come back empty — TikTok restricts the profile video endpoint. Prefer `hashtags`, `search_queries`, or a direct video URL, and treat an empty profile result as a known limit, not a failure to retry endlessly. -- Controlling volume: use `max_items` for the total cap and `results_per_page` per target. -- Requested counts: `max_items` defaults to only 10 — when the task asks for N videos, set `max_items` and `results_per_page` above N. A call that caps below the target can never satisfy it. -- Batch multiple hashtags or search terms into one call rather than many single-term calls. +- Profiles: a creator's `profiles` feed returns the account's metadata (name, followers, bio, verification) reliably, but its video list is often withheld by TikTok — treat an empty video list as a known limit, not a failure to retry endlessly. Prefer `hashtags`, `search_queries`, or a direct video URL for videos. +- Comments on a video: call `tiktok_comments` with the video URL(s) in `video_urls`. +- Finding accounts (not videos): call `tiktok_user_search` with `queries` — this is the reliable path for account discovery (keyword *video* search is login-walled). +- "What's trending now": call `tiktok_trending` (no query needed); set `max_items` for how many. +- Controlling volume: use `max_items` for the total cap and `results_per_page` per target (per-verb equivalents: `comments_per_video`, `results_per_query`). +- Requested counts: `max_items` defaults low — when the task asks for N items, set `max_items` (and the per-target count) above N. A call that caps below the target can never satisfy it. +- Batch multiple hashtags, search terms, queries, or video URLs into one call rather than many single-item calls. - Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, count changes). @@ -59,6 +65,6 @@ Return **only** one JSON object (no markdown/prose): } Route-specific rules: -- `evidence.findings`: one entry per distinct video or delta — a single sentence each; do not paste raw payloads. Max 10 entries, unless the delegated task asks for N items: then return up to N (each backed by a real scraped result, never padded). +- `evidence.findings`: one entry per distinct result (video, comment, or account) or delta — a single sentence each; do not paste raw payloads. Max 10 entries, unless the delegated task asks for N items: then return up to N (each backed by a real scraped result, never padded). - `evidence.sources`: one TikTok URL per finding when applicable, same cap as findings. List each URL once. diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py index eedeaacac..cb5a996eb 100644 --- a/surfsense_backend/scripts/e2e_tiktok_scrape.py +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -15,6 +15,12 @@ What it exercises (everything REAL — live network, live proxy, live browser): Stage 5 — full scrape_tiktok() pipeline on a hashtag. Stage 6 — search via the full pipeline: same graceful-degrade contract as profile (results feed doesn't load for anonymous sessions). + Stage 7 — comments on a real video URL (served anonymously once the panel + opens): real comments OR a single honest ErrorItem. + Stage 8 — user search: the account-discovery XHR that DOES serve anonymous + headless sessions — asserts real account records come back. + Stage 9 — trending: the Explore feed of trending videos — asserts real, + normalized video items come back. On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser suites can pin against real-shaped data without network. @@ -199,6 +205,61 @@ async def stage_search_listing() -> tuple[bool, list[dict[str, Any]]]: return ok, items +async def stage_comments(video_url: str) -> tuple[bool, list[dict[str, Any]]]: + _hr("STAGE 7 — comments graceful-degrade") + print(f" target: {video_url}") + from app.proprietary.platforms.tiktok import scrape_tiktok_comments + + # Comments load over a signed /api/comment/list XHR that TikTok serves to + # anonymous sessions once the panel opens. Pass if real comments come back + # OR a graceful ErrorItem (video has none / disabled / withheld). + items = await scrape_tiktok_comments( + [video_url], per_video=_COUNT, limit=_COUNT + ) + has_comment = any(it.get("id") and not it.get("errorCode") for it in items) + has_error = any(it.get("errorCode") == "no_comments" for it in items) + ok = _check( + "comments yield records or a graceful ErrorItem (never silent empty)", + has_comment or has_error, + f"{len(items)} item(s); comment={has_comment} error={has_error}", + ) + return ok, items + + +async def stage_user_search() -> tuple[bool, list[dict[str, Any]]]: + _hr(f"STAGE 8 — user search (browser): {_PROFILE!r}") + from app.proprietary.platforms.tiktok import search_tiktok_users + + # Unlike keyword *video* search, the account-search XHR serves anonymous + # headless sessions — so this asserts real records, not just degradation. + items = await search_tiktok_users([_PROFILE], per_query=_COUNT, limit=_COUNT) + real = [it for it in items if not it.get("errorCode")] + ok = _check( + "user search returns account records", + bool(real) and bool(real[0].get("uniqueId") or real[0].get("name")), + f"{len(items)} item(s); accounts={len(real)}", + ) + if real: + print(f" sample: @{real[0].get('uniqueId') or real[0].get('name')}") + return ok, items + + +async def stage_trending() -> tuple[bool, list[dict[str, Any]]]: + _hr("STAGE 9 — trending (browser): Explore feed") + from app.proprietary.platforms.tiktok import scrape_tiktok_trending + + items = await scrape_tiktok_trending(count=_COUNT) + real = [it for it in items if not it.get("errorCode")] + ok = _check( + "trending returns normalized video items", + bool(real) and bool(real[0].get("id")) and bool(real[0].get("webVideoUrl")), + f"{len(items)} item(s); videos={len(real)}", + ) + if real: + print(f" sample: {real[0].get('webVideoUrl')} — {real[0].get('text', '')[:60]!r}") + return ok, items + + async def main() -> int: print("TikTok scraper functional e2e — live network + proxy + browser") results: dict[str, bool] = {} @@ -214,8 +275,10 @@ async def main() -> int: if video_url: ok_video, _ = await stage_blob_video(video_url) results["Stage 3 blob video"] = ok_video + ok_comments, _ = await stage_comments(video_url) + results["Stage 7 comments"] = ok_comments else: - print("\n [SKIP] Stage 3 — no captured struct to build a video URL") + print("\n [SKIP] Stage 3/7 — no captured struct to build a video URL") ok_search, _ = await stage_search_listing() results["Stage 6 search listing"] = ok_search @@ -224,6 +287,11 @@ async def main() -> int: results["Stage 2 profile listing"] = ok_profile results["Stage 5 pipeline"] = await stage_pipeline() + ok_users, _ = await stage_user_search() + results["Stage 8 user search"] = ok_users + ok_trending, _ = await stage_trending() + results["Stage 9 trending"] = ok_trending + _hr("SUMMARY") for name, ok in results.items(): print(f" {'PASS' if ok else 'FAIL/SKIP'} — {name}") diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py index 1e5f472ee..aac8efc0e 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py @@ -1,4 +1,4 @@ -"""TikTok scraper tool.""" +"""TikTok scraper tools: scrape (videos), comments, user search, and trending.""" from __future__ import annotations @@ -17,7 +17,7 @@ from ..capability import run_scraper def register( mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext ) -> None: - """Register the TikTok tool.""" + """Register the TikTok tools.""" @mcp.tool( name="surfsense_tiktok_scrape", @@ -86,3 +86,118 @@ def register( workspace=workspace, response_format=response_format, ) + + @mcp.tool( + name="surfsense_tiktok_comments", + title="Scrape TikTok comments", + annotations=SCRAPE, + structured_output=False, + ) + async def tiktok_comments( + video_urls: Annotated[ + list[str], + Field( + description="TikTok video URLs " + "('https://www.tiktok.com/@user/video/123') to pull comments from." + ), + ], + comments_per_video: Annotated[ + int, Field(ge=1, description="Max comments to return per video.") + ] = 20, + max_items: Annotated[ + int, Field(ge=1, description="Maximum comments to return in total.") + ] = 20, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Scrape the public comments of TikTok videos. + + Returns each comment's text, author, like count, and reply count (replies + carry the parent comment id). Example: video_urls=['https://www.tiktok.com/ + @nasa/video/123'], max_items=50. + """ + return await run_scraper( + client, + context, + platform="tiktok", + verb="comments", + payload={ + "video_urls": video_urls, + "comments_per_video": comments_per_video, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_tiktok_user_search", + title="Search TikTok accounts", + annotations=SCRAPE, + structured_output=False, + ) + async def tiktok_user_search( + queries: Annotated[ + list[str], + Field( + description="Keywords to find TikTok accounts by, e.g. " + "['nasa', 'cooking']." + ), + ], + results_per_query: Annotated[ + int, Field(ge=1, description="Max accounts to return per query.") + ] = 10, + max_items: Annotated[ + int, Field(ge=1, description="Maximum accounts to return in total.") + ] = 10, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Find public TikTok accounts by keyword. + + Returns matching profiles with name, followers, bio, and verification — + the reliable account-discovery path (video search is login-walled). + Example: queries=['space agency'], max_items=20. + """ + return await run_scraper( + client, + context, + platform="tiktok", + verb="user_search", + payload={ + "queries": queries, + "results_per_query": results_per_query, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_tiktok_trending", + title="Get trending TikTok videos", + annotations=SCRAPE, + structured_output=False, + ) + async def tiktok_trending( + max_items: Annotated[ + int, + Field(ge=1, description="Max trending videos to return from Explore."), + ] = 20, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Get the current trending TikTok videos from the Explore feed. + + No input needed beyond how many to return; each video comes with caption, + author, stats, music, and its web URL. Example: max_items=30. + """ + return await run_scraper( + client, + context, + platform="tiktok", + verb="trending", + payload={"max_items": max_items}, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/mcp_server/selfcheck.py b/surfsense_mcp/mcp_server/selfcheck.py index e5ac6bd2b..126981e3c 100644 --- a/surfsense_mcp/mcp_server/selfcheck.py +++ b/surfsense_mcp/mcp_server/selfcheck.py @@ -24,6 +24,9 @@ EXPECTED_TOOLS = { "surfsense_youtube_scrape", "surfsense_youtube_comments", "surfsense_tiktok_scrape", + "surfsense_tiktok_comments", + "surfsense_tiktok_user_search", + "surfsense_tiktok_trending", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_list_scraper_runs", diff --git a/surfsense_web/content/docs/connectors/native/tiktok.mdx b/surfsense_web/content/docs/connectors/native/tiktok.mdx index 409df58bd..fc4cf99c3 100644 --- a/surfsense_web/content/docs/connectors/native/tiktok.mdx +++ b/surfsense_web/content/docs/connectors/native/tiktok.mdx @@ -1,19 +1,17 @@ --- title: TikTok -description: Scrape public TikTok videos by URL, profile, hashtag, or search +description: Scrape public TikTok videos, comments, accounts, and trending feeds --- -The TikTok scraper pulls structured public data from TikTok. Give it URLs (a video, a profile, a hashtag, or a search page) and/or profiles, hashtags, or search terms, and it returns videos (caption, author, play/like/comment/share counts, music, hashtags, timestamps, and the web URL). +The TikTok connector pulls structured public data from TikTok across four verbs: **scrape** (videos), **comments**, **user search** (accounts), and **trending**. Every verb returns `{ "items": [...] }` and is billed per returned item; surfaced errors (an `errorCode` field, e.g. a withheld feed) are never charged. -## Endpoint +## Scrape videos ```bash POST /api/v1/workspaces/{workspace_id}/scrapers/tiktok/scrape ``` -## Inputs - -At least one of `urls`, `profiles`, `hashtags`, or `search_queries` is required. +Give it URLs (a video, a profile, a hashtag, or a search page) and/or profiles, hashtags, or search terms; returns videos (caption, author, play/like/comment/share counts, music, hashtags, timestamps, and the web URL). At least one of `urls`, `profiles`, `hashtags`, or `search_queries` is required. | Field | Default | Description | |-------|---------|-------------| @@ -24,22 +22,76 @@ At least one of `urls`, `profiles`, `hashtags`, or `search_queries` is required. | `results_per_page` | `10` | Max videos per profile/hashtag/search target | | `max_items` | `10` | Max total videos returned across all sources (hard cap 100) | -## Example - ```bash curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/scrape" \ -H "Authorization: Bearer $SURFSENSE_API_KEY" \ -H "Content-Type: application/json" \ - -d '{ - "hashtags": ["food"], - "max_items": 25 - }' + -d '{ "hashtags": ["food"], "max_items": 25 }' ``` -The response is `{ "items": [...] }` — one item per video. Billing is per returned item. - -Video, hashtag, and search targets are the reliable paths. TikTok restricts the profile video endpoint for automated clients, so a `profiles` target can return no items even when the account is public — prefer hashtags, search, or a direct video URL. +Video and hashtag targets are the reliable video paths. A `profiles` target returns the account's **metadata** (name, followers, bio, verification) reliably, but TikTok often withholds its **video list** from automated clients — so a profile can return metadata with no videos. Keyword **video** search is login-walled and returns a surfaced error; to find accounts by keyword use **user search** below. -For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → TikTok → Scrape** in your workspace. +## Comments + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/tiktok/comments +``` + +Given TikTok video URLs, returns each video's public comment thread: comment text, author, like count, and reply count (replies carry `repliesToId`, the parent comment id). + +| Field | Default | Description | +|-------|---------|-------------| +| `video_urls` | — | TikTok video URLs (`/@/video/`), max 20 per call | +| `comments_per_video` | `20` | Max comments per video | +| `max_items` | `20` | Max total comments returned (hard cap 100) | + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/comments" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ "video_urls": ["https://www.tiktok.com/@nasa/video/123"], "max_items": 50 }' +``` + +## User search + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/tiktok/user_search +``` + +Finds public TikTok accounts by keyword — the reliable account-discovery path. Returns matching profiles with name, followers, bio, and verification. + +| Field | Default | Description | +|-------|---------|-------------| +| `queries` | — | Keywords to find accounts by, e.g. `["nasa", "cooking"]` (max 20) | +| `results_per_query` | `10` | Max accounts per query | +| `max_items` | `10` | Max total accounts returned (hard cap 100) | + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/user_search" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ "queries": ["space agency"], "max_items": 20 }' +``` + +## Trending + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/tiktok/trending +``` + +Returns the current trending videos from TikTok's Explore feed — no input needed beyond how many to return. Items use the same video shape as **scrape** and bill on the same per-video meter. + +| Field | Default | Description | +|-------|---------|-------------| +| `max_items` | `20` | Max trending videos returned (hard cap 100) | + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/trending" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ "max_items": 30 }' +``` + +For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → TikTok** in your workspace. diff --git a/surfsense_web/lib/connectors-marketing/tiktok.tsx b/surfsense_web/lib/connectors-marketing/tiktok.tsx index a25b71d61..f23b73865 100644 --- a/surfsense_web/lib/connectors-marketing/tiktok.tsx +++ b/surfsense_web/lib/connectors-marketing/tiktok.tsx @@ -8,7 +8,7 @@ export const tiktok: ConnectorPageContent = { metaTitle: "TikTok Scraper API for Trend and Creator Research | SurfSense", metaDescription: - "Scrape public TikTok videos by hashtag, search, or URL with the SurfSense TikTok Scraper API. No approval process or research-API gatekeeping, plus a free tier. Start now.", + "Scrape public TikTok videos, comments, accounts, and trending feeds by hashtag, search, or URL with the SurfSense TikTok Scraper API. No approval process or research-API gatekeeping, plus a free tier. Start now.", keywords: [ "tiktok scraper", "tiktok scraper api", @@ -17,6 +17,9 @@ export const tiktok: ConnectorPageContent = { "scrape tiktok", "tiktok data api", "tiktok hashtag scraper", + "tiktok comments scraper", + "tiktok trending scraper", + "tiktok user search", "tiktok trend tracking", "tiktok mcp", "social listening", @@ -52,7 +55,7 @@ export const tiktok: ConnectorPageContent = { }, extractIntro: - "Every call returns structured video items. Point the API at a hashtag, a search query, a creator profile, or a specific video URL.", + "Every call returns structured items. Scrape videos from a hashtag, search query, creator profile, or video URL — or switch verbs to pull a video's comments, discover accounts by keyword, or fetch the current trending feed.", extractFields: [ { label: "Videos", @@ -101,7 +104,7 @@ export const tiktok: ConnectorPageContent = { { title: "Campaign and sentiment tracking", description: - "Measure how a launch or branded hashtag spreads across TikTok — video count, reach, and engagement over time — and report the momentum, not a vanity view count.", + "Measure how a launch or branded hashtag spreads across TikTok — video count, reach, and engagement over time — then pull the comments on top videos to read how the audience actually reacts, not just a vanity view count.", }, ], @@ -134,7 +137,7 @@ export const tiktok: ConnectorPageContent = { { feature: "Agent-ready", official: "No; you build the harness yourself", - surfsense: "MCP server exposes tiktok.scrape as a native tool", + surfsense: "MCP server exposes scrape, comments, user search, and trending as native tools", }, ], }, @@ -262,7 +265,12 @@ export const tiktok: ConnectorPageContent = { { question: "Can I scrape a specific creator's videos?", answer: - "You can pass profiles or a profile URL, but TikTok restricts its profile video endpoint for automated clients, so a profile target can return no videos even for a public account. For reliable results, scrape by hashtag, by search query, or by a direct video URL.", + "Pass a profile or profile URL and you always get the account's metadata — name, followers, bio, verification. TikTok often withholds the profile video list from automated clients, so that list can come back empty even for a public account; for reliable video results, scrape by hashtag, by search query, or by a direct video URL.", + }, + { + question: "What TikTok data can I scrape?", + answer: + "Four verbs: scrape (videos by hashtag, search, profile, or URL), comments (a video's public comment thread), user search (find accounts by keyword — the reliable discovery path, since keyword video search is login-walled), and trending (the current Explore feed). Each returns structured items and is billed per item returned.", }, ], diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index 4503662f8..d739b9dc7 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -53,7 +53,16 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ id: "tiktok", label: "TikTok", icon: TikTokIcon, - verbs: [{ name: "tiktok.scrape", verb: "scrape", label: "Scrape" }], + verbs: [ + { name: "tiktok.scrape", verb: "scrape", label: "Scrape" }, + { name: "tiktok.comments", verb: "comments", label: "Comments" }, + { + name: "tiktok.user_search", + verb: "user_search", + label: "User Search", + }, + { name: "tiktok.trending", verb: "trending", label: "Trending" }, + ], }, { id: "google_maps", From bcfc3a5c3a79bd1688a476cd9d51da5c6702fb2a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 9 Jul 2026 20:04:30 +0200 Subject: [PATCH 37/56] test(capabilities): expect export_run in the agent tool manifest The run-reader set gained export_run (CSV export of a stored run), but this manifest assertion still expected only read_run/search_run. Add export_run so the agent-door test reflects the shipped readers. --- .../tests/unit/capabilities/access/test_agent_tools.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py b/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py index b2173292f..cdab68bc0 100644 --- a/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py +++ b/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py @@ -78,8 +78,14 @@ async def test_registry_becomes_one_tool_per_verb_plus_readers(isolate): tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=caps) by_name = {t.name: t for t in tools} - # One tool per verb, plus the two shared run-reader tools. - assert set(by_name) == {"web_scrape", "web_discover", "read_run", "search_run"} + # One tool per verb, plus the shared run-reader tools. + assert set(by_name) == { + "web_scrape", + "web_discover", + "read_run", + "search_run", + "export_run", + } assert by_name["web_scrape"].description == "web.scrape does a thing." assert by_name["web_scrape"].args_schema is _EchoInput From c679f2a3ef286162d3223fdce0cdb7c4b2ab3044 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 16:45:45 +0200 Subject: [PATCH 38/56] fix(tiktok): fetch profile video lists headful + dismiss login modal The profile feed (/api/post/item_list) returns an empty 200 to headless sessions but serves data headful on the same proxy IP. Run fetch_item_list headful and dismiss the login modal that blocks mid-scroll. --- .../platforms/tiktok/session/listing.py | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index e94984440..c1f3dfc15 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -11,8 +11,9 @@ The pure response-shape parsing lives in :func:`items_from_response`; this modul is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile -feed (``/api/post/item_list``) is additionally soft-blocked: TikTok returns an -empty 200 even to its own signed request, so profile targets yield nothing. +feed (``/api/post/item_list``) returns an empty 200 to headless sessions, so +:func:`fetch_item_list` runs headful. ponytail: headful needs a display — run it +under Xvfb on a headless server. """ from __future__ import annotations @@ -81,12 +82,33 @@ def _has_mstoken(page: Any) -> bool: return False +def _dismiss_login_modal(page: Any) -> None: + """Close the login modal that blocks scrolling; Escape as fallback. + + Only the modal's own close button, never a generic dialog button (avoids + clicking "Log in"). + """ + try: + closed = page.evaluate( + """() => { + const btn = document.querySelector('[data-e2e="modal-close-inner-button"]'); + if (btn) { btn.click(); return true; } + return false; + }""" + ) + if not closed: + page.keyboard.press("Escape") + except Exception: + pass + + def _scroll_page(page: Any, collected: list[dict[str, Any]], target_count: int) -> None: """Page down a listing feed until enough items are captured or it stops growing.""" last_height = 0 for _ in range(_SCROLL_MAX_ROUNDS): if len(collected) >= target_count: break + _dismiss_login_modal(page) page.evaluate("window.scrollTo(0, document.body.scrollHeight)") page.wait_for_timeout(_SCROLL_SETTLE_MS) height = page.evaluate("document.body.scrollHeight") @@ -207,12 +229,14 @@ def _fetch_sync( markers: tuple[str, ...], extract: ExtractFn, interact: InteractFn, + *, + headless: bool = True, ) -> list[dict[str, Any]]: collected: list[dict[str, Any]] = [] kwargs = build_stealthy_kwargs(get_stealth_config()) StealthyFetcher.fetch( url, - headless=True, + headless=headless, network_idle=False, proxy=get_proxy_url(), page_action=_build_page_action( @@ -224,7 +248,10 @@ def _fetch_sync( async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: - """Return up to ``target_count`` itemStructs from a listing page's XHRs.""" + """Return up to ``target_count`` itemStructs from a listing page's XHRs. + + Runs headful: the profile feed returns an empty 200 to headless sessions. + """ return await asyncio.to_thread( _fetch_sync, page_url, @@ -232,6 +259,7 @@ async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, An _ITEM_LIST_MARKERS, items_from_response, _scroll_page, + headless=False, ) From 44966315f1aade60eb2e843f58dcb0d4cdd93579 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 16:45:53 +0200 Subject: [PATCH 39/56] docs(tiktok): update e2e stage 2 notes for headful profile feed Profile videos now return via the headful item_list capture; reword the stage 2 comments to expect real videos, ErrorItem only as fallback. --- surfsense_backend/scripts/e2e_tiktok_scrape.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py index cb5a996eb..ad3337ef2 100644 --- a/surfsense_backend/scripts/e2e_tiktok_scrape.py +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -7,9 +7,8 @@ Run from the backend directory: What it exercises (everything REAL — live network, live proxy, live browser): Stage 1 — proxy egress proof (informational). - Stage 2 — profile via the full pipeline: TikTok soft-blocks the anonymous - profile feed, so this asserts graceful degradation — real videos OR - a single honest ErrorItem, never a silent empty. + Stage 2 — profile via the full pipeline: asserts real videos come back + (fetch_item_list runs headful), degrading to one ErrorItem if not. Stage 3 — blob video path over HTTP (URL taken from a captured hashtag struct). Stage 4 — hashtag listing via the stealth browser (captures item_list XHRs). Stage 5 — full scrape_tiktok() pipeline on a hashtag. @@ -107,10 +106,8 @@ async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]: _hr(f"STAGE 2 — profile listing graceful-degrade: @{_PROFILE}") from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok - # TikTok soft-blocks the profile feed for anonymous headless sessions - # (empty 200). The shipped contract is a single honest ErrorItem, not a - # silent empty. This stage verifies that graceful degradation, and passes if - # EITHER real videos come back (session got trusted) OR an ErrorItem does. + # fetch_item_list runs headful, so we expect real videos; still accept an + # ErrorItem (never a silent empty) to keep the graceful-degradation contract. items = await scrape_tiktok( TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=_COUNT), limit=_COUNT ) From 678450341454801f4f53550f83d44201d82f089a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:17:27 +0200 Subject: [PATCH 40/56] feat(crawl): add CRAWL_HEADED_XVFB_ENABLED flag Single switch promising an Xvfb display so the stealth browser can run headful (TikTok's profile feed is empty to headless Chromium). Defaults FALSE. --- surfsense_backend/app/config/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 85a913989..1a4d5efe7 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -1147,6 +1147,12 @@ class Config: # round-trip => default FALSE to honor the "no speed regression" bar; flip on # when leak-safety outweighs the marginal latency. CRAWL_DNS_OVER_HTTPS = os.getenv("CRAWL_DNS_OVER_HTTPS", "FALSE").upper() == "TRUE" + # Promises an Xvfb display so the browser can run headful (TikTok's profile + # feed is empty to headless Chromium). Entrypoint starts Xvfb when TRUE; + # FALSE keeps every browser headless. + CRAWL_HEADED_XVFB_ENABLED = ( + os.getenv("CRAWL_HEADED_XVFB_ENABLED", "FALSE").upper() == "TRUE" + ) # Litellm TTS Configuration TTS_SERVICE = os.getenv("TTS_SERVICE") From a551dd7553591ccd65487892b3290be0848d86ed Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:17:36 +0200 Subject: [PATCH 41/56] feat(docker): start Xvfb when CRAWL_HEADED_XVFB_ENABLED is set For every browser-running role (api/worker/all, not migrate) the entrypoint brings up a virtual display and exports DISPLAY, registering Xvfb for graceful shutdown. Off keeps browsers headless. --- .../scripts/docker/entrypoint.sh | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/surfsense_backend/scripts/docker/entrypoint.sh b/surfsense_backend/scripts/docker/entrypoint.sh index 2efd78f94..4e46d184a 100644 --- a/surfsense_backend/scripts/docker/entrypoint.sh +++ b/surfsense_backend/scripts/docker/entrypoint.sh @@ -121,7 +121,28 @@ start_beat() { echo " Celery Beat PID=${PIDS[-1]}" } +# ── Headful browser display ────────────────────────────────── +# Give the stealth browser a virtual display so it can run headful (TikTok's +# profile feed is empty to headless Chromium). Off => every browser stays headless. +start_xvfb() { + if [ "$(echo "${CRAWL_HEADED_XVFB_ENABLED:-false}" | tr '[:upper:]' '[:lower:]')" != "true" ]; then + return + fi + local display="${XVFB_DISPLAY:-:99}" + echo "Starting Xvfb on ${display} (CRAWL_HEADED_XVFB_ENABLED=true)..." + Xvfb "${display}" -screen 0 1920x1080x24 -nolisten tcp >/dev/null 2>&1 & + PIDS+=($!) + export DISPLAY="${display}" + sleep 1 # let the X server accept connections before a browser starts + echo " Xvfb PID=${PIDS[-1]} DISPLAY=${DISPLAY}" +} + # ── Main: run based on role ────────────────────────────────── +# migrate never launches a browser; every other role might, so start the display. +if [ "${SERVICE_ROLE}" != "migrate" ]; then + start_xvfb +fi + case "${SERVICE_ROLE}" in migrate) run_migrations From 6400dc5f0479d14c3681c7b90849a53ba0f8e5d9 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:17:45 +0200 Subject: [PATCH 42/56] fix(tiktok): gate headful profile feed on CRAWL_HEADED_XVFB_ENABLED fetch_item_list goes headful only when the flag promises a display; otherwise it stays headless so the browser launch never fails and the empty feed degrades to an ErrorItem. Verified under xvfb-run: real videos returned on a clean proxy IP, graceful degrade on a flagged one. --- .../proprietary/platforms/tiktok/session/listing.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index c1f3dfc15..dbdf83768 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -11,9 +11,9 @@ The pure response-shape parsing lives in :func:`items_from_response`; this modul is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile -feed (``/api/post/item_list``) returns an empty 200 to headless sessions, so -:func:`fetch_item_list` runs headful. ponytail: headful needs a display — run it -under Xvfb on a headless server. +feed returns an empty 200 to headless sessions, so :func:`fetch_item_list` goes +headful only when ``CRAWL_HEADED_XVFB_ENABLED`` promises an Xvfb display — else it +stays headless and degrades to an ``ErrorItem`` instead of crashing on launch. """ from __future__ import annotations @@ -25,6 +25,7 @@ from typing import Any from scrapling.fetchers import StealthyFetcher +from app.config import config from app.proprietary.web_crawler.stealth import ( build_stealthy_kwargs, get_stealth_config, @@ -250,7 +251,8 @@ def _fetch_sync( async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` itemStructs from a listing page's XHRs. - Runs headful: the profile feed returns an empty 200 to headless sessions. + Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display (the profile feed + is empty to headless sessions); headless otherwise so launch never fails. """ return await asyncio.to_thread( _fetch_sync, @@ -259,7 +261,7 @@ async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, An _ITEM_LIST_MARKERS, items_from_response, _scroll_page, - headless=False, + headless=not config.CRAWL_HEADED_XVFB_ENABLED, ) From d3a6e2cd45f864766b5bd9c997477283e5a335bd Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:17:53 +0200 Subject: [PATCH 43/56] docs(docker): note entrypoint wires xvfb at runtime The xvfb runtime path is now implemented, so drop the stale 'deferred to Slice B' note. --- surfsense_backend/Dockerfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/surfsense_backend/Dockerfile b/surfsense_backend/Dockerfile index fffe2e45d..7140f8809 100644 --- a/surfsense_backend/Dockerfile +++ b/surfsense_backend/Dockerfile @@ -31,11 +31,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ dos2unix \ git \ # ── Phase 3e stealth hardening ────────────────────────────────────────── - # Xvfb: virtual framebuffer so the stealth browser can run headful - # (headless=False) without a real display — many WAFs flag headless Chromium. - # Gated at runtime by CRAWL_HEADED_XVFB_ENABLED (Slice B); installed now so - # the image is ready. Real font packages make canvas/emoji/font-enumeration - # fingerprints resemble a real desktop (set proven against Kasada/Akamai). + # Xvfb: virtual display so the browser can run headful without real hardware + # (entrypoint starts it when CRAWL_HEADED_XVFB_ENABLED; used by TikTok's profile + # feed). Real fonts make canvas/emoji/font fingerprints look like a real desktop. xvfb \ fonts-noto-color-emoji \ fonts-unifont \ From c29b7f4b6bfdd20442d1f3b9292557ea8cbc752a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:17:53 +0200 Subject: [PATCH 44/56] docs(env): document CRAWL_HEADED_XVFB_ENABLED --- surfsense_backend/.env.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index cd250025c..c0e9ff5bd 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -401,6 +401,9 @@ TURNSTILE_SECRET_KEY= # Route DNS via Cloudflare DoH (anti DNS-leak). Adds a DNS round-trip => off by # default to avoid any speed regression; enable for leak-safety-first setups. # CRAWL_DNS_OVER_HTTPS=FALSE +# Run the browser headful on Xvfb — required for TikTok's profile video feed +# (empty to headless Chromium). Entrypoint starts Xvfb when TRUE. +# CRAWL_HEADED_XVFB_ENABLED=FALSE # File Parser Service ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING From 493579913ecab766dc626be785a542fd53bf3319 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:17:53 +0200 Subject: [PATCH 45/56] docs(tiktok): note stage 2 profile feed is headful only when flag is set --- surfsense_backend/scripts/e2e_tiktok_scrape.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py index ad3337ef2..33a5d8e57 100644 --- a/surfsense_backend/scripts/e2e_tiktok_scrape.py +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -7,8 +7,8 @@ Run from the backend directory: What it exercises (everything REAL — live network, live proxy, live browser): Stage 1 — proxy egress proof (informational). - Stage 2 — profile via the full pipeline: asserts real videos come back - (fetch_item_list runs headful), degrading to one ErrorItem if not. + Stage 2 — profile via the full pipeline: real videos when headful + (CRAWL_HEADED_XVFB_ENABLED), else one ErrorItem — never silent empty. Stage 3 — blob video path over HTTP (URL taken from a captured hashtag struct). Stage 4 — hashtag listing via the stealth browser (captures item_list XHRs). Stage 5 — full scrape_tiktok() pipeline on a hashtag. @@ -106,8 +106,8 @@ async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]: _hr(f"STAGE 2 — profile listing graceful-degrade: @{_PROFILE}") from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok - # fetch_item_list runs headful, so we expect real videos; still accept an - # ErrorItem (never a silent empty) to keep the graceful-degradation contract. + # Headful (flag on) yields real videos; still accept an ErrorItem (never a + # silent empty) so the graceful-degradation contract holds either way. items = await scrape_tiktok( TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=_COUNT), limit=_COUNT ) From 15fb50c83e68751daee1aede8d4130e01b9a25c5 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:25:48 +0200 Subject: [PATCH 46/56] feat(tiktok): add TIKTOK_LISTING_MAX_ATTEMPTS knob Bounds the browser-listing retry-on-empty; default 3, set 1 for a static IP. --- surfsense_backend/app/config/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 1a4d5efe7..528f6cc9d 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -728,6 +728,12 @@ class Config: # Comments are the cheapest per-item TikTok data, matching the per-comment # market (and YouTube's comment meter). TIKTOK_MICROS_PER_COMMENT = int(os.getenv("TIKTOK_MICROS_PER_COMMENT", "1500")) + # Browser-listing retries when a feed comes back empty. The profile feed is + # withheld from flagged exit IPs; since the proxy rotates per request, each + # retry lands a fresh IP, turning a bad first draw into a hit. Only spends on + # empty results. ponytail: assumes a rotating proxy — set to 1 for a static IP, + # where retrying just re-hits the same (flagged) exit. + TIKTOK_LISTING_MAX_ATTEMPTS = int(os.getenv("TIKTOK_LISTING_MAX_ATTEMPTS", "3")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. From c8fcf87685c0e7522dd6b38ea4081bcb61b91f01 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:25:48 +0200 Subject: [PATCH 47/56] fix(tiktok): retry empty item_list captures on a fresh exit IP The profile feed is withheld from flagged IPs and the proxy rotates per request, so re-fetching an empty capture (up to TIKTOK_LISTING_MAX_ATTEMPTS) turns a bad first draw into a hit instead of an ErrorItem. --- .../platforms/tiktok/session/listing.py | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index dbdf83768..5668c497a 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -253,16 +253,32 @@ async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, An Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display (the profile feed is empty to headless sessions); headless otherwise so launch never fails. + + Retries on an empty capture up to ``TIKTOK_LISTING_MAX_ATTEMPTS``: the feed is + withheld from flagged exit IPs, and the proxy rotates per request, so each retry + is a fresh IP — turning a bad first draw into a hit instead of an ``ErrorItem``. """ - return await asyncio.to_thread( - _fetch_sync, - page_url, - target_count, - _ITEM_LIST_MARKERS, - items_from_response, - _scroll_page, - headless=not config.CRAWL_HEADED_XVFB_ENABLED, - ) + headless = not config.CRAWL_HEADED_XVFB_ENABLED + attempts = max(1, config.TIKTOK_LISTING_MAX_ATTEMPTS) + for attempt in range(1, attempts + 1): + items = await asyncio.to_thread( + _fetch_sync, + page_url, + target_count, + _ITEM_LIST_MARKERS, + items_from_response, + _scroll_page, + headless=headless, + ) + if items or attempt == attempts: + return items + logger.info( + "[tiktok] empty item_list for %s (attempt %d/%d); retrying on a fresh exit IP", + page_url, + attempt, + attempts, + ) + return [] async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]: From 8f825b7b3c262aa302fb3155d125a761f618069f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:25:48 +0200 Subject: [PATCH 48/56] test(tiktok): cover item_list retry-on-empty loop --- .../platforms/tiktok/test_listing_retry.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_listing_retry.py diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_listing_retry.py b/surfsense_backend/tests/unit/platforms/tiktok/test_listing_retry.py new file mode 100644 index 000000000..2676a2267 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_listing_retry.py @@ -0,0 +1,60 @@ +"""Retry-on-empty for the browser ``item_list`` seam (no browser, fake fetch). + +``fetch_item_list`` re-fetches an empty capture up to ``TIKTOK_LISTING_MAX_ATTEMPTS`` +so a flagged rotating exit IP on the first draw doesn't collapse straight to an +``ErrorItem``. These drive that loop deterministically by faking ``_fetch_sync``. +""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.session import listing + + +class _Fake: + """Returns each queued result once, repeating the last; counts calls.""" + + def __init__(self, results: list[list[dict]]): + self.results = results + self.calls = 0 + + def __call__(self, *args, **kwargs): + out = self.results[min(self.calls, len(self.results) - 1)] + self.calls += 1 + return out + + +def _patch(monkeypatch, fake: _Fake, attempts: int) -> None: + monkeypatch.setattr(listing, "_fetch_sync", fake) + monkeypatch.setattr(listing.config, "TIKTOK_LISTING_MAX_ATTEMPTS", attempts) + + +async def test_returns_first_nonempty_without_retrying(monkeypatch): + fake = _Fake([[{"id": "1"}]]) + _patch(monkeypatch, fake, 3) + items = await listing.fetch_item_list("https://tt/@x", 5) + assert items == [{"id": "1"}] + assert fake.calls == 1 # a draw with items never retries + + +async def test_retries_past_empty_draws_then_hits(monkeypatch): + fake = _Fake([[], [], [{"id": "9"}]]) + _patch(monkeypatch, fake, 3) + items = await listing.fetch_item_list("https://tt/@x", 5) + assert items == [{"id": "9"}] + assert fake.calls == 3 # two empty (flagged-IP) draws retried, third lands + + +async def test_stops_at_attempt_ceiling_when_always_empty(monkeypatch): + fake = _Fake([[]]) + _patch(monkeypatch, fake, 3) + items = await listing.fetch_item_list("https://tt/@x", 5) + assert items == [] + assert fake.calls == 3 # capped; caller then emits the ErrorItem + + +async def test_single_attempt_config_disables_retry(monkeypatch): + fake = _Fake([[]]) + _patch(monkeypatch, fake, 1) + items = await listing.fetch_item_list("https://tt/@x", 5) + assert items == [] + assert fake.calls == 1 # static-IP setups opt out via attempts=1 From 97e250be4b1cdeed31d1ae4b5fc3708e64e0eaf7 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:25:48 +0200 Subject: [PATCH 49/56] docs(env): document TIKTOK_LISTING_MAX_ATTEMPTS --- surfsense_backend/.env.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index c0e9ff5bd..d94acd14e 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -293,6 +293,9 @@ MICROS_PER_PAGE=1000 # TIKTOK_MICROS_PER_VIDEO=3500 # TIKTOK_MICROS_PER_USER=2500 # TIKTOK_MICROS_PER_COMMENT=1500 +# Browser-listing retries when a feed is empty (profile feed is withheld from +# flagged IPs; each retry draws a fresh rotating exit IP). Set to 1 for a static IP. +# TIKTOK_LISTING_MAX_ATTEMPTS=3 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. CREDIT_LOW_BALANCE_WARNING_MICROS=500000 From 01bc3f8de3009f492271f657453fbe5155c0ac84 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:28:47 +0200 Subject: [PATCH 50/56] docs(config): trim narration from xvfb + listing-retry comments --- surfsense_backend/app/config/__init__.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 528f6cc9d..00835dc65 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -728,11 +728,8 @@ class Config: # Comments are the cheapest per-item TikTok data, matching the per-comment # market (and YouTube's comment meter). TIKTOK_MICROS_PER_COMMENT = int(os.getenv("TIKTOK_MICROS_PER_COMMENT", "1500")) - # Browser-listing retries when a feed comes back empty. The profile feed is - # withheld from flagged exit IPs; since the proxy rotates per request, each - # retry lands a fresh IP, turning a bad first draw into a hit. Only spends on - # empty results. ponytail: assumes a rotating proxy — set to 1 for a static IP, - # where retrying just re-hits the same (flagged) exit. + # Retry an empty listing draw on a fresh rotating IP. Set to 1 for a static + # proxy, where every retry re-hits the same exit. TIKTOK_LISTING_MAX_ATTEMPTS = int(os.getenv("TIKTOK_LISTING_MAX_ATTEMPTS", "3")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service @@ -1154,8 +1151,7 @@ class Config: # when leak-safety outweighs the marginal latency. CRAWL_DNS_OVER_HTTPS = os.getenv("CRAWL_DNS_OVER_HTTPS", "FALSE").upper() == "TRUE" # Promises an Xvfb display so the browser can run headful (TikTok's profile - # feed is empty to headless Chromium). Entrypoint starts Xvfb when TRUE; - # FALSE keeps every browser headless. + # feed is empty to headless Chromium). Off keeps every browser headless. CRAWL_HEADED_XVFB_ENABLED = ( os.getenv("CRAWL_HEADED_XVFB_ENABLED", "FALSE").upper() == "TRUE" ) From f74a73efdbd03ad8512112cfbda66f259fdf80f8 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:28:47 +0200 Subject: [PATCH 51/56] docs(tiktok): trim fetch_item_list retry docstring to intent --- .../app/proprietary/platforms/tiktok/session/listing.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index 5668c497a..bc8af56ff 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -253,10 +253,7 @@ async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, An Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display (the profile feed is empty to headless sessions); headless otherwise so launch never fails. - - Retries on an empty capture up to ``TIKTOK_LISTING_MAX_ATTEMPTS``: the feed is - withheld from flagged exit IPs, and the proxy rotates per request, so each retry - is a fresh IP — turning a bad first draw into a hit instead of an ``ErrorItem``. + Retries an empty draw up to ``TIKTOK_LISTING_MAX_ATTEMPTS`` for a fresh exit IP. """ headless = not config.CRAWL_HEADED_XVFB_ENABLED attempts = max(1, config.TIKTOK_LISTING_MAX_ATTEMPTS) From 9793751b44fa5fa2ff7dd6365c2e83a1e4623ec2 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:28:47 +0200 Subject: [PATCH 52/56] docs(tiktok): trim stage 2 comment to the why --- surfsense_backend/scripts/e2e_tiktok_scrape.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py index 33a5d8e57..b399ca3fb 100644 --- a/surfsense_backend/scripts/e2e_tiktok_scrape.py +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -106,8 +106,8 @@ async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]: _hr(f"STAGE 2 — profile listing graceful-degrade: @{_PROFILE}") from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok - # Headful (flag on) yields real videos; still accept an ErrorItem (never a - # silent empty) so the graceful-degradation contract holds either way. + # Accept videos OR an ErrorItem (never a silent empty): the feed degrades + # gracefully when the exit IP is flagged. items = await scrape_tiktok( TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=_COUNT), limit=_COUNT ) From cd82f861dec7c6252bd01ebecf9e8830488f29eb Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:40:04 +0200 Subject: [PATCH 53/56] fix(tiktok-agent): stop steering the subagent to keyword search for videos The playbook listed search_queries as a video source in three places, then contradicted itself. Replace with action-oriented guidance: videos via hashtags/URLs, accounts via user_search. Drop 'login-walled' (the agent has no login path). --- .../subagents/builtins/tiktok/system_prompt.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md index 5b6de993c..03114cd12 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md @@ -6,7 +6,7 @@ Answer the delegated question from live TikTok data gathered with your verb, com -- `tiktok_scrape` — videos from urls / profiles / hashtags / search_queries +- `tiktok_scrape` — videos from a hashtag, a profile, or a TikTok URL - `tiktok_comments` — a video's comment thread, from `video_urls` - `tiktok_user_search` — find accounts by keyword, from `queries` - `tiktok_trending` — the current Explore trending-video feed @@ -14,15 +14,15 @@ Answer the delegated question from live TikTok data gathered with your verb, com -- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#') and/or `search_queries`. +- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#'), or pass a TikTok URL in `urls`. - Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`. -- Profiles: a creator's `profiles` feed returns the account's metadata (name, followers, bio, verification) reliably, but its video list is often withheld by TikTok — treat an empty video list as a known limit, not a failure to retry endlessly. Prefer `hashtags`, `search_queries`, or a direct video URL for videos. +- Profiles: a creator's `profiles` feed returns the account's metadata (name, followers, bio, verification) reliably, but its video list is often withheld by TikTok — treat an empty video list as a known limit, not a failure to retry endlessly. Prefer `hashtags` or a direct video URL for videos. - Comments on a video: call `tiktok_comments` with the video URL(s) in `video_urls`. -- Finding accounts (not videos): call `tiktok_user_search` with `queries` — this is the reliable path for account discovery (keyword *video* search is login-walled). +- Finding accounts by keyword: call `tiktok_user_search` with `queries`. Keyword search returns no videos, so do not use `search_queries` for videos — use `hashtags` or a video URL. - "What's trending now": call `tiktok_trending` (no query needed); set `max_items` for how many. - Controlling volume: use `max_items` for the total cap and `results_per_page` per target (per-verb equivalents: `comments_per_video`, `results_per_query`). - Requested counts: `max_items` defaults low — when the task asks for N items, set `max_items` (and the per-target count) above N. A call that caps below the target can never satisfy it. -- Batch multiple hashtags, search terms, queries, or video URLs into one call rather than many single-item calls. +- Batch multiple hashtags, queries, or video URLs into one call rather than many single-item calls. - Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, count changes). From 1911b783792e8c71c1df3e01f7d0354ab34220b7 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:40:04 +0200 Subject: [PATCH 54/56] fix(tiktok-agent): drop keyword search from the routing description's video sources --- .../multi_agent_chat/subagents/builtins/tiktok/description.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md index d57c551d4..750979348 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md @@ -1,2 +1,2 @@ -TikTok specialist: pulls structured public TikTok data — videos (caption/text, author, play/like/comment/share counts, music, hashtags, timestamps, web URL) from a hashtag feed, a search query, a creator profile, or a known video URL; a video's public comment thread; accounts found by keyword; and the current Explore trending-video feed. Also compares fresh TikTok results against earlier findings in this chat. +TikTok specialist: pulls structured public TikTok data — videos (caption/text, author, play/like/comment/share counts, music, hashtags, timestamps, web URL) from a hashtag feed, a creator profile, or a known video URL; a video's public comment thread; accounts found by keyword; and the current Explore trending-video feed. Also compares fresh TikTok results against earlier findings in this chat. Use whenever the task is to find what is trending or being said on TikTok about a topic, gather a creator's or hashtag's videos, read a video's comments, discover accounts by keyword, or scrape a specific video URL. Triggers include "search TikTok for X", "what's trending on TikTok", "videos with #X", "comments on this TikTok", "find TikTok accounts about X", and "scrape this TikTok video". Not for general web pages (use the web crawling specialist), Google results (use the Google Search specialist), Reddit (use the Reddit specialist), or YouTube (use the YouTube specialist). From 6a0ff1495fbcad084e161e6371b9f3ae7eb9a644 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:40:04 +0200 Subject: [PATCH 55/56] fix(tiktok-mcp): flag that search_queries returns no videos in the scrape tool --- .../features/scrapers/platforms/tiktok.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py index aac8efc0e..c295ec3b6 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py @@ -21,7 +21,7 @@ def register( @mcp.tool( name="surfsense_tiktok_scrape", - title="Search or scrape TikTok", + title="Scrape TikTok videos", annotations=SCRAPE, structured_output=False, ) @@ -51,7 +51,11 @@ def register( ] = None, search_queries: Annotated[ list[str] | None, - Field(description="Terms to search TikTok for, e.g. ['cooking']."), + Field( + description="Keyword search terms. Returns no videos — use " + "hashtags/profiles/urls for videos, or the user-search tool for " + "accounts." + ), ] = None, results_per_page: Annotated[ int, @@ -63,12 +67,13 @@ def register( workspace: WorkspaceParam = None, response_format: ResponseFormatParam = "markdown", ) -> str: - """Search or scrape public TikTok videos. + """Scrape public TikTok videos by hashtag, profile, or URL. - Use this for ANY TikTok research — a creator's videos, a hashtag feed, - a search, or a specific video URL — instead of a generic web search. - Returns videos with text, author, stats, music, and the web URL. - Example: hashtags=['food'], max_items=20. + Use for TikTok video research — a creator's videos, a hashtag feed, or a + specific video/profile/hashtag URL — instead of a generic web search. + Returns videos with text, author, stats, music, and the web URL. For + accounts by keyword use the user-search tool; keyword search returns no + videos. Example: hashtags=['food'], max_items=20. """ return await run_scraper( client, From 903502296ef0f084aebe6c7639d4ea1d75c8359b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:46:08 +0200 Subject: [PATCH 56/56] fix(tiktok-marketing): stop advertising keyword search as a video source The hero, intro, use case, API-reference fields, and two FAQs all listed search as a video path, contradicting the FAQ that admits keyword video search is login-walled. Reconcile every claim to one truth: videos via hashtag/profile/URL; keyword search returns no videos; keyword -> user_search for accounts. --- .../lib/connectors-marketing/tiktok.tsx | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/surfsense_web/lib/connectors-marketing/tiktok.tsx b/surfsense_web/lib/connectors-marketing/tiktok.tsx index f23b73865..d77865633 100644 --- a/surfsense_web/lib/connectors-marketing/tiktok.tsx +++ b/surfsense_web/lib/connectors-marketing/tiktok.tsx @@ -8,7 +8,7 @@ export const tiktok: ConnectorPageContent = { metaTitle: "TikTok Scraper API for Trend and Creator Research | SurfSense", metaDescription: - "Scrape public TikTok videos, comments, accounts, and trending feeds by hashtag, search, or URL with the SurfSense TikTok Scraper API. No approval process or research-API gatekeeping, plus a free tier. Start now.", + "Scrape public TikTok videos, comments, accounts, and trending feeds by hashtag, profile, or URL with the SurfSense TikTok Scraper API. No approval process or research-API gatekeeping, plus a free tier. Start now.", keywords: [ "tiktok scraper", "tiktok scraper api", @@ -29,7 +29,7 @@ export const tiktok: ConnectorPageContent = { h1: "TikTok Scraper API for Trend and Creator Research", heroLede: - "The SurfSense TikTok API extracts public videos by hashtag, search, or URL without TikTok's approval-gated Research API. Give your AI agents a live feed of what your market watches and shares, so you catch a trend while it is still rising.", + "The SurfSense TikTok API extracts public videos by hashtag, creator profile, or URL without TikTok's approval-gated Research API. Give your AI agents a live feed of what your market watches and shares, so you catch a trend while it is still rising.", transcript: { prompt: "Find trending TikToks about meal prep this week", @@ -55,7 +55,7 @@ export const tiktok: ConnectorPageContent = { }, extractIntro: - "Every call returns structured items. Scrape videos from a hashtag, search query, creator profile, or video URL — or switch verbs to pull a video's comments, discover accounts by keyword, or fetch the current trending feed.", + "Every call returns structured items. Scrape videos from a hashtag, creator profile, or video URL — or switch verbs to pull a video's comments, discover accounts by keyword, or fetch the current trending feed.", extractFields: [ { label: "Videos", @@ -89,7 +89,7 @@ export const tiktok: ConnectorPageContent = { { title: "Trend and hashtag monitoring", description: - "Track a hashtag or search term and feed the stream to an agent that flags breakout videos, rising sounds, and formats before they saturate. Catch the wave while it is still rising, not after.", + "Track a hashtag and feed the stream to an agent that flags breakout videos, rising sounds, and formats before they saturate. Catch the wave while it is still rising, not after.", }, { title: "Creator and influencer discovery", @@ -161,7 +161,7 @@ export const tiktok: ConnectorPageContent = { type: "string[]", defaultValue: "[]", description: - "TikTok URLs to scrape: a video, a profile (/@user), a hashtag (/tag/name), or a search URL. Max 20.", + "TikTok URLs to scrape: a video, a profile (/@user), or a hashtag (/tag/name). Max 20.", }, { name: "profiles", @@ -180,13 +180,13 @@ export const tiktok: ConnectorPageContent = { type: "string[]", defaultValue: "[]", description: - "Search terms to run on TikTok. Each returns up to results_per_page videos. Max 20.", + "Keyword search terms. Keyword video search is login-walled and returns no videos — use hashtags/profiles/urls for videos, or user_search for accounts. Max 20.", }, { name: "results_per_page", type: "integer", defaultValue: "10", - description: "Max videos to pull per profile, hashtag, or search target. 1 to 100.", + description: "Max videos to pull per profile or hashtag target. 1 to 100.", }, { name: "max_items", @@ -260,17 +260,17 @@ export const tiktok: ConnectorPageContent = { { question: "What are the rate limits?", answer: - "Each call caps at 100 returned videos across all sources, with up to 20 hashtags, searches, profiles, or URLs per request. SurfSense manages the underlying request budget and request signing for you.", + "Each call caps at 100 returned videos across all sources, with up to 20 hashtags, profiles, or URLs per request. SurfSense manages the underlying request budget and request signing for you.", }, { question: "Can I scrape a specific creator's videos?", answer: - "Pass a profile or profile URL and you always get the account's metadata — name, followers, bio, verification. TikTok often withholds the profile video list from automated clients, so that list can come back empty even for a public account; for reliable video results, scrape by hashtag, by search query, or by a direct video URL.", + "Pass a profile or profile URL and you always get the account's metadata — name, followers, bio, verification. TikTok often withholds the profile video list from automated clients, so that list can come back empty even for a public account; for reliable video results, scrape by hashtag or by a direct video URL.", }, { question: "What TikTok data can I scrape?", answer: - "Four verbs: scrape (videos by hashtag, search, profile, or URL), comments (a video's public comment thread), user search (find accounts by keyword — the reliable discovery path, since keyword video search is login-walled), and trending (the current Explore feed). Each returns structured items and is billed per item returned.", + "Four verbs: scrape (videos by hashtag, profile, or URL), comments (a video's public comment thread), user search (find accounts by keyword — the reliable discovery path, since keyword video search is login-walled), and trending (the current Explore feed). Each returns structured items and is billed per item returned.", }, ],