diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py index b132618d7..c396fceaf 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py @@ -36,6 +36,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = { "google_maps": frozenset(), "google_search": frozenset(), "reddit": frozenset(), + "tiktok": frozenset(), "mcp_discovery": frozenset( { "SLACK_CONNECTOR", diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py new file mode 100644 index 000000000..ec7d5955f --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py @@ -0,0 +1 @@ +"""``tiktok`` builtin subagent: structured public TikTok videos and listings.""" diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py new file mode 100644 index 000000000..2c7dd014b --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py @@ -0,0 +1,43 @@ +"""``tiktok`` route: ``SurfSenseSubagentSpec`` builder for deepagents.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.language_models import BaseChatModel +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) + +from .tools.index import NAME, RULESET, load_tools + + +def build_subagent( + *, + dependencies: dict[str, Any], + model: BaseChatModel | None = None, + middleware_stack: dict[str, Any] | None = None, + mcp_tools: list[BaseTool] | None = None, +) -> SurfSenseSubagentSpec: + tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] + description = ( + read_md_file(__package__, "description").strip() + or "Pulls structured data from public TikTok videos, hashtags, and searches." + ) + system_prompt = read_md_file(__package__, "system_prompt").strip() + return pack_subagent( + name=NAME, + description=description, + system_prompt=system_prompt, + tools=tools, + ruleset=RULESET, + dependencies=dependencies, + model=model, + middleware_stack=middleware_stack, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md new file mode 100644 index 000000000..fc13d864d --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md @@ -0,0 +1,2 @@ +TikTok specialist: pulls structured public TikTok data — videos (caption/text, author, play/like/comment/share counts, music, hashtags, timestamps, web URL) from a hashtag feed, a search query, a creator profile, or a known video URL. Also compares fresh TikTok results against earlier findings in this chat. +Use whenever the task is to find what is trending or being said on TikTok about a topic, gather a creator's or hashtag's videos, or scrape a specific video URL. Triggers include "search TikTok for X", "trending TikTok videos about X", "videos with #X", and "scrape this TikTok video". Not for general web pages (use the web crawling specialist), Google results (use the Google Search specialist), Reddit (use the Reddit specialist), or YouTube (use the YouTube specialist). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md new file mode 100644 index 000000000..49c20164e --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md @@ -0,0 +1,64 @@ +You are the SurfSense TikTok sub-agent. +You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis. + + +Answer the delegated question from live TikTok data gathered with your verb, comparing against earlier results already in this conversation when the task calls for it. + + + +- `tiktok_scrape` +- `read_run` / `search_run` (free readers for stored scrape output) + + + +- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#') and/or `search_queries`. +- Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`. +- Profiles: a creator's `profiles` feed can come back empty — TikTok restricts the profile video endpoint. Prefer `hashtags`, `search_queries`, or a direct video URL, and treat an empty profile result as a known limit, not a failure to retry endlessly. +- Controlling volume: use `max_items` for the total cap and `results_per_page` per target. +- Requested counts: `max_items` defaults to only 10 — when the task asks for N videos, set `max_items` and `results_per_page` above N. A call that caps below the target can never satisfy it. +- Batch multiple hashtags or search terms into one call rather than many single-term calls. + +- Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, count changes). + + + +- Use only tools in ``. +- Report only results present in the tool output. Never invent captions, URLs, authors, or counts. + + + +- Do not read arbitrary web pages — that belongs to the web crawling specialist. +- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on. +- Reddit belongs to the Reddit specialist; YouTube belongs to the YouTube specialist; Google results belong to the Google Search specialist. + + + +- Report uncertainty explicitly when evidence is incomplete or conflicting. +- Never present unverified claims as facts. + + + +- Underspecified request — no usable hashtag, query, or URL — return `status=blocked` with the missing fields. +- Tool failure: return `status=error` with a concise recovery `next_step`. +- No useful evidence: return `status=blocked` with a narrower query or the scope you still need. + + + +Return **only** one JSON object (no markdown/prose): +{ + "status": "success" | "partial" | "blocked" | "error", + "action_summary": string, + "evidence": { + "findings": string[], + "sources": string[], + "confidence": "high" | "medium" | "low" + }, + "next_step": string | null, + "missing_fields": string[] | null, + "assumptions": string[] | null +} + +Route-specific rules: +- `evidence.findings`: one entry per distinct video or delta — a single sentence each; do not paste raw payloads. Max 10 entries, unless the delegated task asks for N items: then return up to N (each backed by a real scraped result, never padded). +- `evidence.sources`: one TikTok URL per finding when applicable, same cap as findings. List each URL once. + diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py new file mode 100644 index 000000000..e790d44f0 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py @@ -0,0 +1,27 @@ +"""``tiktok`` sub-agent tools: the TikTok scrape capability verb.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset +from app.capabilities.core.access.agent import build_capability_tools +from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE + +NAME = "tiktok" + +RULESET = Ruleset(origin=NAME, rules=[]) + +_CI_VERBS = [TIKTOK_SCRAPE] + + +def load_tools( + *, dependencies: dict[str, Any] | None = None, **kwargs: Any +) -> list[BaseTool]: + d = {**(dependencies or {}), **kwargs} + return build_capability_tools( + workspace_id=d.get("workspace_id"), + capabilities=_CI_VERBS, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index 34895a514..71c2e6f3d 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -33,6 +33,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.memory.agent import ( from app.agents.chat.multi_agent_chat.subagents.builtins.reddit.agent import ( build_subagent as build_reddit_subagent, ) +from app.agents.chat.multi_agent_chat.subagents.builtins.tiktok.agent import ( + build_subagent as build_tiktok_subagent, +) from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import ( build_subagent as build_web_crawler_subagent, ) @@ -84,6 +87,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { "memory": build_memory_subagent, "onedrive": build_onedrive_subagent, "reddit": build_reddit_subagent, + "tiktok": build_tiktok_subagent, "web_crawler": build_web_crawler_subagent, "youtube": build_youtube_subagent, } diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py index c3ad04250..66e8f28db 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py @@ -37,6 +37,7 @@ _EXPECTED_SUBAGENTS = frozenset( "memory", "onedrive", "reddit", + "tiktok", "web_crawler", "youtube", } diff --git a/surfsense_mcp/mcp_server/features/scrapers/__init__.py b/surfsense_mcp/mcp_server/features/scrapers/__init__.py index dfa2f3ab2..c0e7dea07 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/__init__.py +++ b/surfsense_mcp/mcp_server/features/scrapers/__init__.py @@ -13,9 +13,9 @@ from mcp.server.fastmcp import FastMCP from ...core.client import SurfSenseClient from ...core.workspace_context import WorkspaceContext from . import run_history -from .platforms import google_maps, google_search, reddit, web, youtube +from .platforms import google_maps, google_search, reddit, tiktok, web, youtube -_REGISTRARS = (web, google_search, reddit, youtube, google_maps, run_history) +_REGISTRARS = (web, google_search, reddit, youtube, tiktok, google_maps, run_history) def register( diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py new file mode 100644 index 000000000..1e5f472ee --- /dev/null +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py @@ -0,0 +1,88 @@ +"""TikTok scraper tool.""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the TikTok tool.""" + + @mcp.tool( + name="surfsense_tiktok_scrape", + title="Search or scrape TikTok", + annotations=SCRAPE, + structured_output=False, + ) + async def tiktok_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="TikTok URLs: a video, a profile " + "('https://www.tiktok.com/@nasa'), a hashtag " + "('https://www.tiktok.com/tag/food'), or a search URL. Provide " + "urls OR profiles/hashtags/search_queries." + ), + ] = None, + profiles: Annotated[ + list[str] | None, + Field( + description="Profile usernames to scrape, with or without a " + "leading '@', e.g. ['nasa']." + ), + ] = None, + hashtags: Annotated[ + list[str] | None, + Field( + description="Hashtag names to scrape, without the '#', e.g. " + "['food']." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field(description="Terms to search TikTok for, e.g. ['cooking']."), + ] = None, + results_per_page: Annotated[ + int, + Field(ge=1, description="Max videos per profile/hashtag/search target."), + ] = 10, + max_items: Annotated[ + int, Field(ge=1, description="Maximum videos to return in total.") + ] = 10, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Search or scrape public TikTok videos. + + Use this for ANY TikTok research — a creator's videos, a hashtag feed, + a search, or a specific video URL — instead of a generic web search. + Returns videos with text, author, stats, music, and the web URL. + Example: hashtags=['food'], max_items=20. + """ + return await run_scraper( + client, + context, + platform="tiktok", + verb="scrape", + payload={ + "urls": urls, + "profiles": profiles, + "hashtags": hashtags, + "search_queries": search_queries, + "results_per_page": results_per_page, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/mcp_server/selfcheck.py b/surfsense_mcp/mcp_server/selfcheck.py index 20224c173..e5ac6bd2b 100644 --- a/surfsense_mcp/mcp_server/selfcheck.py +++ b/surfsense_mcp/mcp_server/selfcheck.py @@ -23,6 +23,7 @@ EXPECTED_TOOLS = { "surfsense_reddit_scrape", "surfsense_youtube_scrape", "surfsense_youtube_comments", + "surfsense_tiktok_scrape", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_list_scraper_runs", diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index 9b508228a..4503662f8 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -3,6 +3,7 @@ import { GoogleMapsIcon, GoogleSearchIcon, RedditIcon, + TikTokIcon, WebIcon, YouTubeIcon, } from "./platform-icons"; @@ -48,6 +49,12 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ { name: "youtube.comments", verb: "comments", label: "Comments" }, ], }, + { + id: "tiktok", + label: "TikTok", + icon: TikTokIcon, + verbs: [{ name: "tiktok.scrape", verb: "scrape", label: "Scrape" }], + }, { id: "google_maps", label: "Google Maps", diff --git a/surfsense_web/lib/playground/platform-icons.tsx b/surfsense_web/lib/playground/platform-icons.tsx index e7a443c2f..474bff317 100644 --- a/surfsense_web/lib/playground/platform-icons.tsx +++ b/surfsense_web/lib/playground/platform-icons.tsx @@ -24,6 +24,7 @@ function brandIcon(src: string, alt: string) { export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit"); export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube"); +export const TikTokIcon = brandIcon("/connectors/tiktok.svg", "TikTok"); export const GoogleMapsIcon = brandIcon("/connectors/google-maps.svg", "Google Maps"); export const GoogleSearchIcon = brandIcon("/connectors/google-search.svg", "Google Search"); export const WebIcon = brandIcon("/connectors/web.svg", "Web"); diff --git a/surfsense_web/public/connectors/tiktok.svg b/surfsense_web/public/connectors/tiktok.svg new file mode 100644 index 000000000..c3c3fe06e --- /dev/null +++ b/surfsense_web/public/connectors/tiktok.svg @@ -0,0 +1,6 @@ + + + + + +