Merge remote-tracking branch 'upstream/dev' into feat/instagram-scraper

surfsense_backend/app/capabilities/core/billing.py

@@ -37,6 +37,9 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = {
     BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT",
     BillingUnit.INSTAGRAM_ITEM: "INSTAGRAM_SCRAPE_MICROS_PER_ITEM",
     BillingUnit.INSTAGRAM_COMMENT: "INSTAGRAM_SCRAPE_MICROS_PER_COMMENT",
+    BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO",
+    BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER",
+    BillingUnit.TIKTOK_COMMENT: "TIKTOK_MICROS_PER_COMMENT",
 }
 
 
@@ -55,6 +58,9 @@ _UNIT_NOUNS: dict[BillingUnit, str] = {
     BillingUnit.YOUTUBE_COMMENT: "comment",
     BillingUnit.INSTAGRAM_ITEM: "item",
     BillingUnit.INSTAGRAM_COMMENT: "comment",
+    BillingUnit.TIKTOK_VIDEO: "video",
+    BillingUnit.TIKTOK_USER: "profile",
+    BillingUnit.TIKTOK_COMMENT: "comment",
 }
 
 

surfsense_backend/app/capabilities/core/types.py

@@ -27,6 +27,9 @@ class BillingUnit(StrEnum):
     YOUTUBE_COMMENT = "youtube_comment"
     INSTAGRAM_ITEM = "instagram_item"
     INSTAGRAM_COMMENT = "instagram_comment"
+    TIKTOK_VIDEO = "tiktok_video"
+    TIKTOK_USER = "tiktok_user"
+    TIKTOK_COMMENT = "tiktok_comment"
 
 
 class BillableInput(Protocol):

surfsense_backend/app/capabilities/tiktok/__init__.py

@@ -0,0 +1,8 @@
+"""``tiktok.*`` namespace: platform-native TikTok data verbs."""
+
+from __future__ import annotations
+
+from app.capabilities.tiktok.comments import definition as _comments  # noqa: F401
+from app.capabilities.tiktok.scrape import definition as _scrape  # noqa: F401
+from app.capabilities.tiktok.trending import definition as _trending  # noqa: F401
+from app.capabilities.tiktok.user_search import definition as _user_search  # noqa: F401

surfsense_backend/app/capabilities/tiktok/comments/__init__.py

@@ -0,0 +1,3 @@
+"""``tiktok.comments``: scrape the public comment thread of TikTok videos."""
+
+from __future__ import annotations

surfsense_backend/app/capabilities/tiktok/comments/definition.py

@@ -0,0 +1,23 @@
+"""``tiktok.comments`` capability registration (billed per comment; see config
+``TIKTOK_MICROS_PER_COMMENT``)."""
+
+from __future__ import annotations
+
+from app.capabilities.core import BillingUnit, Capability, register_capability
+from app.capabilities.tiktok.comments.executor import build_comments_executor
+from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput
+
+TIKTOK_COMMENTS = Capability(
+    name="tiktok.comments",
+    description=(
+        "Scrape the public comments of TikTok videos. Provide video URLs; "
+        "returns comment text, author, likes, and reply counts."
+    ),
+    input_schema=CommentsInput,
+    output_schema=CommentsOutput,
+    executor=build_comments_executor(),
+    billing_unit=BillingUnit.TIKTOK_COMMENT,
+    docs_url="/docs/connectors/native/tiktok",
+)
+
+register_capability(TIKTOK_COMMENTS)

surfsense_backend/app/capabilities/tiktok/comments/executor.py

@@ -0,0 +1,36 @@
+"""``tiktok.comments`` executor: video URLs -> scraper -> TikTok comment items."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+
+from app.capabilities.core import Executor
+from app.capabilities.core.progress import emit_progress
+from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput
+from app.proprietary.platforms.tiktok import scrape_tiktok_comments
+
+CommentsFn = Callable[..., Awaitable[list[dict]]]
+
+
+def build_comments_executor(comments_fn: CommentsFn | None = None) -> Executor:
+    """Bind the executor to a comments fn (defaults to the proprietary actor)."""
+    comments_fn = comments_fn or scrape_tiktok_comments
+
+    async def execute(payload: CommentsInput) -> CommentsOutput:
+        emit_progress(
+            "starting",
+            "Scraping TikTok comments",
+            total=payload.max_items,
+            unit="item",
+        )
+        items = await comments_fn(
+            payload.video_urls,
+            per_video=payload.comments_per_video,
+            limit=payload.max_items,
+        )
+        emit_progress(
+            "done", f"Scraped {len(items)} comment(s)", current=len(items), unit="item"
+        )
+        return CommentsOutput(items=items)
+
+    return execute

surfsense_backend/app/capabilities/tiktok/comments/schemas.py

@@ -0,0 +1,56 @@
+"""``tiktok.comments`` I/O contracts.
+
+URL-only: given TikTok video URLs, return each video's public comment thread.
+Unlike profile-video/general-search feeds, ``/api/comment/list`` is served to
+anonymous sessions once the comments panel opens, so this verb is reliable. Each
+result is a :class:`CommentItem` (top-level comments; replies carry ``repliesToId``).
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from app.capabilities.tiktok.scrape.schemas import (
+    MAX_TIKTOK_ITEMS,
+    MAX_TIKTOK_SOURCES,
+)
+from app.proprietary.platforms.tiktok import CommentItem
+
+
+class CommentsInput(BaseModel):
+    video_urls: list[str] = Field(
+        min_length=1,
+        max_length=MAX_TIKTOK_SOURCES,
+        description="TikTok video URLs (/@<user>/video/<id>) to pull comments from.",
+    )
+    comments_per_video: int = Field(
+        default=20,
+        ge=1,
+        le=MAX_TIKTOK_ITEMS,
+        description="Max comments to return per video.",
+    )
+    max_items: int = Field(
+        default=20,
+        ge=1,
+        le=MAX_TIKTOK_ITEMS,
+        description="Max total comments to return across all videos.",
+    )
+
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable comments for the pre-flight gate: ``max_items`` is a
+        hard cross-video ceiling (le=100), so no call can exceed it."""
+        return self.max_items
+
+
+class CommentsOutput(BaseModel):
+    items: list[CommentItem] = Field(
+        default_factory=list,
+        description="One item per comment returned, in emission order.",
+    )
+
+    @property
+    def billable_units(self) -> int:
+        """One returned comment = one billable unit; ErrorItems (``errorCode`` set,
+        for bad URLs or empty/withheld videos) are surfaced but never charged."""
+        return sum(1 for item in self.items if not getattr(item, "errorCode", None))

surfsense_backend/app/capabilities/tiktok/scrape/__init__.py

@@ -0,0 +1 @@
+"""``tiktok.scrape``: public TikTok videos over the browser-driven scraper."""

surfsense_backend/app/capabilities/tiktok/scrape/definition.py

@@ -0,0 +1,23 @@
+"""``tiktok.scrape`` capability registration (billed per video; see config
+``TIKTOK_MICROS_PER_VIDEO``)."""
+
+from __future__ import annotations
+
+from app.capabilities.core import BillingUnit, Capability, register_capability
+from app.capabilities.tiktok.scrape.executor import build_scrape_executor
+from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
+
+TIKTOK_SCRAPE = Capability(
+    name="tiktok.scrape",
+    description=(
+        "Scrape public TikTok videos. Use urls, profiles, hashtags, or "
+        "search_queries."
+    ),
+    input_schema=ScrapeInput,
+    output_schema=ScrapeOutput,
+    executor=build_scrape_executor(),
+    billing_unit=BillingUnit.TIKTOK_VIDEO,
+    docs_url="/docs/connectors/native/tiktok",
+)
+
+register_capability(TIKTOK_SCRAPE)

surfsense_backend/app/capabilities/tiktok/scrape/executor.py

@@ -0,0 +1,48 @@
+"""``tiktok.scrape`` executor: verb input → scraper → TikTok video items."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+
+from app.capabilities.core import Executor
+from app.capabilities.core.progress import emit_progress
+from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.exceptions import ForbiddenError
+from app.proprietary.platforms.tiktok import (
+    TikTokAccessBlockedError,
+    TikTokScrapeInput,
+    scrape_tiktok,
+)
+
+ScrapeFn = Callable[..., Awaitable[list[dict]]]
+
+
+def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
+    """Bind the executor to a scraper fn (defaults to the proprietary actor)."""
+    scrape_fn = scrape_fn or scrape_tiktok
+
+    async def execute(payload: ScrapeInput) -> ScrapeOutput:
+        actor_input = TikTokScrapeInput(
+            startUrls=[{"url": url} for url in payload.urls],
+            profiles=payload.profiles,
+            hashtags=payload.hashtags,
+            searchQueries=payload.search_queries,
+            resultsPerPage=payload.results_per_page,
+        )
+        emit_progress(
+            "starting", "Resolving TikTok targets", total=payload.max_items, unit="item"
+        )
+        try:
+            items = await scrape_fn(actor_input, limit=payload.max_items)
+        except TikTokAccessBlockedError as exc:
+            # Anonymous-only scraper; a hard block can't be retried with creds.
+            raise ForbiddenError(
+                f"TikTok refused anonymous access: {exc}",
+                code="TIKTOK_ACCESS_BLOCKED",
+            ) from exc
+        emit_progress(
+            "done", f"Scraped {len(items)} item(s)", current=len(items), unit="item"
+        )
+        return ScrapeOutput(items=items)
+
+    return execute

surfsense_backend/app/capabilities/tiktok/scrape/schemas.py

@@ -0,0 +1,87 @@
+"""``tiktok.scrape`` I/O contracts.
+
+A lean, agent-friendly surface over ``TikTokScrapeInput``
+(``app/proprietary/platforms/tiktok``). The executor maps this to the full
+scraper input; the scraper's ``TikTokVideoItem`` is reused verbatim as the
+output element. Any TikTok URL kind (video, profile, hashtag, search) goes in
+``urls``; ``profiles``/``hashtags``/``search_queries`` are typed shortcuts.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field, model_validator
+
+from app.proprietary.platforms.tiktok import TikTokVideoItem
+
+MAX_TIKTOK_SOURCES = 20
+"""Per-call cap on each source list: bounds a synchronous request's fan-out."""
+
+MAX_TIKTOK_ITEMS = 100
+"""Hard ceiling on items returned per call, regardless of the per-target count."""
+
+
+class ScrapeInput(BaseModel):
+    urls: list[str] = Field(
+        default_factory=list,
+        max_length=MAX_TIKTOK_SOURCES,
+        description=(
+            "TikTok URLs to scrape: a video, a profile (/@<user>), a hashtag "
+            "(/tag/<name>), or a search URL. Provide these OR profiles/hashtags/"
+            "search_queries (at least one source is required)."
+        ),
+    )
+    profiles: list[str] = Field(
+        default_factory=list,
+        max_length=MAX_TIKTOK_SOURCES,
+        description="Profile usernames (with or without a leading '@').",
+    )
+    hashtags: list[str] = Field(
+        default_factory=list,
+        max_length=MAX_TIKTOK_SOURCES,
+        description="Hashtag names to scrape, without the leading '#'.",
+    )
+    search_queries: list[str] = Field(
+        default_factory=list,
+        max_length=MAX_TIKTOK_SOURCES,
+        description="Search terms to run on TikTok.",
+    )
+    results_per_page: int = Field(
+        default=10,
+        ge=1,
+        le=MAX_TIKTOK_ITEMS,
+        description="Max videos to pull per profile/hashtag/search target.",
+    )
+    max_items: int = Field(
+        default=10,
+        ge=1,
+        le=MAX_TIKTOK_ITEMS,
+        description="Max total items to return across all sources.",
+    )
+
+    @model_validator(mode="after")
+    def _require_a_source(self) -> ScrapeInput:
+        if not any((self.urls, self.profiles, self.hashtags, self.search_queries)):
+            raise ValueError(
+                "Provide at least one of 'urls', 'profiles', 'hashtags', or "
+                "'search_queries'."
+            )
+        return self
+
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable items for the pre-flight gate: ``max_items`` is a
+        hard cross-source ceiling (le=100), so no call can exceed it."""
+        return self.max_items
+
+
+class ScrapeOutput(BaseModel):
+    items: list[TikTokVideoItem] = Field(
+        default_factory=list,
+        description="One item per video returned, in emission order.",
+    )
+
+    @property
+    def billable_units(self) -> int:
+        """One returned video = one billable unit; ErrorItems (``errorCode`` set,
+        for blocked/empty targets) are surfaced but never charged."""
+        return sum(1 for item in self.items if not getattr(item, "errorCode", None))

surfsense_backend/app/capabilities/tiktok/trending/__init__.py

@@ -0,0 +1,3 @@
+"""``tiktok.trending``: pull the current trending videos from the Explore feed."""
+
+from __future__ import annotations

surfsense_backend/app/capabilities/tiktok/trending/definition.py

@@ -0,0 +1,23 @@
+"""``tiktok.trending`` capability registration (billed per video on the shared
+``TIKTOK_MICROS_PER_VIDEO`` meter)."""
+
+from __future__ import annotations
+
+from app.capabilities.core import BillingUnit, Capability, register_capability
+from app.capabilities.tiktok.trending.executor import build_trending_executor
+from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput
+
+TIKTOK_TRENDING = Capability(
+    name="tiktok.trending",
+    description=(
+        "Get the current trending TikTok videos from the Explore feed. No input "
+        "needed beyond how many to return."
+    ),
+    input_schema=TrendingInput,
+    output_schema=TrendingOutput,
+    executor=build_trending_executor(),
+    billing_unit=BillingUnit.TIKTOK_VIDEO,
+    docs_url="/docs/connectors/native/tiktok",
+)
+
+register_capability(TIKTOK_TRENDING)

surfsense_backend/app/capabilities/tiktok/trending/executor.py

@@ -0,0 +1,32 @@
+"""``tiktok.trending`` executor: Explore feed -> scraper -> TikTok video items."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+
+from app.capabilities.core import Executor
+from app.capabilities.core.progress import emit_progress
+from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput
+from app.proprietary.platforms.tiktok import scrape_tiktok_trending
+
+TrendingFn = Callable[..., Awaitable[list[dict]]]
+
+
+def build_trending_executor(trending_fn: TrendingFn | None = None) -> Executor:
+    """Bind the executor to a trending fn (defaults to the proprietary actor)."""
+    trending_fn = trending_fn or scrape_tiktok_trending
+
+    async def execute(payload: TrendingInput) -> TrendingOutput:
+        emit_progress(
+            "starting",
+            "Fetching TikTok trending videos",
+            total=payload.max_items,
+            unit="item",
+        )
+        items = await trending_fn(count=payload.max_items)
+        emit_progress(
+            "done", f"Fetched {len(items)} video(s)", current=len(items), unit="item"
+        )
+        return TrendingOutput(items=items)
+
+    return execute

surfsense_backend/app/capabilities/tiktok/trending/schemas.py

@@ -0,0 +1,41 @@
+"""``tiktok.trending`` I/O contracts.
+
+The Explore feed (``/api/explore/item_list``) is a single global feed of trending
+videos, served to anonymous sessions. No source input is needed — just how many
+to return. Each result reuses :class:`TikTokVideoItem`, so trending videos bill on
+the same per-video meter as ``tiktok.scrape``.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS
+from app.proprietary.platforms.tiktok import TikTokVideoItem
+
+
+class TrendingInput(BaseModel):
+    max_items: int = Field(
+        default=20,
+        ge=1,
+        le=MAX_TIKTOK_ITEMS,
+        description="Max trending videos to return from the Explore feed.",
+    )
+
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable videos for the pre-flight gate (le=100 ceiling)."""
+        return self.max_items
+
+
+class TrendingOutput(BaseModel):
+    items: list[TikTokVideoItem] = Field(
+        default_factory=list,
+        description="One item per trending video returned, in feed order.",
+    )
+
+    @property
+    def billable_units(self) -> int:
+        """One returned video = one billable unit; an ErrorItem (``errorCode`` set,
+        for an empty/withheld feed) is surfaced but never charged."""
+        return sum(1 for item in self.items if not getattr(item, "errorCode", None))

surfsense_backend/app/capabilities/tiktok/user_search/__init__.py

@@ -0,0 +1,3 @@
+"""``tiktok.user_search``: find public TikTok accounts by keyword."""
+
+from __future__ import annotations

surfsense_backend/app/capabilities/tiktok/user_search/definition.py

@@ -0,0 +1,26 @@
+"""``tiktok.user_search`` capability registration (billed per account; see config
+``TIKTOK_MICROS_PER_USER``)."""
+
+from __future__ import annotations
+
+from app.capabilities.core import BillingUnit, Capability, register_capability
+from app.capabilities.tiktok.user_search.executor import build_user_search_executor
+from app.capabilities.tiktok.user_search.schemas import (
+    UserSearchInput,
+    UserSearchOutput,
+)
+
+TIKTOK_USER_SEARCH = Capability(
+    name="tiktok.user_search",
+    description=(
+        "Find public TikTok accounts by keyword. Returns profile metadata "
+        "(name, followers, bio, verification) per matching account."
+    ),
+    input_schema=UserSearchInput,
+    output_schema=UserSearchOutput,
+    executor=build_user_search_executor(),
+    billing_unit=BillingUnit.TIKTOK_USER,
+    docs_url="/docs/connectors/native/tiktok",
+)
+
+register_capability(TIKTOK_USER_SEARCH)

surfsense_backend/app/capabilities/tiktok/user_search/executor.py

@@ -0,0 +1,39 @@
+"""``tiktok.user_search`` executor: queries -> scraper -> TikTok profile items."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+
+from app.capabilities.core import Executor
+from app.capabilities.core.progress import emit_progress
+from app.capabilities.tiktok.user_search.schemas import (
+    UserSearchInput,
+    UserSearchOutput,
+)
+from app.proprietary.platforms.tiktok import search_tiktok_users
+
+SearchFn = Callable[..., Awaitable[list[dict]]]
+
+
+def build_user_search_executor(search_fn: SearchFn | None = None) -> Executor:
+    """Bind the executor to a search fn (defaults to the proprietary actor)."""
+    search_fn = search_fn or search_tiktok_users
+
+    async def execute(payload: UserSearchInput) -> UserSearchOutput:
+        emit_progress(
+            "starting",
+            "Searching TikTok accounts",
+            total=payload.max_items,
+            unit="item",
+        )
+        items = await search_fn(
+            payload.queries,
+            per_query=payload.results_per_query,
+            limit=payload.max_items,
+        )
+        emit_progress(
+            "done", f"Found {len(items)} account(s)", current=len(items), unit="item"
+        )
+        return UserSearchOutput(items=items)
+
+    return execute

surfsense_backend/app/capabilities/tiktok/user_search/schemas.py

@@ -0,0 +1,56 @@
+"""``tiktok.user_search`` I/O contracts.
+
+Account discovery over ``TikTok``'s Users tab. Where video/general search is
+login-walled for anonymous sessions, ``/api/search/user`` returns public account
+records, so this verb exposes the one reliably-unblocked search path. Each result
+is a :class:`TikTokProfileItem` (the same shape the profile verb emits).
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from app.capabilities.tiktok.scrape.schemas import (
+    MAX_TIKTOK_ITEMS,
+    MAX_TIKTOK_SOURCES,
+)
+from app.proprietary.platforms.tiktok import TikTokProfileItem
+
+
+class UserSearchInput(BaseModel):
+    queries: list[str] = Field(
+        min_length=1,
+        max_length=MAX_TIKTOK_SOURCES,
+        description="Keywords to search for TikTok accounts (e.g. names, brands).",
+    )
+    results_per_query: int = Field(
+        default=10,
+        ge=1,
+        le=MAX_TIKTOK_ITEMS,
+        description="Max accounts to return per query.",
+    )
+    max_items: int = Field(
+        default=10,
+        ge=1,
+        le=MAX_TIKTOK_ITEMS,
+        description="Max total accounts to return across all queries.",
+    )
+
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable accounts for the pre-flight gate: ``max_items`` is a
+        hard cross-query ceiling (le=100), so no call can exceed it."""
+        return self.max_items
+
+
+class UserSearchOutput(BaseModel):
+    items: list[TikTokProfileItem] = Field(
+        default_factory=list,
+        description="One item per account found, in emission order.",
+    )
+
+    @property
+    def billable_units(self) -> int:
+        """One returned account = one billable unit; ErrorItems (``errorCode`` set,
+        for empty/withheld queries) are surfaced but never charged."""
+        return sum(1 for item in self.items if not getattr(item, "errorCode", None))