feat(tiktok): tiktok.scrape capability + billing wire-up

surfsense_backend/app/capabilities/core/billing.py

@@ -35,6 +35,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = {
     BillingUnit.GOOGLE_MAPS_REVIEW: "GOOGLE_MAPS_MICROS_PER_REVIEW",
     BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO",
     BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT",
+    BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO",
 }
 
 
@@ -51,6 +52,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = {
     BillingUnit.GOOGLE_MAPS_REVIEW: "review",
     BillingUnit.YOUTUBE_VIDEO: "video",
     BillingUnit.YOUTUBE_COMMENT: "comment",
+    BillingUnit.TIKTOK_VIDEO: "video",
 }
 
 

surfsense_backend/app/capabilities/core/types.py

@@ -25,6 +25,7 @@ class BillingUnit(StrEnum):
     GOOGLE_MAPS_REVIEW = "google_maps_review"
     YOUTUBE_VIDEO = "youtube_video"
     YOUTUBE_COMMENT = "youtube_comment"
+    TIKTOK_VIDEO = "tiktok_video"
 
 
 class BillableInput(Protocol):

surfsense_backend/app/capabilities/tiktok/__init__.py

@@ -0,0 +1,5 @@
+"""``tiktok.*`` namespace: platform-native TikTok data verbs."""
+
+from __future__ import annotations
+
+from app.capabilities.tiktok.scrape import definition as _scrape  # noqa: F401

surfsense_backend/app/capabilities/tiktok/scrape/__init__.py

@@ -0,0 +1 @@
+"""``tiktok.scrape``: public TikTok videos over the browser-driven scraper."""

surfsense_backend/app/capabilities/tiktok/scrape/definition.py

@@ -0,0 +1,23 @@
+"""``tiktok.scrape`` capability registration (billed per video; see config
+``TIKTOK_MICROS_PER_VIDEO``)."""
+
+from __future__ import annotations
+
+from app.capabilities.core import BillingUnit, Capability, register_capability
+from app.capabilities.tiktok.scrape.executor import build_scrape_executor
+from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
+
+TIKTOK_SCRAPE = Capability(
+    name="tiktok.scrape",
+    description=(
+        "Scrape public TikTok videos. Use urls, profiles, hashtags, or "
+        "search_queries."
+    ),
+    input_schema=ScrapeInput,
+    output_schema=ScrapeOutput,
+    executor=build_scrape_executor(),
+    billing_unit=BillingUnit.TIKTOK_VIDEO,
+    docs_url="/docs/connectors/native/tiktok",
+)
+
+register_capability(TIKTOK_SCRAPE)

surfsense_backend/app/capabilities/tiktok/scrape/executor.py

@@ -0,0 +1,48 @@
+"""``tiktok.scrape`` executor: verb input → scraper → TikTok video items."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+
+from app.capabilities.core import Executor
+from app.capabilities.core.progress import emit_progress
+from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.exceptions import ForbiddenError
+from app.proprietary.platforms.tiktok import (
+    TikTokAccessBlockedError,
+    TikTokScrapeInput,
+    scrape_tiktok,
+)
+
+ScrapeFn = Callable[..., Awaitable[list[dict]]]
+
+
+def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
+    """Bind the executor to a scraper fn (defaults to the proprietary actor)."""
+    scrape_fn = scrape_fn or scrape_tiktok
+
+    async def execute(payload: ScrapeInput) -> ScrapeOutput:
+        actor_input = TikTokScrapeInput(
+            startUrls=[{"url": url} for url in payload.urls],
+            profiles=payload.profiles,
+            hashtags=payload.hashtags,
+            searchQueries=payload.search_queries,
+            resultsPerPage=payload.results_per_page,
+        )
+        emit_progress(
+            "starting", "Resolving TikTok targets", total=payload.max_items, unit="item"
+        )
+        try:
+            items = await scrape_fn(actor_input, limit=payload.max_items)
+        except TikTokAccessBlockedError as exc:
+            # Anonymous-only scraper; a hard block can't be retried with creds.
+            raise ForbiddenError(
+                f"TikTok refused anonymous access: {exc}",
+                code="TIKTOK_ACCESS_BLOCKED",
+            ) from exc
+        emit_progress(
+            "done", f"Scraped {len(items)} item(s)", current=len(items), unit="item"
+        )
+        return ScrapeOutput(items=items)
+
+    return execute

surfsense_backend/app/capabilities/tiktok/scrape/schemas.py

@@ -0,0 +1,86 @@
+"""``tiktok.scrape`` I/O contracts.
+
+A lean, agent-friendly surface over ``TikTokScrapeInput``
+(``app/proprietary/platforms/tiktok``). The executor maps this to the full
+scraper input; the scraper's ``TikTokVideoItem`` is reused verbatim as the
+output element. Any TikTok URL kind (video, profile, hashtag, search) goes in
+``urls``; ``profiles``/``hashtags``/``search_queries`` are typed shortcuts.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field, model_validator
+
+from app.proprietary.platforms.tiktok import TikTokVideoItem
+
+MAX_TIKTOK_SOURCES = 20
+"""Per-call cap on each source list: bounds a synchronous request's fan-out."""
+
+MAX_TIKTOK_ITEMS = 100
+"""Hard ceiling on items returned per call, regardless of the per-target count."""
+
+
+class ScrapeInput(BaseModel):
+    urls: list[str] = Field(
+        default_factory=list,
+        max_length=MAX_TIKTOK_SOURCES,
+        description=(
+            "TikTok URLs to scrape: a video, a profile (/@<user>), a hashtag "
+            "(/tag/<name>), or a search URL. Provide these OR profiles/hashtags/"
+            "search_queries (at least one source is required)."
+        ),
+    )
+    profiles: list[str] = Field(
+        default_factory=list,
+        max_length=MAX_TIKTOK_SOURCES,
+        description="Profile usernames (with or without a leading '@').",
+    )
+    hashtags: list[str] = Field(
+        default_factory=list,
+        max_length=MAX_TIKTOK_SOURCES,
+        description="Hashtag names to scrape, without the leading '#'.",
+    )
+    search_queries: list[str] = Field(
+        default_factory=list,
+        max_length=MAX_TIKTOK_SOURCES,
+        description="Search terms to run on TikTok.",
+    )
+    results_per_page: int = Field(
+        default=10,
+        ge=1,
+        le=MAX_TIKTOK_ITEMS,
+        description="Max videos to pull per profile/hashtag/search target.",
+    )
+    max_items: int = Field(
+        default=10,
+        ge=1,
+        le=MAX_TIKTOK_ITEMS,
+        description="Max total items to return across all sources.",
+    )
+
+    @model_validator(mode="after")
+    def _require_a_source(self) -> ScrapeInput:
+        if not any((self.urls, self.profiles, self.hashtags, self.search_queries)):
+            raise ValueError(
+                "Provide at least one of 'urls', 'profiles', 'hashtags', or "
+                "'search_queries'."
+            )
+        return self
+
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case billable items for the pre-flight gate: ``max_items`` is a
+        hard cross-source ceiling (le=100), so no call can exceed it."""
+        return self.max_items
+
+
+class ScrapeOutput(BaseModel):
+    items: list[TikTokVideoItem] = Field(
+        default_factory=list,
+        description="One item per video returned, in emission order.",
+    )
+
+    @property
+    def billable_units(self) -> int:
+        """One returned item = one billable unit."""
+        return len(self.items)

surfsense_backend/app/config/__init__.py

@@ -714,6 +714,9 @@ class Config:
     # Kept separate from the video rate so comments can be re-tuned toward the
     # cheaper per-comment market ($0.40-2.00/1k) without touching video pricing.
     YOUTUBE_MICROS_PER_COMMENT = int(os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500"))
+    # Browser-driven listings make TikTok heavier per item than the API-backed
+    # video meter, so it sits a touch above YouTube's video rate.
+    TIKTOK_MICROS_PER_VIDEO = int(os.getenv("TIKTOK_MICROS_PER_VIDEO", "3500"))
 
     # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service
     # so the UI can nudge the user to top up / enable auto-reload. $0.50.

surfsense_backend/app/routes/__init__.py

@@ -4,6 +4,7 @@ from fastapi import APIRouter, Depends
 import app.capabilities.google_maps
 import app.capabilities.google_search
 import app.capabilities.reddit
+import app.capabilities.tiktok
 import app.capabilities.web
 import app.capabilities.youtube  # noqa: F401
 from app.automations.api import router as automations_router

surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py

@@ -0,0 +1,79 @@
+"""``tiktok.scrape`` executor: verb input → actor input mapping → typed items.
+
+Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's
+own payload→TikTokScrapeInput mapping and the dict→TikTokVideoItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.tiktok.scrape.executor import build_scrape_executor
+from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.exceptions import ForbiddenError
+from app.proprietary.platforms.tiktok import TikTokAccessBlockedError, TikTokScrapeInput
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeScraper:
+    """Records the actor input + limit it was called with; returns canned items."""
+
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[tuple[TikTokScrapeInput, int | None]] = []
+
+    async def __call__(
+        self, actor_input: TikTokScrapeInput, *, limit: int | None = None
+    ) -> list[dict]:
+        self.calls.append((actor_input, limit))
+        return self._items
+
+
+async def test_maps_urls_to_start_urls_and_wraps_items():
+    scraper = _FakeScraper([{"id": "123", "text": "hello"}])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    out = await execute(ScrapeInput(urls=["https://www.tiktok.com/@nasa/video/123"]))
+
+    assert isinstance(out, ScrapeOutput)
+    assert len(out.items) == 1
+    assert out.items[0].id == "123"
+
+    (actor_input, _limit) = scraper.calls[0]
+    assert [u.url for u in actor_input.startUrls] == [
+        "https://www.tiktok.com/@nasa/video/123"
+    ]
+
+
+async def test_forwards_typed_sources_and_limit():
+    scraper = _FakeScraper([])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    await execute(
+        ScrapeInput(
+            profiles=["nasa"],
+            hashtags=["food"],
+            search_queries=["cats"],
+            results_per_page=7,
+            max_items=25,
+        )
+    )
+
+    (actor_input, limit) = scraper.calls[0]
+    assert actor_input.profiles == ["nasa"]
+    assert actor_input.hashtags == ["food"]
+    assert actor_input.searchQueries == ["cats"]
+    assert actor_input.resultsPerPage == 7
+    # The outer collection limit is the caller's total-item cap.
+    assert limit == 25
+
+
+async def test_access_blocked_maps_to_forbidden():
+    async def _blocked(actor_input: TikTokScrapeInput, *, limit: int | None = None):
+        raise TikTokAccessBlockedError("all IPs refused")
+
+    execute = build_scrape_executor(scrape_fn=_blocked)
+
+    with pytest.raises(ForbiddenError):
+        await execute(ScrapeInput(hashtags=["food"]))

surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py

@@ -0,0 +1,45 @@
+"""``tiktok.scrape`` input guards: a source is required and the batch is bounded."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.tiktok.scrape.schemas import (
+    MAX_TIKTOK_ITEMS,
+    MAX_TIKTOK_SOURCES,
+    ScrapeInput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_rejects_input_with_no_source():
+    with pytest.raises(ValidationError):
+        ScrapeInput()
+
+
+def test_accepts_urls_only():
+    payload = ScrapeInput(urls=["https://www.tiktok.com/@nasa"])
+    assert payload.profiles == []
+
+
+def test_accepts_hashtags_only():
+    payload = ScrapeInput(hashtags=["food"])
+    assert payload.hashtags == ["food"]
+
+
+def test_defaults_and_bounds():
+    payload = ScrapeInput(hashtags=["food"])
+    assert payload.max_items == 10
+    assert payload.results_per_page == 10
+    with pytest.raises(ValidationError):
+        ScrapeInput(hashtags=["food"], max_items=0)
+    with pytest.raises(ValidationError):
+        ScrapeInput(hashtags=["food"], max_items=MAX_TIKTOK_ITEMS + 1)
+
+
+def test_rejects_more_sources_than_the_cap():
+    too_many = [f"tag{i}" for i in range(MAX_TIKTOK_SOURCES + 1)]
+    with pytest.raises(ValidationError):
+        ScrapeInput(hashtags=too_many)

surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py

@@ -0,0 +1,23 @@
+"""The tiktok namespace registers its verb as one Capability the doors/agent read."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities import (
+    tiktok,  # noqa: F401  — importing the namespace registers its verbs
+)
+from app.capabilities.core import BillingUnit
+from app.capabilities.core.store import get_capability
+from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
+
+pytestmark = pytest.mark.unit
+
+
+def test_tiktok_scrape_is_registered_and_billed_per_video():
+    cap = get_capability("tiktok.scrape")
+
+    assert cap.name == "tiktok.scrape"
+    assert cap.input_schema is ScrapeInput
+    assert cap.output_schema is ScrapeOutput
+    assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO