diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 71aa45c58..048797473 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -35,6 +35,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.GOOGLE_MAPS_REVIEW: "GOOGLE_MAPS_MICROS_PER_REVIEW", BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO", BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT", + BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO", } @@ -51,6 +52,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.GOOGLE_MAPS_REVIEW: "review", BillingUnit.YOUTUBE_VIDEO: "video", BillingUnit.YOUTUBE_COMMENT: "comment", + BillingUnit.TIKTOK_VIDEO: "video", } diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index c87601832..b45641a2d 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -25,6 +25,7 @@ class BillingUnit(StrEnum): GOOGLE_MAPS_REVIEW = "google_maps_review" YOUTUBE_VIDEO = "youtube_video" YOUTUBE_COMMENT = "youtube_comment" + TIKTOK_VIDEO = "tiktok_video" class BillableInput(Protocol): diff --git a/surfsense_backend/app/capabilities/tiktok/__init__.py b/surfsense_backend/app/capabilities/tiktok/__init__.py new file mode 100644 index 000000000..ed3f32682 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/__init__.py @@ -0,0 +1,5 @@ +"""``tiktok.*`` namespace: platform-native TikTok data verbs.""" + +from __future__ import annotations + +from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401 diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py b/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py new file mode 100644 index 000000000..9e5acc165 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py @@ -0,0 +1 @@ +"""``tiktok.scrape``: public TikTok videos over the browser-driven scraper.""" diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/definition.py b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py new file mode 100644 index 000000000..9f8632f3b --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py @@ -0,0 +1,23 @@ +"""``tiktok.scrape`` capability registration (billed per video; see config +``TIKTOK_MICROS_PER_VIDEO``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.scrape.executor import build_scrape_executor +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput + +TIKTOK_SCRAPE = Capability( + name="tiktok.scrape", + description=( + "Scrape public TikTok videos. Use urls, profiles, hashtags, or " + "search_queries." + ), + input_schema=ScrapeInput, + output_schema=ScrapeOutput, + executor=build_scrape_executor(), + billing_unit=BillingUnit.TIKTOK_VIDEO, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_SCRAPE) diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/executor.py b/surfsense_backend/app/capabilities/tiktok/scrape/executor.py new file mode 100644 index 000000000..7ef71486d --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/executor.py @@ -0,0 +1,48 @@ +"""``tiktok.scrape`` executor: verb input → scraper → TikTok video items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.tiktok import ( + TikTokAccessBlockedError, + TikTokScrapeInput, + scrape_tiktok, +) + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the executor to a scraper fn (defaults to the proprietary actor).""" + scrape_fn = scrape_fn or scrape_tiktok + + async def execute(payload: ScrapeInput) -> ScrapeOutput: + actor_input = TikTokScrapeInput( + startUrls=[{"url": url} for url in payload.urls], + profiles=payload.profiles, + hashtags=payload.hashtags, + searchQueries=payload.search_queries, + resultsPerPage=payload.results_per_page, + ) + emit_progress( + "starting", "Resolving TikTok targets", total=payload.max_items, unit="item" + ) + try: + items = await scrape_fn(actor_input, limit=payload.max_items) + except TikTokAccessBlockedError as exc: + # Anonymous-only scraper; a hard block can't be retried with creds. + raise ForbiddenError( + f"TikTok refused anonymous access: {exc}", + code="TIKTOK_ACCESS_BLOCKED", + ) from exc + emit_progress( + "done", f"Scraped {len(items)} item(s)", current=len(items), unit="item" + ) + return ScrapeOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py new file mode 100644 index 000000000..697a94043 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py @@ -0,0 +1,86 @@ +"""``tiktok.scrape`` I/O contracts. + +A lean, agent-friendly surface over ``TikTokScrapeInput`` +(``app/proprietary/platforms/tiktok``). The executor maps this to the full +scraper input; the scraper's ``TikTokVideoItem`` is reused verbatim as the +output element. Any TikTok URL kind (video, profile, hashtag, search) goes in +``urls``; ``profiles``/``hashtags``/``search_queries`` are typed shortcuts. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field, model_validator + +from app.proprietary.platforms.tiktok import TikTokVideoItem + +MAX_TIKTOK_SOURCES = 20 +"""Per-call cap on each source list: bounds a synchronous request's fan-out.""" + +MAX_TIKTOK_ITEMS = 100 +"""Hard ceiling on items returned per call, regardless of the per-target count.""" + + +class ScrapeInput(BaseModel): + urls: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description=( + "TikTok URLs to scrape: a video, a profile (/@), a hashtag " + "(/tag/), or a search URL. Provide these OR profiles/hashtags/" + "search_queries (at least one source is required)." + ), + ) + profiles: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description="Profile usernames (with or without a leading '@').", + ) + hashtags: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description="Hashtag names to scrape, without the leading '#'.", + ) + search_queries: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description="Search terms to run on TikTok.", + ) + results_per_page: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max videos to pull per profile/hashtag/search target.", + ) + max_items: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max total items to return across all sources.", + ) + + @model_validator(mode="after") + def _require_a_source(self) -> ScrapeInput: + if not any((self.urls, self.profiles, self.hashtags, self.search_queries)): + raise ValueError( + "Provide at least one of 'urls', 'profiles', 'hashtags', or " + "'search_queries'." + ) + return self + + @property + def estimated_units(self) -> int: + """Worst-case billable items for the pre-flight gate: ``max_items`` is a + hard cross-source ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class ScrapeOutput(BaseModel): + items: list[TikTokVideoItem] = Field( + default_factory=list, + description="One item per video returned, in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned item = one billable unit.""" + return len(self.items) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 5d8e96881..c03ac3598 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -714,6 +714,9 @@ class Config: # Kept separate from the video rate so comments can be re-tuned toward the # cheaper per-comment market ($0.40-2.00/1k) without touching video pricing. YOUTUBE_MICROS_PER_COMMENT = int(os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500")) + # Browser-driven listings make TikTok heavier per item than the API-backed + # video meter, so it sits a touch above YouTube's video rate. + TIKTOK_MICROS_PER_VIDEO = int(os.getenv("TIKTOK_MICROS_PER_VIDEO", "3500")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 1a5b182b8..3c28fd65c 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -4,6 +4,7 @@ from fastapi import APIRouter, Depends import app.capabilities.google_maps import app.capabilities.google_search import app.capabilities.reddit +import app.capabilities.tiktok import app.capabilities.web import app.capabilities.youtube # noqa: F401 from app.automations.api import router as automations_router diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py new file mode 100644 index 000000000..1bf07b0c3 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py @@ -0,0 +1,79 @@ +"""``tiktok.scrape`` executor: verb input → actor input mapping → typed items. + +Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's +own payload→TikTokScrapeInput mapping and the dict→TikTokVideoItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.scrape.executor import build_scrape_executor +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.tiktok import TikTokAccessBlockedError, TikTokScrapeInput + +pytestmark = pytest.mark.unit + + +class _FakeScraper: + """Records the actor input + limit it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[tuple[TikTokScrapeInput, int | None]] = [] + + async def __call__( + self, actor_input: TikTokScrapeInput, *, limit: int | None = None + ) -> list[dict]: + self.calls.append((actor_input, limit)) + return self._items + + +async def test_maps_urls_to_start_urls_and_wraps_items(): + scraper = _FakeScraper([{"id": "123", "text": "hello"}]) + execute = build_scrape_executor(scrape_fn=scraper) + + out = await execute(ScrapeInput(urls=["https://www.tiktok.com/@nasa/video/123"])) + + assert isinstance(out, ScrapeOutput) + assert len(out.items) == 1 + assert out.items[0].id == "123" + + (actor_input, _limit) = scraper.calls[0] + assert [u.url for u in actor_input.startUrls] == [ + "https://www.tiktok.com/@nasa/video/123" + ] + + +async def test_forwards_typed_sources_and_limit(): + scraper = _FakeScraper([]) + execute = build_scrape_executor(scrape_fn=scraper) + + await execute( + ScrapeInput( + profiles=["nasa"], + hashtags=["food"], + search_queries=["cats"], + results_per_page=7, + max_items=25, + ) + ) + + (actor_input, limit) = scraper.calls[0] + assert actor_input.profiles == ["nasa"] + assert actor_input.hashtags == ["food"] + assert actor_input.searchQueries == ["cats"] + assert actor_input.resultsPerPage == 7 + # The outer collection limit is the caller's total-item cap. + assert limit == 25 + + +async def test_access_blocked_maps_to_forbidden(): + async def _blocked(actor_input: TikTokScrapeInput, *, limit: int | None = None): + raise TikTokAccessBlockedError("all IPs refused") + + execute = build_scrape_executor(scrape_fn=_blocked) + + with pytest.raises(ForbiddenError): + await execute(ScrapeInput(hashtags=["food"])) diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py new file mode 100644 index 000000000..e7c978bcc --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py @@ -0,0 +1,45 @@ +"""``tiktok.scrape`` input guards: a source is required and the batch is bounded.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.scrape.schemas import ( + MAX_TIKTOK_ITEMS, + MAX_TIKTOK_SOURCES, + ScrapeInput, +) + +pytestmark = pytest.mark.unit + + +def test_rejects_input_with_no_source(): + with pytest.raises(ValidationError): + ScrapeInput() + + +def test_accepts_urls_only(): + payload = ScrapeInput(urls=["https://www.tiktok.com/@nasa"]) + assert payload.profiles == [] + + +def test_accepts_hashtags_only(): + payload = ScrapeInput(hashtags=["food"]) + assert payload.hashtags == ["food"] + + +def test_defaults_and_bounds(): + payload = ScrapeInput(hashtags=["food"]) + assert payload.max_items == 10 + assert payload.results_per_page == 10 + with pytest.raises(ValidationError): + ScrapeInput(hashtags=["food"], max_items=0) + with pytest.raises(ValidationError): + ScrapeInput(hashtags=["food"], max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_rejects_more_sources_than_the_cap(): + too_many = [f"tag{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] + with pytest.raises(ValidationError): + ScrapeInput(hashtags=too_many) diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py new file mode 100644 index 000000000..3a85fe87f --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py @@ -0,0 +1,23 @@ +"""The tiktok namespace registers its verb as one Capability the doors/agent read.""" + +from __future__ import annotations + +import pytest + +from app.capabilities import ( + tiktok, # noqa: F401 — importing the namespace registers its verbs +) +from app.capabilities.core import BillingUnit +from app.capabilities.core.store import get_capability +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput + +pytestmark = pytest.mark.unit + + +def test_tiktok_scrape_is_registered_and_billed_per_video(): + cap = get_capability("tiktok.scrape") + + assert cap.name == "tiktok.scrape" + assert cap.input_schema is ScrapeInput + assert cap.output_schema is ScrapeOutput + assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO