diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py index 697a94043..2f5005353 100644 --- a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py @@ -82,5 +82,6 @@ class ScrapeOutput(BaseModel): @property def billable_units(self) -> int: - """One returned item = one billable unit.""" - return len(self.items) + """One returned video = one billable unit; ErrorItems (``errorCode`` set, + for blocked/empty targets) are surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py index 44fcc33e2..3e930271f 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py @@ -12,9 +12,19 @@ from typing import Any from ..extraction import parse_video from ..extraction.timestamps import now_iso +from ..schemas import ErrorItem from ..targets.types import TikTokTarget from . import FetchListingFn +# Profile and search feeds are trust-gated: an anonymous headless session gets an +# empty body (profile) or no results XHR (search), while hashtag feeds load. We +# can't tell "genuinely empty" from "blocked" here, so a zero-item listing emits +# one honest ErrorItem instead of vanishing silently. +_EMPTY_LISTING_MESSAGE = ( + "No videos returned. The target may be empty/private/removed, or TikTok " + "withheld this feed from anonymous access (common for profiles and search)." +) + async def iter_listing( target: TikTokTarget, *, cap: int, fetch_listing: FetchListingFn @@ -35,3 +45,11 @@ async def iter_listing( emitted += 1 if emitted >= cap: return + if emitted == 0: + yield ErrorItem( + url=target.url, + input=target.value, + error=_EMPTY_LISTING_MESSAGE, + errorCode="no_items", + scrapedAt=now_iso(), + ).to_output() diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py index 22bebff96..eedeaacac 100644 --- a/surfsense_backend/scripts/e2e_tiktok_scrape.py +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -7,11 +7,14 @@ Run from the backend directory: What it exercises (everything REAL — live network, live proxy, live browser): Stage 1 — proxy egress proof (informational). - Stage 2 — profile listing via the stealth browser (soft-blocked by TikTok; - expected empty until a stronger anti-detection path exists). + Stage 2 — profile via the full pipeline: TikTok soft-blocks the anonymous + profile feed, so this asserts graceful degradation — real videos OR + a single honest ErrorItem, never a silent empty. Stage 3 — blob video path over HTTP (URL taken from a captured hashtag struct). Stage 4 — hashtag listing via the stealth browser (captures item_list XHRs). Stage 5 — full scrape_tiktok() pipeline on a hashtag. + Stage 6 — search via the full pipeline: same graceful-degrade contract as + profile (results feed doesn't load for anonymous sessions). On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser suites can pin against real-shaped data without network. @@ -40,9 +43,11 @@ for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): _FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok" -# Evergreen public targets: a regular high-volume creator and a broad hashtag. +# Evergreen public targets: a regular high-volume creator, a broad hashtag, and +# a common search term. _PROFILE = "nasa" _HASHTAG = "food" +_SEARCH = "meal prep" _COUNT = 5 @@ -93,17 +98,24 @@ async def stage_proxy() -> bool: async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]: - _hr(f"STAGE 2 — profile listing (browser): @{_PROFILE}") - from app.proprietary.platforms.tiktok.session import fetch_item_list + _hr(f"STAGE 2 — profile listing graceful-degrade: @{_PROFILE}") + from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok - url = f"https://www.tiktok.com/@{_PROFILE}" - raw = await fetch_item_list(url, _COUNT) - ok = _check( - "captured itemStructs from item_list XHRs", - len(raw) > 0 and isinstance(raw[0], dict) and bool(raw[0].get("id")), - f"{len(raw)} struct(s)", + # TikTok soft-blocks the profile feed for anonymous headless sessions + # (empty 200). The shipped contract is a single honest ErrorItem, not a + # silent empty. This stage verifies that graceful degradation, and passes if + # EITHER real videos come back (session got trusted) OR an ErrorItem does. + items = await scrape_tiktok( + TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=_COUNT), limit=_COUNT ) - return ok, raw + has_video = any(it.get("id") and not it.get("errorCode") for it in items) + has_error = any(it.get("errorCode") == "no_items" for it in items) + ok = _check( + "profile yields videos or a graceful ErrorItem (never silent empty)", + has_video or has_error, + f"{len(items)} item(s); video={has_video} error={has_error}", + ) + return ok, items async def stage_blob_video(video_url: str) -> tuple[bool, dict[str, Any] | None]: @@ -167,6 +179,26 @@ async def stage_pipeline() -> bool: return ok +async def stage_search_listing() -> tuple[bool, list[dict[str, Any]]]: + _hr(f"STAGE 6 — search listing graceful-degrade: {_SEARCH!r}") + from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok + + # The search results feed doesn't load for anonymous headless sessions + # (results XHR never fires on a cold deep-link). Same contract as profile: + # verify a graceful ErrorItem instead of a silent empty. + items = await scrape_tiktok( + TikTokScrapeInput(searchQueries=[_SEARCH], resultsPerPage=_COUNT), limit=_COUNT + ) + has_video = any(it.get("id") and not it.get("errorCode") for it in items) + has_error = any(it.get("errorCode") == "no_items" for it in items) + ok = _check( + "search yields videos or a graceful ErrorItem (never silent empty)", + has_video or has_error, + f"{len(items)} item(s); video={has_video} error={has_error}", + ) + return ok, items + + async def main() -> int: print("TikTok scraper functional e2e — live network + proxy + browser") results: dict[str, bool] = {} @@ -185,6 +217,9 @@ async def main() -> int: else: print("\n [SKIP] Stage 3 — no captured struct to build a video URL") + ok_search, _ = await stage_search_listing() + results["Stage 6 search listing"] = ok_search + ok_profile, _ = await stage_profile_listing() results["Stage 2 profile listing"] = ok_profile results["Stage 5 pipeline"] = await stage_pipeline() diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py index e7c978bcc..f9cec8c3d 100644 --- a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py +++ b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py @@ -9,6 +9,7 @@ from app.capabilities.tiktok.scrape.schemas import ( MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES, ScrapeInput, + ScrapeOutput, ) pytestmark = pytest.mark.unit @@ -43,3 +44,15 @@ def test_rejects_more_sources_than_the_cap(): too_many = [f"tag{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] with pytest.raises(ValidationError): ScrapeInput(hashtags=too_many) + + +def test_error_items_are_not_billed(): + # Real videos count; ErrorItems (blocked/empty targets) are surfaced free. + out = ScrapeOutput( + items=[ + {"id": "1", "webVideoUrl": "https://tiktok.com/@a/video/1"}, + {"errorCode": "no_items", "input": "nasa", "error": "blocked"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py index 8520a1d85..777ed49de 100644 --- a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py @@ -95,3 +95,18 @@ async def test_listing_dedupes_then_caps_per_target(): TikTokScrapeInput(hashtags=["x"], resultsPerPage=2), fetch_listing=fake_listing ) assert [i["id"] for i in items] == ["1", "2"] + + +async def test_empty_listing_emits_error_item(): + # A trust-gated/empty feed (0 videos) must surface one honest ErrorItem, + # tagged with errorCode, rather than vanishing silently. + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [] + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=["nasa"], resultsPerPage=5), + fetch_listing=fake_listing, + ) + assert len(items) == 1 + assert items[0]["errorCode"] == "no_items" + assert items[0]["input"] == "nasa"