feat(tiktok): emit graceful ErrorItem for blocked/empty listings

Profile and search feeds are trust-gated: an anonymous headless session
gets an empty item_list (profile) or no results XHR (search), while
hashtag feeds load. A zero-item listing now yields one honest ErrorItem
(errorCode="no_items") instead of vanishing silently, and ErrorItems are
excluded from billing so a blocked target is surfaced but never charged.

surfsense_backend/app/capabilities/tiktok/scrape/schemas.py

@@ -82,5 +82,6 @@ class ScrapeOutput(BaseModel):
 
     @property
     def billable_units(self) -> int:
-        """One returned item = one billable unit."""
+        """One returned video = one billable unit; ErrorItems (``errorCode`` set,
-        return len(self.items)
+        for blocked/empty targets) are surfaced but never charged."""
+        return sum(1 for item in self.items if not getattr(item, "errorCode", None))

surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py

@@ -12,9 +12,19 @@ from typing import Any
 
 from ..extraction import parse_video
 from ..extraction.timestamps import now_iso
+from ..schemas import ErrorItem
 from ..targets.types import TikTokTarget
 from . import FetchListingFn
 
+# Profile and search feeds are trust-gated: an anonymous headless session gets an
+# empty body (profile) or no results XHR (search), while hashtag feeds load. We
+# can't tell "genuinely empty" from "blocked" here, so a zero-item listing emits
+# one honest ErrorItem instead of vanishing silently.
+_EMPTY_LISTING_MESSAGE = (
+    "No videos returned. The target may be empty/private/removed, or TikTok "
+    "withheld this feed from anonymous access (common for profiles and search)."
+)
+
 
 async def iter_listing(
     target: TikTokTarget, *, cap: int, fetch_listing: FetchListingFn
@@ -35,3 +45,11 @@ async def iter_listing(
         emitted += 1
         if emitted >= cap:
             return
+    if emitted == 0:
+        yield ErrorItem(
+            url=target.url,
+            input=target.value,
+            error=_EMPTY_LISTING_MESSAGE,
+            errorCode="no_items",
+            scrapedAt=now_iso(),
+        ).to_output()

surfsense_backend/scripts/e2e_tiktok_scrape.py

@@ -7,11 +7,14 @@ Run from the backend directory:
 What it exercises (everything REAL — live network, live proxy, live browser):
 
   Stage 1 — proxy egress proof (informational).
-  Stage 2 — profile listing via the stealth browser (soft-blocked by TikTok;
+  Stage 2 — profile via the full pipeline: TikTok soft-blocks the anonymous
-            expected empty until a stronger anti-detection path exists).
+            profile feed, so this asserts graceful degradation — real videos OR
+            a single honest ErrorItem, never a silent empty.
   Stage 3 — blob video path over HTTP (URL taken from a captured hashtag struct).
   Stage 4 — hashtag listing via the stealth browser (captures item_list XHRs).
   Stage 5 — full scrape_tiktok() pipeline on a hashtag.
+  Stage 6 — search via the full pipeline: same graceful-degrade contract as
+            profile (results feed doesn't load for anonymous sessions).
 
 On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser
 suites can pin against real-shaped data without network.
@@ -40,9 +43,11 @@ for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
 
 _FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok"
 
-# Evergreen public targets: a regular high-volume creator and a broad hashtag.
+# Evergreen public targets: a regular high-volume creator, a broad hashtag, and
+# a common search term.
 _PROFILE = "nasa"
 _HASHTAG = "food"
+_SEARCH = "meal prep"
 _COUNT = 5
 
 
@@ -93,17 +98,24 @@ async def stage_proxy() -> bool:
 
 
 async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]:
-    _hr(f"STAGE 2 — profile listing (browser): @{_PROFILE}")
+    _hr(f"STAGE 2 — profile listing graceful-degrade: @{_PROFILE}")
-    from app.proprietary.platforms.tiktok.session import fetch_item_list
+    from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok
 
-    url = f"https://www.tiktok.com/@{_PROFILE}"
+    # TikTok soft-blocks the profile feed for anonymous headless sessions
-    raw = await fetch_item_list(url, _COUNT)
+    # (empty 200). The shipped contract is a single honest ErrorItem, not a
-    ok = _check(
+    # silent empty. This stage verifies that graceful degradation, and passes if
-        "captured itemStructs from item_list XHRs",
+    # EITHER real videos come back (session got trusted) OR an ErrorItem does.
-        len(raw) > 0 and isinstance(raw[0], dict) and bool(raw[0].get("id")),
+    items = await scrape_tiktok(
-        f"{len(raw)} struct(s)",
+        TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=_COUNT), limit=_COUNT
     )
-    return ok, raw
+    has_video = any(it.get("id") and not it.get("errorCode") for it in items)
+    has_error = any(it.get("errorCode") == "no_items" for it in items)
+    ok = _check(
+        "profile yields videos or a graceful ErrorItem (never silent empty)",
+        has_video or has_error,
+        f"{len(items)} item(s); video={has_video} error={has_error}",
+    )
+    return ok, items
 
 
 async def stage_blob_video(video_url: str) -> tuple[bool, dict[str, Any] | None]:
@@ -167,6 +179,26 @@ async def stage_pipeline() -> bool:
     return ok
 
 
+async def stage_search_listing() -> tuple[bool, list[dict[str, Any]]]:
+    _hr(f"STAGE 6 — search listing graceful-degrade: {_SEARCH!r}")
+    from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok
+
+    # The search results feed doesn't load for anonymous headless sessions
+    # (results XHR never fires on a cold deep-link). Same contract as profile:
+    # verify a graceful ErrorItem instead of a silent empty.
+    items = await scrape_tiktok(
+        TikTokScrapeInput(searchQueries=[_SEARCH], resultsPerPage=_COUNT), limit=_COUNT
+    )
+    has_video = any(it.get("id") and not it.get("errorCode") for it in items)
+    has_error = any(it.get("errorCode") == "no_items" for it in items)
+    ok = _check(
+        "search yields videos or a graceful ErrorItem (never silent empty)",
+        has_video or has_error,
+        f"{len(items)} item(s); video={has_video} error={has_error}",
+    )
+    return ok, items
+
+
 async def main() -> int:
     print("TikTok scraper functional e2e — live network + proxy + browser")
     results: dict[str, bool] = {}
@@ -185,6 +217,9 @@ async def main() -> int:
     else:
         print("\n  [SKIP] Stage 3 — no captured struct to build a video URL")
 
+    ok_search, _ = await stage_search_listing()
+    results["Stage 6 search listing"] = ok_search
+
     ok_profile, _ = await stage_profile_listing()
     results["Stage 2 profile listing"] = ok_profile
     results["Stage 5 pipeline"] = await stage_pipeline()

surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py

@@ -9,6 +9,7 @@ from app.capabilities.tiktok.scrape.schemas import (
     MAX_TIKTOK_ITEMS,
     MAX_TIKTOK_SOURCES,
     ScrapeInput,
+    ScrapeOutput,
 )
 
 pytestmark = pytest.mark.unit
@@ -43,3 +44,15 @@ def test_rejects_more_sources_than_the_cap():
     too_many = [f"tag{i}" for i in range(MAX_TIKTOK_SOURCES + 1)]
     with pytest.raises(ValidationError):
         ScrapeInput(hashtags=too_many)
+
+
+def test_error_items_are_not_billed():
+    # Real videos count; ErrorItems (blocked/empty targets) are surfaced free.
+    out = ScrapeOutput(
+        items=[
+            {"id": "1", "webVideoUrl": "https://tiktok.com/@a/video/1"},
+            {"errorCode": "no_items", "input": "nasa", "error": "blocked"},
+        ]
+    )
+    assert len(out.items) == 2
+    assert out.billable_units == 1

surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py

@@ -95,3 +95,18 @@ async def test_listing_dedupes_then_caps_per_target():
         TikTokScrapeInput(hashtags=["x"], resultsPerPage=2), fetch_listing=fake_listing
     )
     assert [i["id"] for i in items] == ["1", "2"]
+
+
+async def test_empty_listing_emits_error_item():
+    # A trust-gated/empty feed (0 videos) must surface one honest ErrorItem,
+    # tagged with errorCode, rather than vanishing silently.
+    async def fake_listing(_url: str, _count: int) -> list[dict]:
+        return []
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(profiles=["nasa"], resultsPerPage=5),
+        fetch_listing=fake_listing,
+    )
+    assert len(items) == 1
+    assert items[0]["errorCode"] == "no_items"
+    assert items[0]["input"] == "nasa"