test(tiktok): pin live fixtures, trim listing debug scaffolding

surfsense_backend/scripts/e2e_tiktok_scrape.py

@@ -7,10 +7,11 @@ Run from the backend directory:
 What it exercises (everything REAL — live network, live proxy, live browser):
 
   Stage 1 — proxy egress proof (informational).
-  Stage 2 — profile listing via the stealth browser (captures item_list XHRs).
-  Stage 3 — blob video path over HTTP (a video URL derived from Stage 2).
-  Stage 4 — hashtag listing via the stealth browser.
-  Stage 5 — full scrape_tiktok() pipeline on a profile.
+  Stage 2 — profile listing via the stealth browser (soft-blocked by TikTok;
+            expected empty until a stronger anti-detection path exists).
+  Stage 3 — blob video path over HTTP (URL taken from a captured hashtag struct).
+  Stage 4 — hashtag listing via the stealth browser (captures item_list XHRs).
+  Stage 5 — full scrape_tiktok() pipeline on a hashtag.
 
 On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser
 suites can pin against real-shaped data without network.
@@ -39,8 +40,8 @@ for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
 
 _FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok"
 
-# Evergreen public targets: the official account and a broad hashtag.
-_PROFILE = "tiktok"
+# Evergreen public targets: a regular high-volume creator and a broad hashtag.
+_PROFILE = "nasa"
 _HASHTAG = "food"
 _COUNT = 5
 
@@ -70,17 +71,11 @@ def _dump_fixture(name: str, data: Any) -> None:
     print(f"  [fixture] wrote {path.relative_to(_BACKEND_ROOT)}")
 
 
-async def _discover_video_url() -> str | None:
-    """Find a live video URL over plain HTTP (independent of the browser path)."""
-    import re
-
-    from app.proprietary.platforms.tiktok.session import fetch_html
-
-    html = await fetch_html(f"https://www.tiktok.com/@{_PROFILE}")
-    if not html:
-        return None
-    match = re.search(r"/@[\w.]+/video/\d+", html)
-    return f"https://www.tiktok.com{match.group(0)}" if match else None
+def _url_from_struct(struct: dict[str, Any]) -> str | None:
+    """Build a canonical video URL from a captured itemStruct."""
+    author = (struct.get("author") or {}).get("uniqueId")
+    vid = struct.get("id")
+    return f"https://www.tiktok.com/@{author}/video/{vid}" if author and vid else None
 
 
 async def stage_proxy() -> bool:
@@ -108,8 +103,6 @@ async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]:
         len(raw) > 0 and isinstance(raw[0], dict) and bool(raw[0].get("id")),
         f"{len(raw)} struct(s)",
     )
-    if ok:
-        _dump_fixture("listing_item.json", raw[0])
     return ok, raw
 
 
@@ -139,17 +132,20 @@ async def stage_blob_video(video_url: str) -> tuple[bool, dict[str, Any] | None]
     return ok, raw
 
 
-async def stage_hashtag_listing() -> bool:
+async def stage_hashtag_listing() -> tuple[bool, list[dict[str, Any]]]:
     _hr(f"STAGE 4 — hashtag listing (browser): #{_HASHTAG}")
     from app.proprietary.platforms.tiktok.session import fetch_item_list
 
     url = f"https://www.tiktok.com/tag/{_HASHTAG}"
     raw = await fetch_item_list(url, _COUNT)
-    return _check(
+    ok = _check(
         "captured itemStructs for hashtag",
         len(raw) > 0 and bool(raw[0].get("id")),
         f"{len(raw)} struct(s)",
     )
+    if ok:
+        _dump_fixture("listing_item.json", raw[0])
+    return ok, raw
 
 
 async def stage_pipeline() -> bool:
@@ -157,7 +153,7 @@ async def stage_pipeline() -> bool:
     from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok
 
     items = await scrape_tiktok(
-        TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=3), limit=3
+        TikTokScrapeInput(hashtags=[_HASHTAG], resultsPerPage=3), limit=3
     )
     ok = _check(
         "pipeline returns normalized video items",
@@ -177,18 +173,20 @@ async def main() -> int:
 
     results["Stage 1 proxy"] = await stage_proxy()
 
-    # Blob path first: HTTP-only, so it's validated independently of the
-    # proxy-sensitive browser listing path.
-    video_url = await _discover_video_url()
+    # Hashtag listing is the reliable browser path; use one of its captured
+    # structs to build a real video URL for the HTTP blob path.
+    ok_tag, tag_structs = await stage_hashtag_listing()
+    results["Stage 4 hashtag listing"] = ok_tag
+
+    video_url = _url_from_struct(tag_structs[0]) if tag_structs else None
     if video_url:
         ok_video, _ = await stage_blob_video(video_url)
         results["Stage 3 blob video"] = ok_video
     else:
-        print("\n  [SKIP] Stage 3 — could not discover a video URL over HTTP")
+        print("\n  [SKIP] Stage 3 — no captured struct to build a video URL")
 
     ok_profile, _ = await stage_profile_listing()
     results["Stage 2 profile listing"] = ok_profile
-    results["Stage 4 hashtag listing"] = await stage_hashtag_listing()
     results["Stage 5 pipeline"] = await stage_pipeline()
 
     _hr("SUMMARY")