refactor: streamline TikTok and Instagram scraping logic by removing search_queries and enhancing documentation for clarity

surfsense_backend/scripts/e2e_instagram_scraper.py

@@ -54,9 +54,7 @@ from app.proprietary.platforms.instagram.url_resolver import resolve_url  # noqa
 _PROFILE = "natgeo"
 _SEARCH_TERM = "national geographic"
 
-_FIXTURE_DIR = (
-    _BACKEND_ROOT / "tests" / "unit" / "platforms" / "instagram" / "fixtures"
-)
+_FIXTURE_DIR = _BACKEND_ROOT / "tests" / "unit" / "platforms" / "instagram" / "fixtures"
 
 # Fields to strip from dumped fixtures so we never commit PII / volatile tokens.
 _PII_KEYS = frozenset(
@@ -98,7 +96,9 @@ async def step0_probe() -> bool:
         data = await fetch_json(
             "api/v1/users/web_profile_info/", {"username": _PROFILE}
         )
-        user = (data or {}).get("data", {}).get("user") if isinstance(data, dict) else None
+        user = (
+            (data or {}).get("data", {}).get("user") if isinstance(data, dict) else None
+        )
         print(f"    web_profile_info({_PROFILE}) -> user={'yes' if user else 'no'}")
         return _check("sticky web_profile_info", minted and bool(user))
 
@@ -179,9 +179,7 @@ async def step5_search() -> bool:
 
 async def step6_dump_fixtures(post_url: str | None) -> bool:
     _hr("STEP 6 — dump trimmed, anonymized fixtures for offline tests")
-    profile = await fetch_json(
-        "api/v1/users/web_profile_info/", {"username": _PROFILE}
-    )
+    profile = await fetch_json("api/v1/users/web_profile_info/", {"username": _PROFILE})
     _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
     wrote = []
     if isinstance(profile, dict) and profile.get("data", {}).get("user"):

surfsense_backend/scripts/e2e_tiktok_scrape.py

@@ -178,7 +178,9 @@ async def stage_pipeline() -> bool:
         f"{len(items)} item(s)",
     )
     if items:
-        print(f"  sample: {items[0].get('webVideoUrl')} — {items[0].get('text', '')[:60]!r}")
+        print(
+            f"  sample: {items[0].get('webVideoUrl')} — {items[0].get('text', '')[:60]!r}"
+        )
     return ok
 
 
@@ -210,9 +212,7 @@ async def stage_comments(video_url: str) -> tuple[bool, list[dict[str, Any]]]:
     # Comments load over a signed /api/comment/list XHR that TikTok serves to
     # anonymous sessions once the panel opens. Pass if real comments come back
     # OR a graceful ErrorItem (video has none / disabled / withheld).
-    items = await scrape_tiktok_comments(
-        [video_url], per_video=_COUNT, limit=_COUNT
-    )
+    items = await scrape_tiktok_comments([video_url], per_video=_COUNT, limit=_COUNT)
     has_comment = any(it.get("id") and not it.get("errorCode") for it in items)
     has_error = any(it.get("errorCode") == "no_comments" for it in items)
     ok = _check(
@@ -253,7 +253,9 @@ async def stage_trending() -> tuple[bool, list[dict[str, Any]]]:
         f"{len(items)} item(s); videos={len(real)}",
     )
     if real:
-        print(f"  sample: {real[0].get('webVideoUrl')} — {real[0].get('text', '')[:60]!r}")
+        print(
+            f"  sample: {real[0].get('webVideoUrl')} — {real[0].get('text', '')[:60]!r}"
+        )
     return ok, items