Merge remote-tracking branch 'upstream/dev' into feat/instagram-scraper

surfsense_backend/scripts/docker/entrypoint.sh

@@ -121,7 +121,28 @@ start_beat() {
     echo "  Celery Beat PID=${PIDS[-1]}"
 }
 
+# ── Headful browser display ──────────────────────────────────
+# Give the stealth browser a virtual display so it can run headful (TikTok's
+# profile feed is empty to headless Chromium). Off => every browser stays headless.
+start_xvfb() {
+    if [ "$(echo "${CRAWL_HEADED_XVFB_ENABLED:-false}" | tr '[:upper:]' '[:lower:]')" != "true" ]; then
+        return
+    fi
+    local display="${XVFB_DISPLAY:-:99}"
+    echo "Starting Xvfb on ${display} (CRAWL_HEADED_XVFB_ENABLED=true)..."
+    Xvfb "${display}" -screen 0 1920x1080x24 -nolisten tcp >/dev/null 2>&1 &
+    PIDS+=($!)
+    export DISPLAY="${display}"
+    sleep 1  # let the X server accept connections before a browser starts
+    echo "  Xvfb PID=${PIDS[-1]} DISPLAY=${DISPLAY}"
+}
+
 # ── Main: run based on role ──────────────────────────────────
+# migrate never launches a browser; every other role might, so start the display.
+if [ "${SERVICE_ROLE}" != "migrate" ]; then
+    start_xvfb
+fi
+
 case "${SERVICE_ROLE}" in
     migrate)
         run_migrations

surfsense_backend/scripts/e2e_tiktok_scrape.py

@@ -0,0 +1,299 @@
+"""Manual functional e2e for the TikTok scraper (blob + browser-listing seams).
+
+Run from the backend directory:
+    cd surfsense_backend
+    uv run python scripts/e2e_tiktok_scrape.py
+
+What it exercises (everything REAL — live network, live proxy, live browser):
+
+  Stage 1 — proxy egress proof (informational).
+  Stage 2 — profile via the full pipeline: real videos when headful
+            (CRAWL_HEADED_XVFB_ENABLED), else one ErrorItem — never silent empty.
+  Stage 3 — blob video path over HTTP (URL taken from a captured hashtag struct).
+  Stage 4 — hashtag listing via the stealth browser (captures item_list XHRs).
+  Stage 5 — full scrape_tiktok() pipeline on a hashtag.
+  Stage 6 — search via the full pipeline: same graceful-degrade contract as
+            profile (results feed doesn't load for anonymous sessions).
+  Stage 7 — comments on a real video URL (served anonymously once the panel
+            opens): real comments OR a single honest ErrorItem.
+  Stage 8 — user search: the account-discovery XHR that DOES serve anonymous
+            headless sessions — asserts real account records come back.
+  Stage 9 — trending: the Explore feed of trending videos — asserts real,
+            normalized video items come back.
+
+On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser
+suites can pin against real-shaped data without network.
+
+This is NOT a pytest test (it needs a live stack + proxy + a real browser). It is
+the manual counterpart to the unit suites and the truth check for
+``session/listing.py``, which is otherwise unverified.
+"""
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlsplit
+
+from dotenv import load_dotenv
+
+# --- bootstrap: load .env and put the backend root on sys.path before app.* ---
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_BACKEND_ROOT))
+for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
+    if _candidate.exists():
+        load_dotenv(_candidate)
+        break
+
+_FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok"
+
+# Evergreen public targets: a regular high-volume creator, a broad hashtag, and
+# a common search term.
+_PROFILE = "nasa"
+_HASHTAG = "food"
+_SEARCH = "meal prep"
+_COUNT = 5
+
+
+def _mask(url: str | None) -> str:
+    if not url:
+        return "<none>"
+    p = urlsplit(url)
+    creds = "***@" if p.username else ""
+    port = f":{p.port}" if p.port else ""
+    return f"{p.scheme}://{creds}{p.hostname or '?'}{port}"
+
+
+def _hr(title: str) -> None:
+    print(f"\n{'=' * 70}\n{title}\n{'=' * 70}")
+
+
+def _check(label: str, ok: bool, detail: str = "") -> bool:
+    print(f"  [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}")
+    return ok
+
+
+def _dump_fixture(name: str, data: Any) -> None:
+    _FIXTURES.mkdir(parents=True, exist_ok=True)
+    path = _FIXTURES / name
+    path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+    print(f"  [fixture] wrote {path.relative_to(_BACKEND_ROOT)}")
+
+
+def _url_from_struct(struct: dict[str, Any]) -> str | None:
+    """Build a canonical video URL from a captured itemStruct."""
+    author = (struct.get("author") or {}).get("uniqueId")
+    vid = struct.get("id")
+    return f"https://www.tiktok.com/@{author}/video/{vid}" if author and vid else None
+
+
+async def stage_proxy() -> bool:
+    _hr("STAGE 1 — proxy egress (informational)")
+    from app.utils.proxy import get_active_provider, get_proxy_url, is_pool_backed
+
+    provider = get_active_provider()
+    proxy_url = get_proxy_url()
+    print(f"  active proxy provider : {provider.name}")
+    print(f"  proxy url             : {_mask(proxy_url)}")
+    print(f"  pool-backed (rotates) : {is_pool_backed()}")
+    if not proxy_url:
+        print("  [INFO] no proxy configured — TikTok may block anonymous access")
+    return True
+
+
+async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]:
+    _hr(f"STAGE 2 — profile listing graceful-degrade: @{_PROFILE}")
+    from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok
+
+    # Accept videos OR an ErrorItem (never a silent empty): the feed degrades
+    # gracefully when the exit IP is flagged.
+    items = await scrape_tiktok(
+        TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=_COUNT), limit=_COUNT
+    )
+    has_video = any(it.get("id") and not it.get("errorCode") for it in items)
+    has_error = any(it.get("errorCode") == "no_items" for it in items)
+    ok = _check(
+        "profile yields videos or a graceful ErrorItem (never silent empty)",
+        has_video or has_error,
+        f"{len(items)} item(s); video={has_video} error={has_error}",
+    )
+    return ok, items
+
+
+async def stage_blob_video(video_url: str) -> tuple[bool, dict[str, Any] | None]:
+    _hr("STAGE 3 — blob video path (HTTP)")
+    print(f"  target: {video_url}")
+    from app.proprietary.platforms.tiktok.extraction import (
+        extract_rehydration_data,
+        video_item_struct,
+    )
+    from app.proprietary.platforms.tiktok.session import fetch_html
+
+    html = await fetch_html(video_url)
+    if not _check("fetched page HTML", bool(html), f"{len(html or '')} chars"):
+        return False, None
+    data = extract_rehydration_data(html or "")
+    if not _check("extracted rehydration blob", data is not None):
+        return False, None
+    raw = video_item_struct(data or {})
+    ok = _check(
+        "blob carries the video itemStruct",
+        raw is not None and bool(raw.get("id")),
+        f"id={None if raw is None else raw.get('id')}",
+    )
+    if ok and raw is not None:
+        _dump_fixture("video_item_struct.json", raw)
+    return ok, raw
+
+
+async def stage_hashtag_listing() -> tuple[bool, list[dict[str, Any]]]:
+    _hr(f"STAGE 4 — hashtag listing (browser): #{_HASHTAG}")
+    from app.proprietary.platforms.tiktok.session import fetch_item_list
+
+    url = f"https://www.tiktok.com/tag/{_HASHTAG}"
+    raw = await fetch_item_list(url, _COUNT)
+    ok = _check(
+        "captured itemStructs for hashtag",
+        len(raw) > 0 and bool(raw[0].get("id")),
+        f"{len(raw)} struct(s)",
+    )
+    if ok:
+        _dump_fixture("listing_item.json", raw[0])
+    return ok, raw
+
+
+async def stage_pipeline() -> bool:
+    _hr("STAGE 5 — full scrape_tiktok() pipeline")
+    from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(hashtags=[_HASHTAG], resultsPerPage=3), limit=3
+    )
+    ok = _check(
+        "pipeline returns normalized video items",
+        len(items) > 0
+        and bool(items[0].get("id"))
+        and bool(items[0].get("webVideoUrl")),
+        f"{len(items)} item(s)",
+    )
+    if items:
+        print(f"  sample: {items[0].get('webVideoUrl')} — {items[0].get('text', '')[:60]!r}")
+    return ok
+
+
+async def stage_search_listing() -> tuple[bool, list[dict[str, Any]]]:
+    _hr(f"STAGE 6 — search listing graceful-degrade: {_SEARCH!r}")
+    from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok
+
+    # The search results feed doesn't load for anonymous headless sessions
+    # (results XHR never fires on a cold deep-link). Same contract as profile:
+    # verify a graceful ErrorItem instead of a silent empty.
+    items = await scrape_tiktok(
+        TikTokScrapeInput(searchQueries=[_SEARCH], resultsPerPage=_COUNT), limit=_COUNT
+    )
+    has_video = any(it.get("id") and not it.get("errorCode") for it in items)
+    has_error = any(it.get("errorCode") == "no_items" for it in items)
+    ok = _check(
+        "search yields videos or a graceful ErrorItem (never silent empty)",
+        has_video or has_error,
+        f"{len(items)} item(s); video={has_video} error={has_error}",
+    )
+    return ok, items
+
+
+async def stage_comments(video_url: str) -> tuple[bool, list[dict[str, Any]]]:
+    _hr("STAGE 7 — comments graceful-degrade")
+    print(f"  target: {video_url}")
+    from app.proprietary.platforms.tiktok import scrape_tiktok_comments
+
+    # Comments load over a signed /api/comment/list XHR that TikTok serves to
+    # anonymous sessions once the panel opens. Pass if real comments come back
+    # OR a graceful ErrorItem (video has none / disabled / withheld).
+    items = await scrape_tiktok_comments(
+        [video_url], per_video=_COUNT, limit=_COUNT
+    )
+    has_comment = any(it.get("id") and not it.get("errorCode") for it in items)
+    has_error = any(it.get("errorCode") == "no_comments" for it in items)
+    ok = _check(
+        "comments yield records or a graceful ErrorItem (never silent empty)",
+        has_comment or has_error,
+        f"{len(items)} item(s); comment={has_comment} error={has_error}",
+    )
+    return ok, items
+
+
+async def stage_user_search() -> tuple[bool, list[dict[str, Any]]]:
+    _hr(f"STAGE 8 — user search (browser): {_PROFILE!r}")
+    from app.proprietary.platforms.tiktok import search_tiktok_users
+
+    # Unlike keyword *video* search, the account-search XHR serves anonymous
+    # headless sessions — so this asserts real records, not just degradation.
+    items = await search_tiktok_users([_PROFILE], per_query=_COUNT, limit=_COUNT)
+    real = [it for it in items if not it.get("errorCode")]
+    ok = _check(
+        "user search returns account records",
+        bool(real) and bool(real[0].get("uniqueId") or real[0].get("name")),
+        f"{len(items)} item(s); accounts={len(real)}",
+    )
+    if real:
+        print(f"  sample: @{real[0].get('uniqueId') or real[0].get('name')}")
+    return ok, items
+
+
+async def stage_trending() -> tuple[bool, list[dict[str, Any]]]:
+    _hr("STAGE 9 — trending (browser): Explore feed")
+    from app.proprietary.platforms.tiktok import scrape_tiktok_trending
+
+    items = await scrape_tiktok_trending(count=_COUNT)
+    real = [it for it in items if not it.get("errorCode")]
+    ok = _check(
+        "trending returns normalized video items",
+        bool(real) and bool(real[0].get("id")) and bool(real[0].get("webVideoUrl")),
+        f"{len(items)} item(s); videos={len(real)}",
+    )
+    if real:
+        print(f"  sample: {real[0].get('webVideoUrl')} — {real[0].get('text', '')[:60]!r}")
+    return ok, items
+
+
+async def main() -> int:
+    print("TikTok scraper functional e2e — live network + proxy + browser")
+    results: dict[str, bool] = {}
+
+    results["Stage 1 proxy"] = await stage_proxy()
+
+    # Hashtag listing is the reliable browser path; use one of its captured
+    # structs to build a real video URL for the HTTP blob path.
+    ok_tag, tag_structs = await stage_hashtag_listing()
+    results["Stage 4 hashtag listing"] = ok_tag
+
+    video_url = _url_from_struct(tag_structs[0]) if tag_structs else None
+    if video_url:
+        ok_video, _ = await stage_blob_video(video_url)
+        results["Stage 3 blob video"] = ok_video
+        ok_comments, _ = await stage_comments(video_url)
+        results["Stage 7 comments"] = ok_comments
+    else:
+        print("\n  [SKIP] Stage 3/7 — no captured struct to build a video URL")
+
+    ok_search, _ = await stage_search_listing()
+    results["Stage 6 search listing"] = ok_search
+
+    ok_profile, _ = await stage_profile_listing()
+    results["Stage 2 profile listing"] = ok_profile
+    results["Stage 5 pipeline"] = await stage_pipeline()
+
+    ok_users, _ = await stage_user_search()
+    results["Stage 8 user search"] = ok_users
+    ok_trending, _ = await stage_trending()
+    results["Stage 9 trending"] = ok_trending
+
+    _hr("SUMMARY")
+    for name, ok in results.items():
+        print(f"  {'PASS' if ok else 'FAIL/SKIP'} — {name}")
+    return 0 if all(results.values()) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))