From 6400dc5f0479d14c3681c7b90849a53ba0f8e5d9 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:17:45 +0200 Subject: [PATCH] fix(tiktok): gate headful profile feed on CRAWL_HEADED_XVFB_ENABLED fetch_item_list goes headful only when the flag promises a display; otherwise it stays headless so the browser launch never fails and the empty feed degrades to an ErrorItem. Verified under xvfb-run: real videos returned on a clean proxy IP, graceful degrade on a flagged one. --- .../proprietary/platforms/tiktok/session/listing.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index c1f3dfc15..dbdf83768 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -11,9 +11,9 @@ The pure response-shape parsing lives in :func:`items_from_response`; this modul is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile -feed (``/api/post/item_list``) returns an empty 200 to headless sessions, so -:func:`fetch_item_list` runs headful. ponytail: headful needs a display — run it -under Xvfb on a headless server. +feed returns an empty 200 to headless sessions, so :func:`fetch_item_list` goes +headful only when ``CRAWL_HEADED_XVFB_ENABLED`` promises an Xvfb display — else it +stays headless and degrades to an ``ErrorItem`` instead of crashing on launch. """ from __future__ import annotations @@ -25,6 +25,7 @@ from typing import Any from scrapling.fetchers import StealthyFetcher +from app.config import config from app.proprietary.web_crawler.stealth import ( build_stealthy_kwargs, get_stealth_config, @@ -250,7 +251,8 @@ def _fetch_sync( async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` itemStructs from a listing page's XHRs. - Runs headful: the profile feed returns an empty 200 to headless sessions. + Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display (the profile feed + is empty to headless sessions); headless otherwise so launch never fails. """ return await asyncio.to_thread( _fetch_sync, @@ -259,7 +261,7 @@ async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, An _ITEM_LIST_MARKERS, items_from_response, _scroll_page, - headless=False, + headless=not config.CRAWL_HEADED_XVFB_ENABLED, )