fix(tiktok): fetch profile video lists headful + dismiss login modal

The profile feed (/api/post/item_list) returns an empty 200 to headless
sessions but serves data headful on the same proxy IP. Run fetch_item_list
headful and dismiss the login modal that blocks mid-scroll.
This commit is contained in:
CREDO23 2026-07-10 16:45:45 +02:00
parent 867e940911
commit c679f2a3ef

View file

@ -11,8 +11,9 @@ The pure response-shape parsing lives in :func:`items_from_response`; this modul
is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). is the untested browser-I/O glue (covered by the e2e smoke, not unit tests).
Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile
feed (``/api/post/item_list``) is additionally soft-blocked: TikTok returns an feed (``/api/post/item_list``) returns an empty 200 to headless sessions, so
empty 200 even to its own signed request, so profile targets yield nothing. :func:`fetch_item_list` runs headful. ponytail: headful needs a display run it
under Xvfb on a headless server.
""" """
from __future__ import annotations from __future__ import annotations
@ -81,12 +82,33 @@ def _has_mstoken(page: Any) -> bool:
return False return False
def _dismiss_login_modal(page: Any) -> None:
"""Close the login modal that blocks scrolling; Escape as fallback.
Only the modal's own close button, never a generic dialog button (avoids
clicking "Log in").
"""
try:
closed = page.evaluate(
"""() => {
const btn = document.querySelector('[data-e2e="modal-close-inner-button"]');
if (btn) { btn.click(); return true; }
return false;
}"""
)
if not closed:
page.keyboard.press("Escape")
except Exception:
pass
def _scroll_page(page: Any, collected: list[dict[str, Any]], target_count: int) -> None: def _scroll_page(page: Any, collected: list[dict[str, Any]], target_count: int) -> None:
"""Page down a listing feed until enough items are captured or it stops growing.""" """Page down a listing feed until enough items are captured or it stops growing."""
last_height = 0 last_height = 0
for _ in range(_SCROLL_MAX_ROUNDS): for _ in range(_SCROLL_MAX_ROUNDS):
if len(collected) >= target_count: if len(collected) >= target_count:
break break
_dismiss_login_modal(page)
page.evaluate("window.scrollTo(0, document.body.scrollHeight)") page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
page.wait_for_timeout(_SCROLL_SETTLE_MS) page.wait_for_timeout(_SCROLL_SETTLE_MS)
height = page.evaluate("document.body.scrollHeight") height = page.evaluate("document.body.scrollHeight")
@ -207,12 +229,14 @@ def _fetch_sync(
markers: tuple[str, ...], markers: tuple[str, ...],
extract: ExtractFn, extract: ExtractFn,
interact: InteractFn, interact: InteractFn,
*,
headless: bool = True,
) -> list[dict[str, Any]]: ) -> list[dict[str, Any]]:
collected: list[dict[str, Any]] = [] collected: list[dict[str, Any]] = []
kwargs = build_stealthy_kwargs(get_stealth_config()) kwargs = build_stealthy_kwargs(get_stealth_config())
StealthyFetcher.fetch( StealthyFetcher.fetch(
url, url,
headless=True, headless=headless,
network_idle=False, network_idle=False,
proxy=get_proxy_url(), proxy=get_proxy_url(),
page_action=_build_page_action( page_action=_build_page_action(
@ -224,7 +248,10 @@ def _fetch_sync(
async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]:
"""Return up to ``target_count`` itemStructs from a listing page's XHRs.""" """Return up to ``target_count`` itemStructs from a listing page's XHRs.
Runs headful: the profile feed returns an empty 200 to headless sessions.
"""
return await asyncio.to_thread( return await asyncio.to_thread(
_fetch_sync, _fetch_sync,
page_url, page_url,
@ -232,6 +259,7 @@ async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, An
_ITEM_LIST_MARKERS, _ITEM_LIST_MARKERS,
items_from_response, items_from_response,
_scroll_page, _scroll_page,
headless=False,
) )