mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-16 23:01:06 +02:00
fix(tiktok): fetch profile video lists headful + dismiss login modal
The profile feed (/api/post/item_list) returns an empty 200 to headless sessions but serves data headful on the same proxy IP. Run fetch_item_list headful and dismiss the login modal that blocks mid-scroll.
This commit is contained in:
parent
867e940911
commit
c679f2a3ef
1 changed files with 32 additions and 4 deletions
|
|
@ -11,8 +11,9 @@ The pure response-shape parsing lives in :func:`items_from_response`; this modul
|
||||||
is the untested browser-I/O glue (covered by the e2e smoke, not unit tests).
|
is the untested browser-I/O glue (covered by the e2e smoke, not unit tests).
|
||||||
|
|
||||||
Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile
|
Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile
|
||||||
feed (``/api/post/item_list``) is additionally soft-blocked: TikTok returns an
|
feed (``/api/post/item_list``) returns an empty 200 to headless sessions, so
|
||||||
empty 200 even to its own signed request, so profile targets yield nothing.
|
:func:`fetch_item_list` runs headful. ponytail: headful needs a display — run it
|
||||||
|
under Xvfb on a headless server.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
@ -81,12 +82,33 @@ def _has_mstoken(page: Any) -> bool:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _dismiss_login_modal(page: Any) -> None:
|
||||||
|
"""Close the login modal that blocks scrolling; Escape as fallback.
|
||||||
|
|
||||||
|
Only the modal's own close button, never a generic dialog button (avoids
|
||||||
|
clicking "Log in").
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
closed = page.evaluate(
|
||||||
|
"""() => {
|
||||||
|
const btn = document.querySelector('[data-e2e="modal-close-inner-button"]');
|
||||||
|
if (btn) { btn.click(); return true; }
|
||||||
|
return false;
|
||||||
|
}"""
|
||||||
|
)
|
||||||
|
if not closed:
|
||||||
|
page.keyboard.press("Escape")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def _scroll_page(page: Any, collected: list[dict[str, Any]], target_count: int) -> None:
|
def _scroll_page(page: Any, collected: list[dict[str, Any]], target_count: int) -> None:
|
||||||
"""Page down a listing feed until enough items are captured or it stops growing."""
|
"""Page down a listing feed until enough items are captured or it stops growing."""
|
||||||
last_height = 0
|
last_height = 0
|
||||||
for _ in range(_SCROLL_MAX_ROUNDS):
|
for _ in range(_SCROLL_MAX_ROUNDS):
|
||||||
if len(collected) >= target_count:
|
if len(collected) >= target_count:
|
||||||
break
|
break
|
||||||
|
_dismiss_login_modal(page)
|
||||||
page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
||||||
page.wait_for_timeout(_SCROLL_SETTLE_MS)
|
page.wait_for_timeout(_SCROLL_SETTLE_MS)
|
||||||
height = page.evaluate("document.body.scrollHeight")
|
height = page.evaluate("document.body.scrollHeight")
|
||||||
|
|
@ -207,12 +229,14 @@ def _fetch_sync(
|
||||||
markers: tuple[str, ...],
|
markers: tuple[str, ...],
|
||||||
extract: ExtractFn,
|
extract: ExtractFn,
|
||||||
interact: InteractFn,
|
interact: InteractFn,
|
||||||
|
*,
|
||||||
|
headless: bool = True,
|
||||||
) -> list[dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
collected: list[dict[str, Any]] = []
|
collected: list[dict[str, Any]] = []
|
||||||
kwargs = build_stealthy_kwargs(get_stealth_config())
|
kwargs = build_stealthy_kwargs(get_stealth_config())
|
||||||
StealthyFetcher.fetch(
|
StealthyFetcher.fetch(
|
||||||
url,
|
url,
|
||||||
headless=True,
|
headless=headless,
|
||||||
network_idle=False,
|
network_idle=False,
|
||||||
proxy=get_proxy_url(),
|
proxy=get_proxy_url(),
|
||||||
page_action=_build_page_action(
|
page_action=_build_page_action(
|
||||||
|
|
@ -224,7 +248,10 @@ def _fetch_sync(
|
||||||
|
|
||||||
|
|
||||||
async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]:
|
async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]:
|
||||||
"""Return up to ``target_count`` itemStructs from a listing page's XHRs."""
|
"""Return up to ``target_count`` itemStructs from a listing page's XHRs.
|
||||||
|
|
||||||
|
Runs headful: the profile feed returns an empty 200 to headless sessions.
|
||||||
|
"""
|
||||||
return await asyncio.to_thread(
|
return await asyncio.to_thread(
|
||||||
_fetch_sync,
|
_fetch_sync,
|
||||||
page_url,
|
page_url,
|
||||||
|
|
@ -232,6 +259,7 @@ async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, An
|
||||||
_ITEM_LIST_MARKERS,
|
_ITEM_LIST_MARKERS,
|
||||||
items_from_response,
|
items_from_response,
|
||||||
_scroll_page,
|
_scroll_page,
|
||||||
|
headless=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue