mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-16 23:01:06 +02:00
fix(tiktok): retry empty item_list captures on a fresh exit IP
The profile feed is withheld from flagged IPs and the proxy rotates per request, so re-fetching an empty capture (up to TIKTOK_LISTING_MAX_ATTEMPTS) turns a bad first draw into a hit instead of an ErrorItem.
This commit is contained in:
parent
15fb50c83e
commit
c8fcf87685
1 changed files with 25 additions and 9 deletions
|
|
@ -253,16 +253,32 @@ async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, An
|
|||
|
||||
Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display (the profile feed
|
||||
is empty to headless sessions); headless otherwise so launch never fails.
|
||||
|
||||
Retries on an empty capture up to ``TIKTOK_LISTING_MAX_ATTEMPTS``: the feed is
|
||||
withheld from flagged exit IPs, and the proxy rotates per request, so each retry
|
||||
is a fresh IP — turning a bad first draw into a hit instead of an ``ErrorItem``.
|
||||
"""
|
||||
return await asyncio.to_thread(
|
||||
_fetch_sync,
|
||||
page_url,
|
||||
target_count,
|
||||
_ITEM_LIST_MARKERS,
|
||||
items_from_response,
|
||||
_scroll_page,
|
||||
headless=not config.CRAWL_HEADED_XVFB_ENABLED,
|
||||
)
|
||||
headless = not config.CRAWL_HEADED_XVFB_ENABLED
|
||||
attempts = max(1, config.TIKTOK_LISTING_MAX_ATTEMPTS)
|
||||
for attempt in range(1, attempts + 1):
|
||||
items = await asyncio.to_thread(
|
||||
_fetch_sync,
|
||||
page_url,
|
||||
target_count,
|
||||
_ITEM_LIST_MARKERS,
|
||||
items_from_response,
|
||||
_scroll_page,
|
||||
headless=headless,
|
||||
)
|
||||
if items or attempt == attempts:
|
||||
return items
|
||||
logger.info(
|
||||
"[tiktok] empty item_list for %s (attempt %d/%d); retrying on a fresh exit IP",
|
||||
page_url,
|
||||
attempt,
|
||||
attempts,
|
||||
)
|
||||
return []
|
||||
|
||||
|
||||
async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue