mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-16 23:01:06 +02:00
fix(tiktok): retry empty item_list captures on a fresh exit IP
The profile feed is withheld from flagged IPs and the proxy rotates per request, so re-fetching an empty capture (up to TIKTOK_LISTING_MAX_ATTEMPTS) turns a bad first draw into a hit instead of an ErrorItem.
This commit is contained in:
parent
15fb50c83e
commit
c8fcf87685
1 changed files with 25 additions and 9 deletions
|
|
@ -253,16 +253,32 @@ async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, An
|
||||||
|
|
||||||
Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display (the profile feed
|
Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display (the profile feed
|
||||||
is empty to headless sessions); headless otherwise so launch never fails.
|
is empty to headless sessions); headless otherwise so launch never fails.
|
||||||
|
|
||||||
|
Retries on an empty capture up to ``TIKTOK_LISTING_MAX_ATTEMPTS``: the feed is
|
||||||
|
withheld from flagged exit IPs, and the proxy rotates per request, so each retry
|
||||||
|
is a fresh IP — turning a bad first draw into a hit instead of an ``ErrorItem``.
|
||||||
"""
|
"""
|
||||||
return await asyncio.to_thread(
|
headless = not config.CRAWL_HEADED_XVFB_ENABLED
|
||||||
_fetch_sync,
|
attempts = max(1, config.TIKTOK_LISTING_MAX_ATTEMPTS)
|
||||||
page_url,
|
for attempt in range(1, attempts + 1):
|
||||||
target_count,
|
items = await asyncio.to_thread(
|
||||||
_ITEM_LIST_MARKERS,
|
_fetch_sync,
|
||||||
items_from_response,
|
page_url,
|
||||||
_scroll_page,
|
target_count,
|
||||||
headless=not config.CRAWL_HEADED_XVFB_ENABLED,
|
_ITEM_LIST_MARKERS,
|
||||||
)
|
items_from_response,
|
||||||
|
_scroll_page,
|
||||||
|
headless=headless,
|
||||||
|
)
|
||||||
|
if items or attempt == attempts:
|
||||||
|
return items
|
||||||
|
logger.info(
|
||||||
|
"[tiktok] empty item_list for %s (attempt %d/%d); retrying on a fresh exit IP",
|
||||||
|
page_url,
|
||||||
|
attempt,
|
||||||
|
attempts,
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]:
|
async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]:
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue