apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-16 23:01:06 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor(instagram): improve handling of login-gated endpoints in fetch logic

Browse source 
Enhanced the fetch_json function to immediately raise InstagramAccessBlockedError for login-gated endpoints, preventing unnecessary IP rotations. Introduced a new constant for authentication-walled paths to streamline the detection of access blocks, improving the scraper's efficiency in handling Instagram's restrictions.
This commit is contained in:
Anish Sarkar 2026-07-09 20:22:46 +05:30
parent 0309e1d28a
commit f66ab73044
1 changed files with 17 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

21
surfsense_backend/app/proprietary/platforms/instagram/fetch.py
View file

@ -75,6 +75,12 @@ _MAX_ROTATIONS = 3
_MAX_BACKOFFS = 4
_BACKOFF_BASE_S = 5.0
# Endpoints Instagram serves only to logged-in clients (confirmed live). A bare
# 401/403 here is an endpoint auth wall, not a per-IP block, so every rotated IP
# hits the same wall — fail fast instead of burning the pool, exactly like the
# /accounts/login/ redirect branch. Content endpoints (profiles) still rotate.
_AUTH_WALLED_PATHS = ("web/search/topsearch/", "api/v1/tags/web_info/")
# Instagram 429s hard on bursts. Pace each sticky session so a fast IP can't
# burst past the per-IP threshold. ponytail: 1.5s is tuned to residential exits;
# a pool with a stricter per-IP cap may need it raised — watch for 429 log spam.
@ -362,11 +368,18 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N
)
await asyncio.sleep(delay + random.uniform(0, 1))
continue
if status in _ROTATE_STATUSES and attempt < _MAX_ROTATIONS:
attempt += 1
await holder.rotate()
continue
if status in _ROTATE_STATUSES:
# Bare 401/403 on a login-gated endpoint: rotating never clears an
# endpoint auth wall, so fail fast (mirrors the login-redirect
# branch above). Other endpoints rotate — a per-IP 401 recovers.
if any(p in path for p in _AUTH_WALLED_PATHS):
raise InstagramAccessBlockedError(
f"Instagram login wall on {path} (endpoint requires auth)"
)
if attempt < _MAX_ROTATIONS:
attempt += 1
await holder.rotate()
continue
raise InstagramAccessBlockedError(
f"Instagram refused {path} on {attempt} rotated IPs ({status})"
)