diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py index 9f913a579..c3be9c2a8 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py @@ -140,14 +140,18 @@ class IndeedSession: await self._timed_fetch(f"https://{domain}/", google_search=True) self._warmed.add(domain) - async def fetch_html(self, url: str) -> str: + async def fetch_html(self, url: str, *, max_rotations: int | None = None) -> str: """Return a search/company/job page's HTML through the warmed session. Rotates the IP and re-warms on a security-wall bounce or timeout; raises - :class:`IndeedAccessBlockedError` once every rotation is exhausted. + :class:`IndeedAccessBlockedError` once rotations are exhausted. ``max_rotations`` + overrides the default budget: pass ``0`` to fail fast on a systematically + gated page (e.g. anonymous pagination) instead of burning rotations on a + block no fresh IP will clear. """ if self._session is None: await self.start() + budget = _MAX_ROTATIONS if max_rotations is None else max_rotations domain = urlparse(url).hostname or "www.indeed.com" attempt = 0 while True: @@ -163,9 +167,9 @@ class IndeedSession: except Exception as e: logger.warning("[indeed] fetch failed on %s: %s", url, e) - if attempt >= _MAX_ROTATIONS: + if attempt >= budget: raise IndeedAccessBlockedError( - f"Indeed refused {url} on {attempt} rotated IPs" + f"Indeed refused {url} after {attempt + 1} attempt(s)" ) attempt += 1 await self._rotate() diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py index 0c9fcfb16..01371d281 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py @@ -2,7 +2,9 @@ :func:`iter_indeed` streams flat job items from one warmed browser session: ``startUrls`` (search/company pages) or ``queries`` are paged by the ``start`` -offset, deduped by ``jobKey``, and stopped on the first page that adds nothing. +offset, deduped by ``jobKey``, and stopped on the first page that adds nothing +or that Indeed gates (anonymous pagination is bounced past page 1, so a gated +page ends that target and keeps the pages already yielded). :func:`scrape_indeed` collects the stream under a caller ``limit``. Targets run sequentially on a single session — a browser per target would cost @@ -16,7 +18,7 @@ from collections.abc import AsyncIterator from typing import Any from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse -from .fetch import IndeedSession, now_iso, open_session +from .fetch import IndeedAccessBlockedError, IndeedSession, now_iso, open_session from .parsers import ( extract_jobcards_blob, job_results, @@ -60,7 +62,21 @@ async def _paginate( seen: set[str] = set() emitted = 0 for page in range(_MAX_PAGES): - html = await session.fetch_html(_with_start(first_url, page * _PAGE_STEP)) + try: + # Rotate to establish first-page access, but fail fast on deeper + # pages: Indeed gates anonymous pagination to secure.indeed.com, a + # block no fresh IP clears — retrying it only burns the time budget. + # ponytail: deeper pages are dropped rather than retried; lift the + # cap with an async job if full-depth pagination is ever needed. + html = await session.fetch_html( + _with_start(first_url, page * _PAGE_STEP), + max_rotations=None if page == 0 else 0, + ) + except IndeedAccessBlockedError: + if page == 0: + raise # never got in on this target; surface the block + logger.info("[indeed] pagination gated at start=%d; stopping", page * _PAGE_STEP) + return # keep the pages already yielded raws = job_results(extract_jobcards_blob(html)) if not raws: return @@ -127,7 +143,10 @@ async def _enrich(session: IndeedSession, item: dict[str, Any], base_url: str) - if not isinstance(job_url, str): return try: - detail = parse_job_detail(await session.fetch_html(job_url), base_url=base_url) + # Fail fast: enrichment is best-effort, so a gated detail page must not + # rotate IPs and eat the run's time budget for one job's description. + html = await session.fetch_html(job_url, max_rotations=0) + detail = parse_job_detail(html, base_url=base_url) except Exception as exc: logger.warning("[indeed] detail fetch failed for %s: %s", job_url, exc) return diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py index 4eae65f84..258f5154d 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py @@ -85,6 +85,16 @@ async def test_raises_after_exhausting_rotations(): assert session.rotations == 3 +@pytest.mark.asyncio +async def test_max_rotations_zero_fails_fast(): + # A gated page (pagination) must raise on the first block without rotating. + ctrl = _Controller(["BLOCK", "OK"]) + session = IndeedSession(ctrl.factory) + with pytest.raises(IndeedAccessBlockedError): + await session.fetch_html(_URL, max_rotations=0) + assert session.rotations == 0 + + @pytest.mark.asyncio async def test_warms_domain_once_without_rotation(): ctrl = _Controller(["OK", "OK"]) diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py index e5ea23452..acf52296b 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py @@ -7,6 +7,7 @@ from urllib.parse import parse_qs, urlparse import pytest +from app.proprietary.platforms.indeed_jobs.fetch import IndeedAccessBlockedError from app.proprietary.platforms.indeed_jobs.schemas import IndeedScrapeInput from app.proprietary.platforms.indeed_jobs.scraper import iter_indeed, scrape_indeed @@ -38,16 +39,21 @@ def _detail_html(job_key: str) -> str: class _FakeSession: """Returns per-``start`` search pages (or a /viewjob detail) and records URLs.""" - def __init__(self, pages: dict[int, list[str]]) -> None: + def __init__( + self, pages: dict[int, list[str]], blocked_starts: set[int] | None = None + ) -> None: self._pages = pages + self._blocked = blocked_starts or set() self.fetched: list[str] = [] - async def fetch_html(self, url: str) -> str: + async def fetch_html(self, url: str, *, max_rotations: int | None = None) -> str: self.fetched.append(url) query = parse_qs(urlparse(url).query) if "/viewjob" in url: return _detail_html(query.get("jk", [""])[0]) start = int(query.get("start", ["0"])[0]) + if start in self._blocked: + raise IndeedAccessBlockedError(f"gated at start={start}") return _page_html(self._pages.get(start, [])) @@ -77,6 +83,25 @@ async def test_stops_when_a_page_is_all_duplicates(): } +@pytest.mark.asyncio +async def test_pagination_gate_keeps_earlier_pages(): + # Indeed gates anonymous pagination: page 0 serves, start=10 is blocked. + # The already-yielded page-0 items must survive; paging just stops. + session = _FakeSession({0: ["k1", "k2"], 10: ["k3"]}, blocked_starts={10}) + items = await _collect( + IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session + ) + assert [i["jobKey"] for i in items] == ["k1", "k2"] + + +@pytest.mark.asyncio +async def test_first_page_block_propagates(): + # A block on the very first page yielded nothing, so it surfaces as an error. + session = _FakeSession({}, blocked_starts={0}) + with pytest.raises(IndeedAccessBlockedError): + await _collect(IndeedScrapeInput(queries=["dev"]), session) + + @pytest.mark.asyncio async def test_respects_max_items_per_query(): session = _FakeSession({0: ["k1", "k2", "k3", "k4"]})