fix: fail fast on gated indeed pagination, keep collected pages

surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py

@@ -140,14 +140,18 @@ class IndeedSession:
             await self._timed_fetch(f"https://{domain}/", google_search=True)
         self._warmed.add(domain)
 
-    async def fetch_html(self, url: str) -> str:
+    async def fetch_html(self, url: str, *, max_rotations: int | None = None) -> str:
         """Return a search/company/job page's HTML through the warmed session.
 
         Rotates the IP and re-warms on a security-wall bounce or timeout; raises
-        :class:`IndeedAccessBlockedError` once every rotation is exhausted.
+        :class:`IndeedAccessBlockedError` once rotations are exhausted. ``max_rotations``
+        overrides the default budget: pass ``0`` to fail fast on a systematically
+        gated page (e.g. anonymous pagination) instead of burning rotations on a
+        block no fresh IP will clear.
         """
         if self._session is None:
             await self.start()
+        budget = _MAX_ROTATIONS if max_rotations is None else max_rotations
         domain = urlparse(url).hostname or "www.indeed.com"
         attempt = 0
         while True:
@@ -163,9 +167,9 @@ class IndeedSession:
             except Exception as e:
                 logger.warning("[indeed] fetch failed on %s: %s", url, e)
 
-            if attempt >= _MAX_ROTATIONS:
+            if attempt >= budget:
                 raise IndeedAccessBlockedError(
-                    f"Indeed refused {url} on {attempt} rotated IPs"
+                    f"Indeed refused {url} after {attempt + 1} attempt(s)"
                 )
             attempt += 1
             await self._rotate()

surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py

@@ -2,7 +2,9 @@
 
 :func:`iter_indeed` streams flat job items from one warmed browser session:
 ``startUrls`` (search/company pages) or ``queries`` are paged by the ``start``
-offset, deduped by ``jobKey``, and stopped on the first page that adds nothing.
+offset, deduped by ``jobKey``, and stopped on the first page that adds nothing
+or that Indeed gates (anonymous pagination is bounced past page 1, so a gated
+page ends that target and keeps the pages already yielded).
 :func:`scrape_indeed` collects the stream under a caller ``limit``.
 
 Targets run sequentially on a single session — a browser per target would cost
@@ -16,7 +18,7 @@ from collections.abc import AsyncIterator
 from typing import Any
 from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
 
-from .fetch import IndeedSession, now_iso, open_session
+from .fetch import IndeedAccessBlockedError, IndeedSession, now_iso, open_session
 from .parsers import (
     extract_jobcards_blob,
     job_results,
@@ -60,7 +62,21 @@ async def _paginate(
     seen: set[str] = set()
     emitted = 0
     for page in range(_MAX_PAGES):
-        html = await session.fetch_html(_with_start(first_url, page * _PAGE_STEP))
+        try:
+            # Rotate to establish first-page access, but fail fast on deeper
+            # pages: Indeed gates anonymous pagination to secure.indeed.com, a
+            # block no fresh IP clears — retrying it only burns the time budget.
+            # ponytail: deeper pages are dropped rather than retried; lift the
+            # cap with an async job if full-depth pagination is ever needed.
+            html = await session.fetch_html(
+                _with_start(first_url, page * _PAGE_STEP),
+                max_rotations=None if page == 0 else 0,
+            )
+        except IndeedAccessBlockedError:
+            if page == 0:
+                raise  # never got in on this target; surface the block
+            logger.info("[indeed] pagination gated at start=%d; stopping", page * _PAGE_STEP)
+            return  # keep the pages already yielded
         raws = job_results(extract_jobcards_blob(html))
         if not raws:
             return
@@ -127,7 +143,10 @@ async def _enrich(session: IndeedSession, item: dict[str, Any], base_url: str) -
     if not isinstance(job_url, str):
         return
     try:
-        detail = parse_job_detail(await session.fetch_html(job_url), base_url=base_url)
+        # Fail fast: enrichment is best-effort, so a gated detail page must not
+        # rotate IPs and eat the run's time budget for one job's description.
+        html = await session.fetch_html(job_url, max_rotations=0)
+        detail = parse_job_detail(html, base_url=base_url)
     except Exception as exc:
         logger.warning("[indeed] detail fetch failed for %s: %s", job_url, exc)
         return

surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py

@@ -85,6 +85,16 @@ async def test_raises_after_exhausting_rotations():
     assert session.rotations == 3
 
 
+@pytest.mark.asyncio
+async def test_max_rotations_zero_fails_fast():
+    # A gated page (pagination) must raise on the first block without rotating.
+    ctrl = _Controller(["BLOCK", "OK"])
+    session = IndeedSession(ctrl.factory)
+    with pytest.raises(IndeedAccessBlockedError):
+        await session.fetch_html(_URL, max_rotations=0)
+    assert session.rotations == 0
+
+
 @pytest.mark.asyncio
 async def test_warms_domain_once_without_rotation():
     ctrl = _Controller(["OK", "OK"])

surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py

@@ -7,6 +7,7 @@ from urllib.parse import parse_qs, urlparse
 
 import pytest
 
+from app.proprietary.platforms.indeed_jobs.fetch import IndeedAccessBlockedError
 from app.proprietary.platforms.indeed_jobs.schemas import IndeedScrapeInput
 from app.proprietary.platforms.indeed_jobs.scraper import iter_indeed, scrape_indeed
 
@@ -38,16 +39,21 @@ def _detail_html(job_key: str) -> str:
 class _FakeSession:
     """Returns per-``start`` search pages (or a /viewjob detail) and records URLs."""
 
-    def __init__(self, pages: dict[int, list[str]]) -> None:
+    def __init__(
+        self, pages: dict[int, list[str]], blocked_starts: set[int] | None = None
+    ) -> None:
         self._pages = pages
+        self._blocked = blocked_starts or set()
         self.fetched: list[str] = []
 
-    async def fetch_html(self, url: str) -> str:
+    async def fetch_html(self, url: str, *, max_rotations: int | None = None) -> str:
         self.fetched.append(url)
         query = parse_qs(urlparse(url).query)
         if "/viewjob" in url:
             return _detail_html(query.get("jk", [""])[0])
         start = int(query.get("start", ["0"])[0])
+        if start in self._blocked:
+            raise IndeedAccessBlockedError(f"gated at start={start}")
         return _page_html(self._pages.get(start, []))
 
 
@@ -77,6 +83,25 @@ async def test_stops_when_a_page_is_all_duplicates():
     }
 
 
+@pytest.mark.asyncio
+async def test_pagination_gate_keeps_earlier_pages():
+    # Indeed gates anonymous pagination: page 0 serves, start=10 is blocked.
+    # The already-yielded page-0 items must survive; paging just stops.
+    session = _FakeSession({0: ["k1", "k2"], 10: ["k3"]}, blocked_starts={10})
+    items = await _collect(
+        IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session
+    )
+    assert [i["jobKey"] for i in items] == ["k1", "k2"]
+
+
+@pytest.mark.asyncio
+async def test_first_page_block_propagates():
+    # A block on the very first page yielded nothing, so it surfaces as an error.
+    session = _FakeSession({}, blocked_starts={0})
+    with pytest.raises(IndeedAccessBlockedError):
+        await _collect(IndeedScrapeInput(queries=["dev"]), session)
+
+
 @pytest.mark.asyncio
 async def test_respects_max_items_per_query():
     session = _FakeSession({0: ["k1", "k2", "k3", "k4"]})