fix: scrape indeed first page only, steer subagent to one query

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md

@@ -14,10 +14,9 @@ Answer the delegated question from live Indeed job data gathered with your verb,
 - Finding jobs for a role: call `indeed_scrape` with `search_queries`; narrow with `location`, `country`, `job_type`, `level`, `remote`, `radius`, and `from_days`.
 - Scraping a specific Indeed URL: pass a search, company jobs, or single job URL in `urls`.
 - Full descriptions: set `scrape_job_details=true` to fetch each job's detail page (slower: one extra load per job). Leave it false when the listing snippet is enough.
-- Controlling volume: use `max_items` for the total cap and `max_items_per_query` per search.
+- Cost model: Indeed is latency-bound. A cold session spends minutes solving Cloudflare, and each query returns only its first page (~15 jobs) — anonymous pagination is gated, so `max_items` above ~15/query buys nothing. Every extra phrasing adds wait, not depth.
-- Requested counts: `max_items` defaults to only 25 — when the task asks for N jobs, set `max_items` and `max_items_per_query` above N (with headroom for off-topic hits). A call that caps below the target can never satisfy it.
+- Default to ONE focused query. Only add phrasings when the role genuinely needs variety, and then put at most 2–3 in a SINGLE call's `search_queries` (they reuse one warmed session); never make separate `indeed_scrape` calls, which each pay the cold-start cost.
-- Under-delivery: if the first call returns fewer on-topic results than requested, broaden it yourself — more query phrasings, wider `radius`, drop restrictive filters, larger `from_days` — before settling. Return `status=partial` only after the broadened attempt, never after a single narrow call.
+- Prefer returning the first page's on-topic hits promptly over exhaustive coverage. If a single call under-delivers against a large requested N, return `status=partial` noting the ~one-page-per-query ceiling — do not chase it with more calls.
-- Batch multiple search terms into one call rather than many single-term calls.
 <include snippet="run_reader"/>
 - Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, salary/rank changes).
 </playbook>

surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py

@@ -1,14 +1,8 @@
 """Orchestrator for the Indeed scraper.
 
-:func:`iter_indeed` streams flat job items from one warmed browser session:
+:func:`iter_indeed` streams deduped job items from one warmed session; each
-``startUrls`` (search/company pages) or ``queries`` are paged by the ``start``
+search/company target contributes its first page. :func:`scrape_indeed` collects
-offset, deduped by ``jobKey``, and stopped on the first page that adds nothing
+the stream under a caller ``limit``. Targets run sequentially to reuse the session.
-or that Indeed gates (anonymous pagination is bounced past page 1, so a gated
-page ends that target and keeps the pages already yielded).
-:func:`scrape_indeed` collects the stream under a caller ``limit``.
-
-Targets run sequentially on a single session — a browser per target would cost
-far more than the sequential paging it would save.
 """
 
 from __future__ import annotations
@@ -16,9 +10,9 @@ from __future__ import annotations
 import logging
 from collections.abc import AsyncIterator
 from typing import Any
-from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+from urllib.parse import urlparse
 
-from .fetch import IndeedAccessBlockedError, IndeedSession, now_iso, open_session
+from .fetch import IndeedSession, now_iso, open_session
 from .parsers import (
     extract_jobcards_blob,
     job_results,
@@ -32,68 +26,36 @@ logger = logging.getLogger(__name__)
 
 __all__ = ["iter_indeed", "scrape_indeed"]
 
-# Indeed's search pagination increments the ``start`` offset by 10.
-_PAGE_STEP = 10
-# Indeed stops serving useful results past ~1000; cap pages well before that.
-_MAX_PAGES = 10
-
 
 def _emit(partial: dict[str, Any]) -> dict[str, Any]:
     """Stamp ``scrapedAt`` and normalize through the output model."""
     return IndeedItem(**{**partial, "scrapedAt": now_iso()}).to_output()
 
 
-def _with_start(url: str, start: int) -> str:
+async def _search_items(
-    """Return ``url`` with its ``start`` query param set (removed when 0)."""
+    session: IndeedSession, url: str, *, domain: str, max_items: int
-    parsed = urlparse(url)
-    params = [(k, v) for k, v in parse_qsl(parsed.query) if k != "start"]
-    if start:
-        params.append(("start", str(start)))
-    return urlunparse(parsed._replace(query=urlencode(params)))
-
-
-async def _paginate(
-    session: IndeedSession, first_url: str, *, domain: str, max_items: int
 ) -> AsyncIterator[dict[str, Any]]:
-    """Yield items across pages of one search/company URL, deduped by ``jobKey``."""
+    """Yield deduped job cards from one search/company page.
+
+    ponytail: caps a query at its first page (~15 jobs) — anonymous Indeed gates
+    ``start>=10``; deeper depth needs an authenticated session or Indeed's API.
+    """
     if max_items <= 0:
         return
     base_url = f"https://{domain}"
+    html = await session.fetch_html(url)
     seen: set[str] = set()
     emitted = 0
-    for page in range(_MAX_PAGES):
+    for raw in job_results(extract_jobcards_blob(html)):
-        try:
+        item = parse_job(raw, base_url=base_url)
-            # Rotate to establish first-page access, but fail fast on deeper
+        job_key = item.get("jobKey")
-            # pages: Indeed gates anonymous pagination to secure.indeed.com, a
+        if isinstance(job_key, str):
-            # block no fresh IP clears — retrying it only burns the time budget.
+            if job_key in seen:
-            # ponytail: deeper pages are dropped rather than retried; lift the
+                continue
-            # cap with an async job if full-depth pagination is ever needed.
+            seen.add(job_key)
-            html = await session.fetch_html(
+        yield _emit(item)
-                _with_start(first_url, page * _PAGE_STEP),
+        emitted += 1
-                max_rotations=None if page == 0 else 0,
+        if emitted >= max_items:
-            )
-        except IndeedAccessBlockedError:
-            if page == 0:
-                raise  # never got in on this target; surface the block
-            logger.info("[indeed] pagination gated at start=%d; stopping", page * _PAGE_STEP)
-            return  # keep the pages already yielded
-        raws = job_results(extract_jobcards_blob(html))
-        if not raws:
-            return
-        added = 0
-        for raw in raws:
-            item = parse_job(raw, base_url=base_url)
-            job_key = item.get("jobKey")
-            if isinstance(job_key, str):
-                if job_key in seen:
-                    continue
-                seen.add(job_key)
-            yield _emit(item)
-            emitted += 1
-            added += 1
-            if emitted >= max_items:
-                return
-        if added == 0:  # a whole page of duplicates: end of useful results
             return
 
 
@@ -175,7 +137,7 @@ async def iter_indeed(
             if item is not None:
                 yield item
             continue
-        async for item in _paginate(
+        async for item in _search_items(
             session, url, domain=domain, max_items=input_model.maxItemsPerQuery
         ):
             job_key = item.get("jobKey")

surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py

@@ -62,41 +62,29 @@ async def _collect(input_model, session) -> list[dict]:
 
 
 @pytest.mark.asyncio
-async def test_paginates_and_dedupes_across_pages():
+async def test_dedupes_within_page():
-    session = _FakeSession({0: ["k1", "k2", "k3"], 10: ["k3", "k4"], 20: []})
+    session = _FakeSession({0: ["k1", "k2", "k2", "k3"]})
     items = await _collect(
         IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session
     )
-    assert [i["jobKey"] for i in items] == ["k1", "k2", "k3", "k4"]
+    assert [i["jobKey"] for i in items] == ["k1", "k2", "k3"]
     assert all(i["scrapedAt"] for i in items)  # stamped by the orchestrator
 
 
 @pytest.mark.asyncio
-async def test_stops_when_a_page_is_all_duplicates():
+async def test_does_not_fetch_deeper_pages():
-    session = _FakeSession({0: ["k1", "k2"], 10: ["k1", "k2"], 20: ["k9"]})
+    # First page only; ``start>=10`` must never be requested.
+    session = _FakeSession({0: ["k1", "k2"], 10: ["k3"]})
     items = await _collect(
         IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session
     )
     assert [i["jobKey"] for i in items] == ["k1", "k2"]
-    assert 20 not in {
+    assert all("start=" not in u for u in session.fetched)
-        int(parse_qs(urlparse(u).query).get("start", ["0"])[0]) for u in session.fetched
-    }
 
 
 @pytest.mark.asyncio
-async def test_pagination_gate_keeps_earlier_pages():
+async def test_page_block_propagates():
-    # Indeed gates anonymous pagination: page 0 serves, start=10 is blocked.
+    # Nothing yielded before the block, so it surfaces as an error.
-    # The already-yielded page-0 items must survive; paging just stops.
-    session = _FakeSession({0: ["k1", "k2"], 10: ["k3"]}, blocked_starts={10})
-    items = await _collect(
-        IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session
-    )
-    assert [i["jobKey"] for i in items] == ["k1", "k2"]
-
-
-@pytest.mark.asyncio
-async def test_first_page_block_propagates():
-    # A block on the very first page yielded nothing, so it surfaces as an error.
     session = _FakeSession({}, blocked_starts={0})
     with pytest.raises(IndeedAccessBlockedError):
         await _collect(IndeedScrapeInput(queries=["dev"]), session)