diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md index b034b700a..36eb855d2 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md @@ -14,10 +14,9 @@ Answer the delegated question from live Indeed job data gathered with your verb, - Finding jobs for a role: call `indeed_scrape` with `search_queries`; narrow with `location`, `country`, `job_type`, `level`, `remote`, `radius`, and `from_days`. - Scraping a specific Indeed URL: pass a search, company jobs, or single job URL in `urls`. - Full descriptions: set `scrape_job_details=true` to fetch each job's detail page (slower: one extra load per job). Leave it false when the listing snippet is enough. -- Controlling volume: use `max_items` for the total cap and `max_items_per_query` per search. -- Requested counts: `max_items` defaults to only 25 — when the task asks for N jobs, set `max_items` and `max_items_per_query` above N (with headroom for off-topic hits). A call that caps below the target can never satisfy it. -- Under-delivery: if the first call returns fewer on-topic results than requested, broaden it yourself — more query phrasings, wider `radius`, drop restrictive filters, larger `from_days` — before settling. Return `status=partial` only after the broadened attempt, never after a single narrow call. -- Batch multiple search terms into one call rather than many single-term calls. +- Cost model: Indeed is latency-bound. A cold session spends minutes solving Cloudflare, and each query returns only its first page (~15 jobs) — anonymous pagination is gated, so `max_items` above ~15/query buys nothing. Every extra phrasing adds wait, not depth. +- Default to ONE focused query. Only add phrasings when the role genuinely needs variety, and then put at most 2–3 in a SINGLE call's `search_queries` (they reuse one warmed session); never make separate `indeed_scrape` calls, which each pay the cold-start cost. +- Prefer returning the first page's on-topic hits promptly over exhaustive coverage. If a single call under-delivers against a large requested N, return `status=partial` noting the ~one-page-per-query ceiling — do not chase it with more calls. - Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, salary/rank changes). diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py index 01371d281..68b70870a 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py @@ -1,14 +1,8 @@ """Orchestrator for the Indeed scraper. -:func:`iter_indeed` streams flat job items from one warmed browser session: -``startUrls`` (search/company pages) or ``queries`` are paged by the ``start`` -offset, deduped by ``jobKey``, and stopped on the first page that adds nothing -or that Indeed gates (anonymous pagination is bounced past page 1, so a gated -page ends that target and keeps the pages already yielded). -:func:`scrape_indeed` collects the stream under a caller ``limit``. - -Targets run sequentially on a single session — a browser per target would cost -far more than the sequential paging it would save. +:func:`iter_indeed` streams deduped job items from one warmed session; each +search/company target contributes its first page. :func:`scrape_indeed` collects +the stream under a caller ``limit``. Targets run sequentially to reuse the session. """ from __future__ import annotations @@ -16,9 +10,9 @@ from __future__ import annotations import logging from collections.abc import AsyncIterator from typing import Any -from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse +from urllib.parse import urlparse -from .fetch import IndeedAccessBlockedError, IndeedSession, now_iso, open_session +from .fetch import IndeedSession, now_iso, open_session from .parsers import ( extract_jobcards_blob, job_results, @@ -32,68 +26,36 @@ logger = logging.getLogger(__name__) __all__ = ["iter_indeed", "scrape_indeed"] -# Indeed's search pagination increments the ``start`` offset by 10. -_PAGE_STEP = 10 -# Indeed stops serving useful results past ~1000; cap pages well before that. -_MAX_PAGES = 10 - def _emit(partial: dict[str, Any]) -> dict[str, Any]: """Stamp ``scrapedAt`` and normalize through the output model.""" return IndeedItem(**{**partial, "scrapedAt": now_iso()}).to_output() -def _with_start(url: str, start: int) -> str: - """Return ``url`` with its ``start`` query param set (removed when 0).""" - parsed = urlparse(url) - params = [(k, v) for k, v in parse_qsl(parsed.query) if k != "start"] - if start: - params.append(("start", str(start))) - return urlunparse(parsed._replace(query=urlencode(params))) - - -async def _paginate( - session: IndeedSession, first_url: str, *, domain: str, max_items: int +async def _search_items( + session: IndeedSession, url: str, *, domain: str, max_items: int ) -> AsyncIterator[dict[str, Any]]: - """Yield items across pages of one search/company URL, deduped by ``jobKey``.""" + """Yield deduped job cards from one search/company page. + + ponytail: caps a query at its first page (~15 jobs) — anonymous Indeed gates + ``start>=10``; deeper depth needs an authenticated session or Indeed's API. + """ if max_items <= 0: return base_url = f"https://{domain}" + html = await session.fetch_html(url) seen: set[str] = set() emitted = 0 - for page in range(_MAX_PAGES): - try: - # Rotate to establish first-page access, but fail fast on deeper - # pages: Indeed gates anonymous pagination to secure.indeed.com, a - # block no fresh IP clears — retrying it only burns the time budget. - # ponytail: deeper pages are dropped rather than retried; lift the - # cap with an async job if full-depth pagination is ever needed. - html = await session.fetch_html( - _with_start(first_url, page * _PAGE_STEP), - max_rotations=None if page == 0 else 0, - ) - except IndeedAccessBlockedError: - if page == 0: - raise # never got in on this target; surface the block - logger.info("[indeed] pagination gated at start=%d; stopping", page * _PAGE_STEP) - return # keep the pages already yielded - raws = job_results(extract_jobcards_blob(html)) - if not raws: - return - added = 0 - for raw in raws: - item = parse_job(raw, base_url=base_url) - job_key = item.get("jobKey") - if isinstance(job_key, str): - if job_key in seen: - continue - seen.add(job_key) - yield _emit(item) - emitted += 1 - added += 1 - if emitted >= max_items: - return - if added == 0: # a whole page of duplicates: end of useful results + for raw in job_results(extract_jobcards_blob(html)): + item = parse_job(raw, base_url=base_url) + job_key = item.get("jobKey") + if isinstance(job_key, str): + if job_key in seen: + continue + seen.add(job_key) + yield _emit(item) + emitted += 1 + if emitted >= max_items: return @@ -175,7 +137,7 @@ async def iter_indeed( if item is not None: yield item continue - async for item in _paginate( + async for item in _search_items( session, url, domain=domain, max_items=input_model.maxItemsPerQuery ): job_key = item.get("jobKey") diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py index acf52296b..1d815ce98 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py @@ -62,41 +62,29 @@ async def _collect(input_model, session) -> list[dict]: @pytest.mark.asyncio -async def test_paginates_and_dedupes_across_pages(): - session = _FakeSession({0: ["k1", "k2", "k3"], 10: ["k3", "k4"], 20: []}) +async def test_dedupes_within_page(): + session = _FakeSession({0: ["k1", "k2", "k2", "k3"]}) items = await _collect( IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session ) - assert [i["jobKey"] for i in items] == ["k1", "k2", "k3", "k4"] + assert [i["jobKey"] for i in items] == ["k1", "k2", "k3"] assert all(i["scrapedAt"] for i in items) # stamped by the orchestrator @pytest.mark.asyncio -async def test_stops_when_a_page_is_all_duplicates(): - session = _FakeSession({0: ["k1", "k2"], 10: ["k1", "k2"], 20: ["k9"]}) +async def test_does_not_fetch_deeper_pages(): + # First page only; ``start>=10`` must never be requested. + session = _FakeSession({0: ["k1", "k2"], 10: ["k3"]}) items = await _collect( IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session ) assert [i["jobKey"] for i in items] == ["k1", "k2"] - assert 20 not in { - int(parse_qs(urlparse(u).query).get("start", ["0"])[0]) for u in session.fetched - } + assert all("start=" not in u for u in session.fetched) @pytest.mark.asyncio -async def test_pagination_gate_keeps_earlier_pages(): - # Indeed gates anonymous pagination: page 0 serves, start=10 is blocked. - # The already-yielded page-0 items must survive; paging just stops. - session = _FakeSession({0: ["k1", "k2"], 10: ["k3"]}, blocked_starts={10}) - items = await _collect( - IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session - ) - assert [i["jobKey"] for i in items] == ["k1", "k2"] - - -@pytest.mark.asyncio -async def test_first_page_block_propagates(): - # A block on the very first page yielded nothing, so it surfaces as an error. +async def test_page_block_propagates(): + # Nothing yielded before the block, so it surfaces as an error. session = _FakeSession({}, blocked_starts={0}) with pytest.raises(IndeedAccessBlockedError): await _collect(IndeedScrapeInput(queries=["dev"]), session)