apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-26 23:51:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix: scrape indeed first page only, steer subagent to one query

Browse source
This commit is contained in:
CREDO23 2026-07-15 20:54:37 +02:00
parent 6180b96912
commit c485fe4c43
3 changed files with 36 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

30
surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py
View file

@ -62,41 +62,29 @@ async def _collect(input_model, session) -> list[dict]:
@pytest.mark.asyncio
async def test_paginates_and_dedupes_across_pages():
session = _FakeSession({0: ["k1", "k2", "k3"], 10: ["k3", "k4"], 20: []})
async def test_dedupes_within_page():
session = _FakeSession({0: ["k1", "k2", "k2", "k3"]})
items = await _collect(
IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session
)
assert [i["jobKey"] for i in items] == ["k1", "k2", "k3", "k4"]
assert [i["jobKey"] for i in items] == ["k1", "k2", "k3"]
assert all(i["scrapedAt"] for i in items) # stamped by the orchestrator
@pytest.mark.asyncio
async def test_stops_when_a_page_is_all_duplicates():
session = _FakeSession({0: ["k1", "k2"], 10: ["k1", "k2"], 20: ["k9"]})
async def test_does_not_fetch_deeper_pages():
# First page only; ``start>=10`` must never be requested.
session = _FakeSession({0: ["k1", "k2"], 10: ["k3"]})
items = await _collect(
IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session
)
assert [i["jobKey"] for i in items] == ["k1", "k2"]
assert 20 not in {
int(parse_qs(urlparse(u).query).get("start", ["0"])[0]) for u in session.fetched
}
assert all("start=" not in u for u in session.fetched)
@pytest.mark.asyncio
async def test_pagination_gate_keeps_earlier_pages():
# Indeed gates anonymous pagination: page 0 serves, start=10 is blocked.
# The already-yielded page-0 items must survive; paging just stops.
session = _FakeSession({0: ["k1", "k2"], 10: ["k3"]}, blocked_starts={10})
items = await _collect(
IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session
)
assert [i["jobKey"] for i in items] == ["k1", "k2"]
@pytest.mark.asyncio
async def test_first_page_block_propagates():
# A block on the very first page yielded nothing, so it surfaces as an error.
async def test_page_block_propagates():
# Nothing yielded before the block, so it surfaces as an error.
session = _FakeSession({}, blocked_starts={0})
with pytest.raises(IndeedAccessBlockedError):
await _collect(IndeedScrapeInput(queries=["dev"]), session)