mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-10 22:32:16 +02:00
chore: bumped version to 0.0.31
This commit is contained in:
parent
8df8565e0a
commit
1c9ab207ef
56 changed files with 520 additions and 190 deletions
|
|
@ -423,8 +423,12 @@ async def test_run_events_replays_buffer_then_finishes(monkeypatch):
|
|||
|
||||
row = _fake_run_row(status="running")
|
||||
raw = str(row.id)
|
||||
run_event_bus.publish(raw, {"type": "run.progress", "phase": "scraping", "current": 1})
|
||||
run_event_bus.publish(raw, {"type": "run.finished", "status": "success", "item_count": 2})
|
||||
run_event_bus.publish(
|
||||
raw, {"type": "run.progress", "phase": "scraping", "current": 1}
|
||||
)
|
||||
run_event_bus.publish(
|
||||
raw, {"type": "run.finished", "status": "success", "item_count": 2}
|
||||
)
|
||||
|
||||
app = _build_app_with_rows(monkeypatch, [row])
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -43,7 +43,9 @@ async def test_maps_urls_to_start_urls_and_wraps_items():
|
|||
assert out.items[0].dataType == "post"
|
||||
|
||||
(actor_input, _limit) = scraper.calls[0]
|
||||
assert [u.url for u in actor_input.startUrls] == ["https://www.reddit.com/r/python/"]
|
||||
assert [u.url for u in actor_input.startUrls] == [
|
||||
"https://www.reddit.com/r/python/"
|
||||
]
|
||||
assert actor_input.searches == []
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -233,7 +233,9 @@ async def test_gate_reserves_worst_case_captcha_when_solving_enabled(monkeypatch
|
|||
|
||||
with pytest.raises(InsufficientCreditsError):
|
||||
await gate_capability(
|
||||
CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
|
||||
CrawlInput(startUrls=["https://a.com"]),
|
||||
BillingUnit.WEB_CRAWL,
|
||||
_ctx(session),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -64,7 +64,9 @@ def test_preview_is_char_budgeted_and_references_run():
|
|||
per_item = "y" * 500
|
||||
items = [f'{{"i": {i}, "v": "{per_item}"}}' for i in range(500)]
|
||||
body = "\n".join(items)
|
||||
serialized = SerializedOutput(text=body, item_count=len(items), char_count=len(body))
|
||||
serialized = SerializedOutput(
|
||||
text=body, item_count=len(items), char_count=len(body)
|
||||
)
|
||||
|
||||
preview = _build_preview(serialized, run_id="abc")
|
||||
|
||||
|
|
@ -143,8 +145,11 @@ async def test_read_run_paginates(monkeypatch):
|
|||
read_run, _ = _tools()
|
||||
|
||||
out = await read_run.ainvoke(
|
||||
{"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000",
|
||||
"offset": 2, "limit": 3}
|
||||
{
|
||||
"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000",
|
||||
"offset": 2,
|
||||
"limit": 3,
|
||||
}
|
||||
)
|
||||
assert "item_2" in out and "item_3" in out and "item_4" in out
|
||||
assert "item_0" not in out and "item_5" not in out
|
||||
|
|
@ -184,8 +189,7 @@ async def test_search_run_excerpts_huge_matched_line(monkeypatch):
|
|||
_, search_run = _tools()
|
||||
|
||||
out = await search_run.ainvoke(
|
||||
{"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000",
|
||||
"pattern": "NEEDLE"}
|
||||
{"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000", "pattern": "NEEDLE"}
|
||||
)
|
||||
assert "NEEDLE" in out
|
||||
assert "match at char 100000" in out
|
||||
|
|
@ -215,8 +219,10 @@ async def test_search_run_matches(monkeypatch):
|
|||
_patch_session(monkeypatch, _BODY, [])
|
||||
_, search_run = _tools()
|
||||
out = await search_run.ainvoke(
|
||||
{"ref": "spill_" + "0" * 8 + "-0000-0000-0000-000000000000",
|
||||
"pattern": "item_7"}
|
||||
{
|
||||
"ref": "spill_" + "0" * 8 + "-0000-0000-0000-000000000000",
|
||||
"pattern": "item_7",
|
||||
}
|
||||
)
|
||||
assert "item_7" in out
|
||||
assert "item_1" not in out.split("item_7")[0]
|
||||
|
|
@ -232,14 +238,30 @@ _CRAWL_BODY = "\n".join(
|
|||
"url": "https://x.com/team/",
|
||||
"status": "success",
|
||||
"links": [
|
||||
{"url": "https://x.com/author/jane/", "text": "Jane Doe",
|
||||
"context": "Jane Doe General Partner", "kind": "internal"},
|
||||
{"url": "https://x.com/author/bob/", "text": "Bob Roe",
|
||||
"context": "Bob Roe Operations", "kind": "internal"},
|
||||
{
|
||||
"url": "https://x.com/author/jane/",
|
||||
"text": "Jane Doe",
|
||||
"context": "Jane Doe General Partner",
|
||||
"kind": "internal",
|
||||
},
|
||||
{
|
||||
"url": "https://x.com/author/bob/",
|
||||
"text": "Bob Roe",
|
||||
"context": "Bob Roe Operations",
|
||||
"kind": "internal",
|
||||
},
|
||||
# Duplicate of Jane (nav + card) — must dedupe.
|
||||
{"url": "https://x.com/author/jane/", "text": "Jane Doe",
|
||||
"context": "Jane Doe General Partner", "kind": "internal"},
|
||||
{"url": "https://x.com/about/", "text": "About", "kind": "internal"},
|
||||
{
|
||||
"url": "https://x.com/author/jane/",
|
||||
"text": "Jane Doe",
|
||||
"context": "Jane Doe General Partner",
|
||||
"kind": "internal",
|
||||
},
|
||||
{
|
||||
"url": "https://x.com/about/",
|
||||
"text": "About",
|
||||
"kind": "internal",
|
||||
},
|
||||
],
|
||||
}
|
||||
),
|
||||
|
|
|
|||
|
|
@ -101,7 +101,11 @@ async def test_aggregated_contacts_carry_provenance_and_site_wide_flag() -> None
|
|||
class _ContactsEngine:
|
||||
async def crawl_url(self, url: str) -> CrawlOutcome:
|
||||
socials = [footer] + ([person] if url.endswith("/about") else [])
|
||||
links = ["https://e.com/about", "https://e.com/blog"] if url == "https://e.com/" else []
|
||||
links = (
|
||||
["https://e.com/about", "https://e.com/blog"]
|
||||
if url == "https://e.com/"
|
||||
else []
|
||||
)
|
||||
return CrawlOutcome(
|
||||
status=_SUCCESS,
|
||||
result={
|
||||
|
|
|
|||
|
|
@ -43,18 +43,20 @@ def test_estimated_units_for_single_url_is_seed_count() -> None:
|
|||
|
||||
|
||||
def test_estimated_units_for_spider_is_max_pages() -> None:
|
||||
model = CrawlInput(
|
||||
startUrls=["https://a.com"], maxCrawlDepth=2, maxCrawlPages=25
|
||||
)
|
||||
model = CrawlInput(startUrls=["https://a.com"], maxCrawlDepth=2, maxCrawlPages=25)
|
||||
assert model.estimated_units == 25
|
||||
|
||||
|
||||
def test_billable_units_counts_only_successes() -> None:
|
||||
out = CrawlOutput(
|
||||
items=[
|
||||
CrawlItem(url="a", status="success", crawl=CrawlMeta(loadedUrl="a", depth=0)),
|
||||
CrawlItem(
|
||||
url="a", status="success", crawl=CrawlMeta(loadedUrl="a", depth=0)
|
||||
),
|
||||
CrawlItem(url="b", status="empty", crawl=CrawlMeta(loadedUrl="b", depth=1)),
|
||||
CrawlItem(url="c", status="failed", crawl=CrawlMeta(loadedUrl="c", depth=1)),
|
||||
CrawlItem(
|
||||
url="c", status="failed", crawl=CrawlMeta(loadedUrl="c", depth=1)
|
||||
),
|
||||
]
|
||||
)
|
||||
assert out.billable_units == 1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue