feat(web-crawler): add site_crawler spider + url_policy over the fetch engine

2026-07-10 22:32:16 +02:00 · 2026-07-03 10:50:41 +02:00 · 2026-07-03 10:50:41 +02:00 · f82fae3973
commit f82fae3973
parent 5d0bd8125b
7 changed files with 477 additions and 1 deletions
--- a/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector_links.py
+++ b/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector_links.py
@ -0,0 +1,29 @@
+"""``_build_result`` surfaces absolute page links for the spider to enqueue."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.proprietary.web_crawler import WebCrawlerConnector
+
+pytestmark = pytest.mark.unit
+
+
+def test_build_result_includes_absolute_links() -> None:
+    html = (
+        "<html><body>"
+        '<a href="/a">A</a>'
+        '<a href="https://example.com/b">B</a>'
+        "</body></html>"
+    )
+
+    result = WebCrawlerConnector()._build_result(
+        html,
+        "https://example.com/",
+        "scrapling-static",
+        allow_raw_fallback=True,
+    )
+
+    assert result is not None
+    assert "https://example.com/a" in result["links"]
+    assert "https://example.com/b" in result["links"]
--- a/surfsense_backend/tests/unit/proprietary/web_crawler/test_site_crawler.py
+++ b/surfsense_backend/tests/unit/proprietary/web_crawler/test_site_crawler.py
@ -0,0 +1,165 @@
+"""``crawl_site`` BFS behavior: depth, same-site scope, dedupe, and page caps.
+
+Boundary mocked: the engine (a fake ``crawl_url`` serving a canned link graph).
+NOT mocked: the frontier/depth/dedupe/scope logic under test.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.proprietary.web_crawler import CrawlOutcome, CrawlOutcomeStatus
+from app.proprietary.web_crawler.site_crawler import crawl_site
+
+pytestmark = pytest.mark.unit
+
+_SUCCESS = CrawlOutcomeStatus.SUCCESS
+_FAILED = CrawlOutcomeStatus.FAILED
+
+
+class _FakeEngine:
+    """Serves a canned ``(status, links)`` per URL and records fetch order."""
+
+    def __init__(self, graph: dict[str, tuple[CrawlOutcomeStatus, list[str]]]):
+        self._graph = graph
+        self.calls: list[str] = []
+
+    async def crawl_url(self, url: str) -> CrawlOutcome:
+        self.calls.append(url)
+        status, links = self._graph.get(url, (_FAILED, []))
+        if status is _SUCCESS:
+            return CrawlOutcome(
+                status=_SUCCESS,
+                result={
+                    "content": f"C:{url}",
+                    "metadata": {"title": url},
+                    "links": links,
+                },
+            )
+        return CrawlOutcome(status=status, error="boom")
+
+
+async def test_depth_zero_fetches_only_the_seed() -> None:
+    engine = _FakeEngine({"https://e.com/": (_SUCCESS, ["https://e.com/a"])})
+
+    pages = await crawl_site(
+        engine, ["https://e.com/"], max_crawl_depth=0, max_crawl_pages=10
+    )
+
+    assert engine.calls == ["https://e.com/"]
+    assert len(pages) == 1
+    assert pages[0].depth == 0
+    assert pages[0].referrer is None
+    assert pages[0].content == "C:https://e.com/"
+
+
+async def test_depth_one_follows_same_site_links_only() -> None:
+    engine = _FakeEngine(
+        {
+            "https://e.com/": (
+                _SUCCESS,
+                ["https://e.com/a", "https://e.com/b", "https://other.com/x"],
+            ),
+            "https://e.com/a": (_SUCCESS, []),
+            "https://e.com/b": (_SUCCESS, []),
+        }
+    )
+
+    pages = await crawl_site(
+        engine, ["https://e.com/"], max_crawl_depth=1, max_crawl_pages=10
+    )
+
+    assert set(engine.calls) == {
+        "https://e.com/",
+        "https://e.com/a",
+        "https://e.com/b",
+    }
+    depths = {page.url: page.depth for page in pages}
+    assert depths["https://e.com/a"] == 1
+    referrers = {page.url: page.referrer for page in pages}
+    assert referrers["https://e.com/a"] == "https://e.com/"
+
+
+async def test_depth_caps_further_recursion() -> None:
+    engine = _FakeEngine(
+        {
+            "https://e.com/": (_SUCCESS, ["https://e.com/a"]),
+            "https://e.com/a": (_SUCCESS, ["https://e.com/b"]),
+            "https://e.com/b": (_SUCCESS, []),
+        }
+    )
+
+    pages = await crawl_site(
+        engine, ["https://e.com/"], max_crawl_depth=1, max_crawl_pages=10
+    )
+
+    # depth 1 reaches /a but must NOT descend to /b (depth 2).
+    assert set(engine.calls) == {"https://e.com/", "https://e.com/a"}
+    assert all(page.url != "https://e.com/b" for page in pages)
+
+
+async def test_max_pages_caps_total_fetches() -> None:
+    graph: dict[str, tuple[CrawlOutcomeStatus, list[str]]] = {
+        "https://e.com/": (_SUCCESS, [f"https://e.com/{i}" for i in range(10)])
+    }
+    for i in range(10):
+        graph[f"https://e.com/{i}"] = (_SUCCESS, [])
+    engine = _FakeEngine(graph)
+
+    pages = await crawl_site(
+        engine, ["https://e.com/"], max_crawl_depth=1, max_crawl_pages=3
+    )
+
+    assert len(pages) == 3
+    assert len(engine.calls) == 3
+
+
+async def test_dedupes_on_canonical_url() -> None:
+    engine = _FakeEngine(
+        {
+            "https://e.com/": (
+                _SUCCESS,
+                ["https://e.com/a", "https://e.com/a#frag", "https://e.com/a?"],
+            ),
+            "https://e.com/a": (_SUCCESS, ["https://e.com/"]),  # links back to seed
+        }
+    )
+
+    await crawl_site(
+        engine, ["https://e.com/"], max_crawl_depth=3, max_crawl_pages=10
+    )
+
+    assert engine.calls.count("https://e.com/a") == 1
+    assert engine.calls.count("https://e.com/") == 1
+
+
+async def test_failed_page_is_recorded_and_not_expanded() -> None:
+    engine = _FakeEngine({"https://e.com/": (_FAILED, [])})
+
+    pages = await crawl_site(
+        engine, ["https://e.com/"], max_crawl_depth=2, max_crawl_pages=10
+    )
+
+    assert len(pages) == 1
+    assert pages[0].status is _FAILED
+    assert pages[0].content is None
+    assert pages[0].error == "boom"
+
+
+async def test_multiple_seeds_are_all_entry_points() -> None:
+    engine = _FakeEngine(
+        {
+            "https://a.com/": (_SUCCESS, []),
+            "https://b.com/": (_SUCCESS, []),
+        }
+    )
+
+    pages = await crawl_site(
+        engine,
+        ["https://a.com/", "https://b.com/"],
+        max_crawl_depth=0,
+        max_crawl_pages=10,
+    )
+
+    assert {page.url for page in pages} == {"https://a.com/", "https://b.com/"}
+    assert all(page.depth == 0 for page in pages)
--- a/surfsense_backend/tests/unit/proprietary/web_crawler/test_url_policy.py
+++ b/surfsense_backend/tests/unit/proprietary/web_crawler/test_url_policy.py
@ -0,0 +1,89 @@
+"""Pure URL-helper behavior for the site spider (link surfacing + dedupe scope)."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.proprietary.web_crawler.url_policy import (
+    canonicalize_url,
+    extract_links,
+    host_of,
+    same_site,
+)
+
+pytestmark = pytest.mark.unit
+
+_HTML = """
+<html><body>
+  <a href="/about">About (relative)</a>
+  <a href="https://example.com/contact">Contact (absolute)</a>
+  <a href="https://other.com/x">External</a>
+  <a href="#top">Anchor only</a>
+  <a href="mailto:a@b.com">Mail</a>
+  <a href="javascript:void(0)">JS</a>
+  <a href="/about">Duplicate about</a>
+</body></html>
+"""
+
+
+def test_extract_links_absolutizes_relative_hrefs() -> None:
+    links = extract_links(_HTML, "https://example.com/home")
+    assert "https://example.com/about" in links
+    assert "https://example.com/contact" in links
+
+
+def test_extract_links_keeps_only_http_schemes() -> None:
+    links = extract_links(_HTML, "https://example.com/home")
+    assert all(link.startswith(("http://", "https://")) for link in links)
+    assert not any("mailto" in link or "javascript" in link for link in links)
+
+
+def test_extract_links_drops_pure_anchor() -> None:
+    links = extract_links(_HTML, "https://example.com/home")
+    assert "https://example.com/home" not in links  # bare "#top" resolves to self
+
+
+def test_extract_links_dedupes_preserving_first_occurrence() -> None:
+    links = extract_links(_HTML, "https://example.com/home")
+    assert links.count("https://example.com/about") == 1
+
+
+def test_extract_links_strips_fragments() -> None:
+    assert extract_links('<a href="/p#sec">x</a>', "https://e.com") == [
+        "https://e.com/p"
+    ]
+
+
+def test_extract_links_on_empty_or_blank_html_is_empty() -> None:
+    assert extract_links("", "https://e.com") == []
+    assert extract_links("   ", "https://e.com") == []
+    assert extract_links(None, "https://e.com") == []
+
+
+def test_canonicalize_lowercases_host_sorts_query_and_drops_fragment() -> None:
+    assert (
+        canonicalize_url("https://E.com/a?b=2&a=1#frag") == "https://e.com/a?a=1&b=2"
+    )
+
+
+def test_canonicalize_collapses_fragment_and_empty_query_to_one_key() -> None:
+    # The three forms must dedupe to the same visited-set key.
+    canonical = canonicalize_url("https://e.com/a")
+    assert canonicalize_url("https://e.com/a#frag") == canonical
+    assert canonicalize_url("https://e.com/a?") == canonical
+
+
+def test_canonicalize_keeps_nondefault_port() -> None:
+    assert canonicalize_url("https://e.com:8443/x") == "https://e.com:8443/x"
+
+
+def test_host_of_strips_www_and_lowercases() -> None:
+    assert host_of("https://www.Example.com/x") == "example.com"
+    assert host_of("https://Example.com/x") == "example.com"
+
+
+def test_same_site_matches_on_normalized_host() -> None:
+    allowed = {"example.com"}
+    assert same_site("https://www.example.com/a", allowed) is True
+    assert same_site("https://example.com/b", allowed) is True
+    assert same_site("https://other.com/c", allowed) is False