feat(web-crawler): add site_crawler spider + url_policy over the fetch engine

surfsense_backend/app/proprietary/web_crawler/__init__.py

@@ -9,5 +9,12 @@ from app.proprietary.web_crawler.connector import (
     CrawlOutcomeStatus,
     WebCrawlerConnector,
 )
+from app.proprietary.web_crawler.site_crawler import CrawlPage, crawl_site
 
-__all__ = ["CrawlOutcome", "CrawlOutcomeStatus", "WebCrawlerConnector"]
+__all__ = [
+    "CrawlOutcome",
+    "CrawlOutcomeStatus",
+    "CrawlPage",
+    "WebCrawlerConnector",
+    "crawl_site",
+]

surfsense_backend/app/proprietary/web_crawler/connector.py

@@ -34,6 +34,7 @@ from scrapling.engines.toolbelt import is_proxy_error
 from scrapling.fetchers import AsyncFetcher, DynamicFetcher, StealthyFetcher
 
 from app.proprietary.web_crawler.captcha import build_captcha_page_action
+from app.proprietary.web_crawler.url_policy import extract_links
 from app.proprietary.web_crawler.stealth import (
     build_stealthy_kwargs,
     get_stealth_config,
@@ -601,6 +602,9 @@ class WebCrawlerConnector:
             "content": content,
             "metadata": metadata,
             "crawler_type": crawler_type,
+            # Next-hop targets for ``site_crawler.crawl_site``; ignored by
+            # single-URL callers.
+            "links": extract_links(raw_html, url),
         }
 
     @staticmethod

surfsense_backend/app/proprietary/web_crawler/site_crawler.py

@@ -0,0 +1,123 @@
+# SurfSense proprietary crawler engine.
+#
+# Part of the ``app.proprietary`` package; licensed separately from the
+# Apache-2.0 project root (see ``app/proprietary/LICENSE``).
+"""Depth-bounded site crawl built on the single-URL engine (``crawl_url``).
+
+Breadth-first frontier: fetch a page, follow its same-site links one hop deeper,
+dedupe by canonical URL, stop at ``max_crawl_pages``. Every fetch runs through
+``crawl_url`` so tiered fetch, proxy, and captcha handling are reused.
+
+Sequential today; ``_fetch_page`` isolates one unit of work so a bounded worker
+pool can replace the loop later without changing the traversal.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass
+
+from app.proprietary.web_crawler.connector import (
+    CrawlOutcome,
+    CrawlOutcomeStatus,
+    WebCrawlerConnector,
+)
+from app.proprietary.web_crawler.url_policy import canonicalize_url, host_of, same_site
+
+
+@dataclass(frozen=True)
+class CrawlPage:
+    """A fetched page plus its crawl provenance (depth and referrer)."""
+
+    url: str
+    status: CrawlOutcomeStatus
+    depth: int
+    referrer: str | None = None
+    loaded_url: str | None = None
+    content: str | None = None
+    metadata: dict[str, str] | None = None
+    error: str | None = None
+    captcha_attempts: int = 0
+    captcha_solved: bool = False
+
+
+async def crawl_site(
+    engine: WebCrawlerConnector,
+    start_urls: list[str],
+    *,
+    max_crawl_depth: int,
+    max_crawl_pages: int,
+) -> list[CrawlPage]:
+    """Crawl ``start_urls`` up to ``max_crawl_depth`` hops / ``max_crawl_pages`` pages.
+
+    Depth 0 fetches only the start URLs. Links are followed only from successful
+    pages, under the depth cap, and only on a start URL's site. Start URLs count
+    toward ``max_crawl_pages``. Order is BFS from the seeds.
+    """
+    allowed_hosts = {host_of(url) for url in start_urls}
+    visited: set[str] = set()
+    frontier: deque[tuple[str, int, str | None]] = deque()
+    for seed in start_urls:
+        key = canonicalize_url(seed)
+        if key not in visited:
+            visited.add(key)
+            frontier.append((seed, 0, None))
+
+    pages: list[CrawlPage] = []
+    while frontier and len(pages) < max_crawl_pages:
+        url, depth, referrer = frontier.popleft()
+        page, outcome = await _fetch_page(engine, url, depth, referrer)
+        pages.append(page)
+
+        if depth >= max_crawl_depth:
+            continue
+        if outcome.status is not CrawlOutcomeStatus.SUCCESS or not outcome.result:
+            continue
+        for link in outcome.result.get("links", []):
+            key = canonicalize_url(link)
+            if key in visited or not same_site(link, allowed_hosts):
+                continue
+            visited.add(key)
+            frontier.append((link, depth + 1, url))
+    return pages
+
+
+async def _fetch_page(
+    engine: WebCrawlerConnector,
+    url: str,
+    depth: int,
+    referrer: str | None,
+) -> tuple[CrawlPage, CrawlOutcome]:
+    """Fetch one URL and map it to a ``CrawlPage`` (the future concurrency unit)."""
+    outcome = await engine.crawl_url(url)
+    return _to_page(url, outcome, depth, referrer), outcome
+
+
+def _to_page(
+    url: str,
+    outcome: CrawlOutcome,
+    depth: int,
+    referrer: str | None,
+) -> CrawlPage:
+    result = outcome.result or {}
+    if outcome.status is CrawlOutcomeStatus.SUCCESS and result:
+        return CrawlPage(
+            url=url,
+            status=outcome.status,
+            depth=depth,
+            referrer=referrer,
+            loaded_url=result.get("loaded_url") or url,
+            content=result.get("content"),
+            metadata=result.get("metadata"),
+            captcha_attempts=outcome.captcha_attempts,
+            captcha_solved=outcome.captcha_solved,
+        )
+    return CrawlPage(
+        url=url,
+        status=outcome.status,
+        depth=depth,
+        referrer=referrer,
+        error=outcome.error,
+        captcha_attempts=outcome.captcha_attempts,
+        captcha_solved=outcome.captcha_solved,
+    )

surfsense_backend/app/proprietary/web_crawler/url_policy.py

@@ -0,0 +1,59 @@
+# SurfSense proprietary crawler engine.
+#
+# Part of the ``app.proprietary`` package; licensed separately from the
+# Apache-2.0 project root (see ``app/proprietary/LICENSE``).
+"""URL helpers for the site crawler: link extraction, canonical key, same-site scope.
+
+Pure functions (no I/O) so the crawl frontier stays deterministic and testable.
+"""
+
+from __future__ import annotations
+
+from urllib.parse import urldefrag, urljoin, urlsplit
+
+from lxml import html as lxml_html
+from lxml.etree import ParserError
+from w3lib.url import canonicalize_url as _w3lib_canonicalize_url
+
+
+def extract_links(page_html: str | None, base_url: str) -> list[str]:
+    """Absolute, http(s), fragment-free, de-duplicated ``<a href>`` targets.
+
+    Relative hrefs resolve against ``base_url``; the page's own URL is dropped.
+    First-seen order is preserved to keep the frontier stable.
+    """
+    if not page_html or not page_html.strip():
+        return []
+    try:
+        root = lxml_html.fromstring(page_html)
+    except (ParserError, ValueError):
+        return []
+
+    self_url, _ = urldefrag(base_url)
+    seen: set[str] = set()
+    links: list[str] = []
+    for href in root.xpath("//a/@href"):
+        target, _ = urldefrag(urljoin(base_url, href.strip()))
+        if urlsplit(target).scheme not in ("http", "https"):
+            continue
+        if target == self_url or target in seen:
+            continue
+        seen.add(target)
+        links.append(target)
+    return links
+
+
+def canonicalize_url(url: str) -> str:
+    """Stable visited-set key: sorts query, normalizes encoding, drops fragment."""
+    return _w3lib_canonicalize_url(url, keep_fragments=False)
+
+
+def host_of(url: str) -> str:
+    """Lowercased host with a leading ``www.`` removed, for same-site matching."""
+    host = (urlsplit(url).hostname or "").lower()
+    return host[4:] if host.startswith("www.") else host
+
+
+def same_site(url: str, allowed_hosts: set[str]) -> bool:
+    """Whether ``url``'s host (``www.``-normalized) is in ``allowed_hosts``."""
+    return host_of(url) in allowed_hosts