feat: completed init mvp of phase 3

surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py

@@ -12,6 +12,7 @@ from app.proprietary.web_crawler import (
     WebCrawlerConnector,
 )
 from app.proprietary.web_crawler import connector as connector_module
+from app.utils.crawl import BlockType
 
 pytestmark = pytest.mark.unit
 
@@ -40,10 +41,10 @@ async def test_static_tier_success_short_circuits(
     crawler = WebCrawlerConnector()
     later_calls: list[str] = []
 
-    async def _static(_url: str) -> dict:
+    async def _static(_url: str, *_args) -> dict:
         return _result("scrapling-static")
 
-    async def _record_dynamic(_url: str) -> None:
+    async def _record_dynamic(_url: str, *_args) -> None:
         later_calls.append("dynamic")
         return None
 
@@ -70,10 +71,10 @@ async def test_escalates_to_dynamic_on_static_miss(
     """Static empty extraction escalates to the dynamic tier."""
     crawler = WebCrawlerConnector()
 
-    async def _empty(_url: str) -> None:
+    async def _empty(_url: str, *_args) -> None:
         return None
 
-    async def _dynamic(_url: str) -> dict:
+    async def _dynamic(_url: str, *_args) -> dict:
         return _result("scrapling-dynamic")
 
     monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty)
@@ -126,7 +127,7 @@ async def test_proxy_error_rotates_once_when_pool_backed(
     crawler = WebCrawlerConnector()
     calls = {"n": 0}
 
-    async def _flaky(_url: str) -> dict:
+    async def _flaky(_url: str, *_args) -> dict:
         calls["n"] += 1
         if calls["n"] == 1:
             raise RuntimeError("connection refused by upstream proxy")
@@ -149,7 +150,7 @@ async def test_proxy_error_no_retry_when_single_endpoint(
     crawler = WebCrawlerConnector()
     static_calls = {"n": 0}
 
-    async def _proxy_err(_url: str) -> None:
+    async def _proxy_err(_url: str, *_args) -> None:
         static_calls["n"] += 1
         raise RuntimeError("connection refused by upstream proxy")
 
@@ -173,7 +174,7 @@ async def test_captcha_defaults_zero_on_non_stealthy_success(
     """03d: a static success never touches captcha → fields stay 0/False."""
     crawler = WebCrawlerConnector()
 
-    async def _static(_url: str) -> dict:
+    async def _static(_url: str, *_args) -> dict:
         return _result("scrapling-static")
 
     monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _static)
@@ -194,7 +195,7 @@ async def test_captcha_state_surfaced_onto_outcome(
     async def _empty(_url: str, *_args) -> None:
         return None
 
-    async def _stealthy(_url: str, captcha_state: dict) -> dict:
+    async def _stealthy(_url: str, captcha_state: dict, *_args) -> dict:
         # Simulate the page_action having solved a captcha mid-fetch.
         captcha_state["attempts"] = 2
         captcha_state["solved"] = True
@@ -221,7 +222,9 @@ async def test_captcha_attempts_surface_even_when_crawl_fails(
     async def _empty(_url: str, *_args) -> None:
         return None
 
-    async def _stealthy_attempt_then_empty(_url: str, captcha_state: dict) -> None:
+    async def _stealthy_attempt_then_empty(
+        _url: str, captcha_state: dict, *_args
+    ) -> None:
         captcha_state["attempts"] = 1  # solve attempted but crawl still empty
         return None
 
@@ -245,7 +248,7 @@ async def test_non_proxy_error_never_retries(
     crawler = WebCrawlerConnector()
     calls = {"n": 0}
 
-    async def _boom(_url: str) -> None:
+    async def _boom(_url: str, *_args) -> None:
         calls["n"] += 1
         raise RuntimeError("totally unrelated failure")
 
@@ -261,3 +264,104 @@ async def test_non_proxy_error_never_retries(
 
     assert calls["n"] == 1  # not retried (not a proxy error)
     assert outcome.status is CrawlOutcomeStatus.EMPTY
+
+
+async def test_invalid_url_block_type_defaults_unknown() -> None:
+    """03e: a pre-fetch FAILED carries the default UNKNOWN block_type."""
+    outcome = await WebCrawlerConnector().crawl_url("not a url")
+
+    assert outcome.status is CrawlOutcomeStatus.FAILED
+    assert outcome.block_type is BlockType.UNKNOWN
+
+
+async def test_block_type_surfaced_onto_outcome(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """03e: a tier's block classification (in block_state) is stamped onto the
+    outcome — additive only, never gating SUCCESS."""
+    crawler = WebCrawlerConnector()
+
+    async def _empty(_url: str, *_args) -> None:
+        return None
+
+    async def _stealthy_blocked(
+        _url: str, _captcha_state: dict, block_state: dict
+    ) -> None:
+        # Simulate _build_result having classified a Cloudflare interstitial.
+        block_state["block_type"] = BlockType.CLOUDFLARE
+        return None
+
+    monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty)
+    monkeypatch.setattr(crawler, "_crawl_with_dynamic", _empty)
+    monkeypatch.setattr(crawler, "_crawl_with_stealthy", _stealthy_blocked)
+
+    outcome = await crawler.crawl_url("https://example.com")
+
+    assert outcome.status is CrawlOutcomeStatus.EMPTY
+    assert outcome.block_type is BlockType.CLOUDFLARE
+
+
+def test_build_result_classifies_into_block_state() -> None:
+    """03e: _build_result labels the fetched page into the passed block_state."""
+    crawler = WebCrawlerConnector()
+    block_state: dict = {"block_type": BlockType.UNKNOWN}
+
+    cf_html = "<html><head><title>Just a moment...</title></head></html>"
+    result = crawler._build_result(
+        cf_html,
+        "https://example.com",
+        "scrapling-stealthy",
+        allow_raw_fallback=False,
+        status=403,
+        block_state=block_state,
+    )
+
+    # Cloudflare interstitial: no real content extracted (None) but classified.
+    assert result is None
+    assert block_state["block_type"] is BlockType.CLOUDFLARE
+
+
+async def test_static_4xx_is_classified(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """03e: a static-tier 4xx bot-gate is classified before the early return
+    (otherwise the cheapest/first tier's block signal would be lost)."""
+    crawler = WebCrawlerConnector()
+
+    class _Page:
+        status = 403
+        html_content = "<title>Just a moment...</title>"
+
+    class _AsyncFetcher:
+        @staticmethod
+        async def get(*_a, **_k):
+            return _Page()
+
+    monkeypatch.setattr(connector_module, "AsyncFetcher", _AsyncFetcher)
+    monkeypatch.setattr(connector_module, "get_proxy_url", lambda: None)
+
+    block_state: dict = {"block_type": BlockType.UNKNOWN}
+    result = await crawler._crawl_with_async_fetcher(
+        "https://example.com", block_state
+    )
+
+    assert result is None  # 4xx => fall through to next tier
+    assert block_state["block_type"] is BlockType.CLOUDFLARE
+
+
+def test_build_result_ok_on_real_content() -> None:
+    """03e: a normal 200 page with content classifies OK."""
+    crawler = WebCrawlerConnector()
+    block_state: dict = {"block_type": BlockType.UNKNOWN}
+
+    html = "<html><body><article>" + ("Real content. " * 40) + "</article></body></html>"
+    crawler._build_result(
+        html,
+        "https://example.com",
+        "scrapling-static",
+        allow_raw_fallback=False,
+        status=200,
+        block_state=block_state,
+    )
+
+    assert block_state["block_type"] is BlockType.OK

surfsense_backend/tests/unit/proprietary/web_crawler/test_stealth.py

@@ -0,0 +1,113 @@
+"""Unit tests for the Phase 3e stealth kwargs builder (proprietary boundary)."""
+
+import pytest
+
+from app.config import config
+from app.proprietary.web_crawler.stealth import (
+    build_stealthy_kwargs,
+    get_stealth_config,
+    location_to_locale_timezone,
+)
+
+pytestmark = pytest.mark.unit
+
+
+class TestLocationToLocaleTimezone:
+    def test_alpha2_code(self):
+        assert location_to_locale_timezone("us") == ("en-US", "America/New_York")
+        assert location_to_locale_timezone("de") == ("de-DE", "Europe/Berlin")
+
+    def test_case_and_whitespace_insensitive(self):
+        assert location_to_locale_timezone("  US  ") == (
+            "en-US",
+            "America/New_York",
+        )
+
+    def test_full_country_name_alias(self):
+        assert location_to_locale_timezone("Germany") == (
+            "de-DE",
+            "Europe/Berlin",
+        )
+        assert location_to_locale_timezone("united kingdom") == (
+            "en-GB",
+            "Europe/London",
+        )
+
+    def test_leading_token_of_vendor_string(self):
+        # Vendor strings like "us:nyc" / "de-rotating" still resolve on the head.
+        assert location_to_locale_timezone("us:nyc") == (
+            "en-US",
+            "America/New_York",
+        )
+        assert location_to_locale_timezone("de-rotating") == (
+            "de-DE",
+            "Europe/Berlin",
+        )
+
+    def test_empty_and_unknown_return_none(self):
+        assert location_to_locale_timezone("") == (None, None)
+        assert location_to_locale_timezone(None) == (None, None)
+        assert location_to_locale_timezone("atlantis") == (None, None)
+
+
+class TestBuildStealthyKwargs:
+    def test_defaults_have_no_geoip_keys(self, monkeypatch):
+        # geoip off => locale/timezone_id absent (browser keeps system default).
+        monkeypatch.setattr(config, "CRAWL_GEOIP_MATCH_ENABLED", False)
+        monkeypatch.setattr(config, "CRAWL_BLOCK_WEBRTC", True)
+        monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", False)
+        monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", True)
+        monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", False)
+        monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "us")
+
+        kwargs = build_stealthy_kwargs(get_stealth_config())
+
+        assert kwargs == {
+            "block_webrtc": True,
+            "hide_canvas": False,
+            "google_search": True,
+            "dns_over_https": False,
+        }
+        assert "locale" not in kwargs
+        assert "timezone_id" not in kwargs
+
+    def test_flags_reflect_config(self, monkeypatch):
+        monkeypatch.setattr(config, "CRAWL_GEOIP_MATCH_ENABLED", False)
+        monkeypatch.setattr(config, "CRAWL_BLOCK_WEBRTC", False)
+        monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", True)
+        monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", False)
+        monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", True)
+        monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "")
+
+        kwargs = build_stealthy_kwargs(get_stealth_config())
+
+        assert kwargs["block_webrtc"] is False
+        assert kwargs["hide_canvas"] is True
+        assert kwargs["google_search"] is False
+        assert kwargs["dns_over_https"] is True
+
+    def test_geoip_on_adds_locale_timezone(self, monkeypatch):
+        monkeypatch.setattr(config, "CRAWL_GEOIP_MATCH_ENABLED", True)
+        monkeypatch.setattr(config, "CRAWL_BLOCK_WEBRTC", True)
+        monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", False)
+        monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", True)
+        monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", False)
+        monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "de")
+
+        kwargs = build_stealthy_kwargs(get_stealth_config())
+
+        assert kwargs["locale"] == "de-DE"
+        assert kwargs["timezone_id"] == "Europe/Berlin"
+
+    def test_geoip_on_but_unknown_location_skips(self, monkeypatch):
+        monkeypatch.setattr(config, "CRAWL_GEOIP_MATCH_ENABLED", True)
+        monkeypatch.setattr(config, "CRAWL_BLOCK_WEBRTC", True)
+        monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", False)
+        monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", True)
+        monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", False)
+        monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "atlantis")
+
+        kwargs = build_stealthy_kwargs(get_stealth_config())
+
+        assert "locale" not in kwargs
+        assert "timezone_id" not in kwargs

surfsense_backend/tests/unit/utils/test_crawl_classifier.py

@@ -0,0 +1,75 @@
+"""Unit tests for the Phase 3e block classifier (Apache-2 layer)."""
+
+import pytest
+
+from app.utils.crawl import BlockType, classify_block
+
+pytestmark = pytest.mark.unit
+
+
+class TestClassifyBlock:
+    def test_ok_on_plain_content(self):
+        assert classify_block(200, "<html><body>hello</body></html>") is BlockType.OK
+
+    def test_none_body_is_empty(self):
+        assert classify_block(200, None) is BlockType.EMPTY
+        assert classify_block(200, "   ") is BlockType.EMPTY
+
+    def test_rate_limited_takes_precedence(self):
+        # 429 wins even if a challenge marker is present in the body.
+        assert (
+            classify_block(429, '<div class="cf-turnstile"></div>')
+            is BlockType.RATE_LIMITED
+        )
+
+    @pytest.mark.parametrize(
+        "html",
+        [
+            "<title>Just a moment...</title>",
+            "<h1>Checking your browser before accessing</h1>",
+            "please enable javascript and cookies to continue",
+            "verify you are human",
+            '<div id="challenge-running"></div>',
+            '<div id="turnstile-wrapper"></div>',
+            '<div class="cf-turnstile"></div>',
+            "blocked by ddos-guard.net",
+        ],
+    )
+    def test_cloudflare_markers(self, html):
+        assert classify_block(403, html) is BlockType.CLOUDFLARE
+
+    def test_hcaptcha(self):
+        assert (
+            classify_block(200, '<div class="h-captcha"></div>')
+            is BlockType.CAPTCHA_HCAPTCHA
+        )
+
+    def test_recaptcha(self):
+        assert (
+            classify_block(200, '<div class="g-recaptcha"></div>')
+            is BlockType.CAPTCHA_RECAPTCHA
+        )
+
+    def test_datadome(self):
+        assert (
+            classify_block(403, "var dd={'host':'geo.captcha-delivery.com'}")
+            is BlockType.DATADOME
+        )
+
+    def test_kasada(self):
+        assert classify_block(200, "window.KPSDK.configure()") is BlockType.KASADA
+
+    def test_bot_gate_status_without_marker_is_unknown(self):
+        # 202/403 with no recognized challenge marker => generic blocked-ish.
+        assert classify_block(202, "<html><body>x</body></html>") is BlockType.UNKNOWN
+        assert classify_block(403, "<html><body>x</body></html>") is BlockType.UNKNOWN
+
+    def test_bot_gate_status_empty_body_is_unknown(self):
+        assert classify_block(403, None) is BlockType.UNKNOWN
+
+    def test_cloudflare_marker_beats_generic_403(self):
+        # A 403 Cloudflare page is CLOUDFLARE, not UNKNOWN.
+        assert (
+            classify_block(403, "<title>Just a moment...</title>")
+            is BlockType.CLOUDFLARE
+        )