feat: completed init mvp of phase 3

surfsense_backend/.env.example

@@ -369,6 +369,22 @@ TURNSTILE_SECRET_KEY=
 # CAPTCHA_V3_MIN_SCORE=0.7
 # CAPTCHA_V3_ACTION=verify
 
+# =====================================================================
+# Stealth hardening (Phase 3e, Slice A) — runtime/config-level levers on the
+# stealth browser tier. Consumed by app/proprietary/web_crawler/stealth.py.
+# Defaults add no crawl-speed regression and preserve today's behavior.
+# Match browser locale/timezone to RESIDENTIAL_PROXY_LOCATION (no exit-IP lookup).
+# CRAWL_GEOIP_MATCH_ENABLED=FALSE
+# WebRTC respects the proxy (anti local-IP leak). Cheap + safe.
+# CRAWL_BLOCK_WEBRTC=TRUE
+# Random canvas noise (an unstable canvas hash is itself a tell) — opt-in only.
+# CRAWL_HIDE_CANVAS=FALSE
+# Send a Google referer so the first hit looks like organic arrival.
+# CRAWL_GOOGLE_SEARCH_REFERER=TRUE
+# Route DNS via Cloudflare DoH (anti DNS-leak). Adds a DNS round-trip => off by
+# default to avoid any speed regression; enable for leak-safety-first setups.
+# CRAWL_DNS_OVER_HTTPS=FALSE
+
 # File Parser Service
 ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING
 UNSTRUCTURED_API_KEY=Tpu3P0U8iy

surfsense_backend/Dockerfile

@@ -30,6 +30,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libxrender1 \
     dos2unix \
     git \
+    # ── Phase 3e stealth hardening ──────────────────────────────────────────
+    # Xvfb: virtual framebuffer so the stealth browser can run headful
+    # (headless=False) without a real display — many WAFs flag headless Chromium.
+    # Gated at runtime by CRAWL_HEADED_XVFB_ENABLED (Slice B); installed now so
+    # the image is ready. Real font packages make canvas/emoji/font-enumeration
+    # fingerprints resemble a real desktop (set proven against Kasada/Akamai).
+    xvfb \
+    fonts-noto-color-emoji \
+    fonts-unifont \
+    fonts-ipafont-gothic \
+    fonts-wqy-zenhei \
+    fonts-tlwg-loma-otf \
+    fonts-dejavu \
+    fonts-liberation \
     && rm -rf /var/lib/apt/lists/*
 
 RUN which ffmpeg && ffmpeg -version

surfsense_backend/app/config/__init__.py

@@ -1086,6 +1086,38 @@ class Config:
     CAPTCHA_V3_MIN_SCORE = float(os.getenv("CAPTCHA_V3_MIN_SCORE", "0.7"))
     CAPTCHA_V3_ACTION = os.getenv("CAPTCHA_V3_ACTION", "verify")
 
+    # =====================================================================
+    # Phase 3e — Stealth hardening (Slice A): runtime/config-level levers
+    # layered on Scrapling's patchright-Chromium StealthyFetcher tier. All are
+    # consumed by the centralized kwargs builder in
+    # app/proprietary/web_crawler/stealth.py (proprietary — bypass tuning), which
+    # is the single source of truth imported by the crawler AND the 03f harness
+    # (no test-vs-prod drift). Defaults preserve today's behavior /
+    # introduce no crawl-speed regression. See plans/backend/03e-stealth-hardening.md.
+    # =====================================================================
+    # Map the configured proxy region (RESIDENTIAL_PROXY_LOCATION) -> browser
+    # locale/timezone so the fingerprint coheres with the proxy exit geo. No
+    # exit-IP lookup (zero added latency); unknown/empty region => skip.
+    CRAWL_GEOIP_MATCH_ENABLED = (
+        os.getenv("CRAWL_GEOIP_MATCH_ENABLED", "FALSE").upper() == "TRUE"
+    )
+    # Force WebRTC to respect the proxy (prevents real-local-IP leak). Cheap +
+    # safe => default TRUE.
+    CRAWL_BLOCK_WEBRTC = os.getenv("CRAWL_BLOCK_WEBRTC", "TRUE").upper() == "TRUE"
+    # Random canvas noise. An UNSTABLE canvas hash is itself a fingerprint tell,
+    # so default FALSE (opt-in + 03f-validated). See 03e §2.
+    CRAWL_HIDE_CANVAS = os.getenv("CRAWL_HIDE_CANVAS", "FALSE").upper() == "TRUE"
+    # Set a Google referer so the first hit looks like organic arrival.
+    CRAWL_GOOGLE_SEARCH_REFERER = (
+        os.getenv("CRAWL_GOOGLE_SEARCH_REFERER", "TRUE").upper() == "TRUE"
+    )
+    # Route DNS via Cloudflare DoH (anti DNS-leak behind proxies). Adds a DNS
+    # round-trip => default FALSE to honor the "no speed regression" bar; flip on
+    # when leak-safety outweighs the marginal latency.
+    CRAWL_DNS_OVER_HTTPS = (
+        os.getenv("CRAWL_DNS_OVER_HTTPS", "FALSE").upper() == "TRUE"
+    )
+
     # Litellm TTS Configuration
     TTS_SERVICE = os.getenv("TTS_SERVICE")
     TTS_SERVICE_API_BASE = os.getenv("TTS_SERVICE_API_BASE")

surfsense_backend/app/proprietary/web_crawler/connector.py

@@ -34,7 +34,12 @@ from scrapling.engines.toolbelt import is_proxy_error
 from scrapling.fetchers import AsyncFetcher, DynamicFetcher, StealthyFetcher
 
 from app.proprietary.web_crawler.captcha import build_captcha_page_action
+from app.proprietary.web_crawler.stealth import (
+    build_stealthy_kwargs,
+    get_stealth_config,
+)
 from app.utils.captcha import captcha_enabled, get_captcha_config
+from app.utils.crawl import BlockType, classify_block
 from app.utils.proxy import get_proxy_url, is_pool_backed
 
 logger = logging.getLogger(__name__)
@@ -64,6 +69,11 @@ class CrawlOutcome:
     them off the outcome regardless of crawl SUCCESS (the solver charges per
     *attempt*). They are populated only by the StealthyFetcher tier when captcha
     solving is enabled; every other path leaves the defaults (0 / False).
+
+    Phase 3e ``block_type`` is purely *additive* telemetry: the block classifier
+    labels the last fetched page (Cloudflare / captcha / DataDome / rate-limited
+    / ...) for tuning + future escalation routing. It does NOT influence the
+    billable ``SUCCESS`` predicate above.
     """
 
     status: CrawlOutcomeStatus
@@ -72,6 +82,7 @@ class CrawlOutcome:
     tier: str | None = None
     captcha_attempts: int = 0
     captcha_solved: bool = False
+    block_type: BlockType = BlockType.UNKNOWN
 
 
 class WebCrawlerConnector:
@@ -102,11 +113,17 @@ class WebCrawlerConnector:
         # billing can read it regardless of crawl SUCCESS. Per-call (not on
         # ``self``) => safe under concurrent ``crawl_url`` calls.
         captcha_state: dict[str, Any] = {"attempts": 0, "solved": False}
+        # Per-call block-classifier telemetry (03e). ``_build_result`` (the one
+        # place with raw_html + status) classifies each fetched page into here;
+        # crawl_url stamps it onto every outcome. Additive only — never gates
+        # SUCCESS. Per-call (not on ``self``) => concurrency-safe.
+        block_state: dict[str, Any] = {"block_type": BlockType.UNKNOWN}
         try:
             if not validators.url(url):
                 return CrawlOutcome(
                     status=CrawlOutcomeStatus.FAILED,
                     error=f"Invalid URL: {url}",
+                    block_type=block_state["block_type"],
                 )
 
             errors: list[str] = []
@@ -120,7 +137,8 @@ class WebCrawlerConnector:
             try:
                 logger.info(f"[webcrawler] Using Scrapling AsyncFetcher for: {url}")
                 result = await self._run_tier_with_proxy_retry(
-                    "scrapling-static", lambda: self._crawl_with_async_fetcher(url)
+                    "scrapling-static",
+                    lambda: self._crawl_with_async_fetcher(url, block_state),
                 )
                 if result:
                     self._log_tier_outcome(
@@ -131,6 +149,7 @@ class WebCrawlerConnector:
                         status=CrawlOutcomeStatus.SUCCESS,
                         result=result,
                         tier="scrapling-static",
+                        block_type=block_state["block_type"],
                     )
                 reached_without_content = True
                 errors.append("Scrapling static: empty extraction")
@@ -146,7 +165,8 @@ class WebCrawlerConnector:
             try:
                 logger.info(f"[webcrawler] Using Scrapling DynamicFetcher for: {url}")
                 result = await self._run_tier_with_proxy_retry(
-                    "scrapling-dynamic", lambda: self._crawl_with_dynamic(url)
+                    "scrapling-dynamic",
+                    lambda: self._crawl_with_dynamic(url, block_state),
                 )
                 if result:
                     self._log_tier_outcome(
@@ -157,6 +177,7 @@ class WebCrawlerConnector:
                         status=CrawlOutcomeStatus.SUCCESS,
                         result=result,
                         tier="scrapling-dynamic",
+                        block_type=block_state["block_type"],
                     )
                 reached_without_content = True
                 errors.append("Scrapling dynamic: empty extraction")
@@ -181,7 +202,9 @@ class WebCrawlerConnector:
                 logger.info(f"[webcrawler] Using Scrapling StealthyFetcher for: {url}")
                 result = await self._run_tier_with_proxy_retry(
                     "scrapling-stealthy",
-                    lambda: self._crawl_with_stealthy(url, captcha_state),
+                    lambda: self._crawl_with_stealthy(
+                        url, captcha_state, block_state
+                    ),
                 )
                 if result:
                     self._log_tier_outcome(
@@ -194,6 +217,7 @@ class WebCrawlerConnector:
                         tier="scrapling-stealthy",
                         captcha_attempts=captcha_state["attempts"],
                         captcha_solved=captcha_state["solved"],
+                        block_type=block_state["block_type"],
                     )
                 reached_without_content = True
                 errors.append("Scrapling stealthy: empty extraction")
@@ -219,12 +243,14 @@ class WebCrawlerConnector:
                     error=f"No content extracted for {url}. {'; '.join(errors)}",
                     captcha_attempts=captcha_state["attempts"],
                     captcha_solved=captcha_state["solved"],
+                    block_type=block_state["block_type"],
                 )
             return CrawlOutcome(
                 status=CrawlOutcomeStatus.FAILED,
                 error=f"All crawl methods failed for {url}. {'; '.join(errors)}",
                 captcha_attempts=captcha_state["attempts"],
                 captcha_solved=captcha_state["solved"],
+                block_type=block_state["block_type"],
             )
 
         except Exception as e:
@@ -234,6 +260,7 @@ class WebCrawlerConnector:
                 error=f"Error crawling URL {url}: {e!s}",
                 captcha_attempts=captcha_state["attempts"],
                 captcha_solved=captcha_state["solved"],
+                block_type=block_state["block_type"],
             )
 
     async def _run_tier_with_proxy_retry(
@@ -307,7 +334,9 @@ class WebCrawlerConnector:
             total_ms,
         )
 
-    async def _crawl_with_async_fetcher(self, url: str) -> dict[str, Any] | None:
+    async def _crawl_with_async_fetcher(
+        self, url: str, block_state: dict[str, Any] | None = None
+    ) -> dict[str, Any] | None:
         """
         Crawl URL using Scrapling's AsyncFetcher (static HTTP) + Trafilatura.
 
@@ -331,6 +360,13 @@ class WebCrawlerConnector:
 
         status = getattr(page, "status", None)
         if status is not None and status >= 400:
+            # 03e: classify here too — this early return skips _build_result, and
+            # the static tier is the first/cheapest hit, so the 403/429 bot-gate
+            # (the most common block signal) would otherwise never be labeled.
+            if block_state is not None:
+                block_state["block_type"] = classify_block(
+                    status, getattr(page, "html_content", None)
+                )
             logger.info(
                 "%s tier=scrapling-static url=%s fetch_ms=%.1f status=%s outcome=http_error",
                 _PERF,
@@ -347,18 +383,25 @@ class WebCrawlerConnector:
             allow_raw_fallback=False,
             fetch_ms=fetch_ms,
             status=status,
+            block_state=block_state,
         )
 
-    async def _crawl_with_dynamic(self, url: str) -> dict[str, Any] | None:
+    async def _crawl_with_dynamic(
+        self, url: str, block_state: dict[str, Any] | None = None
+    ) -> dict[str, Any] | None:
         """
         Crawl URL using Scrapling's DynamicFetcher (full browser) + Trafilatura.
 
         Runs the sync fetch in a worker thread so it works on any event loop,
         including Windows ``SelectorEventLoop`` which cannot spawn subprocesses.
         """
-        return await asyncio.to_thread(self._crawl_with_dynamic_sync, url)
+        return await asyncio.to_thread(
+            self._crawl_with_dynamic_sync, url, block_state
+        )
 
-    def _crawl_with_dynamic_sync(self, url: str) -> dict[str, Any] | None:
+    def _crawl_with_dynamic_sync(
+        self, url: str, block_state: dict[str, Any] | None = None
+    ) -> dict[str, Any] | None:
         """Synchronous DynamicFetcher crawl executed in a worker thread."""
         fetch_start = time.perf_counter()
         page = DynamicFetcher.fetch(
@@ -376,10 +419,14 @@ class WebCrawlerConnector:
             allow_raw_fallback=False,
             fetch_ms=fetch_ms,
             status=getattr(page, "status", None),
+            block_state=block_state,
         )
 
     async def _crawl_with_stealthy(
-        self, url: str, captcha_state: dict[str, Any] | None = None
+        self,
+        url: str,
+        captcha_state: dict[str, Any] | None = None,
+        block_state: dict[str, Any] | None = None,
     ) -> dict[str, Any] | None:
         """
         Crawl URL using Scrapling's StealthyFetcher (patchright-Chromium) + Trafilatura.
@@ -390,13 +437,18 @@ class WebCrawlerConnector:
 
         ``captcha_state`` (03d) is mutated in place by the captcha page_action
         (attempts/solved) so ``crawl_url`` can surface it on the outcome.
+        ``block_state`` (03e) is populated by ``_build_result`` with the block
+        classification of the fetched page.
         """
         return await asyncio.to_thread(
-            self._crawl_with_stealthy_sync, url, captcha_state
+            self._crawl_with_stealthy_sync, url, captcha_state, block_state
         )
 
     def _crawl_with_stealthy_sync(
-        self, url: str, captcha_state: dict[str, Any] | None = None
+        self,
+        url: str,
+        captcha_state: dict[str, Any] | None = None,
+        block_state: dict[str, Any] | None = None,
     ) -> dict[str, Any] | None:
         """Synchronous StealthyFetcher crawl executed in a worker thread."""
         fetch_start = time.perf_counter()
@@ -425,6 +477,11 @@ class WebCrawlerConnector:
             "solve_cloudflare": True,
             "proxy": proxy,
         }
+        # 03e Slice A: merge config-driven stealth levers (block_webrtc,
+        # hide_canvas, google_search, dns_over_https, geoip locale/timezone).
+        # Keys never collide with the core kwargs above; defaults preserve
+        # today's behavior and add no crawl-speed regression.
+        fetch_kwargs.update(build_stealthy_kwargs(get_stealth_config()))
         if page_action is not None:
             fetch_kwargs["page_action"] = page_action
         page = StealthyFetcher.fetch(url, **fetch_kwargs)
@@ -436,6 +493,7 @@ class WebCrawlerConnector:
             allow_raw_fallback=True,
             fetch_ms=fetch_ms,
             status=getattr(page, "status", None),
+            block_state=block_state,
         )
 
     def _build_result(
@@ -447,6 +505,7 @@ class WebCrawlerConnector:
         allow_raw_fallback: bool,
         fetch_ms: float | None = None,
         status: int | None = None,
+        block_state: dict[str, Any] | None = None,
     ) -> dict[str, Any] | None:
         """
         Extract markdown + metadata from raw HTML using Trafilatura.
@@ -465,6 +524,12 @@ class WebCrawlerConnector:
         Returns:
             Result dict (content/metadata/crawler_type) or ``None``.
         """
+        # 03e: classify the fetched page (additive telemetry/routing only — never
+        # gates SUCCESS). Done before the early returns so EMPTY/no-extraction
+        # pages still get labeled. Last tier to fetch wins in block_state.
+        if block_state is not None:
+            block_state["block_type"] = classify_block(status, raw_html)
+
         html_len = len(raw_html) if raw_html else 0
 
         if not raw_html or len(raw_html.strip()) == 0:

surfsense_backend/app/proprietary/web_crawler/stealth.py

@@ -0,0 +1,216 @@
+# SurfSense proprietary crawler engine.
+#
+# This module is part of the ``app.proprietary`` package and is licensed
+# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
+# Do not relicense or redistribute this file under Apache-2.0.
+"""Stealth-hardening config + the centralized per-tier kwargs builder (Phase 3e).
+
+This is **bypass-specific tuning** (hence proprietary): it decides the
+anti-detection lever set the StealthyFetcher tier runs with (WebRTC/canvas/DNS
+handling, organic-referer, and proxy-geo-coherent ``locale``/``timezone_id``).
+It is the **single source of truth** that turns config -> ``StealthyFetcher``
+keyword arguments, imported by both the crawler (``connector.py``) and the 03f
+manual harness, so the scorecard grades the exact browser we ship (no
+test-vs-prod drift).
+
+The generic, vendor-agnostic block *classifier* (passive telemetry, public
+markers) stays Apache-2.0 in ``app/utils/crawl/``. The further bypass logic
+(WebGL spoof JS, humanize choreography) is deferred to later 03e slices and will
+also live here.
+
+Defaults preserve today's behavior and add **no crawl-speed regression**:
+``dns_over_https`` (the only lever with a latency cost) is off unless explicitly
+enabled, and geoip resolution is a pure in-process dict lookup (no exit-IP call).
+"""
+
+from dataclasses import dataclass
+from typing import Any
+
+from app.config import config
+
+# Coarse region -> (locale, IANA timezone) map. Country granularity only: per the
+# 03e "Geoip accuracy" risk, wrong-but-coherent beats a default mismatch and
+# per-city precision isn't worth it. Keyed by ISO-3166 alpha-2; common full names
+# are aliased below. Unknown/empty => (None, None) => leave Scrapling's default.
+_REGION_TO_LOCALE_TZ: dict[str, tuple[str, str]] = {
+    "us": ("en-US", "America/New_York"),
+    "ca": ("en-CA", "America/Toronto"),
+    "gb": ("en-GB", "Europe/London"),
+    "ie": ("en-IE", "Europe/Dublin"),
+    "au": ("en-AU", "Australia/Sydney"),
+    "nz": ("en-NZ", "Pacific/Auckland"),
+    "de": ("de-DE", "Europe/Berlin"),
+    "fr": ("fr-FR", "Europe/Paris"),
+    "es": ("es-ES", "Europe/Madrid"),
+    "it": ("it-IT", "Europe/Rome"),
+    "nl": ("nl-NL", "Europe/Amsterdam"),
+    "be": ("nl-BE", "Europe/Brussels"),
+    "ch": ("de-CH", "Europe/Zurich"),
+    "at": ("de-AT", "Europe/Vienna"),
+    "se": ("sv-SE", "Europe/Stockholm"),
+    "no": ("nb-NO", "Europe/Oslo"),
+    "dk": ("da-DK", "Europe/Copenhagen"),
+    "fi": ("fi-FI", "Europe/Helsinki"),
+    "pl": ("pl-PL", "Europe/Warsaw"),
+    "pt": ("pt-PT", "Europe/Lisbon"),
+    "ru": ("ru-RU", "Europe/Moscow"),
+    "ua": ("uk-UA", "Europe/Kyiv"),
+    "tr": ("tr-TR", "Europe/Istanbul"),
+    "in": ("en-IN", "Asia/Kolkata"),
+    "jp": ("ja-JP", "Asia/Tokyo"),
+    "kr": ("ko-KR", "Asia/Seoul"),
+    "cn": ("zh-CN", "Asia/Shanghai"),
+    "hk": ("zh-HK", "Asia/Hong_Kong"),
+    "tw": ("zh-TW", "Asia/Taipei"),
+    "sg": ("en-SG", "Asia/Singapore"),
+    "id": ("id-ID", "Asia/Jakarta"),
+    "ph": ("en-PH", "Asia/Manila"),
+    "th": ("th-TH", "Asia/Bangkok"),
+    "vn": ("vi-VN", "Asia/Ho_Chi_Minh"),
+    "ae": ("ar-AE", "Asia/Dubai"),
+    "il": ("he-IL", "Asia/Jerusalem"),
+    "za": ("en-ZA", "Africa/Johannesburg"),
+    "br": ("pt-BR", "America/Sao_Paulo"),
+    "mx": ("es-MX", "America/Mexico_City"),
+    "ar": ("es-AR", "America/Argentina/Buenos_Aires"),
+    "cl": ("es-CL", "America/Santiago"),
+}
+
+# Full-name / synonym aliases -> alpha-2 key above.
+_REGION_ALIASES: dict[str, str] = {
+    "usa": "us",
+    "united states": "us",
+    "united states of america": "us",
+    "america": "us",
+    "uk": "gb",
+    "united kingdom": "gb",
+    "great britain": "gb",
+    "england": "gb",
+    "canada": "ca",
+    "australia": "au",
+    "germany": "de",
+    "deutschland": "de",
+    "france": "fr",
+    "spain": "es",
+    "italy": "it",
+    "netherlands": "nl",
+    "holland": "nl",
+    "sweden": "se",
+    "norway": "no",
+    "denmark": "dk",
+    "finland": "fi",
+    "poland": "pl",
+    "portugal": "pt",
+    "russia": "ru",
+    "ukraine": "ua",
+    "turkey": "tr",
+    "india": "in",
+    "japan": "jp",
+    "korea": "kr",
+    "south korea": "kr",
+    "china": "cn",
+    "hong kong": "hk",
+    "taiwan": "tw",
+    "singapore": "sg",
+    "indonesia": "id",
+    "philippines": "ph",
+    "thailand": "th",
+    "vietnam": "vn",
+    "israel": "il",
+    "south africa": "za",
+    "brazil": "br",
+    "brasil": "br",
+    "mexico": "mx",
+    "argentina": "ar",
+    "chile": "cl",
+}
+
+
+def location_to_locale_timezone(
+    location: str | None,
+) -> tuple[str | None, str | None]:
+    """Map a free-form proxy region string -> (locale, timezone_id).
+
+    ``location`` is the vendor-specific ``RESIDENTIAL_PROXY_LOCATION`` value
+    (``03b``). Best-effort: matches an ISO-3166 alpha-2 code (e.g. ``"us"``) or a
+    common country name (e.g. ``"Germany"``); only the leading token is honored,
+    so vendor strings like ``"us:nyc"`` or ``"de region"`` still resolve. Returns
+    ``(None, None)`` for empty/unknown input (caller then leaves the browser
+    default).
+    """
+    if not location:
+        return (None, None)
+
+    normalized = location.strip().lower()
+    if not normalized:
+        return (None, None)
+
+    # Direct alpha-2 hit.
+    if normalized in _REGION_TO_LOCALE_TZ:
+        return _REGION_TO_LOCALE_TZ[normalized]
+
+    # Full-name / synonym hit.
+    if normalized in _REGION_ALIASES:
+        return _REGION_TO_LOCALE_TZ[_REGION_ALIASES[normalized]]
+
+    # Leading token (handles "us:nyc", "de-rotating", "united states (east)").
+    head = normalized.replace("-", " ").replace("_", " ").replace(":", " ").split()
+    if head:
+        token = head[0]
+        if token in _REGION_TO_LOCALE_TZ:
+            return _REGION_TO_LOCALE_TZ[token]
+        if token in _REGION_ALIASES:
+            return _REGION_TO_LOCALE_TZ[_REGION_ALIASES[token]]
+
+    return (None, None)
+
+
+@dataclass(frozen=True)
+class StealthConfig:
+    """Immutable snapshot of the Phase 3e Slice-A stealth levers for one crawl."""
+
+    geoip_match_enabled: bool
+    proxy_location: str
+    block_webrtc: bool
+    hide_canvas: bool
+    google_search: bool
+    dns_over_https: bool
+
+
+def get_stealth_config() -> StealthConfig:
+    """Build a :class:`StealthConfig` from the current process config/env."""
+    return StealthConfig(
+        geoip_match_enabled=config.CRAWL_GEOIP_MATCH_ENABLED,
+        proxy_location=config.RESIDENTIAL_PROXY_LOCATION or "",
+        block_webrtc=config.CRAWL_BLOCK_WEBRTC,
+        hide_canvas=config.CRAWL_HIDE_CANVAS,
+        google_search=config.CRAWL_GOOGLE_SEARCH_REFERER,
+        dns_over_https=config.CRAWL_DNS_OVER_HTTPS,
+    )
+
+
+def build_stealthy_kwargs(cfg: StealthConfig) -> dict[str, Any]:
+    """Return the config-derived ``StealthyFetcher.fetch`` keyword arguments.
+
+    Only the Slice-A stealth levers are returned; the caller merges these into
+    the tier's own kwargs (``headless``/``network_idle``/``block_ads``/
+    ``solve_cloudflare``/``proxy``/captcha ``page_action``), which never collide
+    with the keys here. ``locale``/``timezone_id`` are added only when geoip
+    matching is enabled *and* the proxy region resolves — otherwise the browser
+    keeps its system default.
+    """
+    kwargs: dict[str, Any] = {
+        "block_webrtc": cfg.block_webrtc,
+        "hide_canvas": cfg.hide_canvas,
+        "google_search": cfg.google_search,
+        "dns_over_https": cfg.dns_over_https,
+    }
+
+    if cfg.geoip_match_enabled:
+        locale, timezone_id = location_to_locale_timezone(cfg.proxy_location)
+        if locale:
+            kwargs["locale"] = locale
+        if timezone_id:
+            kwargs["timezone_id"] = timezone_id
+
+    return kwargs

surfsense_backend/app/proprietary/web_crawler/testbench/README.md

@@ -0,0 +1,83 @@
+# Crawler testbench — manual undetectability & extraction scorecard (Phase 3f)
+
+A **manual, repeatable scorecard** for the Universal WebURL Crawler. It answers
+one question with evidence: *how undetectable (and how correct) is the crawler
+right now?* — so we know when the free-stack ceiling (`03e`) is reached and the
+deferred paid-unblocker tier is worth flipping.
+
+This is **dev/operator tooling**, not a product code path and **not** part of the
+pytest suite (it hits the live internet and needs proxy creds). See
+`plans/backend/03f-undetectability-testing.md` for the full design.
+
+## What it measures (two axes, two suites)
+
+- **Suite S — stealth / anti-bot:** drives the real `StealthyFetcher` tier
+  (built from the **same** `app/proprietary/web_crawler/stealth.py` builder the
+  crawler ships, so no test-vs-prod drift) against the standard bot-detection
+  sites — sannysoft, deviceandbrowserinfo, reCAPTCHA v3, CreepJS, BrowserScan,
+  incolumitas, fingerprint-scan, FingerprintJS Pro, a **Cloudflare-challenge
+  canary** (the only row that exercises `solve_cloudflare`), and iphey — plus the
+  HTTP-tier TLS fingerprint (`tls.peet.ws`), exit-IP echo, and manual
+  per-property links (browserleaks). Every detection site is **auto-graded** from
+  its real DOM verdict (no screenshot reading required).
+- **Suite E — extraction correctness:** drives the real `crawl_url` ladder
+  against ToS-safe scraping sandboxes (`toscrape`, `scrapethissite`) and asserts
+  known content appears in the extracted markdown.
+
+## Run it
+
+From the backend directory (`surfsense_backend/`):
+
+```bash
+uv run python -m app.proprietary.web_crawler.testbench --suite all
+# or
+.\.venv\Scripts\python.exe -m app.proprietary.web_crawler.testbench --suite all
+```
+
+Flags:
+
+| Flag | Meaning |
+|---|---|
+| `--suite S\|E\|all` | Which suite(s) to run (default `all`). |
+| `--proxy URL` | Override the proxy for Suite S (default: the app proxy provider from `.env`). |
+| `--headed` | Run the browser tier headful (`headless=False`). |
+| `--no-screenshots` | Skip per-site screenshots. |
+
+Captcha solving is **forced OFF** every run — Suite S measures the *unaided*
+stealth score.
+
+## Reading the output
+
+Each row prints `PASS / FAIL / ERR / INFO` with the tier and (where one exists) a
+numeric. The harness is **tolerant**: a parse miss is `ERROR`, never a crash.
+Detection sites are auto-graded from their DOM verdict; `INFO` is reserved for
+purely informational rows (TLS fingerprint, exit IP) and the manual
+per-property links (browserleaks canvas/webgl/fonts/webrtc). Screenshots are
+still captured for every browser row as a backstop.
+
+Outputs land in `results/`, which is **entirely gitignored** (run-local):
+
+- `scorecard-<ts>.json` / `.md` — timestamped scorecard + readable report.
+- `latest.json` — convenience pointer.
+- `screenshots/` — per-site full-page captures.
+
+Every run prints **drift vs the last baseline** (the previous on-disk scorecard)
+so you can see the trend (our stealth improving, or a WAF tightening).
+
+## The aspirational bars (from CloakBrowser, recorded as targets)
+
+`sannysoft` 0 failed cells · `deviceandbrowserinfo` isBot=false ·
+reCAPTCHA v3 `>= 0.7` · CreepJS headless `<= 30%`, no spoof tells · FingerprintJS
+demo not blocked · Cloudflare challenge bypassed · iphey `Trustworthy`. We
+**record our actual numbers as the baseline** — these are targets, not build
+gates (the run never blocks anything).
+
+## Caveats
+
+- **Proxy required for realism.** Without residential egress the hard rows fail
+  by design (datacenter IP); a red scorecard from no-proxy is expected.
+- **Sites change.** Detection sites move their DOM; if an auto-parser starts
+  returning `INFO`/`ERROR`, read the screenshot and (optionally) tighten the
+  parser — each site is one entry in `suite_stealth.py`.
+- **Not a guarantee.** Passing sannysoft/CreepJS ≠ beating DataDome/Kasada; the
+  value is *trend + ceiling visibility*, not a green checkmark.

surfsense_backend/app/proprietary/web_crawler/testbench/__init__.py

@@ -0,0 +1,25 @@
+# SurfSense proprietary crawler engine.
+#
+# This module is part of the ``app.proprietary`` package and is licensed
+# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
+# Do not relicense or redistribute this file under Apache-2.0.
+"""Phase 3f — manual undetectability & extraction test harness (dev tooling).
+
+A repeatable, **manual** scorecard that grades the Universal WebURL Crawler on two
+axes:
+
+- **Suite S (stealth / anti-bot):** drives the real Scrapling tiers against the
+  industry-standard bot-detection + fingerprint sites and parses each verdict.
+- **Suite E (extraction correctness):** drives the real ``crawl_url`` ladder
+  against ToS-safe scraping sandboxes and asserts known values.
+
+It is **not** collected by pytest (it hits the live internet and needs proxy
+creds). Run it from the backend directory:
+
+    uv run python -m app.proprietary.web_crawler.testbench --suite all
+    # or: .\\.venv\\Scripts\\python.exe -m app.proprietary.web_crawler.testbench --suite all
+
+See ``README.md`` for the runbook and ``plans/backend/03f-undetectability-testing.md``
+for the design. This package is dev/operator tooling, untouched by the product
+rename, and only *measures* what 03a–03e built.
+"""

surfsense_backend/app/proprietary/web_crawler/testbench/__main__.py

@@ -0,0 +1,141 @@
+# SurfSense proprietary crawler engine.
+#
+# This module is part of the ``app.proprietary`` package and is licensed
+# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
+# Do not relicense or redistribute this file under Apache-2.0.
+"""CLI for the 03f manual scorecard. Run from the backend directory:
+
+    uv run python -m app.proprietary.web_crawler.testbench --suite all
+    uv run python -m app.proprietary.web_crawler.testbench --suite S --headed
+    uv run python -m app.proprietary.web_crawler.testbench --suite E --no-screenshots
+
+Mirrors CloakBrowser's ``bin/cloaktest`` ergonomics. Writes a timestamped
+scorecard (JSON + markdown) under ``results/`` and diffs against the last
+baseline. Captcha solving is forced OFF so the scorecard measures the *unaided*
+stealth ceiling (03f §Harness).
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import os
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+# --- bootstrap: load .env + put backend root on sys.path BEFORE importing app.* ---
+# __file__ = app/proprietary/web_crawler/testbench/__main__.py -> backend root is 4 up.
+_BACKEND_ROOT = Path(__file__).resolve().parents[4]
+if str(_BACKEND_ROOT) not in sys.path:
+    sys.path.insert(0, str(_BACKEND_ROOT))
+for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
+    if _candidate.exists():
+        load_dotenv(_candidate)
+        break
+
+# Force captcha solving OFF: Suite S measures the unaided score and we never want
+# the 03d injector firing paid solves against the reCAPTCHA-demo row. Must be set
+# before app.config is imported (config snapshots env at import).
+os.environ["CAPTCHA_SOLVING_ENABLED"] = "false"
+
+
+def _parse_args(argv: list[str]) -> argparse.Namespace:
+    p = argparse.ArgumentParser(
+        prog="python -m app.proprietary.web_crawler.testbench",
+        description="Manual undetectability & extraction scorecard (Phase 3f).",
+    )
+    p.add_argument(
+        "--suite",
+        choices=["S", "E", "all"],
+        default="all",
+        help="S=stealth/anti-bot, E=extraction correctness, all=both.",
+    )
+    p.add_argument(
+        "--proxy",
+        default=None,
+        help="Override proxy URL for Suite S (default: the app proxy provider).",
+    )
+    p.add_argument(
+        "--headed",
+        action="store_true",
+        help="Run the browser tier headful (headless=False).",
+    )
+    p.add_argument(
+        "--no-screenshots",
+        action="store_true",
+        help="Skip per-site screenshots (faster, no results/screenshots writes).",
+    )
+    return p.parse_args(argv)
+
+
+async def _amain(args: argparse.Namespace) -> int:
+    # Imports happen here (after bootstrap + env override) so app.config sees the
+    # forced CAPTCHA_SOLVING_ENABLED=false and the resolved .env.
+    from app.utils.proxy import get_proxy_url
+
+    from .core import (
+        RunMeta,
+        diff_against_baseline,
+        load_last_baseline,
+        render_console,
+        scrapling_version,
+        write_scorecard,
+    )
+
+    proxy = args.proxy or get_proxy_url()
+    screenshots = not args.no_screenshots
+    results = []
+
+    print(
+        f"== crawler scorecard == suite={args.suite} "
+        f"headed={args.headed} proxy={'set' if proxy else 'NONE'} "
+        f"screenshots={screenshots}"
+    )
+    if not proxy:
+        print(
+            "  WARNING: no proxy configured — the hard anti-bot rows are expected "
+            "to fail from a datacenter IP (see README)."
+        )
+
+    baseline = load_last_baseline()
+
+    if args.suite in ("S", "all"):
+        from .suite_stealth import run_suite_s
+
+        results += await run_suite_s(
+            proxy=proxy, headed=args.headed, screenshots=screenshots
+        )
+
+    if args.suite in ("E", "all"):
+        from .suite_extraction import run_suite_e
+
+        results += await run_suite_e()
+
+    render_console(results)
+
+    meta = RunMeta.now(
+        suites=args.suite,
+        proxy=proxy,
+        headed=args.headed,
+        scrapling_version=scrapling_version(),
+    )
+    json_path, md_path = write_scorecard(results, meta)
+
+    print("\n--- drift vs last baseline ---")
+    for line in diff_against_baseline(results, baseline):
+        print(f"  {line}")
+
+    print(f"\nscorecard JSON: {json_path}")
+    print(f"scorecard MD:   {md_path}")
+    return 0
+
+
+def main() -> None:
+    args = _parse_args(sys.argv[1:])
+    raise SystemExit(asyncio.run(_amain(args)))
+
+
+if __name__ == "__main__":
+    main()

surfsense_backend/app/proprietary/web_crawler/testbench/core.py

@@ -0,0 +1,325 @@
+# SurfSense proprietary crawler engine.
+#
+# This module is part of the ``app.proprietary`` package and is licensed
+# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
+# Do not relicense or redistribute this file under Apache-2.0.
+"""Shared primitives for the 03f scorecard: result model, the closure-cell
+``page_action`` verdict-extractor (the 03d-shared mechanism), and the scorecard
+snapshot writer / baseline-differ / console renderer.
+
+Stdlib-only on purpose (the plan's "no new prod dependency" bar). Everything here
+is tolerant: a parse miss yields an ``ERROR`` row, never a crash — detection sites
+change their DOM and the harness is manual.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from enum import Enum
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlsplit
+
+# Results live next to the harness; screenshots are gitignored, scorecard JSON is
+# committed so runs diff against the last baseline (see README + plan §Scorecard).
+_PKG_DIR = Path(__file__).resolve().parent
+RESULTS_DIR = _PKG_DIR / "results"
+SCREENSHOTS_DIR = RESULTS_DIR / "screenshots"
+
+
+class CheckStatus(str, Enum):
+    """Outcome of a single scorecard row."""
+
+    PASS = "PASS"  # met the aspirational bar
+    FAIL = "FAIL"  # ran, did not meet the bar
+    ERROR = "ERROR"  # could not run / parse (never fatal to the run)
+    INFO = "INFO"  # recorded, no pass/fail bar (e.g. TLS JA3, manual links)
+    SKIP = "SKIP"  # intentionally not run this invocation
+
+
+@dataclass
+class CheckResult:
+    """One row of the scorecard."""
+
+    suite: str  # "S" (stealth) | "E" (extraction)
+    name: str  # site / check key
+    tier: str  # scrapling-static | scrapling-stealthy | crawl_url | n/a
+    status: CheckStatus
+    bar: str  # human-readable aspirational threshold
+    detail: str = ""  # short human summary of what was observed
+    numeric: float | None = None  # comparable metric where one exists
+    screenshot: str | None = None  # path, when captured
+
+    def to_row(self) -> dict[str, Any]:
+        d = asdict(self)
+        d["status"] = self.status.value
+        return d
+
+
+@dataclass
+class RunMeta:
+    """Provenance for a scorecard snapshot so baselines are comparable."""
+
+    timestamp: str
+    suites: str
+    proxy: str  # masked
+    headed: bool
+    scrapling_version: str
+    captcha_disabled: bool = True
+    notes: str = ""
+
+    @staticmethod
+    def now(
+        *, suites: str, proxy: str | None, headed: bool, scrapling_version: str
+    ) -> RunMeta:
+        return RunMeta(
+            timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"),
+            suites=suites,
+            proxy=mask_proxy(proxy),
+            headed=headed,
+            scrapling_version=scrapling_version,
+        )
+
+
+# --- closure-cell page_action (the mechanism shared with 03d) ----------------
+
+
+@dataclass
+class EvalCell:
+    """A mutable cell a ``page_action`` writes its ``page.evaluate`` result into.
+
+    Scrapling discards a ``page_action``'s return value, so the only way to get a
+    JS-object verdict (CreepJS ``window.Fingerprint``, a Castle.js score node) out
+    of the browser is to stash it in a closure variable. This is the exact same
+    plumbing 03d's captcha-token injector uses — factored once here.
+    """
+
+    value: Any = None
+    error: str | None = None
+    captured: bool = False
+
+
+def make_page_action(
+    *,
+    evaluate_js: str | None = None,
+    screenshot_path: str | None = None,
+    pre_wait_ms: int = 0,
+) -> tuple[Callable[[Any], Any], EvalCell]:
+    """Build a sync ``page_action`` + the :class:`EvalCell` it writes into.
+
+    The action (in priority order) optionally sleeps ``pre_wait_ms`` (lets async
+    scores settle), optionally evaluates ``evaluate_js`` into the cell, and
+    optionally writes a full-page screenshot. Each step is independently guarded
+    so one failure (e.g. a site without the JS object) never aborts the fetch.
+    Returns the page unchanged (Scrapling re-reads its DOM afterwards).
+    """
+    cell = EvalCell()
+
+    def _action(page: Any) -> Any:
+        if pre_wait_ms > 0:
+            try:
+                page.wait_for_timeout(pre_wait_ms)
+            except Exception:
+                pass
+        if evaluate_js is not None:
+            try:
+                cell.value = page.evaluate(evaluate_js)
+                cell.captured = True
+            except Exception as exc:  # noqa: BLE001 - tolerant by design
+                cell.error = f"{type(exc).__name__}: {exc}"
+        if screenshot_path is not None:
+            try:
+                # The screenshot dir must exist *now* (the page action runs mid-suite,
+                # before the end-of-run ``ensure_dirs``); Scrapling/patchright will not
+                # create it for us, so a missing dir silently drops every screenshot.
+                Path(screenshot_path).parent.mkdir(parents=True, exist_ok=True)
+                page.screenshot(path=screenshot_path, full_page=True)
+            except Exception as exc:  # noqa: BLE001 - tolerant, but surface the cause
+                if cell.error is None:
+                    cell.error = f"screenshot: {type(exc).__name__}: {exc}"
+        return page
+
+    return _action, cell
+
+
+# --- helpers -----------------------------------------------------------------
+
+
+def mask_proxy(url: str | None) -> str:
+    """Mask credentials in a proxy URL for logs/snapshots."""
+    if not url:
+        return "<none>"
+    try:
+        p = urlsplit(url)
+        host = p.hostname or "?"
+        port = f":{p.port}" if p.port else ""
+        return f"{p.scheme}://***@{host}{port}"
+    except Exception:
+        return "<set>"
+
+
+def ensure_dirs() -> None:
+    RESULTS_DIR.mkdir(parents=True, exist_ok=True)
+    SCREENSHOTS_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def scrapling_version() -> str:
+    try:
+        import scrapling  # noqa: PLC0415
+
+        return getattr(scrapling, "__version__", "unknown")
+    except Exception:
+        return "unavailable"
+
+
+# --- scorecard I/O -----------------------------------------------------------
+
+
+def write_scorecard(
+    results: list[CheckResult], meta: RunMeta
+) -> tuple[Path, Path]:
+    """Write the JSON snapshot (committed, baseline) + a readable markdown report.
+
+    The JSON filename is timestamped so prior runs are kept as the baseline trail;
+    ``latest.json`` is overwritten as the convenience pointer used by the differ.
+    """
+    ensure_dirs()
+    stamp = meta.timestamp.replace(":", "").replace("-", "")
+    payload = {
+        "meta": asdict(meta),
+        "summary": summarize(results),
+        "results": [r.to_row() for r in results],
+    }
+    json_path = RESULTS_DIR / f"scorecard-{stamp}.json"
+    json_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    (RESULTS_DIR / "latest.json").write_text(
+        json.dumps(payload, indent=2), encoding="utf-8"
+    )
+    md_path = RESULTS_DIR / f"scorecard-{stamp}.md"
+    md_path.write_text(_render_markdown(results, meta), encoding="utf-8")
+    return json_path, md_path
+
+
+def load_last_baseline() -> dict[str, Any] | None:
+    """Load the most recent prior scorecard JSON (excluding ``latest.json``)."""
+    if not RESULTS_DIR.exists():
+        return None
+    snaps = sorted(RESULTS_DIR.glob("scorecard-*.json"))
+    if not snaps:
+        return None
+    try:
+        return json.loads(snaps[-1].read_text(encoding="utf-8"))
+    except Exception:
+        return None
+
+
+def summarize(results: list[CheckResult]) -> dict[str, dict[str, int]]:
+    """Per-suite ``{passed, failed, error, info, total}`` counts."""
+    out: dict[str, dict[str, int]] = {}
+    for r in results:
+        s = out.setdefault(
+            r.suite,
+            {"passed": 0, "failed": 0, "error": 0, "info": 0, "skip": 0, "total": 0},
+        )
+        s["total"] += 1
+        key = {
+            CheckStatus.PASS: "passed",
+            CheckStatus.FAIL: "failed",
+            CheckStatus.ERROR: "error",
+            CheckStatus.INFO: "info",
+            CheckStatus.SKIP: "skip",
+        }[r.status]
+        s[key] += 1
+    return out
+
+
+def diff_against_baseline(
+    results: list[CheckResult], baseline: dict[str, Any] | None
+) -> list[str]:
+    """Human-readable drift lines comparing this run to the last baseline."""
+    if not baseline:
+        return ["(no prior baseline — this run becomes the baseline)"]
+    prior = {(r["suite"], r["name"]): r for r in baseline.get("results", [])}
+    lines: list[str] = []
+    for r in results:
+        was = prior.get((r.suite, r.name))
+        if was is None:
+            lines.append(f"+ NEW  [{r.suite}] {r.name}: {r.status.value}")
+            continue
+        if was["status"] != r.status.value:
+            lines.append(
+                f"~ {r.name}: status {was['status']} -> {r.status.value}"
+            )
+        old_n, new_n = was.get("numeric"), r.numeric
+        if old_n is not None and new_n is not None and abs(old_n - new_n) > 1e-9:
+            lines.append(f"~ {r.name}: numeric {old_n} -> {new_n}")
+    seen = {(r.suite, r.name) for r in results}
+    for key, was in prior.items():
+        if key not in seen:
+            lines.append(f"- GONE [{key[0]}] {key[1]} (was {was['status']})")
+    return lines or ["(no changes vs last baseline)"]
+
+
+# --- rendering ---------------------------------------------------------------
+
+_ICON = {
+    CheckStatus.PASS: "PASS",
+    CheckStatus.FAIL: "FAIL",
+    CheckStatus.ERROR: "ERR ",
+    CheckStatus.INFO: "INFO",
+    CheckStatus.SKIP: "SKIP",
+}
+
+
+def render_console(results: list[CheckResult]) -> None:
+    """Print the grouped scorecard + per-suite totals to stdout."""
+    by_suite: dict[str, list[CheckResult]] = {}
+    for r in results:
+        by_suite.setdefault(r.suite, []).append(r)
+
+    for suite in sorted(by_suite):
+        label = "Suite S - stealth/anti-bot" if suite == "S" else (
+            "Suite E - extraction" if suite == "E" else f"Suite {suite}"
+        )
+        print(f"\n=== {label} ===")
+        for r in by_suite[suite]:
+            num = f" [{r.numeric:g}]" if r.numeric is not None else ""
+            print(f"  {_ICON[r.status]}  {r.name:<34} {r.tier:<18}{num}")
+            if r.detail:
+                print(f"         {r.detail}")
+
+    print("\n--- summary ---")
+    for suite, s in summarize(results).items():
+        print(
+            f"  Suite {suite}: {s['passed']}/{s['total']} passed "
+            f"(fail={s['failed']} err={s['error']} info={s['info']})"
+        )
+
+
+def _render_markdown(results: list[CheckResult], meta: RunMeta) -> str:
+    lines = [
+        f"# Crawler scorecard — {meta.timestamp}",
+        "",
+        f"- proxy: `{meta.proxy}`  headed: `{meta.headed}`  "
+        f"captcha-disabled: `{meta.captcha_disabled}`",
+        f"- scrapling: `{meta.scrapling_version}`  suites: `{meta.suites}`",
+        "",
+        "| Suite | Check | Tier | Status | Numeric | Detail |",
+        "|---|---|---|---|---|---|",
+    ]
+    for r in results:
+        num = "" if r.numeric is None else f"{r.numeric:g}"
+        detail = r.detail.replace("|", "\\|")
+        lines.append(
+            f"| {r.suite} | {r.name} | {r.tier} | {r.status.value} | {num} | {detail} |"
+        )
+    lines.append("")
+    for suite, s in summarize(results).items():
+        lines.append(
+            f"- **Suite {suite}**: {s['passed']}/{s['total']} passed "
+            f"(fail={s['failed']}, err={s['error']}, info={s['info']})"
+        )
+    return "\n".join(lines) + "\n"

surfsense_backend/app/proprietary/web_crawler/testbench/results/.gitignore

@@ -0,0 +1,5 @@
+# Everything under results/ is run-local output (timestamped scorecards,
+# latest.json pointer, screenshots, ad-hoc dumps). None of it is committed —
+# the dir is kept tracked only via this file so the harness has a write target.
+*
+!.gitignore

surfsense_backend/app/proprietary/web_crawler/testbench/suite_extraction.py

@@ -0,0 +1,123 @@
+# SurfSense proprietary crawler engine.
+#
+# This module is part of the ``app.proprietary`` package and is licensed
+# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
+# Do not relicense or redistribute this file under Apache-2.0.
+"""Suite E — extraction correctness (separate axis from stealth).
+
+Unlike Suite S, this drives the **real ``crawl_url`` ladder end-to-end** — the
+auto-tier + Trafilatura markdown path is exactly the production behavior we want
+to assert. Targets are purpose-built, ToS-safe scraping sandboxes with known
+content, so extraction regressions are caught deterministically (a missing
+expected string => FAIL). The ``/js`` cases exercise the DynamicFetcher (browser)
+tier; the static catalogs exercise the HTTP tier.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from app.proprietary.web_crawler import CrawlOutcomeStatus, WebCrawlerConnector
+
+from .core import CheckResult, CheckStatus
+
+
+@dataclass
+class ExtractionCase:
+    """A sandbox URL + strings that must appear in the extracted markdown."""
+
+    name: str
+    url: str
+    must_contain: list[str] = field(default_factory=list)
+
+
+_CASES: list[ExtractionCase] = [
+    ExtractionCase(
+        name="books_static",
+        url="https://books.toscrape.com/",
+        must_contain=["A Light in the Attic", "Tipping the Velvet"],
+    ),
+    ExtractionCase(
+        name="quotes_static",
+        url="https://quotes.toscrape.com/",
+        must_contain=["Albert Einstein", "world as we have created it"],
+    ),
+    ExtractionCase(
+        name="quotes_js_rendered",
+        url="https://quotes.toscrape.com/js/",
+        must_contain=["Albert Einstein", "world as we have created it"],
+    ),
+    ExtractionCase(
+        name="scrapethissite_simple",
+        url="https://www.scrapethissite.com/pages/simple/",
+        must_contain=["Andorra", "Afghanistan"],
+    ),
+]
+
+
+def _content_of(outcome) -> str:
+    result = getattr(outcome, "result", None) or {}
+    if isinstance(result, dict):
+        return str(result.get("content") or "")
+    return ""
+
+
+async def _run_case(
+    connector: WebCrawlerConnector, case: ExtractionCase
+) -> CheckResult:
+    bar = f"SUCCESS + contains {len(case.must_contain)} marker(s)"
+    try:
+        outcome = await connector.crawl_url(case.url)
+    except Exception as exc:  # noqa: BLE001 - never crash the run
+        return CheckResult(
+            suite="E",
+            name=case.name,
+            tier="crawl_url",
+            status=CheckStatus.ERROR,
+            bar=bar,
+            detail=f"{type(exc).__name__}: {exc}",
+        )
+
+    tier = getattr(outcome, "tier", None) or "crawl_url"
+    if outcome.status is not CrawlOutcomeStatus.SUCCESS:
+        return CheckResult(
+            suite="E",
+            name=case.name,
+            tier=tier,
+            status=CheckStatus.FAIL,
+            bar=bar,
+            detail=f"status={outcome.status.value}: {outcome.error or ''}"[:160],
+        )
+
+    content = _content_of(outcome)
+    lowered = content.lower()
+    missing = [m for m in case.must_contain if m.lower() not in lowered]
+    if missing:
+        return CheckResult(
+            suite="E",
+            name=case.name,
+            tier=tier,
+            status=CheckStatus.FAIL,
+            bar=bar,
+            detail=f"missing {missing} (len={len(content)})",
+            numeric=float(len(content)),
+        )
+    return CheckResult(
+        suite="E",
+        name=case.name,
+        tier=tier,
+        status=CheckStatus.PASS,
+        bar=bar,
+        detail=f"all markers present (tier={tier}, len={len(content)})",
+        numeric=float(len(content)),
+    )
+
+
+async def run_suite_e() -> list[CheckResult]:
+    """Run the extraction-correctness cases against the live sandboxes."""
+    connector = WebCrawlerConnector()
+    results: list[CheckResult] = []
+    for case in _CASES:
+        print(f"  [E] {case.name} -> {case.url}")
+        results.append(await _run_case(connector, case))
+    return results

surfsense_backend/app/proprietary/web_crawler/testbench/suite_stealth.py

@@ -0,0 +1,509 @@
+# SurfSense proprietary crawler engine.
+#
+# This module is part of the ``app.proprietary`` package and is licensed
+# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
+# Do not relicense or redistribute this file under Apache-2.0.
+"""Suite S — stealth / anti-bot scorecard.
+
+Drives the **browser tier** (``StealthyFetcher``) against the standard
+bot-detection sites and the **HTTP tier** (``AsyncFetcher``) against the TLS /
+proxy-leak JSON endpoints. Critically, the browser tier is built from the **same
+centralized stealth builder the crawler ships** (``build_stealthy_kwargs`` /
+``get_stealth_config`` in ``app/proprietary/web_crawler/stealth.py``) so the scorecard grades
+the exact browser we run in production — no test-vs-prod drift (03f §Harness).
+
+Verdict extraction is best-effort and tolerant: machine-readable signals
+(reCAPTCHA score text, are-you-a-bot text, sannysoft failed-cell count, TLS/proxy
+JSON) get a real PASS/FAIL/numeric; DOM-heavy fingerprint pages are recorded as
+INFO with a screenshot + captured text for the operator to read. Tightening any
+parser later is a one-function change (the spec list is the extension point).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import re
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+from scrapling.fetchers import AsyncFetcher, StealthyFetcher
+
+from app.proprietary.web_crawler.stealth import (
+    build_stealthy_kwargs,
+    get_stealth_config,
+)
+
+from .core import (
+    SCREENSHOTS_DIR,
+    CheckResult,
+    CheckStatus,
+    EvalCell,
+    make_page_action,
+)
+
+# Parser: (page, eval_cell) -> (status, numeric, detail).
+Parser = Callable[[Any, EvalCell], "tuple[CheckStatus, float | None, str]"]
+
+
+@dataclass
+class StealthSite:
+    """One browser-tier detection site + how to read its verdict."""
+
+    name: str
+    url: str
+    bar: str
+    parse: Parser
+    settle_ms: int = 0  # extra in-page wait for async-computed scores
+    evaluate_js: str | None = None  # for window.* / JS-object verdicts
+
+
+def _text(page: Any, limit: int = 4000) -> str:
+    """Best-effort visible text of a fetched page (short, for fallback details)."""
+    return _full_text(page, limit)
+
+
+def _full_text(page: Any, limit: int = 200_000) -> str:
+    """Untruncated visible text — detector verdicts live deep in long pages."""
+    try:
+        txt = page.get_all_text()
+        if txt:
+            return str(txt)[:limit]
+    except Exception:
+        pass
+    try:
+        return str(page.html_content or "")[:limit]
+    except Exception:
+        return ""
+
+
+# --- per-site parsers (all written against real DOM dumps, no screenshots) ---
+
+
+def _parse_sannysoft(page: Any, _cell: EvalCell):
+    """Count failed result cells (JS sets class='failed' on red rows)."""
+    html = ""
+    try:
+        html = page.html_content or ""
+    except Exception:
+        pass
+    fails = len(re.findall(r'class="[^"]*\bfailed\b[^"]*"', html))
+    if not html:
+        return (CheckStatus.ERROR, None, "no html returned")
+    status = CheckStatus.PASS if fails == 0 else CheckStatus.FAIL
+    return (status, float(fails), f"{fails} failed cell(s) (bar: 0)")
+
+
+# deviceandbrowserinfo renders a JSON verdict block: {"isBot": false, ...}.
+_ISBOT_RE = re.compile(r'"isbot"\s*:\s*(true|false)', re.IGNORECASE)
+
+
+def _parse_areyouabot(page: Any, _cell: EvalCell):
+    txt = _full_text(page)
+    m = _ISBOT_RE.search(txt)
+    if m:
+        is_bot = m.group(1).lower() == "true"
+        return (
+            CheckStatus.FAIL if is_bot else CheckStatus.PASS,
+            1.0 if is_bot else 0.0,
+            f"isBot={is_bot}",
+        )
+    low = txt.lower()
+    if "you are human" in low or "not a bot" in low:
+        return (CheckStatus.PASS, 0.0, "verdict: human")
+    if "you are a bot" in low:
+        return (CheckStatus.FAIL, 1.0, "verdict: BOT")
+    return (CheckStatus.INFO, None, f"unparsed: {low[:120]!r}")
+
+
+# The reCAPTCHA demo echoes the server verify response: {"success":true,"score":0.9}.
+_SCORE_JSON_RE = re.compile(r'"score"\s*:\s*([0-9]*\.?[0-9]+)')
+
+
+def _parse_recaptcha_v3(page: Any, _cell: EvalCell):
+    txt = _full_text(page)
+    m = _SCORE_JSON_RE.search(txt)
+    if not m:  # fall back to looser prose match
+        m = re.search(r"score[^0-9]{0,12}([01](?:\.\d+)?)", txt, re.IGNORECASE)
+    if not m:
+        return (CheckStatus.INFO, None, f"no score parsed: {txt[:120]!r}")
+    score = float(m.group(1))
+    status = CheckStatus.PASS if score >= 0.7 else CheckStatus.FAIL
+    return (status, score, f"reCAPTCHA v3 score={score} (bar: >=0.7)")
+
+
+# CreepJS prints category percentages ("33% headless") + boolean tells inline.
+_CREEPJS_TELLS = (
+    "webDriverIsOn",
+    "hasHeadlessUA",
+    "hasHeadlessWorkerUA",
+    "hasBadWebGL",
+    "hasSwiftShader",
+)
+
+
+def _parse_creepjs(page: Any, _cell: EvalCell):
+    """Grade CreepJS by its headless-similarity % + the boolean spoof tells."""
+    txt = _full_text(page)
+
+    def _pct(label: str) -> int | None:
+        m = re.search(r"(\d+)%\s*" + label, txt, re.IGNORECASE)
+        return int(m.group(1)) if m else None
+
+    headless = _pct(r"headless")
+    stealth = _pct(r"stealth")
+    tells = [
+        flag
+        for flag in _CREEPJS_TELLS
+        if re.search(re.escape(flag) + r"\s*:\s*true", txt, re.IGNORECASE)
+    ]
+    if headless is None and not tells:
+        return (CheckStatus.INFO, None, f"unparsed: {txt[:120]!r}")
+    bad = bool(tells) or (headless is not None and headless > 30)
+    detail = f"headless={headless}% stealth={stealth}% tells={tells or 'none'}"
+    return (
+        CheckStatus.FAIL if bad else CheckStatus.PASS,
+        float(headless) if headless is not None else None,
+        detail,
+    )
+
+
+# incolumitas emits JSON test blocks ("WEBDRIVER":"FAIL") + an IP classification.
+_FAIL_KEY_RE = re.compile(r'"([A-Za-z_]+)"\s*:\s*"FAIL"')
+_DC_RE = re.compile(r'"is_datacenter"\s*:\s*(true|false)', re.IGNORECASE)
+_BEHAV_RE = re.compile(r"Your Behavioral Score:\s*([0-9.]+)", re.IGNORECASE)
+
+
+def _parse_incolumitas(page: Any, _cell: EvalCell):
+    txt = _full_text(page)
+    fails = sorted(set(_FAIL_KEY_RE.findall(txt)))
+    dc = _DC_RE.search(txt)
+    datacenter = dc.group(1) if dc else "?"
+    behav = _BEHAV_RE.search(txt)
+    bscore = behav.group(1) if behav else "n/a (no synthetic input)"
+    if not fails and dc is None:
+        return (CheckStatus.INFO, None, f"unparsed: {txt[:120]!r}")
+    detail = (
+        f"fpscanner FAIL={fails or 'none'} datacenter={datacenter} "
+        f"behavioral={bscore}"
+    )
+    return (
+        CheckStatus.PASS if not fails else CheckStatus.FAIL,
+        float(len(fails)),
+        detail,
+    )
+
+
+# fingerprint-scan.com: "Bot Risk Score: 35/100"; the site flags >50 as bot.
+_RISK_RE = re.compile(r"Bot Risk Score:\s*(\d+)\s*/\s*100", re.IGNORECASE)
+
+
+def _parse_fingerprint_scan(page: Any, _cell: EvalCell):
+    txt = _full_text(page)
+    m = _RISK_RE.search(txt)
+    if not m:
+        return (CheckStatus.INFO, None, f"no risk score: {txt[:120]!r}")
+    score = int(m.group(1))
+    status = CheckStatus.PASS if score < 50 else CheckStatus.FAIL
+    return (status, float(score), f"bot risk {score}/100 (bar: <50)")
+
+
+def _parse_fingerprintjs(page: Any, _cell: EvalCell):
+    """FingerprintJS Pro Smart Signals: a block message means we were detected."""
+    low = _full_text(page).lower()
+    if "access denied" in low or "tampering detected" in low:
+        return (CheckStatus.FAIL, 1.0, "blocked: anti-detect tampering / access denied")
+    if "search for today's flights" in low or "flight" in low:
+        return (CheckStatus.PASS, 0.0, "not blocked (flight results served)")
+    return (CheckStatus.INFO, None, f"unparsed: {low[:120]!r}")
+
+
+def _parse_browserscan(page: Any, _cell: EvalCell):
+    """BrowserScan grades each probe Normal/Abnormal; any Abnormal = detected."""
+    txt = _full_text(page)
+    abnormal = len(re.findall(r"\bAbnormal\b", txt))
+    if abnormal == 0 and "Test Results:" in txt and "Normal" in txt:
+        return (CheckStatus.PASS, 0.0, "Test Results: Normal (0 Abnormal)")
+    if abnormal > 0:
+        return (CheckStatus.FAIL, float(abnormal), f"{abnormal} Abnormal signal(s)")
+    return (CheckStatus.INFO, None, f"unparsed: {txt[:120]!r}")
+
+
+# The scrapingcourse CF canary prints an explicit success line once solved; a
+# failure leaves the interstitial ("Just a moment...") in the DOM. This is the
+# ONLY site here that exercises solve_cloudflare (the others present no challenge).
+_CF_PASS = "you bypassed the cloudflare challenge"
+_CF_BLOCK = ("just a moment", "attention required", "verify you are human")
+
+
+def _parse_cloudflare(page: Any, _cell: EvalCell):
+    low = _full_text(page).lower()
+    if _CF_PASS in low:
+        return (CheckStatus.PASS, 0.0, "bypassed Cloudflare challenge")
+    if any(m in low for m in _CF_BLOCK):
+        return (CheckStatus.FAIL, 1.0, "stuck on Cloudflare interstitial")
+    return (CheckStatus.INFO, None, f"unparsed: {low[:120]!r}")
+
+
+# iphey renders a masthead verdict "Your Digital Identity Looks <Trustworthy|
+# Unreliable|Suspicious>" after an async fingerprint+IP correlation (slow → big
+# settle). Only "Trustworthy" is a clean pass.
+_IPHEY_RE = re.compile(r"Your Digital Identity Looks\s+([A-Za-z]+)", re.IGNORECASE)
+
+
+def _parse_iphey(page: Any, _cell: EvalCell):
+    txt = _full_text(page)
+    m = _IPHEY_RE.search(txt)
+    if not m:
+        return (CheckStatus.INFO, None, f"verdict not loaded: {txt[:120]!r}")
+    verdict = m.group(1)
+    ok = verdict.lower() == "trustworthy"
+    return (
+        CheckStatus.PASS if ok else CheckStatus.FAIL,
+        0.0 if ok else 1.0,
+        f"iphey verdict: {verdict}",
+    )
+
+
+_BROWSER_SITES: list[StealthSite] = [
+    StealthSite(
+        name="sannysoft",
+        url="https://bot.sannysoft.com/",
+        bar="0 failed cells",
+        parse=_parse_sannysoft,
+        settle_ms=3000,
+    ),
+    StealthSite(
+        name="deviceandbrowserinfo",
+        url="https://deviceandbrowserinfo.com/are_you_a_bot",
+        bar="isBot=false",
+        parse=_parse_areyouabot,
+        settle_ms=3000,
+    ),
+    StealthSite(
+        name="recaptcha_v3_score",
+        url="https://recaptcha-demo.appspot.com/recaptcha-v3-request-scores.php",
+        bar="score >= 0.7",
+        parse=_parse_recaptcha_v3,
+        # v3 runs grecaptcha.execute then round-trips a server verify; too-short a
+        # wait reads the page before the {"score":..} JSON lands (false 0.0).
+        settle_ms=12000,
+    ),
+    StealthSite(
+        name="creepjs",
+        url="https://abrahamjuliot.github.io/creepjs/",
+        bar="headless <=30%, no spoof tells",
+        parse=_parse_creepjs,
+        settle_ms=30000,
+    ),
+    StealthSite(
+        name="browserscan",
+        url="https://www.browserscan.net/bot-detection",
+        bar="0 Abnormal",
+        parse=_parse_browserscan,
+        settle_ms=8000,
+    ),
+    StealthSite(
+        name="incolumitas",
+        url="https://bot.incolumitas.com/",
+        bar="0 fpscanner FAIL",
+        parse=_parse_incolumitas,
+        settle_ms=12000,
+    ),
+    StealthSite(
+        name="fingerprint_scan",
+        url="https://fingerprint-scan.com/",
+        bar="bot risk < 50/100",
+        parse=_parse_fingerprint_scan,
+        settle_ms=20000,
+    ),
+    StealthSite(
+        name="fingerprintjs_demo",
+        url="https://demo.fingerprint.com/web-scraping",
+        bar="not blocked",
+        parse=_parse_fingerprintjs,
+        settle_ms=6000,
+    ),
+    StealthSite(
+        name="cloudflare_challenge",
+        url="https://www.scrapingcourse.com/cloudflare-challenge",
+        bar="bypass CF challenge (exercises solve_cloudflare)",
+        parse=_parse_cloudflare,
+        settle_ms=15000,
+    ),
+    StealthSite(
+        name="iphey",
+        url="https://iphey.com/",
+        bar="verdict Trustworthy",
+        parse=_parse_iphey,
+        # iphey's verdict loads via a slow async correlation; <~20s reads the
+        # "Temporary value" placeholder (false INFO).
+        settle_ms=25000,
+    ),
+]
+
+# S2 — per-property fingerprint pages: too visual to auto-grade; emit as manual
+# links so the operator can confirm the 03e levers (canvas/webgl/fonts/webrtc).
+_MANUAL_LINKS: list[tuple[str, str]] = [
+    ("browserleaks_canvas", "https://browserleaks.com/canvas"),
+    ("browserleaks_webgl", "https://browserleaks.com/webgl"),
+    ("browserleaks_fonts", "https://browserleaks.com/fonts"),
+    ("browserleaks_webrtc", "https://browserleaks.com/webrtc"),
+]
+
+
+def _run_browser_site(
+    site: StealthSite, *, proxy: str | None, headed: bool, screenshots: bool
+) -> CheckResult:
+    shot = str(SCREENSHOTS_DIR / f"S_{site.name}.png") if screenshots else None
+    action, cell = make_page_action(
+        evaluate_js=site.evaluate_js,
+        screenshot_path=shot,
+        pre_wait_ms=site.settle_ms,
+    )
+    kwargs: dict[str, Any] = {
+        "headless": not headed,
+        "network_idle": True,
+        "block_ads": True,
+        "solve_cloudflare": True,
+        "proxy": proxy,
+        "timeout": 120000,
+        "page_action": action,
+    }
+    # Single source of truth — the exact levers the production crawler ships.
+    kwargs.update(build_stealthy_kwargs(get_stealth_config()))
+
+    try:
+        page = StealthyFetcher.fetch(site.url, **kwargs)
+    except Exception as exc:  # noqa: BLE001 - never crash the run
+        return CheckResult(
+            suite="S",
+            name=site.name,
+            tier="scrapling-stealthy",
+            status=CheckStatus.ERROR,
+            bar=site.bar,
+            detail=f"fetch failed: {type(exc).__name__}: {exc}",
+            screenshot=shot,
+        )
+
+    try:
+        status, numeric, detail = site.parse(page, cell)
+    except Exception as exc:  # noqa: BLE001
+        status, numeric, detail = (
+            CheckStatus.ERROR,
+            None,
+            f"parse failed: {type(exc).__name__}: {exc}",
+        )
+    return CheckResult(
+        suite="S",
+        name=site.name,
+        tier="scrapling-stealthy",
+        status=status,
+        bar=site.bar,
+        detail=detail,
+        numeric=numeric,
+        screenshot=shot,
+    )
+
+
+async def _run_tls(proxy: str | None) -> CheckResult:
+    """S3 — JA3/JA4/PeetPrint of the impersonated static (curl_cffi) tier."""
+    try:
+        page = await AsyncFetcher.get(
+            "https://tls.peet.ws/api/all",
+            stealthy_headers=True,
+            impersonate="chrome",
+            proxy=proxy,
+            timeout=30,
+        )
+        data = page.json()
+        tls = data.get("tls", {}) if isinstance(data, dict) else {}
+        ja3 = tls.get("ja3_hash") or tls.get("ja3")
+        ja4 = tls.get("ja4")
+        peet = tls.get("peetprint_hash")
+        return CheckResult(
+            suite="S",
+            name="tls_fingerprint",
+            tier="scrapling-static",
+            status=CheckStatus.INFO,
+            bar="informational (diff vs real Chrome)",
+            detail=f"ja3={ja3} ja4={ja4} peet={peet}",
+        )
+    except Exception as exc:  # noqa: BLE001
+        return CheckResult(
+            suite="S",
+            name="tls_fingerprint",
+            tier="scrapling-static",
+            status=CheckStatus.ERROR,
+            bar="informational",
+            detail=f"{type(exc).__name__}: {exc}",
+        )
+
+
+async def _run_proxy_leak(proxy: str | None) -> CheckResult:
+    """S4 — record the exit IP seen by an echo endpoint (proves egress path)."""
+    try:
+        page = await AsyncFetcher.get(
+            "https://api.ipify.org?format=json",
+            impersonate="chrome",
+            proxy=proxy,
+            timeout=30,
+        )
+        data = page.json()
+        ip = data.get("ip") if isinstance(data, dict) else None
+        note = "via proxy" if proxy else "DIRECT (no proxy — datacenter IP)"
+        return CheckResult(
+            suite="S",
+            name="exit_ip",
+            tier="scrapling-static",
+            status=CheckStatus.INFO,
+            bar="not your real/datacenter IP",
+            detail=f"exit_ip={ip} ({note})",
+        )
+    except Exception as exc:  # noqa: BLE001
+        return CheckResult(
+            suite="S",
+            name="exit_ip",
+            tier="scrapling-static",
+            status=CheckStatus.ERROR,
+            bar="n/a",
+            detail=f"{type(exc).__name__}: {exc}",
+        )
+
+
+async def run_suite_s(
+    *, proxy: str | None, headed: bool, screenshots: bool
+) -> list[CheckResult]:
+    """Run the full stealth suite (browser tier sites + TLS + proxy + manual links)."""
+    results: list[CheckResult] = []
+
+    # Browser sites are sync (spin up a real browser) → offload per-site so one
+    # slow score doesn't stall the loop and the event loop stays clean.
+    for site in _BROWSER_SITES:
+        print(f"  [S] {site.name} ... (settle {site.settle_ms}ms)")
+        res = await asyncio.to_thread(
+            _run_browser_site,
+            site,
+            proxy=proxy,
+            headed=headed,
+            screenshots=screenshots,
+        )
+        results.append(res)
+
+    print("  [S] tls_fingerprint + exit_ip ...")
+    results.append(await _run_tls(proxy))
+    results.append(await _run_proxy_leak(proxy))
+
+    for name, url in _MANUAL_LINKS:
+        results.append(
+            CheckResult(
+                suite="S",
+                name=name,
+                tier="manual",
+                status=CheckStatus.INFO,
+                bar="open in a real headed run + compare",
+                detail=url,
+            )
+        )
+
+    return results

surfsense_backend/app/utils/crawl/__init__.py

@@ -0,0 +1,18 @@
+"""App-wide crawler block classification (Apache-2.0, generic).
+
+Phase 3e (Slice A). Mirrors ``app/utils/proxy`` and ``app/utils/captcha``: this
+package holds only the **generic, vendor-agnostic** glue — here, a pure block
+classifier (passive telemetry from public anti-bot markers). It is consumed by
+the separately licensed proprietary crawler to label ``CrawlOutcome.block_type``.
+
+The **bypass-specific tuning** (the stealth kwargs builder / geoip coherence, and
+the deferred WebGL spoof + humanize choreography) is NOT generic and lives under
+the proprietary boundary in ``app/proprietary/web_crawler/`` (``stealth.py``).
+"""
+
+from app.utils.crawl.classifier import BlockType, classify_block
+
+__all__ = [
+    "BlockType",
+    "classify_block",
+]

surfsense_backend/app/utils/crawl/classifier.py

@@ -0,0 +1,94 @@
+"""Block classifier — labels a fetched page so the ladder can learn (Phase 3e).
+
+Pure, additive, status + body-marker based. It attaches
+:class:`~app.proprietary.web_crawler.connector.CrawlOutcome.block_type` for
+telemetry / future escalation routing (per-domain memory, `03d` captcha routing)
+and **never** changes *when* a crawl is ``SUCCESS`` — `03c` bills on
+``status == SUCCESS``, so this stays read-only metadata.
+
+Markers mirror the proven set in the Camoufox-based FlareSolverr alternative
+``references/trawl-dev/packages/tiers/src/detect.ts`` (plus DataDome/Kasada for
+our enterprise targets). Regexes are compiled once at import (hot-path hygiene).
+"""
+
+import re
+from enum import Enum
+
+
+class BlockType(str, Enum):
+    """Coarse label for what a fetched page represents."""
+
+    OK = "ok"  # usable content, no challenge detected
+    CLOUDFLARE = "cloudflare"  # CF interstitial / Turnstile / DDoS-Guard
+    CAPTCHA_RECAPTCHA = "captcha_recaptcha"
+    CAPTCHA_HCAPTCHA = "captcha_hcaptcha"
+    DATADOME = "datadome"
+    KASADA = "kasada"
+    RATE_LIMITED = "rate_limited"
+    EMPTY = "empty"  # fetched but no HTML body
+    UNKNOWN = "unknown"  # blocking-ish status with no recognized marker
+
+
+# --- compiled marker patterns (hoisted; see Vercel rule 7.9) ---
+_CLOUDFLARE = re.compile(
+    r"just a moment"
+    r"|checking your browser"
+    r"|enable javascript and cookies to continue"
+    r"|verify you are human"
+    r"|cf-mitigated"
+    r"|id=\"(?:cf-)?challenge-running\""
+    r"|id=\"turnstile-wrapper\""
+    r"|ddos-guard",
+    re.IGNORECASE,
+)
+_TURNSTILE = re.compile(
+    r"class=\"cf-turnstile\""
+    r"|challenges\.cloudflare\.com/turnstile"
+    r"|cdn-cgi/challenge-platform[^\"']*turnstile",
+    re.IGNORECASE,
+)
+_HCAPTCHA = re.compile(r"class=\"h-captcha\"|hcaptcha\.com/1/api", re.IGNORECASE)
+_RECAPTCHA = re.compile(
+    r"class=\"g-recaptcha\"|google\.com/recaptcha|recaptcha\.net/recaptcha",
+    re.IGNORECASE,
+)
+_DATADOME = re.compile(r"datadome|geo\.captcha-delivery\.com", re.IGNORECASE)
+_KASADA = re.compile(r"kpsdk|x-kpsdk|kasada", re.IGNORECASE)
+
+# Statuses some CDNs use as a bot-gate before the real response (detect.ts:isBlocked).
+_BOT_GATE_STATUSES = frozenset({202, 403})
+
+
+def classify_block(status: int | None, html: str | None) -> BlockType:
+    """Label a fetched page from its HTTP status and HTML body.
+
+    Precedence: explicit rate-limit status, then specific anti-bot/captcha body
+    markers, then a generic bot-gate status with no marker, else ``OK`` (or
+    ``EMPTY`` when there is no body). Marker checks run before the generic
+    bot-gate so a ``403`` Cloudflare challenge is labeled ``CLOUDFLARE``, not
+    ``UNKNOWN``.
+    """
+    if status == 429:
+        return BlockType.RATE_LIMITED
+
+    if not html or not html.strip():
+        # No body: distinguish a blocking-ish status from a genuinely empty 200.
+        if status in _BOT_GATE_STATUSES:
+            return BlockType.UNKNOWN
+        return BlockType.EMPTY
+
+    if _CLOUDFLARE.search(html) or _TURNSTILE.search(html):
+        return BlockType.CLOUDFLARE
+    if _DATADOME.search(html):
+        return BlockType.DATADOME
+    if _KASADA.search(html):
+        return BlockType.KASADA
+    if _HCAPTCHA.search(html):
+        return BlockType.CAPTCHA_HCAPTCHA
+    if _RECAPTCHA.search(html):
+        return BlockType.CAPTCHA_RECAPTCHA
+
+    if status in _BOT_GATE_STATUSES:
+        return BlockType.UNKNOWN
+
+    return BlockType.OK

surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py

@@ -12,6 +12,7 @@ from app.proprietary.web_crawler import (
     WebCrawlerConnector,
 )
 from app.proprietary.web_crawler import connector as connector_module
+from app.utils.crawl import BlockType
 
 pytestmark = pytest.mark.unit
 
@@ -40,10 +41,10 @@ async def test_static_tier_success_short_circuits(
     crawler = WebCrawlerConnector()
     later_calls: list[str] = []
 
-    async def _static(_url: str) -> dict:
+    async def _static(_url: str, *_args) -> dict:
         return _result("scrapling-static")
 
-    async def _record_dynamic(_url: str) -> None:
+    async def _record_dynamic(_url: str, *_args) -> None:
         later_calls.append("dynamic")
         return None
 
@@ -70,10 +71,10 @@ async def test_escalates_to_dynamic_on_static_miss(
     """Static empty extraction escalates to the dynamic tier."""
     crawler = WebCrawlerConnector()
 
-    async def _empty(_url: str) -> None:
+    async def _empty(_url: str, *_args) -> None:
         return None
 
-    async def _dynamic(_url: str) -> dict:
+    async def _dynamic(_url: str, *_args) -> dict:
         return _result("scrapling-dynamic")
 
     monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty)
@@ -126,7 +127,7 @@ async def test_proxy_error_rotates_once_when_pool_backed(
     crawler = WebCrawlerConnector()
     calls = {"n": 0}
 
-    async def _flaky(_url: str) -> dict:
+    async def _flaky(_url: str, *_args) -> dict:
         calls["n"] += 1
         if calls["n"] == 1:
             raise RuntimeError("connection refused by upstream proxy")
@@ -149,7 +150,7 @@ async def test_proxy_error_no_retry_when_single_endpoint(
     crawler = WebCrawlerConnector()
     static_calls = {"n": 0}
 
-    async def _proxy_err(_url: str) -> None:
+    async def _proxy_err(_url: str, *_args) -> None:
         static_calls["n"] += 1
         raise RuntimeError("connection refused by upstream proxy")
 
@@ -173,7 +174,7 @@ async def test_captcha_defaults_zero_on_non_stealthy_success(
     """03d: a static success never touches captcha → fields stay 0/False."""
     crawler = WebCrawlerConnector()
 
-    async def _static(_url: str) -> dict:
+    async def _static(_url: str, *_args) -> dict:
         return _result("scrapling-static")
 
     monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _static)
@@ -194,7 +195,7 @@ async def test_captcha_state_surfaced_onto_outcome(
     async def _empty(_url: str, *_args) -> None:
         return None
 
-    async def _stealthy(_url: str, captcha_state: dict) -> dict:
+    async def _stealthy(_url: str, captcha_state: dict, *_args) -> dict:
         # Simulate the page_action having solved a captcha mid-fetch.
         captcha_state["attempts"] = 2
         captcha_state["solved"] = True
@@ -221,7 +222,9 @@ async def test_captcha_attempts_surface_even_when_crawl_fails(
     async def _empty(_url: str, *_args) -> None:
         return None
 
-    async def _stealthy_attempt_then_empty(_url: str, captcha_state: dict) -> None:
+    async def _stealthy_attempt_then_empty(
+        _url: str, captcha_state: dict, *_args
+    ) -> None:
         captcha_state["attempts"] = 1  # solve attempted but crawl still empty
         return None
 
@@ -245,7 +248,7 @@ async def test_non_proxy_error_never_retries(
     crawler = WebCrawlerConnector()
     calls = {"n": 0}
 
-    async def _boom(_url: str) -> None:
+    async def _boom(_url: str, *_args) -> None:
         calls["n"] += 1
         raise RuntimeError("totally unrelated failure")
 
@@ -261,3 +264,104 @@ async def test_non_proxy_error_never_retries(
 
     assert calls["n"] == 1  # not retried (not a proxy error)
     assert outcome.status is CrawlOutcomeStatus.EMPTY
+
+
+async def test_invalid_url_block_type_defaults_unknown() -> None:
+    """03e: a pre-fetch FAILED carries the default UNKNOWN block_type."""
+    outcome = await WebCrawlerConnector().crawl_url("not a url")
+
+    assert outcome.status is CrawlOutcomeStatus.FAILED
+    assert outcome.block_type is BlockType.UNKNOWN
+
+
+async def test_block_type_surfaced_onto_outcome(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """03e: a tier's block classification (in block_state) is stamped onto the
+    outcome — additive only, never gating SUCCESS."""
+    crawler = WebCrawlerConnector()
+
+    async def _empty(_url: str, *_args) -> None:
+        return None
+
+    async def _stealthy_blocked(
+        _url: str, _captcha_state: dict, block_state: dict
+    ) -> None:
+        # Simulate _build_result having classified a Cloudflare interstitial.
+        block_state["block_type"] = BlockType.CLOUDFLARE
+        return None
+
+    monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty)
+    monkeypatch.setattr(crawler, "_crawl_with_dynamic", _empty)
+    monkeypatch.setattr(crawler, "_crawl_with_stealthy", _stealthy_blocked)
+
+    outcome = await crawler.crawl_url("https://example.com")
+
+    assert outcome.status is CrawlOutcomeStatus.EMPTY
+    assert outcome.block_type is BlockType.CLOUDFLARE
+
+
+def test_build_result_classifies_into_block_state() -> None:
+    """03e: _build_result labels the fetched page into the passed block_state."""
+    crawler = WebCrawlerConnector()
+    block_state: dict = {"block_type": BlockType.UNKNOWN}
+
+    cf_html = "<html><head><title>Just a moment...</title></head></html>"
+    result = crawler._build_result(
+        cf_html,
+        "https://example.com",
+        "scrapling-stealthy",
+        allow_raw_fallback=False,
+        status=403,
+        block_state=block_state,
+    )
+
+    # Cloudflare interstitial: no real content extracted (None) but classified.
+    assert result is None
+    assert block_state["block_type"] is BlockType.CLOUDFLARE
+
+
+async def test_static_4xx_is_classified(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """03e: a static-tier 4xx bot-gate is classified before the early return
+    (otherwise the cheapest/first tier's block signal would be lost)."""
+    crawler = WebCrawlerConnector()
+
+    class _Page:
+        status = 403
+        html_content = "<title>Just a moment...</title>"
+
+    class _AsyncFetcher:
+        @staticmethod
+        async def get(*_a, **_k):
+            return _Page()
+
+    monkeypatch.setattr(connector_module, "AsyncFetcher", _AsyncFetcher)
+    monkeypatch.setattr(connector_module, "get_proxy_url", lambda: None)
+
+    block_state: dict = {"block_type": BlockType.UNKNOWN}
+    result = await crawler._crawl_with_async_fetcher(
+        "https://example.com", block_state
+    )
+
+    assert result is None  # 4xx => fall through to next tier
+    assert block_state["block_type"] is BlockType.CLOUDFLARE
+
+
+def test_build_result_ok_on_real_content() -> None:
+    """03e: a normal 200 page with content classifies OK."""
+    crawler = WebCrawlerConnector()
+    block_state: dict = {"block_type": BlockType.UNKNOWN}
+
+    html = "<html><body><article>" + ("Real content. " * 40) + "</article></body></html>"
+    crawler._build_result(
+        html,
+        "https://example.com",
+        "scrapling-static",
+        allow_raw_fallback=False,
+        status=200,
+        block_state=block_state,
+    )
+
+    assert block_state["block_type"] is BlockType.OK

surfsense_backend/tests/unit/proprietary/web_crawler/test_stealth.py

@@ -0,0 +1,113 @@
+"""Unit tests for the Phase 3e stealth kwargs builder (proprietary boundary)."""
+
+import pytest
+
+from app.config import config
+from app.proprietary.web_crawler.stealth import (
+    build_stealthy_kwargs,
+    get_stealth_config,
+    location_to_locale_timezone,
+)
+
+pytestmark = pytest.mark.unit
+
+
+class TestLocationToLocaleTimezone:
+    def test_alpha2_code(self):
+        assert location_to_locale_timezone("us") == ("en-US", "America/New_York")
+        assert location_to_locale_timezone("de") == ("de-DE", "Europe/Berlin")
+
+    def test_case_and_whitespace_insensitive(self):
+        assert location_to_locale_timezone("  US  ") == (
+            "en-US",
+            "America/New_York",
+        )
+
+    def test_full_country_name_alias(self):
+        assert location_to_locale_timezone("Germany") == (
+            "de-DE",
+            "Europe/Berlin",
+        )
+        assert location_to_locale_timezone("united kingdom") == (
+            "en-GB",
+            "Europe/London",
+        )
+
+    def test_leading_token_of_vendor_string(self):
+        # Vendor strings like "us:nyc" / "de-rotating" still resolve on the head.
+        assert location_to_locale_timezone("us:nyc") == (
+            "en-US",
+            "America/New_York",
+        )
+        assert location_to_locale_timezone("de-rotating") == (
+            "de-DE",
+            "Europe/Berlin",
+        )
+
+    def test_empty_and_unknown_return_none(self):
+        assert location_to_locale_timezone("") == (None, None)
+        assert location_to_locale_timezone(None) == (None, None)
+        assert location_to_locale_timezone("atlantis") == (None, None)
+
+
+class TestBuildStealthyKwargs:
+    def test_defaults_have_no_geoip_keys(self, monkeypatch):
+        # geoip off => locale/timezone_id absent (browser keeps system default).
+        monkeypatch.setattr(config, "CRAWL_GEOIP_MATCH_ENABLED", False)
+        monkeypatch.setattr(config, "CRAWL_BLOCK_WEBRTC", True)
+        monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", False)
+        monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", True)
+        monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", False)
+        monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "us")
+
+        kwargs = build_stealthy_kwargs(get_stealth_config())
+
+        assert kwargs == {
+            "block_webrtc": True,
+            "hide_canvas": False,
+            "google_search": True,
+            "dns_over_https": False,
+        }
+        assert "locale" not in kwargs
+        assert "timezone_id" not in kwargs
+
+    def test_flags_reflect_config(self, monkeypatch):
+        monkeypatch.setattr(config, "CRAWL_GEOIP_MATCH_ENABLED", False)
+        monkeypatch.setattr(config, "CRAWL_BLOCK_WEBRTC", False)
+        monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", True)
+        monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", False)
+        monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", True)
+        monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "")
+
+        kwargs = build_stealthy_kwargs(get_stealth_config())
+
+        assert kwargs["block_webrtc"] is False
+        assert kwargs["hide_canvas"] is True
+        assert kwargs["google_search"] is False
+        assert kwargs["dns_over_https"] is True
+
+    def test_geoip_on_adds_locale_timezone(self, monkeypatch):
+        monkeypatch.setattr(config, "CRAWL_GEOIP_MATCH_ENABLED", True)
+        monkeypatch.setattr(config, "CRAWL_BLOCK_WEBRTC", True)
+        monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", False)
+        monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", True)
+        monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", False)
+        monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "de")
+
+        kwargs = build_stealthy_kwargs(get_stealth_config())
+
+        assert kwargs["locale"] == "de-DE"
+        assert kwargs["timezone_id"] == "Europe/Berlin"
+
+    def test_geoip_on_but_unknown_location_skips(self, monkeypatch):
+        monkeypatch.setattr(config, "CRAWL_GEOIP_MATCH_ENABLED", True)
+        monkeypatch.setattr(config, "CRAWL_BLOCK_WEBRTC", True)
+        monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", False)
+        monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", True)
+        monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", False)
+        monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "atlantis")
+
+        kwargs = build_stealthy_kwargs(get_stealth_config())
+
+        assert "locale" not in kwargs
+        assert "timezone_id" not in kwargs

surfsense_backend/tests/unit/utils/test_crawl_classifier.py

@@ -0,0 +1,75 @@
+"""Unit tests for the Phase 3e block classifier (Apache-2 layer)."""
+
+import pytest
+
+from app.utils.crawl import BlockType, classify_block
+
+pytestmark = pytest.mark.unit
+
+
+class TestClassifyBlock:
+    def test_ok_on_plain_content(self):
+        assert classify_block(200, "<html><body>hello</body></html>") is BlockType.OK
+
+    def test_none_body_is_empty(self):
+        assert classify_block(200, None) is BlockType.EMPTY
+        assert classify_block(200, "   ") is BlockType.EMPTY
+
+    def test_rate_limited_takes_precedence(self):
+        # 429 wins even if a challenge marker is present in the body.
+        assert (
+            classify_block(429, '<div class="cf-turnstile"></div>')
+            is BlockType.RATE_LIMITED
+        )
+
+    @pytest.mark.parametrize(
+        "html",
+        [
+            "<title>Just a moment...</title>",
+            "<h1>Checking your browser before accessing</h1>",
+            "please enable javascript and cookies to continue",
+            "verify you are human",
+            '<div id="challenge-running"></div>',
+            '<div id="turnstile-wrapper"></div>',
+            '<div class="cf-turnstile"></div>',
+            "blocked by ddos-guard.net",
+        ],
+    )
+    def test_cloudflare_markers(self, html):
+        assert classify_block(403, html) is BlockType.CLOUDFLARE
+
+    def test_hcaptcha(self):
+        assert (
+            classify_block(200, '<div class="h-captcha"></div>')
+            is BlockType.CAPTCHA_HCAPTCHA
+        )
+
+    def test_recaptcha(self):
+        assert (
+            classify_block(200, '<div class="g-recaptcha"></div>')
+            is BlockType.CAPTCHA_RECAPTCHA
+        )
+
+    def test_datadome(self):
+        assert (
+            classify_block(403, "var dd={'host':'geo.captcha-delivery.com'}")
+            is BlockType.DATADOME
+        )
+
+    def test_kasada(self):
+        assert classify_block(200, "window.KPSDK.configure()") is BlockType.KASADA
+
+    def test_bot_gate_status_without_marker_is_unknown(self):
+        # 202/403 with no recognized challenge marker => generic blocked-ish.
+        assert classify_block(202, "<html><body>x</body></html>") is BlockType.UNKNOWN
+        assert classify_block(403, "<html><body>x</body></html>") is BlockType.UNKNOWN
+
+    def test_bot_gate_status_empty_body_is_unknown(self):
+        assert classify_block(403, None) is BlockType.UNKNOWN
+
+    def test_cloudflare_marker_beats_generic_403(self):
+        # A 403 Cloudflare page is CLOUDFLARE, not UNKNOWN.
+        assert (
+            classify_block(403, "<title>Just a moment...</title>")
+            is BlockType.CLOUDFLARE
+        )