apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-06 22:12:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat: completed init mvp of phase 3

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-06-30 03:02:40 -07:00
parent bdacea8b6e
commit 15fd0b08d6
20 changed files with 2107 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

32
surfsense_backend/app/config/__init__.py
View file

@ -1086,6 +1086,38 @@ class Config:
CAPTCHA_V3_MIN_SCORE = float(os.getenv("CAPTCHA_V3_MIN_SCORE", "0.7"))
CAPTCHA_V3_ACTION = os.getenv("CAPTCHA_V3_ACTION", "verify")
# =====================================================================
# Phase 3e — Stealth hardening (Slice A): runtime/config-level levers
# layered on Scrapling's patchright-Chromium StealthyFetcher tier. All are
# consumed by the centralized kwargs builder in
# app/proprietary/web_crawler/stealth.py (proprietary — bypass tuning), which
# is the single source of truth imported by the crawler AND the 03f harness
# (no test-vs-prod drift). Defaults preserve today's behavior /
# introduce no crawl-speed regression. See plans/backend/03e-stealth-hardening.md.
# =====================================================================
# Map the configured proxy region (RESIDENTIAL_PROXY_LOCATION) -> browser
# locale/timezone so the fingerprint coheres with the proxy exit geo. No
# exit-IP lookup (zero added latency); unknown/empty region => skip.
CRAWL_GEOIP_MATCH_ENABLED = (
os.getenv("CRAWL_GEOIP_MATCH_ENABLED", "FALSE").upper() == "TRUE"
)
# Force WebRTC to respect the proxy (prevents real-local-IP leak). Cheap +
# safe => default TRUE.
CRAWL_BLOCK_WEBRTC = os.getenv("CRAWL_BLOCK_WEBRTC", "TRUE").upper() == "TRUE"
# Random canvas noise. An UNSTABLE canvas hash is itself a fingerprint tell,
# so default FALSE (opt-in + 03f-validated). See 03e §2.
CRAWL_HIDE_CANVAS = os.getenv("CRAWL_HIDE_CANVAS", "FALSE").upper() == "TRUE"
# Set a Google referer so the first hit looks like organic arrival.
CRAWL_GOOGLE_SEARCH_REFERER = (
os.getenv("CRAWL_GOOGLE_SEARCH_REFERER", "TRUE").upper() == "TRUE"
)
# Route DNS via Cloudflare DoH (anti DNS-leak behind proxies). Adds a DNS
# round-trip => default FALSE to honor the "no speed regression" bar; flip on
# when leak-safety outweighs the marginal latency.
CRAWL_DNS_OVER_HTTPS = (
os.getenv("CRAWL_DNS_OVER_HTTPS", "FALSE").upper() == "TRUE"
)
# Litellm TTS Configuration
TTS_SERVICE = os.getenv("TTS_SERVICE")
TTS_SERVICE_API_BASE = os.getenv("TTS_SERVICE_API_BASE")

85
surfsense_backend/app/proprietary/web_crawler/connector.py
View file

@ -34,7 +34,12 @@ from scrapling.engines.toolbelt import is_proxy_error
from scrapling.fetchers import AsyncFetcher, DynamicFetcher, StealthyFetcher
from app.proprietary.web_crawler.captcha import build_captcha_page_action
from app.proprietary.web_crawler.stealth import (
build_stealthy_kwargs,
get_stealth_config,
)
from app.utils.captcha import captcha_enabled, get_captcha_config
from app.utils.crawl import BlockType, classify_block
from app.utils.proxy import get_proxy_url, is_pool_backed
logger = logging.getLogger(__name__)
@ -64,6 +69,11 @@ class CrawlOutcome:
them off the outcome regardless of crawl SUCCESS (the solver charges per
*attempt*). They are populated only by the StealthyFetcher tier when captcha
solving is enabled; every other path leaves the defaults (0 / False).
Phase 3e ``block_type`` is purely *additive* telemetry: the block classifier
labels the last fetched page (Cloudflare / captcha / DataDome / rate-limited
/ ...) for tuning + future escalation routing. It does NOT influence the
billable ``SUCCESS`` predicate above.
"""
status: CrawlOutcomeStatus
@ -72,6 +82,7 @@ class CrawlOutcome:
tier: str | None = None
captcha_attempts: int = 0
captcha_solved: bool = False
block_type: BlockType = BlockType.UNKNOWN
class WebCrawlerConnector:
@ -102,11 +113,17 @@ class WebCrawlerConnector:
# billing can read it regardless of crawl SUCCESS. Per-call (not on
# ``self``) => safe under concurrent ``crawl_url`` calls.
captcha_state: dict[str, Any] = {"attempts": 0, "solved": False}
# Per-call block-classifier telemetry (03e). ``_build_result`` (the one
# place with raw_html + status) classifies each fetched page into here;
# crawl_url stamps it onto every outcome. Additive only — never gates
# SUCCESS. Per-call (not on ``self``) => concurrency-safe.
block_state: dict[str, Any] = {"block_type": BlockType.UNKNOWN}
try:
if not validators.url(url):
return CrawlOutcome(
status=CrawlOutcomeStatus.FAILED,
error=f"Invalid URL: {url}",
block_type=block_state["block_type"],
)
errors: list[str] = []
@ -120,7 +137,8 @@ class WebCrawlerConnector:
try:
logger.info(f"[webcrawler] Using Scrapling AsyncFetcher for: {url}")
result = await self._run_tier_with_proxy_retry(
"scrapling-static", lambda: self._crawl_with_async_fetcher(url)
"scrapling-static",
lambda: self._crawl_with_async_fetcher(url, block_state),
)
if result:
self._log_tier_outcome(
@ -131,6 +149,7 @@ class WebCrawlerConnector:
status=CrawlOutcomeStatus.SUCCESS,
result=result,
tier="scrapling-static",
block_type=block_state["block_type"],
)
reached_without_content = True
errors.append("Scrapling static: empty extraction")
@ -146,7 +165,8 @@ class WebCrawlerConnector:
try:
logger.info(f"[webcrawler] Using Scrapling DynamicFetcher for: {url}")
result = await self._run_tier_with_proxy_retry(
"scrapling-dynamic", lambda: self._crawl_with_dynamic(url)
"scrapling-dynamic",
lambda: self._crawl_with_dynamic(url, block_state),
)
if result:
self._log_tier_outcome(
@ -157,6 +177,7 @@ class WebCrawlerConnector:
status=CrawlOutcomeStatus.SUCCESS,
result=result,
tier="scrapling-dynamic",
block_type=block_state["block_type"],
)
reached_without_content = True
errors.append("Scrapling dynamic: empty extraction")
@ -181,7 +202,9 @@ class WebCrawlerConnector:
logger.info(f"[webcrawler] Using Scrapling StealthyFetcher for: {url}")
result = await self._run_tier_with_proxy_retry(
"scrapling-stealthy",
lambda: self._crawl_with_stealthy(url, captcha_state),
lambda: self._crawl_with_stealthy(
url, captcha_state, block_state
),
)
if result:
self._log_tier_outcome(
@ -194,6 +217,7 @@ class WebCrawlerConnector:
tier="scrapling-stealthy",
captcha_attempts=captcha_state["attempts"],
captcha_solved=captcha_state["solved"],
block_type=block_state["block_type"],
)
reached_without_content = True
errors.append("Scrapling stealthy: empty extraction")
@ -219,12 +243,14 @@ class WebCrawlerConnector:
error=f"No content extracted for {url}. {'; '.join(errors)}",
captcha_attempts=captcha_state["attempts"],
captcha_solved=captcha_state["solved"],
block_type=block_state["block_type"],
)
return CrawlOutcome(
status=CrawlOutcomeStatus.FAILED,
error=f"All crawl methods failed for {url}. {'; '.join(errors)}",
captcha_attempts=captcha_state["attempts"],
captcha_solved=captcha_state["solved"],
block_type=block_state["block_type"],
)
except Exception as e:
@ -234,6 +260,7 @@ class WebCrawlerConnector:
error=f"Error crawling URL {url}: {e!s}",
captcha_attempts=captcha_state["attempts"],
captcha_solved=captcha_state["solved"],
block_type=block_state["block_type"],
)
async def _run_tier_with_proxy_retry(
@ -307,7 +334,9 @@ class WebCrawlerConnector:
total_ms,
)
async def _crawl_with_async_fetcher(self, url: str) -> dict[str, Any] | None:
async def _crawl_with_async_fetcher(
self, url: str, block_state: dict[str, Any] | None = None
) -> dict[str, Any] | None:
"""
Crawl URL using Scrapling's AsyncFetcher (static HTTP) + Trafilatura.
@ -331,6 +360,13 @@ class WebCrawlerConnector:
status = getattr(page, "status", None)
if status is not None and status >= 400:
# 03e: classify here too — this early return skips _build_result, and
# the static tier is the first/cheapest hit, so the 403/429 bot-gate
# (the most common block signal) would otherwise never be labeled.
if block_state is not None:
block_state["block_type"] = classify_block(
status, getattr(page, "html_content", None)
)
logger.info(
"%s tier=scrapling-static url=%s fetch_ms=%.1f status=%s outcome=http_error",
_PERF,
@ -347,18 +383,25 @@ class WebCrawlerConnector:
allow_raw_fallback=False,
fetch_ms=fetch_ms,
status=status,
block_state=block_state,
)
async def _crawl_with_dynamic(self, url: str) -> dict[str, Any] | None:
async def _crawl_with_dynamic(
self, url: str, block_state: dict[str, Any] | None = None
) -> dict[str, Any] | None:
"""
Crawl URL using Scrapling's DynamicFetcher (full browser) + Trafilatura.
Runs the sync fetch in a worker thread so it works on any event loop,
including Windows ``SelectorEventLoop`` which cannot spawn subprocesses.
"""
return await asyncio.to_thread(self._crawl_with_dynamic_sync, url)
return await asyncio.to_thread(
self._crawl_with_dynamic_sync, url, block_state
)
def _crawl_with_dynamic_sync(self, url: str) -> dict[str, Any] | None:
def _crawl_with_dynamic_sync(
self, url: str, block_state: dict[str, Any] | None = None
) -> dict[str, Any] | None:
"""Synchronous DynamicFetcher crawl executed in a worker thread."""
fetch_start = time.perf_counter()
page = DynamicFetcher.fetch(
@ -376,10 +419,14 @@ class WebCrawlerConnector:
allow_raw_fallback=False,
fetch_ms=fetch_ms,
status=getattr(page, "status", None),
block_state=block_state,
)
async def _crawl_with_stealthy(
self, url: str, captcha_state: dict[str, Any] | None = None
self,
url: str,
captcha_state: dict[str, Any] | None = None,
block_state: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""
Crawl URL using Scrapling's StealthyFetcher (patchright-Chromium) + Trafilatura.
@ -390,13 +437,18 @@ class WebCrawlerConnector:
``captcha_state`` (03d) is mutated in place by the captcha page_action
(attempts/solved) so ``crawl_url`` can surface it on the outcome.
``block_state`` (03e) is populated by ``_build_result`` with the block
classification of the fetched page.
"""
return await asyncio.to_thread(
self._crawl_with_stealthy_sync, url, captcha_state
self._crawl_with_stealthy_sync, url, captcha_state, block_state
)
def _crawl_with_stealthy_sync(
self, url: str, captcha_state: dict[str, Any] | None = None
self,
url: str,
captcha_state: dict[str, Any] | None = None,
block_state: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Synchronous StealthyFetcher crawl executed in a worker thread."""
fetch_start = time.perf_counter()
@ -425,6 +477,11 @@ class WebCrawlerConnector:
"solve_cloudflare": True,
"proxy": proxy,
}
# 03e Slice A: merge config-driven stealth levers (block_webrtc,
# hide_canvas, google_search, dns_over_https, geoip locale/timezone).
# Keys never collide with the core kwargs above; defaults preserve
# today's behavior and add no crawl-speed regression.
fetch_kwargs.update(build_stealthy_kwargs(get_stealth_config()))
if page_action is not None:
fetch_kwargs["page_action"] = page_action
page = StealthyFetcher.fetch(url, **fetch_kwargs)
@ -436,6 +493,7 @@ class WebCrawlerConnector:
allow_raw_fallback=True,
fetch_ms=fetch_ms,
status=getattr(page, "status", None),
block_state=block_state,
)
def _build_result(
@ -447,6 +505,7 @@ class WebCrawlerConnector:
allow_raw_fallback: bool,
fetch_ms: float | None = None,
status: int | None = None,
block_state: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""
Extract markdown + metadata from raw HTML using Trafilatura.
@ -465,6 +524,12 @@ class WebCrawlerConnector:
Returns:
Result dict (content/metadata/crawler_type) or ``None``.
"""
# 03e: classify the fetched page (additive telemetry/routing only — never
# gates SUCCESS). Done before the early returns so EMPTY/no-extraction
# pages still get labeled. Last tier to fetch wins in block_state.
if block_state is not None:
block_state["block_type"] = classify_block(status, raw_html)
html_len = len(raw_html) if raw_html else 0
if not raw_html or len(raw_html.strip()) == 0:

216
surfsense_backend/app/proprietary/web_crawler/stealth.py Normal file
View file

@ -0,0 +1,216 @@
# SurfSense proprietary crawler engine.
#
# This module is part of the ``app.proprietary`` package and is licensed
# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
# Do not relicense or redistribute this file under Apache-2.0.
"""Stealth-hardening config + the centralized per-tier kwargs builder (Phase 3e).
This is **bypass-specific tuning** (hence proprietary): it decides the
anti-detection lever set the StealthyFetcher tier runs with (WebRTC/canvas/DNS
handling, organic-referer, and proxy-geo-coherent ``locale``/``timezone_id``).
It is the **single source of truth** that turns config -> ``StealthyFetcher``
keyword arguments, imported by both the crawler (``connector.py``) and the 03f
manual harness, so the scorecard grades the exact browser we ship (no
test-vs-prod drift).
The generic, vendor-agnostic block *classifier* (passive telemetry, public
markers) stays Apache-2.0 in ``app/utils/crawl/``. The further bypass logic
(WebGL spoof JS, humanize choreography) is deferred to later 03e slices and will
also live here.
Defaults preserve today's behavior and add **no crawl-speed regression**:
``dns_over_https`` (the only lever with a latency cost) is off unless explicitly
enabled, and geoip resolution is a pure in-process dict lookup (no exit-IP call).
"""
from dataclasses import dataclass
from typing import Any
from app.config import config
# Coarse region -> (locale, IANA timezone) map. Country granularity only: per the
# 03e "Geoip accuracy" risk, wrong-but-coherent beats a default mismatch and
# per-city precision isn't worth it. Keyed by ISO-3166 alpha-2; common full names
# are aliased below. Unknown/empty => (None, None) => leave Scrapling's default.
_REGION_TO_LOCALE_TZ: dict[str, tuple[str, str]] = {
"us": ("en-US", "America/New_York"),
"ca": ("en-CA", "America/Toronto"),
"gb": ("en-GB", "Europe/London"),
"ie": ("en-IE", "Europe/Dublin"),
"au": ("en-AU", "Australia/Sydney"),
"nz": ("en-NZ", "Pacific/Auckland"),
"de": ("de-DE", "Europe/Berlin"),
"fr": ("fr-FR", "Europe/Paris"),
"es": ("es-ES", "Europe/Madrid"),
"it": ("it-IT", "Europe/Rome"),
"nl": ("nl-NL", "Europe/Amsterdam"),
"be": ("nl-BE", "Europe/Brussels"),
"ch": ("de-CH", "Europe/Zurich"),
"at": ("de-AT", "Europe/Vienna"),
"se": ("sv-SE", "Europe/Stockholm"),
"no": ("nb-NO", "Europe/Oslo"),
"dk": ("da-DK", "Europe/Copenhagen"),
"fi": ("fi-FI", "Europe/Helsinki"),
"pl": ("pl-PL", "Europe/Warsaw"),
"pt": ("pt-PT", "Europe/Lisbon"),
"ru": ("ru-RU", "Europe/Moscow"),
"ua": ("uk-UA", "Europe/Kyiv"),
"tr": ("tr-TR", "Europe/Istanbul"),
"in": ("en-IN", "Asia/Kolkata"),
"jp": ("ja-JP", "Asia/Tokyo"),
"kr": ("ko-KR", "Asia/Seoul"),
"cn": ("zh-CN", "Asia/Shanghai"),
"hk": ("zh-HK", "Asia/Hong_Kong"),
"tw": ("zh-TW", "Asia/Taipei"),
"sg": ("en-SG", "Asia/Singapore"),
"id": ("id-ID", "Asia/Jakarta"),
"ph": ("en-PH", "Asia/Manila"),
"th": ("th-TH", "Asia/Bangkok"),
"vn": ("vi-VN", "Asia/Ho_Chi_Minh"),
"ae": ("ar-AE", "Asia/Dubai"),
"il": ("he-IL", "Asia/Jerusalem"),
"za": ("en-ZA", "Africa/Johannesburg"),
"br": ("pt-BR", "America/Sao_Paulo"),
"mx": ("es-MX", "America/Mexico_City"),
"ar": ("es-AR", "America/Argentina/Buenos_Aires"),
"cl": ("es-CL", "America/Santiago"),
}
# Full-name / synonym aliases -> alpha-2 key above.
_REGION_ALIASES: dict[str, str] = {
"usa": "us",
"united states": "us",
"united states of america": "us",
"america": "us",
"uk": "gb",
"united kingdom": "gb",
"great britain": "gb",
"england": "gb",
"canada": "ca",
"australia": "au",
"germany": "de",
"deutschland": "de",
"france": "fr",
"spain": "es",
"italy": "it",
"netherlands": "nl",
"holland": "nl",
"sweden": "se",
"norway": "no",
"denmark": "dk",
"finland": "fi",
"poland": "pl",
"portugal": "pt",
"russia": "ru",
"ukraine": "ua",
"turkey": "tr",
"india": "in",
"japan": "jp",
"korea": "kr",
"south korea": "kr",
"china": "cn",
"hong kong": "hk",
"taiwan": "tw",
"singapore": "sg",
"indonesia": "id",
"philippines": "ph",
"thailand": "th",
"vietnam": "vn",
"israel": "il",
"south africa": "za",
"brazil": "br",
"brasil": "br",
"mexico": "mx",
"argentina": "ar",
"chile": "cl",
}
def location_to_locale_timezone(
location: str | None,
) -> tuple[str | None, str | None]:
"""Map a free-form proxy region string -> (locale, timezone_id).
``location`` is the vendor-specific ``RESIDENTIAL_PROXY_LOCATION`` value
(``03b``). Best-effort: matches an ISO-3166 alpha-2 code (e.g. ``"us"``) or a
common country name (e.g. ``"Germany"``); only the leading token is honored,
so vendor strings like ``"us:nyc"`` or ``"de region"`` still resolve. Returns
``(None, None)`` for empty/unknown input (caller then leaves the browser
default).
"""
if not location:
return (None, None)
normalized = location.strip().lower()
if not normalized:
return (None, None)
# Direct alpha-2 hit.
if normalized in _REGION_TO_LOCALE_TZ:
return _REGION_TO_LOCALE_TZ[normalized]
# Full-name / synonym hit.
if normalized in _REGION_ALIASES:
return _REGION_TO_LOCALE_TZ[_REGION_ALIASES[normalized]]
# Leading token (handles "us:nyc", "de-rotating", "united states (east)").
head = normalized.replace("-", " ").replace("_", " ").replace(":", " ").split()
if head:
token = head[0]
if token in _REGION_TO_LOCALE_TZ:
return _REGION_TO_LOCALE_TZ[token]
if token in _REGION_ALIASES:
return _REGION_TO_LOCALE_TZ[_REGION_ALIASES[token]]
return (None, None)
@dataclass(frozen=True)
class StealthConfig:
"""Immutable snapshot of the Phase 3e Slice-A stealth levers for one crawl."""
geoip_match_enabled: bool
proxy_location: str
block_webrtc: bool
hide_canvas: bool
google_search: bool
dns_over_https: bool
def get_stealth_config() -> StealthConfig:
"""Build a :class:`StealthConfig` from the current process config/env."""
return StealthConfig(
geoip_match_enabled=config.CRAWL_GEOIP_MATCH_ENABLED,
proxy_location=config.RESIDENTIAL_PROXY_LOCATION or "",
block_webrtc=config.CRAWL_BLOCK_WEBRTC,
hide_canvas=config.CRAWL_HIDE_CANVAS,
google_search=config.CRAWL_GOOGLE_SEARCH_REFERER,
dns_over_https=config.CRAWL_DNS_OVER_HTTPS,
)
def build_stealthy_kwargs(cfg: StealthConfig) -> dict[str, Any]:
"""Return the config-derived ``StealthyFetcher.fetch`` keyword arguments.
Only the Slice-A stealth levers are returned; the caller merges these into
the tier's own kwargs (``headless``/``network_idle``/``block_ads``/
``solve_cloudflare``/``proxy``/captcha ``page_action``), which never collide
with the keys here. ``locale``/``timezone_id`` are added only when geoip
matching is enabled *and* the proxy region resolves — otherwise the browser
keeps its system default.
"""
kwargs: dict[str, Any] = {
"block_webrtc": cfg.block_webrtc,
"hide_canvas": cfg.hide_canvas,
"google_search": cfg.google_search,
"dns_over_https": cfg.dns_over_https,
}
if cfg.geoip_match_enabled:
locale, timezone_id = location_to_locale_timezone(cfg.proxy_location)
if locale:
kwargs["locale"] = locale
if timezone_id:
kwargs["timezone_id"] = timezone_id
return kwargs

83
surfsense_backend/app/proprietary/web_crawler/testbench/README.md Normal file
View file

@ -0,0 +1,83 @@
# Crawler testbench — manual undetectability & extraction scorecard (Phase 3f)
A **manual, repeatable scorecard** for the Universal WebURL Crawler. It answers
one question with evidence: *how undetectable (and how correct) is the crawler
right now?* — so we know when the free-stack ceiling (`03e`) is reached and the
deferred paid-unblocker tier is worth flipping.
This is **dev/operator tooling**, not a product code path and **not** part of the
pytest suite (it hits the live internet and needs proxy creds). See
`plans/backend/03f-undetectability-testing.md` for the full design.
## What it measures (two axes, two suites)
- **Suite S — stealth / anti-bot:** drives the real `StealthyFetcher` tier
(built from the **same** `app/proprietary/web_crawler/stealth.py` builder the
crawler ships, so no test-vs-prod drift) against the standard bot-detection
sites — sannysoft, deviceandbrowserinfo, reCAPTCHA v3, CreepJS, BrowserScan,
incolumitas, fingerprint-scan, FingerprintJS Pro, a **Cloudflare-challenge
canary** (the only row that exercises `solve_cloudflare`), and iphey — plus the
HTTP-tier TLS fingerprint (`tls.peet.ws`), exit-IP echo, and manual
per-property links (browserleaks). Every detection site is **auto-graded** from
its real DOM verdict (no screenshot reading required).
- **Suite E — extraction correctness:** drives the real `crawl_url` ladder
against ToS-safe scraping sandboxes (`toscrape`, `scrapethissite`) and asserts
known content appears in the extracted markdown.
## Run it
From the backend directory (`surfsense_backend/`):
```bash
uv run python -m app.proprietary.web_crawler.testbench --suite all
# or
.\.venv\Scripts\python.exe -m app.proprietary.web_crawler.testbench --suite all
```
Flags:
| Flag | Meaning |
|---|---|
| `--suite S\|E\|all` | Which suite(s) to run (default `all`). |
| `--proxy URL` | Override the proxy for Suite S (default: the app proxy provider from `.env`). |
| `--headed` | Run the browser tier headful (`headless=False`). |
| `--no-screenshots` | Skip per-site screenshots. |
Captcha solving is **forced OFF** every run — Suite S measures the *unaided*
stealth score.
## Reading the output
Each row prints `PASS / FAIL / ERR / INFO` with the tier and (where one exists) a
numeric. The harness is **tolerant**: a parse miss is `ERROR`, never a crash.
Detection sites are auto-graded from their DOM verdict; `INFO` is reserved for
purely informational rows (TLS fingerprint, exit IP) and the manual
per-property links (browserleaks canvas/webgl/fonts/webrtc). Screenshots are
still captured for every browser row as a backstop.
Outputs land in `results/`, which is **entirely gitignored** (run-local):
- `scorecard-<ts>.json` / `.md` — timestamped scorecard + readable report.
- `latest.json` — convenience pointer.
- `screenshots/` — per-site full-page captures.
Every run prints **drift vs the last baseline** (the previous on-disk scorecard)
so you can see the trend (our stealth improving, or a WAF tightening).
## The aspirational bars (from CloakBrowser, recorded as targets)
`sannysoft` 0 failed cells · `deviceandbrowserinfo` isBot=false ·
reCAPTCHA v3 `>= 0.7` · CreepJS headless `<= 30%`, no spoof tells · FingerprintJS
demo not blocked · Cloudflare challenge bypassed · iphey `Trustworthy`. We
**record our actual numbers as the baseline** — these are targets, not build
gates (the run never blocks anything).
## Caveats
- **Proxy required for realism.** Without residential egress the hard rows fail
by design (datacenter IP); a red scorecard from no-proxy is expected.
- **Sites change.** Detection sites move their DOM; if an auto-parser starts
returning `INFO`/`ERROR`, read the screenshot and (optionally) tighten the
parser — each site is one entry in `suite_stealth.py`.
- **Not a guarantee.** Passing sannysoft/CreepJS ≠ beating DataDome/Kasada; the
value is *trend + ceiling visibility*, not a green checkmark.

25
surfsense_backend/app/proprietary/web_crawler/testbench/__init__.py Normal file
View file

@ -0,0 +1,25 @@
# SurfSense proprietary crawler engine.
#
# This module is part of the ``app.proprietary`` package and is licensed
# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
# Do not relicense or redistribute this file under Apache-2.0.
"""Phase 3f — manual undetectability & extraction test harness (dev tooling).
A repeatable, **manual** scorecard that grades the Universal WebURL Crawler on two
axes:
- **Suite S (stealth / anti-bot):** drives the real Scrapling tiers against the
industry-standard bot-detection + fingerprint sites and parses each verdict.
- **Suite E (extraction correctness):** drives the real ``crawl_url`` ladder
against ToS-safe scraping sandboxes and asserts known values.
It is **not** collected by pytest (it hits the live internet and needs proxy
creds). Run it from the backend directory:
uv run python -m app.proprietary.web_crawler.testbench --suite all
# or: .\\.venv\\Scripts\\python.exe -m app.proprietary.web_crawler.testbench --suite all
See ``README.md`` for the runbook and ``plans/backend/03f-undetectability-testing.md``
for the design. This package is dev/operator tooling, untouched by the product
rename, and only *measures* what 03a–03e built.
"""

141
surfsense_backend/app/proprietary/web_crawler/testbench/__main__.py Normal file
View file

@ -0,0 +1,141 @@
# SurfSense proprietary crawler engine.
#
# This module is part of the ``app.proprietary`` package and is licensed
# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
# Do not relicense or redistribute this file under Apache-2.0.
"""CLI for the 03f manual scorecard. Run from the backend directory:
uv run python -m app.proprietary.web_crawler.testbench --suite all
uv run python -m app.proprietary.web_crawler.testbench --suite S --headed
uv run python -m app.proprietary.web_crawler.testbench --suite E --no-screenshots
Mirrors CloakBrowser's ``bin/cloaktest`` ergonomics. Writes a timestamped
scorecard (JSON + markdown) under ``results/`` and diffs against the last
baseline. Captcha solving is forced OFF so the scorecard measures the *unaided*
stealth ceiling (03f §Harness).
"""
from __future__ import annotations
import argparse
import asyncio
import os
import sys
from pathlib import Path
from dotenv import load_dotenv
# --- bootstrap: load .env + put backend root on sys.path BEFORE importing app.* ---
# __file__ = app/proprietary/web_crawler/testbench/__main__.py -> backend root is 4 up.
_BACKEND_ROOT = Path(__file__).resolve().parents[4]
if str(_BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(_BACKEND_ROOT))
for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
if _candidate.exists():
load_dotenv(_candidate)
break
# Force captcha solving OFF: Suite S measures the unaided score and we never want
# the 03d injector firing paid solves against the reCAPTCHA-demo row. Must be set
# before app.config is imported (config snapshots env at import).
os.environ["CAPTCHA_SOLVING_ENABLED"] = "false"
def _parse_args(argv: list[str]) -> argparse.Namespace:
p = argparse.ArgumentParser(
prog="python -m app.proprietary.web_crawler.testbench",
description="Manual undetectability & extraction scorecard (Phase 3f).",
)
p.add_argument(
"--suite",
choices=["S", "E", "all"],
default="all",
help="S=stealth/anti-bot, E=extraction correctness, all=both.",
)
p.add_argument(
"--proxy",
default=None,
help="Override proxy URL for Suite S (default: the app proxy provider).",
)
p.add_argument(
"--headed",
action="store_true",
help="Run the browser tier headful (headless=False).",
)
p.add_argument(
"--no-screenshots",
action="store_true",
help="Skip per-site screenshots (faster, no results/screenshots writes).",
)
return p.parse_args(argv)
async def _amain(args: argparse.Namespace) -> int:
# Imports happen here (after bootstrap + env override) so app.config sees the
# forced CAPTCHA_SOLVING_ENABLED=false and the resolved .env.
from app.utils.proxy import get_proxy_url
from .core import (
RunMeta,
diff_against_baseline,
load_last_baseline,
render_console,
scrapling_version,
write_scorecard,
)
proxy = args.proxy or get_proxy_url()
screenshots = not args.no_screenshots
results = []
print(
f"== crawler scorecard == suite={args.suite} "
f"headed={args.headed} proxy={'set' if proxy else 'NONE'} "
f"screenshots={screenshots}"
)
if not proxy:
print(
" WARNING: no proxy configured — the hard anti-bot rows are expected "
"to fail from a datacenter IP (see README)."
)
baseline = load_last_baseline()
if args.suite in ("S", "all"):
from .suite_stealth import run_suite_s
results += await run_suite_s(
proxy=proxy, headed=args.headed, screenshots=screenshots
)
if args.suite in ("E", "all"):
from .suite_extraction import run_suite_e
results += await run_suite_e()
render_console(results)
meta = RunMeta.now(
suites=args.suite,
proxy=proxy,
headed=args.headed,
scrapling_version=scrapling_version(),
)
json_path, md_path = write_scorecard(results, meta)
print("\n--- drift vs last baseline ---")
for line in diff_against_baseline(results, baseline):
print(f" {line}")
print(f"\nscorecard JSON: {json_path}")
print(f"scorecard MD: {md_path}")
return 0
def main() -> None:
args = _parse_args(sys.argv[1:])
raise SystemExit(asyncio.run(_amain(args)))
if __name__ == "__main__":
main()

325
surfsense_backend/app/proprietary/web_crawler/testbench/core.py Normal file
View file

@ -0,0 +1,325 @@
# SurfSense proprietary crawler engine.
#
# This module is part of the ``app.proprietary`` package and is licensed
# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
# Do not relicense or redistribute this file under Apache-2.0.
"""Shared primitives for the 03f scorecard: result model, the closure-cell
``page_action`` verdict-extractor (the 03d-shared mechanism), and the scorecard
snapshot writer / baseline-differ / console renderer.
Stdlib-only on purpose (the plan's "no new prod dependency" bar). Everything here
is tolerant: a parse miss yields an ``ERROR`` row, never a crash — detection sites
change their DOM and the harness is manual.
"""
from __future__ import annotations
import json
from collections.abc import Callable
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from enum import Enum
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
# Results live next to the harness; screenshots are gitignored, scorecard JSON is
# committed so runs diff against the last baseline (see README + plan §Scorecard).
_PKG_DIR = Path(__file__).resolve().parent
RESULTS_DIR = _PKG_DIR / "results"
SCREENSHOTS_DIR = RESULTS_DIR / "screenshots"
class CheckStatus(str, Enum):
"""Outcome of a single scorecard row."""
PASS = "PASS" # met the aspirational bar
FAIL = "FAIL" # ran, did not meet the bar
ERROR = "ERROR" # could not run / parse (never fatal to the run)
INFO = "INFO" # recorded, no pass/fail bar (e.g. TLS JA3, manual links)
SKIP = "SKIP" # intentionally not run this invocation
@dataclass
class CheckResult:
"""One row of the scorecard."""
suite: str # "S" (stealth) | "E" (extraction)
name: str # site / check key
tier: str # scrapling-static | scrapling-stealthy | crawl_url | n/a
status: CheckStatus
bar: str # human-readable aspirational threshold
detail: str = "" # short human summary of what was observed
numeric: float | None = None # comparable metric where one exists
screenshot: str | None = None # path, when captured
def to_row(self) -> dict[str, Any]:
d = asdict(self)
d["status"] = self.status.value
return d
@dataclass
class RunMeta:
"""Provenance for a scorecard snapshot so baselines are comparable."""
timestamp: str
suites: str
proxy: str # masked
headed: bool
scrapling_version: str
captcha_disabled: bool = True
notes: str = ""
@staticmethod
def now(
*, suites: str, proxy: str | None, headed: bool, scrapling_version: str
) -> RunMeta:
return RunMeta(
timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"),
suites=suites,
proxy=mask_proxy(proxy),
headed=headed,
scrapling_version=scrapling_version,
)
# --- closure-cell page_action (the mechanism shared with 03d) ----------------
@dataclass
class EvalCell:
"""A mutable cell a ``page_action`` writes its ``page.evaluate`` result into.
Scrapling discards a ``page_action``'s return value, so the only way to get a
JS-object verdict (CreepJS ``window.Fingerprint``, a Castle.js score node) out
of the browser is to stash it in a closure variable. This is the exact same
plumbing 03d's captcha-token injector uses — factored once here.
"""
value: Any = None
error: str | None = None
captured: bool = False
def make_page_action(
*,
evaluate_js: str | None = None,
screenshot_path: str | None = None,
pre_wait_ms: int = 0,
) -> tuple[Callable[[Any], Any], EvalCell]:
"""Build a sync ``page_action`` + the :class:`EvalCell` it writes into.
The action (in priority order) optionally sleeps ``pre_wait_ms`` (lets async
scores settle), optionally evaluates ``evaluate_js`` into the cell, and
optionally writes a full-page screenshot. Each step is independently guarded
so one failure (e.g. a site without the JS object) never aborts the fetch.
Returns the page unchanged (Scrapling re-reads its DOM afterwards).
"""
cell = EvalCell()
def _action(page: Any) -> Any:
if pre_wait_ms > 0:
try:
page.wait_for_timeout(pre_wait_ms)
except Exception:
pass
if evaluate_js is not None:
try:
cell.value = page.evaluate(evaluate_js)
cell.captured = True
except Exception as exc: # noqa: BLE001 - tolerant by design
cell.error = f"{type(exc).__name__}: {exc}"
if screenshot_path is not None:
try:
# The screenshot dir must exist *now* (the page action runs mid-suite,
# before the end-of-run ``ensure_dirs``); Scrapling/patchright will not
# create it for us, so a missing dir silently drops every screenshot.
Path(screenshot_path).parent.mkdir(parents=True, exist_ok=True)
page.screenshot(path=screenshot_path, full_page=True)
except Exception as exc: # noqa: BLE001 - tolerant, but surface the cause
if cell.error is None:
cell.error = f"screenshot: {type(exc).__name__}: {exc}"
return page
return _action, cell
# --- helpers -----------------------------------------------------------------
def mask_proxy(url: str | None) -> str:
"""Mask credentials in a proxy URL for logs/snapshots."""
if not url:
return "<none>"
try:
p = urlsplit(url)
host = p.hostname or "?"
port = f":{p.port}" if p.port else ""
return f"{p.scheme}://***@{host}{port}"
except Exception:
return "<set>"
def ensure_dirs() -> None:
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
SCREENSHOTS_DIR.mkdir(parents=True, exist_ok=True)
def scrapling_version() -> str:
try:
import scrapling # noqa: PLC0415
return getattr(scrapling, "__version__", "unknown")
except Exception:
return "unavailable"
# --- scorecard I/O -----------------------------------------------------------
def write_scorecard(
results: list[CheckResult], meta: RunMeta
) -> tuple[Path, Path]:
"""Write the JSON snapshot (committed, baseline) + a readable markdown report.
The JSON filename is timestamped so prior runs are kept as the baseline trail;
``latest.json`` is overwritten as the convenience pointer used by the differ.
"""
ensure_dirs()
stamp = meta.timestamp.replace(":", "").replace("-", "")
payload = {
"meta": asdict(meta),
"summary": summarize(results),
"results": [r.to_row() for r in results],
}
json_path = RESULTS_DIR / f"scorecard-{stamp}.json"
json_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
(RESULTS_DIR / "latest.json").write_text(
json.dumps(payload, indent=2), encoding="utf-8"
)
md_path = RESULTS_DIR / f"scorecard-{stamp}.md"
md_path.write_text(_render_markdown(results, meta), encoding="utf-8")
return json_path, md_path
def load_last_baseline() -> dict[str, Any] | None:
"""Load the most recent prior scorecard JSON (excluding ``latest.json``)."""
if not RESULTS_DIR.exists():
return None
snaps = sorted(RESULTS_DIR.glob("scorecard-*.json"))
if not snaps:
return None
try:
return json.loads(snaps[-1].read_text(encoding="utf-8"))
except Exception:
return None
def summarize(results: list[CheckResult]) -> dict[str, dict[str, int]]:
"""Per-suite ``{passed, failed, error, info, total}`` counts."""
out: dict[str, dict[str, int]] = {}
for r in results:
s = out.setdefault(
r.suite,
{"passed": 0, "failed": 0, "error": 0, "info": 0, "skip": 0, "total": 0},
)
s["total"] += 1
key = {
CheckStatus.PASS: "passed",
CheckStatus.FAIL: "failed",
CheckStatus.ERROR: "error",
CheckStatus.INFO: "info",
CheckStatus.SKIP: "skip",
}[r.status]
s[key] += 1
return out
def diff_against_baseline(
results: list[CheckResult], baseline: dict[str, Any] | None
) -> list[str]:
"""Human-readable drift lines comparing this run to the last baseline."""
if not baseline:
return ["(no prior baseline — this run becomes the baseline)"]
prior = {(r["suite"], r["name"]): r for r in baseline.get("results", [])}
lines: list[str] = []
for r in results:
was = prior.get((r.suite, r.name))
if was is None:
lines.append(f"+ NEW [{r.suite}] {r.name}: {r.status.value}")
continue
if was["status"] != r.status.value:
lines.append(
f"~ {r.name}: status {was['status']} -> {r.status.value}"
)
old_n, new_n = was.get("numeric"), r.numeric
if old_n is not None and new_n is not None and abs(old_n - new_n) > 1e-9:
lines.append(f"~ {r.name}: numeric {old_n} -> {new_n}")
seen = {(r.suite, r.name) for r in results}
for key, was in prior.items():
if key not in seen:
lines.append(f"- GONE [{key[0]}] {key[1]} (was {was['status']})")
return lines or ["(no changes vs last baseline)"]
# --- rendering ---------------------------------------------------------------
_ICON = {
CheckStatus.PASS: "PASS",
CheckStatus.FAIL: "FAIL",
CheckStatus.ERROR: "ERR ",
CheckStatus.INFO: "INFO",
CheckStatus.SKIP: "SKIP",
}
def render_console(results: list[CheckResult]) -> None:
"""Print the grouped scorecard + per-suite totals to stdout."""
by_suite: dict[str, list[CheckResult]] = {}
for r in results:
by_suite.setdefault(r.suite, []).append(r)
for suite in sorted(by_suite):
label = "Suite S - stealth/anti-bot" if suite == "S" else (
"Suite E - extraction" if suite == "E" else f"Suite {suite}"
)
print(f"\n=== {label} ===")
for r in by_suite[suite]:
num = f" [{r.numeric:g}]" if r.numeric is not None else ""
print(f" {_ICON[r.status]} {r.name:<34} {r.tier:<18}{num}")
if r.detail:
print(f" {r.detail}")
print("\n--- summary ---")
for suite, s in summarize(results).items():
print(
f" Suite {suite}: {s['passed']}/{s['total']} passed "
f"(fail={s['failed']} err={s['error']} info={s['info']})"
)
def _render_markdown(results: list[CheckResult], meta: RunMeta) -> str:
lines = [
f"# Crawler scorecard — {meta.timestamp}",
"",
f"- proxy: `{meta.proxy}` headed: `{meta.headed}` "
f"captcha-disabled: `{meta.captcha_disabled}`",
f"- scrapling: `{meta.scrapling_version}` suites: `{meta.suites}`",
"",
"| Suite | Check | Tier | Status | Numeric | Detail |",
"|---|---|---|---|---|---|",
]
for r in results:
num = "" if r.numeric is None else f"{r.numeric:g}"
detail = r.detail.replace("|", "\\|")
lines.append(
f"| {r.suite} | {r.name} | {r.tier} | {r.status.value} | {num} | {detail} |"
)
lines.append("")
for suite, s in summarize(results).items():
lines.append(
f"- **Suite {suite}**: {s['passed']}/{s['total']} passed "
f"(fail={s['failed']}, err={s['error']}, info={s['info']})"
)
return "\n".join(lines) + "\n"

5
surfsense_backend/app/proprietary/web_crawler/testbench/results/.gitignore vendored Normal file
View file

@ -0,0 +1,5 @@
# Everything under results/ is run-local output (timestamped scorecards,
# latest.json pointer, screenshots, ad-hoc dumps). None of it is committed —
# the dir is kept tracked only via this file so the harness has a write target.
*
!.gitignore

123
surfsense_backend/app/proprietary/web_crawler/testbench/suite_extraction.py Normal file
View file

@ -0,0 +1,123 @@
# SurfSense proprietary crawler engine.
#
# This module is part of the ``app.proprietary`` package and is licensed
# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
# Do not relicense or redistribute this file under Apache-2.0.
"""Suite E — extraction correctness (separate axis from stealth).
Unlike Suite S, this drives the **real ``crawl_url`` ladder end-to-end** — the
auto-tier + Trafilatura markdown path is exactly the production behavior we want
to assert. Targets are purpose-built, ToS-safe scraping sandboxes with known
content, so extraction regressions are caught deterministically (a missing
expected string => FAIL). The ``/js`` cases exercise the DynamicFetcher (browser)
tier; the static catalogs exercise the HTTP tier.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from app.proprietary.web_crawler import CrawlOutcomeStatus, WebCrawlerConnector
from .core import CheckResult, CheckStatus
@dataclass
class ExtractionCase:
"""A sandbox URL + strings that must appear in the extracted markdown."""
name: str
url: str
must_contain: list[str] = field(default_factory=list)
_CASES: list[ExtractionCase] = [
ExtractionCase(
name="books_static",
url="https://books.toscrape.com/",
must_contain=["A Light in the Attic", "Tipping the Velvet"],
),
ExtractionCase(
name="quotes_static",
url="https://quotes.toscrape.com/",
must_contain=["Albert Einstein", "world as we have created it"],
),
ExtractionCase(
name="quotes_js_rendered",
url="https://quotes.toscrape.com/js/",
must_contain=["Albert Einstein", "world as we have created it"],
),
ExtractionCase(
name="scrapethissite_simple",
url="https://www.scrapethissite.com/pages/simple/",
must_contain=["Andorra", "Afghanistan"],
),
]
def _content_of(outcome) -> str:
result = getattr(outcome, "result", None) or {}
if isinstance(result, dict):
return str(result.get("content") or "")
return ""
async def _run_case(
connector: WebCrawlerConnector, case: ExtractionCase
) -> CheckResult:
bar = f"SUCCESS + contains {len(case.must_contain)} marker(s)"
try:
outcome = await connector.crawl_url(case.url)
except Exception as exc: # noqa: BLE001 - never crash the run
return CheckResult(
suite="E",
name=case.name,
tier="crawl_url",
status=CheckStatus.ERROR,
bar=bar,
detail=f"{type(exc).__name__}: {exc}",
)
tier = getattr(outcome, "tier", None) or "crawl_url"
if outcome.status is not CrawlOutcomeStatus.SUCCESS:
return CheckResult(
suite="E",
name=case.name,
tier=tier,
status=CheckStatus.FAIL,
bar=bar,
detail=f"status={outcome.status.value}: {outcome.error or ''}"[:160],
)
content = _content_of(outcome)
lowered = content.lower()
missing = [m for m in case.must_contain if m.lower() not in lowered]
if missing:
return CheckResult(
suite="E",
name=case.name,
tier=tier,
status=CheckStatus.FAIL,
bar=bar,
detail=f"missing {missing} (len={len(content)})",
numeric=float(len(content)),
)
return CheckResult(
suite="E",
name=case.name,
tier=tier,
status=CheckStatus.PASS,
bar=bar,
detail=f"all markers present (tier={tier}, len={len(content)})",
numeric=float(len(content)),
)
async def run_suite_e() -> list[CheckResult]:
"""Run the extraction-correctness cases against the live sandboxes."""
connector = WebCrawlerConnector()
results: list[CheckResult] = []
for case in _CASES:
print(f" [E] {case.name} -> {case.url}")
results.append(await _run_case(connector, case))
return results

509
surfsense_backend/app/proprietary/web_crawler/testbench/suite_stealth.py Normal file
View file

@ -0,0 +1,509 @@
# SurfSense proprietary crawler engine.
#
# This module is part of the ``app.proprietary`` package and is licensed
# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
# Do not relicense or redistribute this file under Apache-2.0.
"""Suite S — stealth / anti-bot scorecard.
Drives the **browser tier** (``StealthyFetcher``) against the standard
bot-detection sites and the **HTTP tier** (``AsyncFetcher``) against the TLS /
proxy-leak JSON endpoints. Critically, the browser tier is built from the **same
centralized stealth builder the crawler ships** (``build_stealthy_kwargs`` /
``get_stealth_config`` in ``app/proprietary/web_crawler/stealth.py``) so the scorecard grades
the exact browser we run in production — no test-vs-prod drift (03f §Harness).
Verdict extraction is best-effort and tolerant: machine-readable signals
(reCAPTCHA score text, are-you-a-bot text, sannysoft failed-cell count, TLS/proxy
JSON) get a real PASS/FAIL/numeric; DOM-heavy fingerprint pages are recorded as
INFO with a screenshot + captured text for the operator to read. Tightening any
parser later is a one-function change (the spec list is the extension point).
"""
from __future__ import annotations
import asyncio
import re
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from scrapling.fetchers import AsyncFetcher, StealthyFetcher
from app.proprietary.web_crawler.stealth import (
build_stealthy_kwargs,
get_stealth_config,
)
from .core import (
SCREENSHOTS_DIR,
CheckResult,
CheckStatus,
EvalCell,
make_page_action,
)
# Parser: (page, eval_cell) -> (status, numeric, detail).
Parser = Callable[[Any, EvalCell], "tuple[CheckStatus, float | None, str]"]
@dataclass
class StealthSite:
"""One browser-tier detection site + how to read its verdict."""
name: str
url: str
bar: str
parse: Parser
settle_ms: int = 0 # extra in-page wait for async-computed scores
evaluate_js: str | None = None # for window.* / JS-object verdicts
def _text(page: Any, limit: int = 4000) -> str:
"""Best-effort visible text of a fetched page (short, for fallback details)."""
return _full_text(page, limit)
def _full_text(page: Any, limit: int = 200_000) -> str:
"""Untruncated visible text — detector verdicts live deep in long pages."""
try:
txt = page.get_all_text()
if txt:
return str(txt)[:limit]
except Exception:
pass
try:
return str(page.html_content or "")[:limit]
except Exception:
return ""
# --- per-site parsers (all written against real DOM dumps, no screenshots) ---
def _parse_sannysoft(page: Any, _cell: EvalCell):
"""Count failed result cells (JS sets class='failed' on red rows)."""
html = ""
try:
html = page.html_content or ""
except Exception:
pass
fails = len(re.findall(r'class="[^"]*\bfailed\b[^"]*"', html))
if not html:
return (CheckStatus.ERROR, None, "no html returned")
status = CheckStatus.PASS if fails == 0 else CheckStatus.FAIL
return (status, float(fails), f"{fails} failed cell(s) (bar: 0)")
# deviceandbrowserinfo renders a JSON verdict block: {"isBot": false, ...}.
_ISBOT_RE = re.compile(r'"isbot"\s*:\s*(true|false)', re.IGNORECASE)
def _parse_areyouabot(page: Any, _cell: EvalCell):
txt = _full_text(page)
m = _ISBOT_RE.search(txt)
if m:
is_bot = m.group(1).lower() == "true"
return (
CheckStatus.FAIL if is_bot else CheckStatus.PASS,
1.0 if is_bot else 0.0,
f"isBot={is_bot}",
)
low = txt.lower()
if "you are human" in low or "not a bot" in low:
return (CheckStatus.PASS, 0.0, "verdict: human")
if "you are a bot" in low:
return (CheckStatus.FAIL, 1.0, "verdict: BOT")
return (CheckStatus.INFO, None, f"unparsed: {low[:120]!r}")
# The reCAPTCHA demo echoes the server verify response: {"success":true,"score":0.9}.
_SCORE_JSON_RE = re.compile(r'"score"\s*:\s*([0-9]*\.?[0-9]+)')
def _parse_recaptcha_v3(page: Any, _cell: EvalCell):
txt = _full_text(page)
m = _SCORE_JSON_RE.search(txt)
if not m: # fall back to looser prose match
m = re.search(r"score[^0-9]{0,12}([01](?:\.\d+)?)", txt, re.IGNORECASE)
if not m:
return (CheckStatus.INFO, None, f"no score parsed: {txt[:120]!r}")
score = float(m.group(1))
status = CheckStatus.PASS if score >= 0.7 else CheckStatus.FAIL
return (status, score, f"reCAPTCHA v3 score={score} (bar: >=0.7)")
# CreepJS prints category percentages ("33% headless") + boolean tells inline.
_CREEPJS_TELLS = (
"webDriverIsOn",
"hasHeadlessUA",
"hasHeadlessWorkerUA",
"hasBadWebGL",
"hasSwiftShader",
)
def _parse_creepjs(page: Any, _cell: EvalCell):
"""Grade CreepJS by its headless-similarity % + the boolean spoof tells."""
txt = _full_text(page)
def _pct(label: str) -> int | None:
m = re.search(r"(\d+)%\s*" + label, txt, re.IGNORECASE)
return int(m.group(1)) if m else None
headless = _pct(r"headless")
stealth = _pct(r"stealth")
tells = [
flag
for flag in _CREEPJS_TELLS
if re.search(re.escape(flag) + r"\s*:\s*true", txt, re.IGNORECASE)
]
if headless is None and not tells:
return (CheckStatus.INFO, None, f"unparsed: {txt[:120]!r}")
bad = bool(tells) or (headless is not None and headless > 30)
detail = f"headless={headless}% stealth={stealth}% tells={tells or 'none'}"
return (
CheckStatus.FAIL if bad else CheckStatus.PASS,
float(headless) if headless is not None else None,
detail,
)
# incolumitas emits JSON test blocks ("WEBDRIVER":"FAIL") + an IP classification.
_FAIL_KEY_RE = re.compile(r'"([A-Za-z_]+)"\s*:\s*"FAIL"')
_DC_RE = re.compile(r'"is_datacenter"\s*:\s*(true|false)', re.IGNORECASE)
_BEHAV_RE = re.compile(r"Your Behavioral Score:\s*([0-9.]+)", re.IGNORECASE)
def _parse_incolumitas(page: Any, _cell: EvalCell):
txt = _full_text(page)
fails = sorted(set(_FAIL_KEY_RE.findall(txt)))
dc = _DC_RE.search(txt)
datacenter = dc.group(1) if dc else "?"
behav = _BEHAV_RE.search(txt)
bscore = behav.group(1) if behav else "n/a (no synthetic input)"
if not fails and dc is None:
return (CheckStatus.INFO, None, f"unparsed: {txt[:120]!r}")
detail = (
f"fpscanner FAIL={fails or 'none'} datacenter={datacenter} "
f"behavioral={bscore}"
)
return (
CheckStatus.PASS if not fails else CheckStatus.FAIL,
float(len(fails)),
detail,
)
# fingerprint-scan.com: "Bot Risk Score: 35/100"; the site flags >50 as bot.
_RISK_RE = re.compile(r"Bot Risk Score:\s*(\d+)\s*/\s*100", re.IGNORECASE)
def _parse_fingerprint_scan(page: Any, _cell: EvalCell):
txt = _full_text(page)
m = _RISK_RE.search(txt)
if not m:
return (CheckStatus.INFO, None, f"no risk score: {txt[:120]!r}")
score = int(m.group(1))
status = CheckStatus.PASS if score < 50 else CheckStatus.FAIL
return (status, float(score), f"bot risk {score}/100 (bar: <50)")
def _parse_fingerprintjs(page: Any, _cell: EvalCell):
"""FingerprintJS Pro Smart Signals: a block message means we were detected."""
low = _full_text(page).lower()
if "access denied" in low or "tampering detected" in low:
return (CheckStatus.FAIL, 1.0, "blocked: anti-detect tampering / access denied")
if "search for today's flights" in low or "flight" in low:
return (CheckStatus.PASS, 0.0, "not blocked (flight results served)")
return (CheckStatus.INFO, None, f"unparsed: {low[:120]!r}")
def _parse_browserscan(page: Any, _cell: EvalCell):
"""BrowserScan grades each probe Normal/Abnormal; any Abnormal = detected."""
txt = _full_text(page)
abnormal = len(re.findall(r"\bAbnormal\b", txt))
if abnormal == 0 and "Test Results:" in txt and "Normal" in txt:
return (CheckStatus.PASS, 0.0, "Test Results: Normal (0 Abnormal)")
if abnormal > 0:
return (CheckStatus.FAIL, float(abnormal), f"{abnormal} Abnormal signal(s)")
return (CheckStatus.INFO, None, f"unparsed: {txt[:120]!r}")
# The scrapingcourse CF canary prints an explicit success line once solved; a
# failure leaves the interstitial ("Just a moment...") in the DOM. This is the
# ONLY site here that exercises solve_cloudflare (the others present no challenge).
_CF_PASS = "you bypassed the cloudflare challenge"
_CF_BLOCK = ("just a moment", "attention required", "verify you are human")
def _parse_cloudflare(page: Any, _cell: EvalCell):
low = _full_text(page).lower()
if _CF_PASS in low:
return (CheckStatus.PASS, 0.0, "bypassed Cloudflare challenge")
if any(m in low for m in _CF_BLOCK):
return (CheckStatus.FAIL, 1.0, "stuck on Cloudflare interstitial")
return (CheckStatus.INFO, None, f"unparsed: {low[:120]!r}")
# iphey renders a masthead verdict "Your Digital Identity Looks <Trustworthy|
# Unreliable|Suspicious>" after an async fingerprint+IP correlation (slow → big
# settle). Only "Trustworthy" is a clean pass.
_IPHEY_RE = re.compile(r"Your Digital Identity Looks\s+([A-Za-z]+)", re.IGNORECASE)
def _parse_iphey(page: Any, _cell: EvalCell):
txt = _full_text(page)
m = _IPHEY_RE.search(txt)
if not m:
return (CheckStatus.INFO, None, f"verdict not loaded: {txt[:120]!r}")
verdict = m.group(1)
ok = verdict.lower() == "trustworthy"
return (
CheckStatus.PASS if ok else CheckStatus.FAIL,
0.0 if ok else 1.0,
f"iphey verdict: {verdict}",
)
_BROWSER_SITES: list[StealthSite] = [
StealthSite(
name="sannysoft",
url="https://bot.sannysoft.com/",
bar="0 failed cells",
parse=_parse_sannysoft,
settle_ms=3000,
),
StealthSite(
name="deviceandbrowserinfo",
url="https://deviceandbrowserinfo.com/are_you_a_bot",
bar="isBot=false",
parse=_parse_areyouabot,
settle_ms=3000,
),
StealthSite(
name="recaptcha_v3_score",
url="https://recaptcha-demo.appspot.com/recaptcha-v3-request-scores.php",
bar="score >= 0.7",
parse=_parse_recaptcha_v3,
# v3 runs grecaptcha.execute then round-trips a server verify; too-short a
# wait reads the page before the {"score":..} JSON lands (false 0.0).
settle_ms=12000,
),
StealthSite(
name="creepjs",
url="https://abrahamjuliot.github.io/creepjs/",
bar="headless <=30%, no spoof tells",
parse=_parse_creepjs,
settle_ms=30000,
),
StealthSite(
name="browserscan",
url="https://www.browserscan.net/bot-detection",
bar="0 Abnormal",
parse=_parse_browserscan,
settle_ms=8000,
),
StealthSite(
name="incolumitas",
url="https://bot.incolumitas.com/",
bar="0 fpscanner FAIL",
parse=_parse_incolumitas,
settle_ms=12000,
),
StealthSite(
name="fingerprint_scan",
url="https://fingerprint-scan.com/",
bar="bot risk < 50/100",
parse=_parse_fingerprint_scan,
settle_ms=20000,
),
StealthSite(
name="fingerprintjs_demo",
url="https://demo.fingerprint.com/web-scraping",
bar="not blocked",
parse=_parse_fingerprintjs,
settle_ms=6000,
),
StealthSite(
name="cloudflare_challenge",
url="https://www.scrapingcourse.com/cloudflare-challenge",
bar="bypass CF challenge (exercises solve_cloudflare)",
parse=_parse_cloudflare,
settle_ms=15000,
),
StealthSite(
name="iphey",
url="https://iphey.com/",
bar="verdict Trustworthy",
parse=_parse_iphey,
# iphey's verdict loads via a slow async correlation; <~20s reads the
# "Temporary value" placeholder (false INFO).
settle_ms=25000,
),
]
# S2 — per-property fingerprint pages: too visual to auto-grade; emit as manual
# links so the operator can confirm the 03e levers (canvas/webgl/fonts/webrtc).
_MANUAL_LINKS: list[tuple[str, str]] = [
("browserleaks_canvas", "https://browserleaks.com/canvas"),
("browserleaks_webgl", "https://browserleaks.com/webgl"),
("browserleaks_fonts", "https://browserleaks.com/fonts"),
("browserleaks_webrtc", "https://browserleaks.com/webrtc"),
]
def _run_browser_site(
site: StealthSite, *, proxy: str | None, headed: bool, screenshots: bool
) -> CheckResult:
shot = str(SCREENSHOTS_DIR / f"S_{site.name}.png") if screenshots else None
action, cell = make_page_action(
evaluate_js=site.evaluate_js,
screenshot_path=shot,
pre_wait_ms=site.settle_ms,
)
kwargs: dict[str, Any] = {
"headless": not headed,
"network_idle": True,
"block_ads": True,
"solve_cloudflare": True,
"proxy": proxy,
"timeout": 120000,
"page_action": action,
}
# Single source of truth — the exact levers the production crawler ships.
kwargs.update(build_stealthy_kwargs(get_stealth_config()))
try:
page = StealthyFetcher.fetch(site.url, **kwargs)
except Exception as exc: # noqa: BLE001 - never crash the run
return CheckResult(
suite="S",
name=site.name,
tier="scrapling-stealthy",
status=CheckStatus.ERROR,
bar=site.bar,
detail=f"fetch failed: {type(exc).__name__}: {exc}",
screenshot=shot,
)
try:
status, numeric, detail = site.parse(page, cell)
except Exception as exc: # noqa: BLE001
status, numeric, detail = (
CheckStatus.ERROR,
None,
f"parse failed: {type(exc).__name__}: {exc}",
)
return CheckResult(
suite="S",
name=site.name,
tier="scrapling-stealthy",
status=status,
bar=site.bar,
detail=detail,
numeric=numeric,
screenshot=shot,
)
async def _run_tls(proxy: str | None) -> CheckResult:
"""S3 — JA3/JA4/PeetPrint of the impersonated static (curl_cffi) tier."""
try:
page = await AsyncFetcher.get(
"https://tls.peet.ws/api/all",
stealthy_headers=True,
impersonate="chrome",
proxy=proxy,
timeout=30,
)
data = page.json()
tls = data.get("tls", {}) if isinstance(data, dict) else {}
ja3 = tls.get("ja3_hash") or tls.get("ja3")
ja4 = tls.get("ja4")
peet = tls.get("peetprint_hash")
return CheckResult(
suite="S",
name="tls_fingerprint",
tier="scrapling-static",
status=CheckStatus.INFO,
bar="informational (diff vs real Chrome)",
detail=f"ja3={ja3} ja4={ja4} peet={peet}",
)
except Exception as exc: # noqa: BLE001
return CheckResult(
suite="S",
name="tls_fingerprint",
tier="scrapling-static",
status=CheckStatus.ERROR,
bar="informational",
detail=f"{type(exc).__name__}: {exc}",
)
async def _run_proxy_leak(proxy: str | None) -> CheckResult:
"""S4 — record the exit IP seen by an echo endpoint (proves egress path)."""
try:
page = await AsyncFetcher.get(
"https://api.ipify.org?format=json",
impersonate="chrome",
proxy=proxy,
timeout=30,
)
data = page.json()
ip = data.get("ip") if isinstance(data, dict) else None
note = "via proxy" if proxy else "DIRECT (no proxy — datacenter IP)"
return CheckResult(
suite="S",
name="exit_ip",
tier="scrapling-static",
status=CheckStatus.INFO,
bar="not your real/datacenter IP",
detail=f"exit_ip={ip} ({note})",
)
except Exception as exc: # noqa: BLE001
return CheckResult(
suite="S",
name="exit_ip",
tier="scrapling-static",
status=CheckStatus.ERROR,
bar="n/a",
detail=f"{type(exc).__name__}: {exc}",
)
async def run_suite_s(
*, proxy: str | None, headed: bool, screenshots: bool
) -> list[CheckResult]:
"""Run the full stealth suite (browser tier sites + TLS + proxy + manual links)."""
results: list[CheckResult] = []
# Browser sites are sync (spin up a real browser) → offload per-site so one
# slow score doesn't stall the loop and the event loop stays clean.
for site in _BROWSER_SITES:
print(f" [S] {site.name} ... (settle {site.settle_ms}ms)")
res = await asyncio.to_thread(
_run_browser_site,
site,
proxy=proxy,
headed=headed,
screenshots=screenshots,
)
results.append(res)
print(" [S] tls_fingerprint + exit_ip ...")
results.append(await _run_tls(proxy))
results.append(await _run_proxy_leak(proxy))
for name, url in _MANUAL_LINKS:
results.append(
CheckResult(
suite="S",
name=name,
tier="manual",
status=CheckStatus.INFO,
bar="open in a real headed run + compare",
detail=url,
)
)
return results

18
surfsense_backend/app/utils/crawl/__init__.py Normal file
View file

@ -0,0 +1,18 @@
"""App-wide crawler block classification (Apache-2.0, generic).
Phase 3e (Slice A). Mirrors ``app/utils/proxy`` and ``app/utils/captcha``: this
package holds only the **generic, vendor-agnostic** glue — here, a pure block
classifier (passive telemetry from public anti-bot markers). It is consumed by
the separately licensed proprietary crawler to label ``CrawlOutcome.block_type``.
The **bypass-specific tuning** (the stealth kwargs builder / geoip coherence, and
the deferred WebGL spoof + humanize choreography) is NOT generic and lives under
the proprietary boundary in ``app/proprietary/web_crawler/`` (``stealth.py``).
"""
from app.utils.crawl.classifier import BlockType, classify_block
__all__ = [
"BlockType",
"classify_block",
]

94
surfsense_backend/app/utils/crawl/classifier.py Normal file
View file

@ -0,0 +1,94 @@
"""Block classifier — labels a fetched page so the ladder can learn (Phase 3e).
Pure, additive, status + body-marker based. It attaches
:class:`~app.proprietary.web_crawler.connector.CrawlOutcome.block_type` for
telemetry / future escalation routing (per-domain memory, `03d` captcha routing)
and **never** changes *when* a crawl is ``SUCCESS`` — `03c` bills on
``status == SUCCESS``, so this stays read-only metadata.
Markers mirror the proven set in the Camoufox-based FlareSolverr alternative
``references/trawl-dev/packages/tiers/src/detect.ts`` (plus DataDome/Kasada for
our enterprise targets). Regexes are compiled once at import (hot-path hygiene).
"""
import re
from enum import Enum
class BlockType(str, Enum):
"""Coarse label for what a fetched page represents."""
OK = "ok" # usable content, no challenge detected
CLOUDFLARE = "cloudflare" # CF interstitial / Turnstile / DDoS-Guard
CAPTCHA_RECAPTCHA = "captcha_recaptcha"
CAPTCHA_HCAPTCHA = "captcha_hcaptcha"
DATADOME = "datadome"
KASADA = "kasada"
RATE_LIMITED = "rate_limited"
EMPTY = "empty" # fetched but no HTML body
UNKNOWN = "unknown" # blocking-ish status with no recognized marker
# --- compiled marker patterns (hoisted; see Vercel rule 7.9) ---
_CLOUDFLARE = re.compile(
r"just a moment"
r"|checking your browser"
r"|enable javascript and cookies to continue"
r"|verify you are human"
r"|cf-mitigated"
r"|id=\"(?:cf-)?challenge-running\""
r"|id=\"turnstile-wrapper\""
r"|ddos-guard",
re.IGNORECASE,
)
_TURNSTILE = re.compile(
r"class=\"cf-turnstile\""
r"|challenges\.cloudflare\.com/turnstile"
r"|cdn-cgi/challenge-platform[^\"']*turnstile",
re.IGNORECASE,
)
_HCAPTCHA = re.compile(r"class=\"h-captcha\"|hcaptcha\.com/1/api", re.IGNORECASE)
_RECAPTCHA = re.compile(
r"class=\"g-recaptcha\"|google\.com/recaptcha|recaptcha\.net/recaptcha",
re.IGNORECASE,
)
_DATADOME = re.compile(r"datadome|geo\.captcha-delivery\.com", re.IGNORECASE)
_KASADA = re.compile(r"kpsdk|x-kpsdk|kasada", re.IGNORECASE)
# Statuses some CDNs use as a bot-gate before the real response (detect.ts:isBlocked).
_BOT_GATE_STATUSES = frozenset({202, 403})
def classify_block(status: int | None, html: str | None) -> BlockType:
"""Label a fetched page from its HTTP status and HTML body.
Precedence: explicit rate-limit status, then specific anti-bot/captcha body
markers, then a generic bot-gate status with no marker, else ``OK`` (or
``EMPTY`` when there is no body). Marker checks run before the generic
bot-gate so a ``403`` Cloudflare challenge is labeled ``CLOUDFLARE``, not
``UNKNOWN``.
"""
if status == 429:
return BlockType.RATE_LIMITED
if not html or not html.strip():
# No body: distinguish a blocking-ish status from a genuinely empty 200.
if status in _BOT_GATE_STATUSES:
return BlockType.UNKNOWN
return BlockType.EMPTY
if _CLOUDFLARE.search(html) or _TURNSTILE.search(html):
return BlockType.CLOUDFLARE
if _DATADOME.search(html):
return BlockType.DATADOME
if _KASADA.search(html):
return BlockType.KASADA
if _HCAPTCHA.search(html):
return BlockType.CAPTCHA_HCAPTCHA
if _RECAPTCHA.search(html):
return BlockType.CAPTCHA_RECAPTCHA
if status in _BOT_GATE_STATUSES:
return BlockType.UNKNOWN
return BlockType.OK