mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-06 22:12:12 +02:00
94 lines
3.4 KiB
Python
94 lines
3.4 KiB
Python
"""Block classifier — labels a fetched page so the ladder can learn (Phase 3e).
|
|
|
|
Pure, additive, status + body-marker based. It attaches
|
|
:class:`~app.proprietary.web_crawler.connector.CrawlOutcome.block_type` for
|
|
telemetry / future escalation routing (per-domain memory, `03d` captcha routing)
|
|
and **never** changes *when* a crawl is ``SUCCESS`` — `03c` bills on
|
|
``status == SUCCESS``, so this stays read-only metadata.
|
|
|
|
Markers mirror the proven set in the Camoufox-based FlareSolverr alternative
|
|
``references/trawl-dev/packages/tiers/src/detect.ts`` (plus DataDome/Kasada for
|
|
our enterprise targets). Regexes are compiled once at import (hot-path hygiene).
|
|
"""
|
|
|
|
import re
|
|
from enum import Enum
|
|
|
|
|
|
class BlockType(str, Enum):
|
|
"""Coarse label for what a fetched page represents."""
|
|
|
|
OK = "ok" # usable content, no challenge detected
|
|
CLOUDFLARE = "cloudflare" # CF interstitial / Turnstile / DDoS-Guard
|
|
CAPTCHA_RECAPTCHA = "captcha_recaptcha"
|
|
CAPTCHA_HCAPTCHA = "captcha_hcaptcha"
|
|
DATADOME = "datadome"
|
|
KASADA = "kasada"
|
|
RATE_LIMITED = "rate_limited"
|
|
EMPTY = "empty" # fetched but no HTML body
|
|
UNKNOWN = "unknown" # blocking-ish status with no recognized marker
|
|
|
|
|
|
# --- compiled marker patterns (hoisted; see Vercel rule 7.9) ---
|
|
_CLOUDFLARE = re.compile(
|
|
r"just a moment"
|
|
r"|checking your browser"
|
|
r"|enable javascript and cookies to continue"
|
|
r"|verify you are human"
|
|
r"|cf-mitigated"
|
|
r"|id=\"(?:cf-)?challenge-running\""
|
|
r"|id=\"turnstile-wrapper\""
|
|
r"|ddos-guard",
|
|
re.IGNORECASE,
|
|
)
|
|
_TURNSTILE = re.compile(
|
|
r"class=\"cf-turnstile\""
|
|
r"|challenges\.cloudflare\.com/turnstile"
|
|
r"|cdn-cgi/challenge-platform[^\"']*turnstile",
|
|
re.IGNORECASE,
|
|
)
|
|
_HCAPTCHA = re.compile(r"class=\"h-captcha\"|hcaptcha\.com/1/api", re.IGNORECASE)
|
|
_RECAPTCHA = re.compile(
|
|
r"class=\"g-recaptcha\"|google\.com/recaptcha|recaptcha\.net/recaptcha",
|
|
re.IGNORECASE,
|
|
)
|
|
_DATADOME = re.compile(r"datadome|geo\.captcha-delivery\.com", re.IGNORECASE)
|
|
_KASADA = re.compile(r"kpsdk|x-kpsdk|kasada", re.IGNORECASE)
|
|
|
|
# Statuses some CDNs use as a bot-gate before the real response (detect.ts:isBlocked).
|
|
_BOT_GATE_STATUSES = frozenset({202, 403})
|
|
|
|
|
|
def classify_block(status: int | None, html: str | None) -> BlockType:
|
|
"""Label a fetched page from its HTTP status and HTML body.
|
|
|
|
Precedence: explicit rate-limit status, then specific anti-bot/captcha body
|
|
markers, then a generic bot-gate status with no marker, else ``OK`` (or
|
|
``EMPTY`` when there is no body). Marker checks run before the generic
|
|
bot-gate so a ``403`` Cloudflare challenge is labeled ``CLOUDFLARE``, not
|
|
``UNKNOWN``.
|
|
"""
|
|
if status == 429:
|
|
return BlockType.RATE_LIMITED
|
|
|
|
if not html or not html.strip():
|
|
# No body: distinguish a blocking-ish status from a genuinely empty 200.
|
|
if status in _BOT_GATE_STATUSES:
|
|
return BlockType.UNKNOWN
|
|
return BlockType.EMPTY
|
|
|
|
if _CLOUDFLARE.search(html) or _TURNSTILE.search(html):
|
|
return BlockType.CLOUDFLARE
|
|
if _DATADOME.search(html):
|
|
return BlockType.DATADOME
|
|
if _KASADA.search(html):
|
|
return BlockType.KASADA
|
|
if _HCAPTCHA.search(html):
|
|
return BlockType.CAPTCHA_HCAPTCHA
|
|
if _RECAPTCHA.search(html):
|
|
return BlockType.CAPTCHA_RECAPTCHA
|
|
|
|
if status in _BOT_GATE_STATUSES:
|
|
return BlockType.UNKNOWN
|
|
|
|
return BlockType.OK
|