apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-06 22:12:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(crawler): captcha solving + per-attempt billing (Phase 3d)

Browse source 
Wire captchatools as the StealthyFetcher-tier page_action to detect, harvest
(egressing from the crawl's own proxy IP), inject, and submit reCAPTCHA v2/v3
and hCaptcha tokens. Opt-in and off by default (zero attempts, zero cost).

Licensing split:
- Apache-2 app/utils/captcha/ holds the generic, vendor-agnostic config
  (CaptchaConfig + captcha_enabled() = flag AND key present).
- Proprietary app/proprietary/web_crawler/captcha.py holds the bypass logic
  (detect/harvest/inject) plus a process-wide solver latch that halts solving
  on unrecoverable errors (no balance / bad key).

Crawler: CrawlOutcome gains captcha_attempts/captcha_solved, surfaced via a
per-call captcha_state dict threaded crawl_url -> _crawl_with_stealthy(_sync)
and stamped onto every stealth terminal outcome. The stealth tier captures the
proxy once and reuses it for both the fetch and the solver (IP-coherence).

Billing: WebCrawlCreditService gains captcha_billing_enabled,
captcha_solves_to_micros, charge_captcha, and a generic check_balance, sharing
a single _apply_debit path. The indexer accumulates attempts (even on failed
crawls), runs a combined crawl+captcha pre-flight, and posts a per-attempt
owner charge as usage_type="web_crawl_captcha". The captcha worst-case is only
reserved when solving is actually enabled, so a solving-off deployment is never
blocked for captcha that can never run. Both chat scrape tools fold attempts
into the current turn before the success/fail branch.

Fully config-driven prices; no migration. New unit tests cover the config,
factory (detection/latch/timeout/cap), credit service, indexer wiring, and the
chat fold.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-06-29 22:58:21 -07:00
parent aad366ba13
commit bdacea8b6e
19 changed files with 1494 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

33
surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/scrape_webpage.py
View file

@ -55,6 +55,36 @@ def _bill_successful_scrape() -> None:
)
def _bill_captcha_attempts(outcome) -> None:
"""Fold captcha solve attempts (Phase 3d) into the current chat turn's bill.
Per *attempt*, not per success: a solve that didn't rescue the crawl still
cost real solver money, so this runs before the success/failure branch.
Mirrors :func:`_bill_successful_scrape` (rides the turn accumulator, settles
at ``finalize_credit``; record-only on free/anonymous turns). No-op when
captcha billing is off, there were no attempts, or no turn is active.
"""
from app.services.token_tracking_service import get_current_accumulator
from app.services.web_crawl_credit_service import WebCrawlCreditService
if not WebCrawlCreditService.captcha_billing_enabled():
return
attempts = getattr(outcome, "captcha_attempts", 0) or 0
if attempts <= 0:
return
acc = get_current_accumulator()
if acc is None:
return
acc.add(
model="web_crawl_captcha",
prompt_tokens=0,
completion_tokens=0,
total_tokens=0,
cost_micros=WebCrawlCreditService.captcha_solves_to_micros(attempts),
call_kind="web_crawl_captcha",
)
def extract_domain(url: str) -> str:
"""Extract the domain from a URL."""
try:
@ -258,6 +288,9 @@ def create_scrape_webpage_tool():
connector = WebCrawlerConnector()
outcome = await connector.crawl_url(url)
# 03d: bill any captcha attempts (even if the crawl ultimately failed).
_bill_captcha_attempts(outcome)
if outcome.status is not CrawlOutcomeStatus.SUCCESS or not outcome.result:
return {
"id": scrape_id,

33
surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py
View file

@ -49,6 +49,36 @@ def _bill_successful_scrape() -> None:
)
def _bill_captcha_attempts(outcome) -> None:
"""Fold captcha solve attempts (Phase 3d) into the current chat turn's bill.
Per *attempt*, not per success: a solve that didn't rescue the crawl still
cost real solver money, so this runs before the success/failure branch.
Mirrors :func:`_bill_successful_scrape` (rides the turn accumulator, settles
at ``finalize_credit``; record-only on free/anonymous turns). No-op when
captcha billing is off, there were no attempts, or no turn is active.
"""
from app.services.token_tracking_service import get_current_accumulator
from app.services.web_crawl_credit_service import WebCrawlCreditService
if not WebCrawlCreditService.captcha_billing_enabled():
return
attempts = getattr(outcome, "captcha_attempts", 0) or 0
if attempts <= 0:
return
acc = get_current_accumulator()
if acc is None:
return
acc.add(
model="web_crawl_captcha",
prompt_tokens=0,
completion_tokens=0,
total_tokens=0,
cost_micros=WebCrawlCreditService.captcha_solves_to_micros(attempts),
call_kind="web_crawl_captcha",
)
def extract_domain(url: str) -> str:
"""Extract the domain from a URL."""
try:
@ -252,6 +282,9 @@ def create_scrape_webpage_tool():
connector = WebCrawlerConnector()
outcome = await connector.crawl_url(url)
# 03d: bill any captcha attempts (even if the crawl ultimately failed).
_bill_captcha_attempts(outcome)
if outcome.status is not CrawlOutcomeStatus.SUCCESS or not outcome.result:
return {
"id": scrape_id,

45
surfsense_backend/app/config/__init__.py
View file

@ -670,6 +670,21 @@ class Config:
os.getenv("WEB_CRAWL_MICROS_PER_SUCCESS", "1000")
)
# Phase 3d captcha-solve billing. Captcha can't ride the per-success crawl
# meter above: the solver charges per *attempt* regardless of whether the
# crawl ultimately succeeds, so solves are metered as a SEPARATE per-attempt
# unit (usage_type="web_crawl_captcha"). Off by default; independent of the
# crawl-billing flag. Price is config-driven (no hardcoded rate):
# WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE = round(USD_per_1000_solves * 1_000)
# $3/1000 -> 3000 (default) | $5/1000 -> 5000
# Set with margin over the solver vendor's per-attempt price.
WEB_CRAWL_CAPTCHA_BILLING_ENABLED = (
os.getenv("WEB_CRAWL_CAPTCHA_BILLING_ENABLED", "FALSE").upper() == "TRUE"
)
WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE = int(
os.getenv("WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE", "3000")
)
# Low-balance WARNING threshold (micro-USD). Surfaced by the quota service
# so the UI can nudge the user to top up / enable auto-reload. $0.50.
CREDIT_LOW_BALANCE_WARNING_MICROS = int(
@ -1041,6 +1056,36 @@ class Config:
CUSTOM_PROXY_URL = os.getenv("CUSTOM_PROXY_URL")
CUSTOM_PROXY_URLS = os.getenv("CUSTOM_PROXY_URLS")
# =====================================================================
# Phase 3d — Captcha solving (reCAPTCHA v2/v3, hCaptcha) via captchatools.
# The LAST-resort bypass tier: only fires on the StealthyFetcher browser
# tier, only when a sitekey is detected, and only when explicitly enabled.
# Cloudflare Turnstile is already handled free in-framework (03a), NOT here.
# One app-wide config (mirrors the single PROXY_PROVIDER model) — no
# per-connector config. Off by default => zero solve attempts, zero cost.
# Solving may violate a target site's ToS; treat as opt-in/owner-acknowledged
# and public-data only (no logged-in bypass).
# =====================================================================
CAPTCHA_SOLVING_ENABLED = (
os.getenv("CAPTCHA_SOLVING_ENABLED", "FALSE").upper() == "TRUE"
)
# captchatools "solving_site": capmonster | 2captcha | anticaptcha |
# capsolver | captchaai. captchatools is itself the provider registry, so we
# do not rebuild a vendor hierarchy.
CAPTCHA_SOLVER_PROVIDER = os.getenv("CAPTCHA_SOLVER_PROVIDER", "capsolver")
CAPTCHA_SOLVER_API_KEY = os.getenv("CAPTCHA_SOLVER_API_KEY")
# Per-URL solve cap so one hostile page can't burn unbounded solver credit.
CAPTCHA_MAX_ATTEMPTS_PER_URL = int(
os.getenv("CAPTCHA_MAX_ATTEMPTS_PER_URL", "1")
)
# Abort a single solve after this many seconds (solves take 10-60s).
CAPTCHA_SOLVE_TIMEOUT_S = int(os.getenv("CAPTCHA_SOLVE_TIMEOUT_S", "120"))
# Default captcha type when detection is ambiguous: v2 | v3 | hcaptcha.
CAPTCHA_TYPE_DEFAULT = os.getenv("CAPTCHA_TYPE_DEFAULT", "v2")
# reCAPTCHA v3 tuning (only used for v3 challenges).
CAPTCHA_V3_MIN_SCORE = float(os.getenv("CAPTCHA_V3_MIN_SCORE", "0.7"))
CAPTCHA_V3_ACTION = os.getenv("CAPTCHA_V3_ACTION", "verify")
# Litellm TTS Configuration
TTS_SERVICE = os.getenv("TTS_SERVICE")
TTS_SERVICE_API_BASE = os.getenv("TTS_SERVICE_API_BASE")

326
surfsense_backend/app/proprietary/web_crawler/captcha.py Normal file
View file

@ -0,0 +1,326 @@
# SurfSense proprietary crawler engine.
#
# This module is part of the ``app.proprietary`` package and is licensed
# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
# Do not relicense or redistribute this file under Apache-2.0.
"""Captcha detection + token-injection ``page_action`` (Phase 3d).
This is the **bypass logic** (hence proprietary); the generic, vendor-agnostic
config lives in the Apache-2.0 ``app/utils/captcha/`` package. ``captchatools``
is the multi-vendor solver registry — we only detect the challenge, harvest a
token egressing from **the crawl's own proxy IP** (token IP-binding), inject it,
and submit.
Why a closure cell: Scrapling runs ``page_action`` after navigation but
**swallows its exceptions and discards its return value** (see
``references/Scrapling`` engine ``_stealth.py``). So the only way to surface
"did we solve / how many attempts" back to the crawler (for per-attempt billing
and the retry cap) is to mutate a caller-owned ``state`` dict.
Why a process latch: the solver README warns that hammering it while out of
balance can get the IP **temporarily banned**, so ``ErrNoBalance`` /
``ErrWrongAPIKey`` latch solving OFF for the rest of the process (config/balance
won't change without a restart). ``reset_solver_latch()`` clears it for tests.
"""
import logging
import re
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout
from typing import Any
from urllib.parse import urlsplit
from app.utils.captcha import CaptchaConfig
logger = logging.getLogger(__name__)
_CAPTCHA_LOG = "[webcrawler][captcha]"
# Process-wide kill switch tripped on unrecoverable solver errors (no balance /
# bad key). Module-level by design: it must persist across the per-URL
# ``page_action`` instances created during one crawl run.
_solver_latched = False
def reset_solver_latch() -> None:
"""Clear the process solver latch (test seam / explicit re-enable)."""
global _solver_latched
_solver_latched = False
def solver_latched() -> bool:
return _solver_latched
def _latch_solver(reason: str) -> None:
global _solver_latched
_solver_latched = True
logger.warning("%s solver latched OFF for this process: %s", _CAPTCHA_LOG, reason)
# --- detection -------------------------------------------------------------
# (challenge_type, css_selector_for_widget). Order = detection priority.
_WIDGET_SELECTORS = (
("v2", ".g-recaptcha[data-sitekey]"),
("hcaptcha", ".h-captcha[data-sitekey]"),
)
_IFRAME_PATTERNS = (
("v2", re.compile(r"google\.com/recaptcha", re.I)),
("hcaptcha", re.compile(r"hcaptcha\.com", re.I)),
)
_SITEKEY_IN_SRC = re.compile(r"[?&]k=([\w-]+)")
_RENDER_SITEKEY = re.compile(r"render=([\w-]+)")
def detect_challenge(page: Any, cfg: CaptchaConfig) -> tuple[str, str] | None:
"""Return ``(challenge_type, sitekey)`` if a solvable challenge is present.
``challenge_type`` is one of ``v2`` / ``v3`` / ``hcaptcha``. Returns
``None`` when nothing solvable is found (the crawler then proceeds to
``wait_selector`` / extraction unchanged). Detection is defensive: any DOM
access raising is treated as "not found" rather than aborting the fetch.
"""
try:
for ctype, selector in _WIDGET_SELECTORS:
el = page.query_selector(selector)
if el is not None:
sitekey = el.get_attribute("data-sitekey")
if sitekey:
return ctype, sitekey
# Iframe fallback: widget rendered inside an iframe whose src carries
# the sitekey as ``?k=...``.
for frame_el in page.query_selector_all("iframe[src]") or []:
src = frame_el.get_attribute("src") or ""
for ctype, pat in _IFRAME_PATTERNS:
if pat.search(src):
m = _SITEKEY_IN_SRC.search(src)
if m:
return ctype, m.group(1)
# reCAPTCHA v3: no widget; sitekey rides the api.js ``?render=<key>``.
if cfg.captcha_type_default == "v3":
for script_el in page.query_selector_all("script[src]") or []:
src = script_el.get_attribute("src") or ""
if "recaptcha" in src.lower():
m = _RENDER_SITEKEY.search(src)
if m and m.group(1) != "explicit":
return "v3", m.group(1)
except Exception as exc: # noqa: BLE001
logger.debug("%s detection error (treated as no challenge): %s", _CAPTCHA_LOG, exc)
return None
# --- proxy / harvest -------------------------------------------------------
def proxy_url_to_captchatools(proxy_url: str | None) -> str | None:
"""Reformat ``http://user:pass@host:port`` -> ``host:port:user:pass``.
captchatools wants the colon-delimited form. Returns ``None`` for a missing
or unparseable proxy so the solver harvests proxyless (the token may then be
IP-mismatched, but that's better than crashing the fetch).
"""
if not proxy_url:
return None
try:
p = urlsplit(proxy_url)
if not p.hostname or not p.port:
return None
if p.username and p.password:
return f"{p.hostname}:{p.port}:{p.username}:{p.password}"
return f"{p.hostname}:{p.port}"
except Exception: # noqa: BLE001
return None
def _harvest_token(
cfg: CaptchaConfig,
challenge_type: str,
sitekey: str,
page_url: str,
proxy: str | None,
user_agent: str | None,
) -> str | None:
"""Harvest a token from the solver. Raises on solver errors so the caller
can latch on the unrecoverable ones; returns ``None`` on an empty token.
"""
import captchatools
harvester = captchatools.new_harvester(
api_key=cfg.api_key,
solving_site=cfg.solving_site,
sitekey=sitekey,
captcha_url=page_url,
captcha_type=challenge_type,
min_score=cfg.v3_min_score,
action=cfg.v3_action,
)
token = harvester.get_token(
proxy=proxy,
proxy_type="HTTP",
user_agent=user_agent,
)
return token or None
# --- injection -------------------------------------------------------------
# Sets the response token into the right textarea(s) and fires the JS callbacks
# registered by the widget, then submits the closest form. Best-effort: returns
# nothing; success is judged by the crawler's post-fetch extraction.
_INJECT_JS = r"""
(args) => {
const { token, ctype } = args;
const names = ctype === 'hcaptcha'
? ['h-captcha-response', 'g-recaptcha-response']
: ['g-recaptcha-response'];
for (const name of names) {
let ta = document.querySelector('textarea[name="' + name + '"]')
|| document.getElementById(name);
if (!ta) {
ta = document.createElement('textarea');
ta.name = name; ta.id = name;
ta.style.display = 'none';
document.body.appendChild(ta);
}
ta.value = token; ta.innerHTML = token;
ta.dispatchEvent(new Event('input', { bubbles: true }));
ta.dispatchEvent(new Event('change', { bubbles: true }));
}
// Fire grecaptcha client callbacks when present (v2 invisible / explicit).
try {
const cfg = window.___grecaptcha_cfg;
if (cfg && cfg.clients) {
for (const id in cfg.clients) {
const client = cfg.clients[id];
for (const k in client) {
const o = client[k];
if (o && typeof o === 'object') {
for (const kk in o) {
const cb = o[kk];
if (cb && cb.callback && typeof cb.callback === 'function') {
try { cb.callback(token); } catch (e) {}
}
}
}
}
}
}
} catch (e) {}
// Submit the form containing the response field, if any.
const field = document.querySelector('textarea[name="g-recaptcha-response"], textarea[name="h-captcha-response"]');
const form = field ? field.closest('form') : document.querySelector('form');
if (form) { try { form.requestSubmit ? form.requestSubmit() : form.submit(); } catch (e) {} }
}
"""
def _inject_and_submit(page: Any, challenge_type: str, token: str) -> None:
try:
page.evaluate(_INJECT_JS, {"token": token, "ctype": challenge_type})
try:
page.wait_for_load_state("networkidle", timeout=15000)
except Exception: # noqa: BLE001
pass
except Exception as exc: # noqa: BLE001
logger.warning("%s injection error: %s", _CAPTCHA_LOG, exc)
# --- factory ---------------------------------------------------------------
def build_captcha_page_action(
state: dict[str, Any],
proxy_url: str | None,
cfg: CaptchaConfig,
) -> Callable[[Any], Any] | None:
"""Build the sync ``page_action`` for ``StealthyFetcher.fetch``.
``state`` is mutated in place: ``attempts`` (int) and ``solved`` (bool). The
crawler reads it back after the fetch to bill per attempt and enforce the
per-URL cap. Returns ``None`` when solving is disabled/latched so the caller
can skip wiring it entirely (keeping the stealth tier unchanged).
"""
if not cfg.enabled or solver_latched():
return None
captcha_proxy = proxy_url_to_captchatools(proxy_url)
def page_action(page: Any) -> Any:
# Hard cap: never exceed the per-URL budget even across proxy-retry
# re-runs that share this ``state``.
if state.get("attempts", 0) >= cfg.max_attempts_per_url:
return page
if solver_latched():
return page
detected = detect_challenge(page, cfg)
if detected is None:
return page # nothing to solve; let extraction proceed
challenge_type, sitekey = detected
page_url = getattr(page, "url", "") or ""
try:
user_agent = page.evaluate("() => navigator.userAgent")
except Exception: # noqa: BLE001
user_agent = None
# This counts as an attempt the moment we call the (paid) solver.
state["attempts"] = state.get("attempts", 0) + 1
logger.info(
"%s solving type=%s site=%s attempt=%d",
_CAPTCHA_LOG,
challenge_type,
sitekey[:12],
state["attempts"],
)
# Run the (blocking) solver in a worker so the timeout can actually
# ABANDON a slow solve. NOTE: do NOT use ``with ThreadPoolExecutor`` —
# its ``__exit__`` joins the worker (``wait=True``), which would block
# for the full solve and defeat the timeout. Shut down non-blocking.
executor = ThreadPoolExecutor(max_workers=1)
future = executor.submit(
_harvest_token,
cfg,
challenge_type,
sitekey,
page_url,
captcha_proxy,
user_agent,
)
try:
token = future.result(timeout=cfg.timeout_s)
except FuturesTimeout:
logger.warning("%s solve timed out after %ss", _CAPTCHA_LOG, cfg.timeout_s)
return page
except Exception as exc: # noqa: BLE001
if _is_unrecoverable(exc):
_latch_solver(repr(exc))
else:
logger.warning("%s solve error: %s", _CAPTCHA_LOG, exc)
return page
finally:
executor.shutdown(wait=False, cancel_futures=True)
if not token:
return page
_inject_and_submit(page, challenge_type, token)
state["solved"] = True
logger.info("%s solved type=%s site=%s", _CAPTCHA_LOG, challenge_type, sitekey[:12])
return page
return page_action
def _is_unrecoverable(exc: Exception) -> bool:
"""True for solver errors that must latch solving off (no balance / bad key).
Matched by class name to avoid a hard import of ``captchatools.exceptions``
at module load (the dep is optional / lazily imported).
"""
name = type(exc).__name__.lower()
return "nobalance" in name or "wrongapikey" in name or "apikey" in name

79
surfsense_backend/app/proprietary/web_crawler/connector.py
View file

@ -33,6 +33,8 @@ import validators
from scrapling.engines.toolbelt import is_proxy_error
from scrapling.fetchers import AsyncFetcher, DynamicFetcher, StealthyFetcher
from app.proprietary.web_crawler.captcha import build_captcha_page_action
from app.utils.captcha import captcha_enabled, get_captcha_config
from app.utils.proxy import get_proxy_url, is_pool_backed
logger = logging.getLogger(__name__)
@ -56,14 +58,20 @@ class CrawlOutcome:
The **billable success predicate is single-sourced**:
``status == CrawlOutcomeStatus.SUCCESS`` (Phase 3c meters on it). Picking a
dataclass over a tuple lets later subplans append fields without breaking
callers (03d adds ``captcha_attempts``/``captcha_solved`` for per-attempt
billing; 03e's block classifier can attach a ``block_type``).
callers (03e's block classifier can attach a ``block_type``).
Phase 3d captcha fields are surfaced here so per-attempt billing can read
them off the outcome regardless of crawl SUCCESS (the solver charges per
*attempt*). They are populated only by the StealthyFetcher tier when captcha
solving is enabled; every other path leaves the defaults (0 / False).
"""
status: CrawlOutcomeStatus
result: dict[str, Any] | None = None
error: str | None = None
tier: str | None = None
captcha_attempts: int = 0
captcha_solved: bool = False
class WebCrawlerConnector:
@ -89,6 +97,11 @@ class WebCrawlerConnector:
- crawler_type: Identifier of the tier that produced the content
"""
total_start = time.perf_counter()
# Per-call captcha telemetry (03d). Mutated only by the StealthyFetcher
# tier's page_action; stamped onto the returned outcome so per-attempt
# billing can read it regardless of crawl SUCCESS. Per-call (not on
# ``self``) => safe under concurrent ``crawl_url`` calls.
captcha_state: dict[str, Any] = {"attempts": 0, "solved": False}
try:
if not validators.url(url):
return CrawlOutcome(
@ -167,7 +180,8 @@ class WebCrawlerConnector:
try:
logger.info(f"[webcrawler] Using Scrapling StealthyFetcher for: {url}")
result = await self._run_tier_with_proxy_retry(
"scrapling-stealthy", lambda: self._crawl_with_stealthy(url)
"scrapling-stealthy",
lambda: self._crawl_with_stealthy(url, captcha_state),
)
if result:
self._log_tier_outcome(
@ -178,6 +192,8 @@ class WebCrawlerConnector:
status=CrawlOutcomeStatus.SUCCESS,
result=result,
tier="scrapling-stealthy",
captcha_attempts=captcha_state["attempts"],
captcha_solved=captcha_state["solved"],
)
reached_without_content = True
errors.append("Scrapling stealthy: empty extraction")
@ -201,10 +217,14 @@ class WebCrawlerConnector:
return CrawlOutcome(
status=CrawlOutcomeStatus.EMPTY,
error=f"No content extracted for {url}. {'; '.join(errors)}",
captcha_attempts=captcha_state["attempts"],
captcha_solved=captcha_state["solved"],
)
return CrawlOutcome(
status=CrawlOutcomeStatus.FAILED,
error=f"All crawl methods failed for {url}. {'; '.join(errors)}",
captcha_attempts=captcha_state["attempts"],
captcha_solved=captcha_state["solved"],
)
except Exception as e:
@ -212,6 +232,8 @@ class WebCrawlerConnector:
return CrawlOutcome(
status=CrawlOutcomeStatus.FAILED,
error=f"Error crawling URL {url}: {e!s}",
captcha_attempts=captcha_state["attempts"],
captcha_solved=captcha_state["solved"],
)
async def _run_tier_with_proxy_retry(
@ -356,29 +378,56 @@ class WebCrawlerConnector:
status=getattr(page, "status", None),
)
async def _crawl_with_stealthy(self, url: str) -> dict[str, Any] | None:
async def _crawl_with_stealthy(
self, url: str, captcha_state: dict[str, Any] | None = None
) -> dict[str, Any] | None:
"""
Crawl URL using Scrapling's StealthyFetcher (patchright-Chromium) + Trafilatura.
Last-resort tier with anti-bot features. Runs the sync fetch in a worker
thread for the same event-loop-safety reasons as DynamicFetcher. Falls
back to the raw HTML when Trafilatura extraction is empty.
"""
return await asyncio.to_thread(self._crawl_with_stealthy_sync, url)
def _crawl_with_stealthy_sync(self, url: str) -> dict[str, Any] | None:
``captcha_state`` (03d) is mutated in place by the captcha page_action
(attempts/solved) so ``crawl_url`` can surface it on the outcome.
"""
return await asyncio.to_thread(
self._crawl_with_stealthy_sync, url, captcha_state
)
def _crawl_with_stealthy_sync(
self, url: str, captcha_state: dict[str, Any] | None = None
) -> dict[str, Any] | None:
"""Synchronous StealthyFetcher crawl executed in a worker thread."""
fetch_start = time.perf_counter()
# Capture the proxy endpoint ONCE so the captcha solver egresses from the
# SAME IP as this fetch (tokens are IP-bound). Re-calling get_proxy_url()
# inside the page_action would rotate a pool-backed provider to a
# different IP and invalidate the token (03d proxy-coherence caveat).
proxy = get_proxy_url()
# Build the captcha page_action only when solving is enabled (and not
# process-latched). ``None`` => stealth tier behaves exactly as before.
page_action = None
if captcha_state is not None and captcha_enabled():
page_action = build_captcha_page_action(
captcha_state, proxy, get_captcha_config()
)
# ``solve_cloudflare=True`` runs the full Turnstile/Interstitial challenge
# loop; scoped to this last-resort tier only (it spins up the browser).
page = StealthyFetcher.fetch(
url,
headless=True,
network_idle=True,
block_ads=True,
solve_cloudflare=True,
proxy=get_proxy_url(),
)
# Scrapling runs solve_cloudflare BEFORE page_action, so Cloudflare is
# cleared first, then our captcha injector runs.
fetch_kwargs: dict[str, Any] = {
"headless": True,
"network_idle": True,
"block_ads": True,
"solve_cloudflare": True,
"proxy": proxy,
}
if page_action is not None:
fetch_kwargs["page_action"] = page_action
page = StealthyFetcher.fetch(url, **fetch_kwargs)
fetch_ms = (time.perf_counter() - fetch_start) * 1000
return self._build_result(
page.html_content,

111
surfsense_backend/app/services/web_crawl_credit_service.py
View file

@ -51,15 +51,31 @@ class WebCrawlCreditService:
"""
return int(successes) * config.WEB_CRAWL_MICROS_PER_SUCCESS
async def get_available_micros(self, user_id: str | UUID) -> int | None:
"""Return spendable credit in micro-USD (``balance - reserved``).
@staticmethod
def captcha_billing_enabled() -> bool:
"""Phase 3d: whether captcha *solves* are metered.
Returns ``None`` when crawl billing is disabled, which callers treat as
"unlimited" (no blocking, no charge).
Independent of crawl billing: a deployment may bill solves without
billing crawls (or vice-versa). Off by default.
"""
if not config.WEB_CRAWL_CREDIT_BILLING_ENABLED:
return None
return config.WEB_CRAWL_CAPTCHA_BILLING_ENABLED
@staticmethod
def captcha_solves_to_micros(attempts: int) -> int:
"""Convert a captcha *attempt* count to USD micro-credits.
Reads ``config.WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE`` (single source of
truth). Charged per attempt — not per success — because the solver
vendor bills every attempt regardless of crawl outcome.
"""
return int(attempts) * config.WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE
async def _spendable_micros(self, user_id: str | UUID) -> int:
"""Raw ``balance - reserved`` read, **ungated** by any billing flag.
Used by :meth:`check_balance` for combined (crawl + captcha) pre-flight,
where the relevant gate is decided by the caller, not by a single flag.
"""
from app.db import User
result = await self.session.execute(
@ -74,6 +90,39 @@ class WebCrawlCreditService:
balance, reserved = row
return balance - reserved
async def get_available_micros(self, user_id: str | UUID) -> int | None:
"""Return spendable credit in micro-USD (``balance - reserved``).
Returns ``None`` when crawl billing is disabled, which callers treat as
"unlimited" (no blocking, no charge).
"""
if not config.WEB_CRAWL_CREDIT_BILLING_ENABLED:
return None
return await self._spendable_micros(user_id)
async def check_balance(self, user_id: str | UUID, required_micros: int) -> None:
"""Raise :class:`InsufficientCreditsError` if the wallet can't cover
``required_micros`` (a combined crawl + worst-case captcha estimate).
Generic and **ungated** — the caller computes ``required_micros`` from
whichever billers are enabled and only calls this when at least one is.
No-op for a non-positive requirement.
"""
if required_micros <= 0:
return
available = await self._spendable_micros(user_id)
if required_micros > available:
raise InsufficientCreditsError(
message=(
"This run would exceed your available credit. "
f"Available: ${available / 1_000_000:.2f}, "
f"estimated need: ${required_micros / 1_000_000:.2f}. "
"Add more credits to continue."
),
balance_micros=available,
required_micros=required_micros,
)
async def check_credits(
self, user_id: str | UUID, estimated_successes: int = 1
) -> None:
@ -104,20 +153,14 @@ class WebCrawlCreditService:
required_micros=required,
)
async def charge_credits(
self, user_id: str | UUID, successes: int
) -> int | None:
"""Debit the wallet for ``successes`` successful crawls.
async def _apply_debit(self, user_id: str | UUID, cost_micros: int) -> int | None:
"""Debit ``cost_micros`` from the wallet and commit (shared by all
charge paths). Flushes any audit row the caller staged before this.
Commits the session (mirroring ``EtlCreditService.charge_credits``),
which also flushes any audit row staged by the caller before this call.
No-op when billing is disabled or ``successes <= 0``.
Returns the new balance in micros, or ``None`` when nothing was charged.
Mirrors ``EtlCreditService.charge_credits``' commit-then-refresh +
best-effort auto-reload. No-op for a non-positive cost.
"""
if not config.WEB_CRAWL_CREDIT_BILLING_ENABLED:
return None
if successes <= 0:
if cost_micros <= 0:
return None
from app.db import User
@ -127,8 +170,7 @@ class WebCrawlCreditService:
if not user:
raise ValueError(f"User with ID {user_id} not found")
cost = self.successes_to_micros(successes)
user.credit_micros_balance -= cost
user.credit_micros_balance -= cost_micros
await self.session.commit()
await self.session.refresh(user)
@ -141,3 +183,32 @@ class WebCrawlCreditService:
pass
return user.credit_micros_balance
async def charge_credits(
self, user_id: str | UUID, successes: int
) -> int | None:
"""Debit the wallet for ``successes`` successful crawls.
No-op when crawl billing is disabled or ``successes <= 0``. Returns the
new balance in micros, or ``None`` when nothing was charged.
"""
if not config.WEB_CRAWL_CREDIT_BILLING_ENABLED:
return None
if successes <= 0:
return None
return await self._apply_debit(user_id, self.successes_to_micros(successes))
async def charge_captcha(
self, user_id: str | UUID, attempts: int
) -> int | None:
"""Debit the wallet for ``attempts`` captcha solves (Phase 3d).
Per-attempt (not per-success): the solver charges for every attempt even
when the crawl ultimately fails. No-op when captcha billing is disabled
or ``attempts <= 0``.
"""
if not config.WEB_CRAWL_CAPTCHA_BILLING_ENABLED:
return None
if attempts <= 0:
return None
return await self._apply_debit(user_id, self.captcha_solves_to_micros(attempts))

61
surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py
View file

@ -17,11 +17,13 @@ from app.proprietary.web_crawler import (
CrawlOutcomeStatus,
WebCrawlerConnector,
)
from app.config import config
from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType
from app.services.etl_credit_service import InsufficientCreditsError
from app.services.task_logging_service import TaskLoggingService
from app.services.token_tracking_service import record_token_usage
from app.services.web_crawl_credit_service import WebCrawlCreditService
from app.utils.captcha import captcha_enabled
from app.utils.document_converters import (
create_document_chunks,
embed_text,
@ -175,19 +177,31 @@ async def index_crawled_urls(
return 0, "No URLs provided for indexing"
# =======================================================================
# Phase 3c: crawl-credit pre-flight. Bill the workspace OWNER. ``len(urls)``
# is a safe upper bound on successes, so a passing check guarantees the
# wallet can't go negative from this run. No-op when billing is disabled
# (self-hosted/OSS) — we also skip the owner lookup in that case.
# Phase 3c/3d: crawl-credit pre-flight. Bill the workspace OWNER. The
# estimate is a safe UPPER BOUND so the wallet can't strand mid-batch:
# crawl = len(urls) successes (3c; actual successes are <=)
# captcha = len(urls) * MAX_ATTEMPTS (3d; per-attempt, worst case)
# No-op (and no owner lookup) when both billers are disabled.
# =======================================================================
credit_service = WebCrawlCreditService(session)
owner_user_id = user_id
if credit_service.billing_enabled():
if credit_service.billing_enabled() or credit_service.captcha_billing_enabled():
owner_user_id = (
await _resolve_workspace_owner(session, workspace_id) or user_id
)
required_micros = 0
if credit_service.billing_enabled():
required_micros += credit_service.successes_to_micros(len(urls))
# Only reserve captcha budget when solving is actually enabled;
# with solving off, attempts can never happen, so reserving would
# wrongly block runs for captcha that will never be attempted.
if credit_service.captcha_billing_enabled() and captcha_enabled():
worst_case_attempts = len(urls) * config.CAPTCHA_MAX_ATTEMPTS_PER_URL
required_micros += credit_service.captcha_solves_to_micros(
worst_case_attempts
)
try:
await credit_service.check_credits(owner_user_id, len(urls))
await credit_service.check_balance(owner_user_id, required_micros)
except InsufficientCreditsError as credit_error:
await task_logger.log_task_failure(
log_entry,
@ -218,6 +232,11 @@ async def index_crawled_urls(
# extracted content, independent of downstream KB dedupe/unchanged
# bucketing. This is the billable unit Phase 3c meters on.
crawls_succeeded = 0
# Phase 3d: per-attempt captcha telemetry summed across the batch. Only
# accumulated when captcha billing is on (avoids touching the outcome
# field otherwise — and keeps MagicMock outcomes in tests arithmetic-safe).
total_captcha_attempts = 0
total_captcha_solved = 0
# Heartbeat tracking - update notification periodically to prevent appearing stuck
last_heartbeat_time = time.time()
@ -345,6 +364,13 @@ async def index_crawled_urls(
# Crawl the URL
outcome = await crawler.crawl_url(url)
# Phase 3d: count captcha attempts BEFORE the success branch — a
# failed solve still cost real solver money and must bill.
if credit_service.captcha_billing_enabled():
total_captcha_attempts += outcome.captcha_attempts
if outcome.captcha_solved:
total_captcha_solved += 1
if (
outcome.status is not CrawlOutcomeStatus.SUCCESS
or not outcome.result
@ -526,6 +552,29 @@ async def index_crawled_urls(
)
await credit_service.charge_credits(owner_user_id, crawls_succeeded)
# =======================================================================
# Phase 3d: charge the workspace owner per captcha *attempt* (separate
# unit; charged even when the crawl ultimately failed). Same stage +
# owner as the crawl charge above.
# =======================================================================
if credit_service.captcha_billing_enabled() and total_captcha_attempts > 0:
await record_token_usage(
session,
usage_type="web_crawl_captcha",
workspace_id=workspace_id,
user_id=owner_user_id,
cost_micros=credit_service.captcha_solves_to_micros(
total_captcha_attempts
),
call_details={
"attempts": total_captcha_attempts,
"solved": total_captcha_solved,
"connector_id": connector_id,
},
message_id=None,
)
await credit_service.charge_captcha(owner_user_id, total_captcha_attempts)
# Build warning message if there were issues
warning_parts = []
if duplicate_content_count > 0:

22
surfsense_backend/app/utils/captcha/__init__.py Normal file
View file

@ -0,0 +1,22 @@
"""App-wide captcha-solver configuration (Apache-2.0, generic glue).
This package holds only the **generic, vendor-agnostic** config resolution for
captcha solving — mirroring ``app/utils/proxy/`` (which stays Apache-2.0 even
though the crawler that consumes it is proprietary). The actual bypass logic
(challenge detection + token injection ``page_action``) lives in the separately
licensed ``app/proprietary/web_crawler/captcha.py``.
``captchatools`` is itself the multi-vendor registry (capmonster / 2captcha /
anticaptcha / capsolver / captchaai), so there is no provider hierarchy here —
just one env-driven :class:`CaptchaConfig` and a cheap ``captcha_enabled()``
gate callers check before constructing anything (mirrors
``WebCrawlCreditService.billing_enabled()``).
"""
from app.utils.captcha.config import (
CaptchaConfig,
captcha_enabled,
get_captcha_config,
)
__all__ = ["CaptchaConfig", "captcha_enabled", "get_captcha_config"]

54
surfsense_backend/app/utils/captcha/config.py Normal file
View file

@ -0,0 +1,54 @@
"""Env-resolved captcha-solver config (one app-wide config; no per-connector).
Resolved from :data:`app.config.config` only, mirroring ``03b``'s single
``PROXY_PROVIDER`` model. Off by default: ``captcha_enabled()`` is ``False``
unless both ``CAPTCHA_SOLVING_ENABLED=TRUE`` and an API key are present, so a
misconfigured deployment (flag on, key missing) never attempts a paid solve.
"""
from dataclasses import dataclass
from app.config import config
@dataclass(frozen=True)
class CaptchaConfig:
"""Immutable snapshot of the captcha-solver settings for one crawl."""
enabled: bool
solving_site: str
api_key: str | None
max_attempts_per_url: int
timeout_s: int
captcha_type_default: str
v3_min_score: float
v3_action: str
def get_captcha_config() -> CaptchaConfig:
"""Build a :class:`CaptchaConfig` from the current process config/env."""
api_key = config.CAPTCHA_SOLVER_API_KEY
flag_on = config.CAPTCHA_SOLVING_ENABLED
return CaptchaConfig(
# Effective-enabled requires a key too: a flag with no key would only
# produce ``ErrWrongAPIKey`` per attempt (and still risk an IP ban from
# the solver), so treat key-less config as disabled.
enabled=bool(flag_on and api_key),
solving_site=config.CAPTCHA_SOLVER_PROVIDER,
api_key=api_key,
max_attempts_per_url=config.CAPTCHA_MAX_ATTEMPTS_PER_URL,
timeout_s=config.CAPTCHA_SOLVE_TIMEOUT_S,
captcha_type_default=config.CAPTCHA_TYPE_DEFAULT,
v3_min_score=config.CAPTCHA_V3_MIN_SCORE,
v3_action=config.CAPTCHA_V3_ACTION,
)
def captcha_enabled() -> bool:
"""Cheap gate: is captcha solving effectively configured (flag + key)?
Callers check this before building the ``page_action`` or capturing the
crawl's proxy endpoint for the solver, so the stealth tier stays unchanged
when solving is off.
"""
return bool(config.CAPTCHA_SOLVING_ENABLED and config.CAPTCHA_SOLVER_API_KEY)