mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-06 22:12:12 +02:00
feat(crawler): captcha solving + per-attempt billing (Phase 3d)
Wire captchatools as the StealthyFetcher-tier page_action to detect, harvest (egressing from the crawl's own proxy IP), inject, and submit reCAPTCHA v2/v3 and hCaptcha tokens. Opt-in and off by default (zero attempts, zero cost). Licensing split: - Apache-2 app/utils/captcha/ holds the generic, vendor-agnostic config (CaptchaConfig + captcha_enabled() = flag AND key present). - Proprietary app/proprietary/web_crawler/captcha.py holds the bypass logic (detect/harvest/inject) plus a process-wide solver latch that halts solving on unrecoverable errors (no balance / bad key). Crawler: CrawlOutcome gains captcha_attempts/captcha_solved, surfaced via a per-call captcha_state dict threaded crawl_url -> _crawl_with_stealthy(_sync) and stamped onto every stealth terminal outcome. The stealth tier captures the proxy once and reuses it for both the fetch and the solver (IP-coherence). Billing: WebCrawlCreditService gains captcha_billing_enabled, captcha_solves_to_micros, charge_captcha, and a generic check_balance, sharing a single _apply_debit path. The indexer accumulates attempts (even on failed crawls), runs a combined crawl+captcha pre-flight, and posts a per-attempt owner charge as usage_type="web_crawl_captcha". The captcha worst-case is only reserved when solving is actually enabled, so a solving-off deployment is never blocked for captcha that can never run. Both chat scrape tools fold attempts into the current turn before the success/fail branch. Fully config-driven prices; no migration. New unit tests cover the config, factory (detection/latch/timeout/cap), credit service, indexer wiring, and the chat fold. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
aad366ba13
commit
bdacea8b6e
19 changed files with 1494 additions and 51 deletions
|
|
@ -266,6 +266,15 @@ MICROS_PER_PAGE=1000
|
|||
# WEB_CRAWL_CREDIT_BILLING_ENABLED=FALSE
|
||||
# WEB_CRAWL_MICROS_PER_SUCCESS=1000
|
||||
|
||||
# Phase 3d: bill captcha solves as a SEPARATE per-attempt unit (the solver
|
||||
# charges per attempt regardless of crawl success). Independent of the crawl
|
||||
# flag above. Price config-driven (no hardcoded rate):
|
||||
# WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE = round(USD_per_1000_solves * 1_000)
|
||||
# 3000 == $3 / 1000 solves (default) | 5000 == $5/1000
|
||||
# Set with margin over your solver vendor's per-attempt price.
|
||||
# WEB_CRAWL_CAPTCHA_BILLING_ENABLED=FALSE
|
||||
# WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000
|
||||
|
||||
# Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50.
|
||||
CREDIT_LOW_BALANCE_WARNING_MICROS=500000
|
||||
|
||||
|
|
@ -340,6 +349,26 @@ TURNSTILE_SECRET_KEY=
|
|||
# CUSTOM_PROXY_URL=http://user:pass@host:port
|
||||
# CUSTOM_PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port
|
||||
|
||||
# =====================================================================
|
||||
# Captcha solving (Phase 3d) — LAST-resort bypass tier via captchatools.
|
||||
# Only fires on the stealth browser tier when a sitekey is detected AND
|
||||
# CAPTCHA_SOLVING_ENABLED=TRUE. Cloudflare Turnstile is already solved free
|
||||
# in-framework (no config needed). Off by default => zero attempts, zero cost.
|
||||
# NOTE: automated solving may violate a target site's ToS — opt-in, public
|
||||
# data only (no logged-in bypass). captchatools is itself the vendor registry.
|
||||
# CAPTCHA_SOLVING_ENABLED=FALSE
|
||||
# solving_site: capmonster | 2captcha | anticaptcha | capsolver | captchaai
|
||||
# CAPTCHA_SOLVER_PROVIDER=capsolver
|
||||
# CAPTCHA_SOLVER_API_KEY=
|
||||
# Per-URL solve cap (bounds solver spend on a hostile page).
|
||||
# CAPTCHA_MAX_ATTEMPTS_PER_URL=1
|
||||
# CAPTCHA_SOLVE_TIMEOUT_S=120
|
||||
# Default type when detection is ambiguous: v2 | v3 | hcaptcha
|
||||
# CAPTCHA_TYPE_DEFAULT=v2
|
||||
# reCAPTCHA v3 tuning (only used for v3 challenges).
|
||||
# CAPTCHA_V3_MIN_SCORE=0.7
|
||||
# CAPTCHA_V3_ACTION=verify
|
||||
|
||||
# File Parser Service
|
||||
ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING
|
||||
UNSTRUCTURED_API_KEY=Tpu3P0U8iy
|
||||
|
|
|
|||
|
|
@ -55,6 +55,36 @@ def _bill_successful_scrape() -> None:
|
|||
)
|
||||
|
||||
|
||||
def _bill_captcha_attempts(outcome) -> None:
|
||||
"""Fold captcha solve attempts (Phase 3d) into the current chat turn's bill.
|
||||
|
||||
Per *attempt*, not per success: a solve that didn't rescue the crawl still
|
||||
cost real solver money, so this runs before the success/failure branch.
|
||||
Mirrors :func:`_bill_successful_scrape` (rides the turn accumulator, settles
|
||||
at ``finalize_credit``; record-only on free/anonymous turns). No-op when
|
||||
captcha billing is off, there were no attempts, or no turn is active.
|
||||
"""
|
||||
from app.services.token_tracking_service import get_current_accumulator
|
||||
from app.services.web_crawl_credit_service import WebCrawlCreditService
|
||||
|
||||
if not WebCrawlCreditService.captcha_billing_enabled():
|
||||
return
|
||||
attempts = getattr(outcome, "captcha_attempts", 0) or 0
|
||||
if attempts <= 0:
|
||||
return
|
||||
acc = get_current_accumulator()
|
||||
if acc is None:
|
||||
return
|
||||
acc.add(
|
||||
model="web_crawl_captcha",
|
||||
prompt_tokens=0,
|
||||
completion_tokens=0,
|
||||
total_tokens=0,
|
||||
cost_micros=WebCrawlCreditService.captcha_solves_to_micros(attempts),
|
||||
call_kind="web_crawl_captcha",
|
||||
)
|
||||
|
||||
|
||||
def extract_domain(url: str) -> str:
|
||||
"""Extract the domain from a URL."""
|
||||
try:
|
||||
|
|
@ -258,6 +288,9 @@ def create_scrape_webpage_tool():
|
|||
connector = WebCrawlerConnector()
|
||||
outcome = await connector.crawl_url(url)
|
||||
|
||||
# 03d: bill any captcha attempts (even if the crawl ultimately failed).
|
||||
_bill_captcha_attempts(outcome)
|
||||
|
||||
if outcome.status is not CrawlOutcomeStatus.SUCCESS or not outcome.result:
|
||||
return {
|
||||
"id": scrape_id,
|
||||
|
|
|
|||
|
|
@ -49,6 +49,36 @@ def _bill_successful_scrape() -> None:
|
|||
)
|
||||
|
||||
|
||||
def _bill_captcha_attempts(outcome) -> None:
|
||||
"""Fold captcha solve attempts (Phase 3d) into the current chat turn's bill.
|
||||
|
||||
Per *attempt*, not per success: a solve that didn't rescue the crawl still
|
||||
cost real solver money, so this runs before the success/failure branch.
|
||||
Mirrors :func:`_bill_successful_scrape` (rides the turn accumulator, settles
|
||||
at ``finalize_credit``; record-only on free/anonymous turns). No-op when
|
||||
captcha billing is off, there were no attempts, or no turn is active.
|
||||
"""
|
||||
from app.services.token_tracking_service import get_current_accumulator
|
||||
from app.services.web_crawl_credit_service import WebCrawlCreditService
|
||||
|
||||
if not WebCrawlCreditService.captcha_billing_enabled():
|
||||
return
|
||||
attempts = getattr(outcome, "captcha_attempts", 0) or 0
|
||||
if attempts <= 0:
|
||||
return
|
||||
acc = get_current_accumulator()
|
||||
if acc is None:
|
||||
return
|
||||
acc.add(
|
||||
model="web_crawl_captcha",
|
||||
prompt_tokens=0,
|
||||
completion_tokens=0,
|
||||
total_tokens=0,
|
||||
cost_micros=WebCrawlCreditService.captcha_solves_to_micros(attempts),
|
||||
call_kind="web_crawl_captcha",
|
||||
)
|
||||
|
||||
|
||||
def extract_domain(url: str) -> str:
|
||||
"""Extract the domain from a URL."""
|
||||
try:
|
||||
|
|
@ -252,6 +282,9 @@ def create_scrape_webpage_tool():
|
|||
connector = WebCrawlerConnector()
|
||||
outcome = await connector.crawl_url(url)
|
||||
|
||||
# 03d: bill any captcha attempts (even if the crawl ultimately failed).
|
||||
_bill_captcha_attempts(outcome)
|
||||
|
||||
if outcome.status is not CrawlOutcomeStatus.SUCCESS or not outcome.result:
|
||||
return {
|
||||
"id": scrape_id,
|
||||
|
|
|
|||
|
|
@ -670,6 +670,21 @@ class Config:
|
|||
os.getenv("WEB_CRAWL_MICROS_PER_SUCCESS", "1000")
|
||||
)
|
||||
|
||||
# Phase 3d captcha-solve billing. Captcha can't ride the per-success crawl
|
||||
# meter above: the solver charges per *attempt* regardless of whether the
|
||||
# crawl ultimately succeeds, so solves are metered as a SEPARATE per-attempt
|
||||
# unit (usage_type="web_crawl_captcha"). Off by default; independent of the
|
||||
# crawl-billing flag. Price is config-driven (no hardcoded rate):
|
||||
# WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE = round(USD_per_1000_solves * 1_000)
|
||||
# $3/1000 -> 3000 (default) | $5/1000 -> 5000
|
||||
# Set with margin over the solver vendor's per-attempt price.
|
||||
WEB_CRAWL_CAPTCHA_BILLING_ENABLED = (
|
||||
os.getenv("WEB_CRAWL_CAPTCHA_BILLING_ENABLED", "FALSE").upper() == "TRUE"
|
||||
)
|
||||
WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE = int(
|
||||
os.getenv("WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE", "3000")
|
||||
)
|
||||
|
||||
# Low-balance WARNING threshold (micro-USD). Surfaced by the quota service
|
||||
# so the UI can nudge the user to top up / enable auto-reload. $0.50.
|
||||
CREDIT_LOW_BALANCE_WARNING_MICROS = int(
|
||||
|
|
@ -1041,6 +1056,36 @@ class Config:
|
|||
CUSTOM_PROXY_URL = os.getenv("CUSTOM_PROXY_URL")
|
||||
CUSTOM_PROXY_URLS = os.getenv("CUSTOM_PROXY_URLS")
|
||||
|
||||
# =====================================================================
|
||||
# Phase 3d — Captcha solving (reCAPTCHA v2/v3, hCaptcha) via captchatools.
|
||||
# The LAST-resort bypass tier: only fires on the StealthyFetcher browser
|
||||
# tier, only when a sitekey is detected, and only when explicitly enabled.
|
||||
# Cloudflare Turnstile is already handled free in-framework (03a), NOT here.
|
||||
# One app-wide config (mirrors the single PROXY_PROVIDER model) — no
|
||||
# per-connector config. Off by default => zero solve attempts, zero cost.
|
||||
# Solving may violate a target site's ToS; treat as opt-in/owner-acknowledged
|
||||
# and public-data only (no logged-in bypass).
|
||||
# =====================================================================
|
||||
CAPTCHA_SOLVING_ENABLED = (
|
||||
os.getenv("CAPTCHA_SOLVING_ENABLED", "FALSE").upper() == "TRUE"
|
||||
)
|
||||
# captchatools "solving_site": capmonster | 2captcha | anticaptcha |
|
||||
# capsolver | captchaai. captchatools is itself the provider registry, so we
|
||||
# do not rebuild a vendor hierarchy.
|
||||
CAPTCHA_SOLVER_PROVIDER = os.getenv("CAPTCHA_SOLVER_PROVIDER", "capsolver")
|
||||
CAPTCHA_SOLVER_API_KEY = os.getenv("CAPTCHA_SOLVER_API_KEY")
|
||||
# Per-URL solve cap so one hostile page can't burn unbounded solver credit.
|
||||
CAPTCHA_MAX_ATTEMPTS_PER_URL = int(
|
||||
os.getenv("CAPTCHA_MAX_ATTEMPTS_PER_URL", "1")
|
||||
)
|
||||
# Abort a single solve after this many seconds (solves take 10-60s).
|
||||
CAPTCHA_SOLVE_TIMEOUT_S = int(os.getenv("CAPTCHA_SOLVE_TIMEOUT_S", "120"))
|
||||
# Default captcha type when detection is ambiguous: v2 | v3 | hcaptcha.
|
||||
CAPTCHA_TYPE_DEFAULT = os.getenv("CAPTCHA_TYPE_DEFAULT", "v2")
|
||||
# reCAPTCHA v3 tuning (only used for v3 challenges).
|
||||
CAPTCHA_V3_MIN_SCORE = float(os.getenv("CAPTCHA_V3_MIN_SCORE", "0.7"))
|
||||
CAPTCHA_V3_ACTION = os.getenv("CAPTCHA_V3_ACTION", "verify")
|
||||
|
||||
# Litellm TTS Configuration
|
||||
TTS_SERVICE = os.getenv("TTS_SERVICE")
|
||||
TTS_SERVICE_API_BASE = os.getenv("TTS_SERVICE_API_BASE")
|
||||
|
|
|
|||
326
surfsense_backend/app/proprietary/web_crawler/captcha.py
Normal file
326
surfsense_backend/app/proprietary/web_crawler/captcha.py
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
# SurfSense proprietary crawler engine.
|
||||
#
|
||||
# This module is part of the ``app.proprietary`` package and is licensed
|
||||
# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
|
||||
# Do not relicense or redistribute this file under Apache-2.0.
|
||||
"""Captcha detection + token-injection ``page_action`` (Phase 3d).
|
||||
|
||||
This is the **bypass logic** (hence proprietary); the generic, vendor-agnostic
|
||||
config lives in the Apache-2.0 ``app/utils/captcha/`` package. ``captchatools``
|
||||
is the multi-vendor solver registry — we only detect the challenge, harvest a
|
||||
token egressing from **the crawl's own proxy IP** (token IP-binding), inject it,
|
||||
and submit.
|
||||
|
||||
Why a closure cell: Scrapling runs ``page_action`` after navigation but
|
||||
**swallows its exceptions and discards its return value** (see
|
||||
``references/Scrapling`` engine ``_stealth.py``). So the only way to surface
|
||||
"did we solve / how many attempts" back to the crawler (for per-attempt billing
|
||||
and the retry cap) is to mutate a caller-owned ``state`` dict.
|
||||
|
||||
Why a process latch: the solver README warns that hammering it while out of
|
||||
balance can get the IP **temporarily banned**, so ``ErrNoBalance`` /
|
||||
``ErrWrongAPIKey`` latch solving OFF for the rest of the process (config/balance
|
||||
won't change without a restart). ``reset_solver_latch()`` clears it for tests.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from app.utils.captcha import CaptchaConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CAPTCHA_LOG = "[webcrawler][captcha]"
|
||||
|
||||
# Process-wide kill switch tripped on unrecoverable solver errors (no balance /
|
||||
# bad key). Module-level by design: it must persist across the per-URL
|
||||
# ``page_action`` instances created during one crawl run.
|
||||
_solver_latched = False
|
||||
|
||||
|
||||
def reset_solver_latch() -> None:
|
||||
"""Clear the process solver latch (test seam / explicit re-enable)."""
|
||||
global _solver_latched
|
||||
_solver_latched = False
|
||||
|
||||
|
||||
def solver_latched() -> bool:
|
||||
return _solver_latched
|
||||
|
||||
|
||||
def _latch_solver(reason: str) -> None:
|
||||
global _solver_latched
|
||||
_solver_latched = True
|
||||
logger.warning("%s solver latched OFF for this process: %s", _CAPTCHA_LOG, reason)
|
||||
|
||||
|
||||
# --- detection -------------------------------------------------------------
|
||||
|
||||
# (challenge_type, css_selector_for_widget). Order = detection priority.
|
||||
_WIDGET_SELECTORS = (
|
||||
("v2", ".g-recaptcha[data-sitekey]"),
|
||||
("hcaptcha", ".h-captcha[data-sitekey]"),
|
||||
)
|
||||
_IFRAME_PATTERNS = (
|
||||
("v2", re.compile(r"google\.com/recaptcha", re.I)),
|
||||
("hcaptcha", re.compile(r"hcaptcha\.com", re.I)),
|
||||
)
|
||||
_SITEKEY_IN_SRC = re.compile(r"[?&]k=([\w-]+)")
|
||||
_RENDER_SITEKEY = re.compile(r"render=([\w-]+)")
|
||||
|
||||
|
||||
def detect_challenge(page: Any, cfg: CaptchaConfig) -> tuple[str, str] | None:
|
||||
"""Return ``(challenge_type, sitekey)`` if a solvable challenge is present.
|
||||
|
||||
``challenge_type`` is one of ``v2`` / ``v3`` / ``hcaptcha``. Returns
|
||||
``None`` when nothing solvable is found (the crawler then proceeds to
|
||||
``wait_selector`` / extraction unchanged). Detection is defensive: any DOM
|
||||
access raising is treated as "not found" rather than aborting the fetch.
|
||||
"""
|
||||
try:
|
||||
for ctype, selector in _WIDGET_SELECTORS:
|
||||
el = page.query_selector(selector)
|
||||
if el is not None:
|
||||
sitekey = el.get_attribute("data-sitekey")
|
||||
if sitekey:
|
||||
return ctype, sitekey
|
||||
|
||||
# Iframe fallback: widget rendered inside an iframe whose src carries
|
||||
# the sitekey as ``?k=...``.
|
||||
for frame_el in page.query_selector_all("iframe[src]") or []:
|
||||
src = frame_el.get_attribute("src") or ""
|
||||
for ctype, pat in _IFRAME_PATTERNS:
|
||||
if pat.search(src):
|
||||
m = _SITEKEY_IN_SRC.search(src)
|
||||
if m:
|
||||
return ctype, m.group(1)
|
||||
|
||||
# reCAPTCHA v3: no widget; sitekey rides the api.js ``?render=<key>``.
|
||||
if cfg.captcha_type_default == "v3":
|
||||
for script_el in page.query_selector_all("script[src]") or []:
|
||||
src = script_el.get_attribute("src") or ""
|
||||
if "recaptcha" in src.lower():
|
||||
m = _RENDER_SITEKEY.search(src)
|
||||
if m and m.group(1) != "explicit":
|
||||
return "v3", m.group(1)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("%s detection error (treated as no challenge): %s", _CAPTCHA_LOG, exc)
|
||||
return None
|
||||
|
||||
|
||||
# --- proxy / harvest -------------------------------------------------------
|
||||
|
||||
|
||||
def proxy_url_to_captchatools(proxy_url: str | None) -> str | None:
|
||||
"""Reformat ``http://user:pass@host:port`` -> ``host:port:user:pass``.
|
||||
|
||||
captchatools wants the colon-delimited form. Returns ``None`` for a missing
|
||||
or unparseable proxy so the solver harvests proxyless (the token may then be
|
||||
IP-mismatched, but that's better than crashing the fetch).
|
||||
"""
|
||||
if not proxy_url:
|
||||
return None
|
||||
try:
|
||||
p = urlsplit(proxy_url)
|
||||
if not p.hostname or not p.port:
|
||||
return None
|
||||
if p.username and p.password:
|
||||
return f"{p.hostname}:{p.port}:{p.username}:{p.password}"
|
||||
return f"{p.hostname}:{p.port}"
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
def _harvest_token(
|
||||
cfg: CaptchaConfig,
|
||||
challenge_type: str,
|
||||
sitekey: str,
|
||||
page_url: str,
|
||||
proxy: str | None,
|
||||
user_agent: str | None,
|
||||
) -> str | None:
|
||||
"""Harvest a token from the solver. Raises on solver errors so the caller
|
||||
can latch on the unrecoverable ones; returns ``None`` on an empty token.
|
||||
"""
|
||||
import captchatools
|
||||
|
||||
harvester = captchatools.new_harvester(
|
||||
api_key=cfg.api_key,
|
||||
solving_site=cfg.solving_site,
|
||||
sitekey=sitekey,
|
||||
captcha_url=page_url,
|
||||
captcha_type=challenge_type,
|
||||
min_score=cfg.v3_min_score,
|
||||
action=cfg.v3_action,
|
||||
)
|
||||
token = harvester.get_token(
|
||||
proxy=proxy,
|
||||
proxy_type="HTTP",
|
||||
user_agent=user_agent,
|
||||
)
|
||||
return token or None
|
||||
|
||||
|
||||
# --- injection -------------------------------------------------------------
|
||||
|
||||
# Sets the response token into the right textarea(s) and fires the JS callbacks
|
||||
# registered by the widget, then submits the closest form. Best-effort: returns
|
||||
# nothing; success is judged by the crawler's post-fetch extraction.
|
||||
_INJECT_JS = r"""
|
||||
(args) => {
|
||||
const { token, ctype } = args;
|
||||
const names = ctype === 'hcaptcha'
|
||||
? ['h-captcha-response', 'g-recaptcha-response']
|
||||
: ['g-recaptcha-response'];
|
||||
for (const name of names) {
|
||||
let ta = document.querySelector('textarea[name="' + name + '"]')
|
||||
|| document.getElementById(name);
|
||||
if (!ta) {
|
||||
ta = document.createElement('textarea');
|
||||
ta.name = name; ta.id = name;
|
||||
ta.style.display = 'none';
|
||||
document.body.appendChild(ta);
|
||||
}
|
||||
ta.value = token; ta.innerHTML = token;
|
||||
ta.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
ta.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}
|
||||
// Fire grecaptcha client callbacks when present (v2 invisible / explicit).
|
||||
try {
|
||||
const cfg = window.___grecaptcha_cfg;
|
||||
if (cfg && cfg.clients) {
|
||||
for (const id in cfg.clients) {
|
||||
const client = cfg.clients[id];
|
||||
for (const k in client) {
|
||||
const o = client[k];
|
||||
if (o && typeof o === 'object') {
|
||||
for (const kk in o) {
|
||||
const cb = o[kk];
|
||||
if (cb && cb.callback && typeof cb.callback === 'function') {
|
||||
try { cb.callback(token); } catch (e) {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {}
|
||||
// Submit the form containing the response field, if any.
|
||||
const field = document.querySelector('textarea[name="g-recaptcha-response"], textarea[name="h-captcha-response"]');
|
||||
const form = field ? field.closest('form') : document.querySelector('form');
|
||||
if (form) { try { form.requestSubmit ? form.requestSubmit() : form.submit(); } catch (e) {} }
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def _inject_and_submit(page: Any, challenge_type: str, token: str) -> None:
|
||||
try:
|
||||
page.evaluate(_INJECT_JS, {"token": token, "ctype": challenge_type})
|
||||
try:
|
||||
page.wait_for_load_state("networkidle", timeout=15000)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("%s injection error: %s", _CAPTCHA_LOG, exc)
|
||||
|
||||
|
||||
# --- factory ---------------------------------------------------------------
|
||||
|
||||
|
||||
def build_captcha_page_action(
|
||||
state: dict[str, Any],
|
||||
proxy_url: str | None,
|
||||
cfg: CaptchaConfig,
|
||||
) -> Callable[[Any], Any] | None:
|
||||
"""Build the sync ``page_action`` for ``StealthyFetcher.fetch``.
|
||||
|
||||
``state`` is mutated in place: ``attempts`` (int) and ``solved`` (bool). The
|
||||
crawler reads it back after the fetch to bill per attempt and enforce the
|
||||
per-URL cap. Returns ``None`` when solving is disabled/latched so the caller
|
||||
can skip wiring it entirely (keeping the stealth tier unchanged).
|
||||
"""
|
||||
if not cfg.enabled or solver_latched():
|
||||
return None
|
||||
|
||||
captcha_proxy = proxy_url_to_captchatools(proxy_url)
|
||||
|
||||
def page_action(page: Any) -> Any:
|
||||
# Hard cap: never exceed the per-URL budget even across proxy-retry
|
||||
# re-runs that share this ``state``.
|
||||
if state.get("attempts", 0) >= cfg.max_attempts_per_url:
|
||||
return page
|
||||
if solver_latched():
|
||||
return page
|
||||
|
||||
detected = detect_challenge(page, cfg)
|
||||
if detected is None:
|
||||
return page # nothing to solve; let extraction proceed
|
||||
challenge_type, sitekey = detected
|
||||
|
||||
page_url = getattr(page, "url", "") or ""
|
||||
try:
|
||||
user_agent = page.evaluate("() => navigator.userAgent")
|
||||
except Exception: # noqa: BLE001
|
||||
user_agent = None
|
||||
|
||||
# This counts as an attempt the moment we call the (paid) solver.
|
||||
state["attempts"] = state.get("attempts", 0) + 1
|
||||
logger.info(
|
||||
"%s solving type=%s site=%s attempt=%d",
|
||||
_CAPTCHA_LOG,
|
||||
challenge_type,
|
||||
sitekey[:12],
|
||||
state["attempts"],
|
||||
)
|
||||
|
||||
# Run the (blocking) solver in a worker so the timeout can actually
|
||||
# ABANDON a slow solve. NOTE: do NOT use ``with ThreadPoolExecutor`` —
|
||||
# its ``__exit__`` joins the worker (``wait=True``), which would block
|
||||
# for the full solve and defeat the timeout. Shut down non-blocking.
|
||||
executor = ThreadPoolExecutor(max_workers=1)
|
||||
future = executor.submit(
|
||||
_harvest_token,
|
||||
cfg,
|
||||
challenge_type,
|
||||
sitekey,
|
||||
page_url,
|
||||
captcha_proxy,
|
||||
user_agent,
|
||||
)
|
||||
try:
|
||||
token = future.result(timeout=cfg.timeout_s)
|
||||
except FuturesTimeout:
|
||||
logger.warning("%s solve timed out after %ss", _CAPTCHA_LOG, cfg.timeout_s)
|
||||
return page
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if _is_unrecoverable(exc):
|
||||
_latch_solver(repr(exc))
|
||||
else:
|
||||
logger.warning("%s solve error: %s", _CAPTCHA_LOG, exc)
|
||||
return page
|
||||
finally:
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
if not token:
|
||||
return page
|
||||
|
||||
_inject_and_submit(page, challenge_type, token)
|
||||
state["solved"] = True
|
||||
logger.info("%s solved type=%s site=%s", _CAPTCHA_LOG, challenge_type, sitekey[:12])
|
||||
return page
|
||||
|
||||
return page_action
|
||||
|
||||
|
||||
def _is_unrecoverable(exc: Exception) -> bool:
|
||||
"""True for solver errors that must latch solving off (no balance / bad key).
|
||||
|
||||
Matched by class name to avoid a hard import of ``captchatools.exceptions``
|
||||
at module load (the dep is optional / lazily imported).
|
||||
"""
|
||||
name = type(exc).__name__.lower()
|
||||
return "nobalance" in name or "wrongapikey" in name or "apikey" in name
|
||||
|
|
@ -33,6 +33,8 @@ import validators
|
|||
from scrapling.engines.toolbelt import is_proxy_error
|
||||
from scrapling.fetchers import AsyncFetcher, DynamicFetcher, StealthyFetcher
|
||||
|
||||
from app.proprietary.web_crawler.captcha import build_captcha_page_action
|
||||
from app.utils.captcha import captcha_enabled, get_captcha_config
|
||||
from app.utils.proxy import get_proxy_url, is_pool_backed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -56,14 +58,20 @@ class CrawlOutcome:
|
|||
The **billable success predicate is single-sourced**:
|
||||
``status == CrawlOutcomeStatus.SUCCESS`` (Phase 3c meters on it). Picking a
|
||||
dataclass over a tuple lets later subplans append fields without breaking
|
||||
callers (03d adds ``captcha_attempts``/``captcha_solved`` for per-attempt
|
||||
billing; 03e's block classifier can attach a ``block_type``).
|
||||
callers (03e's block classifier can attach a ``block_type``).
|
||||
|
||||
Phase 3d captcha fields are surfaced here so per-attempt billing can read
|
||||
them off the outcome regardless of crawl SUCCESS (the solver charges per
|
||||
*attempt*). They are populated only by the StealthyFetcher tier when captcha
|
||||
solving is enabled; every other path leaves the defaults (0 / False).
|
||||
"""
|
||||
|
||||
status: CrawlOutcomeStatus
|
||||
result: dict[str, Any] | None = None
|
||||
error: str | None = None
|
||||
tier: str | None = None
|
||||
captcha_attempts: int = 0
|
||||
captcha_solved: bool = False
|
||||
|
||||
|
||||
class WebCrawlerConnector:
|
||||
|
|
@ -89,6 +97,11 @@ class WebCrawlerConnector:
|
|||
- crawler_type: Identifier of the tier that produced the content
|
||||
"""
|
||||
total_start = time.perf_counter()
|
||||
# Per-call captcha telemetry (03d). Mutated only by the StealthyFetcher
|
||||
# tier's page_action; stamped onto the returned outcome so per-attempt
|
||||
# billing can read it regardless of crawl SUCCESS. Per-call (not on
|
||||
# ``self``) => safe under concurrent ``crawl_url`` calls.
|
||||
captcha_state: dict[str, Any] = {"attempts": 0, "solved": False}
|
||||
try:
|
||||
if not validators.url(url):
|
||||
return CrawlOutcome(
|
||||
|
|
@ -167,7 +180,8 @@ class WebCrawlerConnector:
|
|||
try:
|
||||
logger.info(f"[webcrawler] Using Scrapling StealthyFetcher for: {url}")
|
||||
result = await self._run_tier_with_proxy_retry(
|
||||
"scrapling-stealthy", lambda: self._crawl_with_stealthy(url)
|
||||
"scrapling-stealthy",
|
||||
lambda: self._crawl_with_stealthy(url, captcha_state),
|
||||
)
|
||||
if result:
|
||||
self._log_tier_outcome(
|
||||
|
|
@ -178,6 +192,8 @@ class WebCrawlerConnector:
|
|||
status=CrawlOutcomeStatus.SUCCESS,
|
||||
result=result,
|
||||
tier="scrapling-stealthy",
|
||||
captcha_attempts=captcha_state["attempts"],
|
||||
captcha_solved=captcha_state["solved"],
|
||||
)
|
||||
reached_without_content = True
|
||||
errors.append("Scrapling stealthy: empty extraction")
|
||||
|
|
@ -201,10 +217,14 @@ class WebCrawlerConnector:
|
|||
return CrawlOutcome(
|
||||
status=CrawlOutcomeStatus.EMPTY,
|
||||
error=f"No content extracted for {url}. {'; '.join(errors)}",
|
||||
captcha_attempts=captcha_state["attempts"],
|
||||
captcha_solved=captcha_state["solved"],
|
||||
)
|
||||
return CrawlOutcome(
|
||||
status=CrawlOutcomeStatus.FAILED,
|
||||
error=f"All crawl methods failed for {url}. {'; '.join(errors)}",
|
||||
captcha_attempts=captcha_state["attempts"],
|
||||
captcha_solved=captcha_state["solved"],
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
|
@ -212,6 +232,8 @@ class WebCrawlerConnector:
|
|||
return CrawlOutcome(
|
||||
status=CrawlOutcomeStatus.FAILED,
|
||||
error=f"Error crawling URL {url}: {e!s}",
|
||||
captcha_attempts=captcha_state["attempts"],
|
||||
captcha_solved=captcha_state["solved"],
|
||||
)
|
||||
|
||||
async def _run_tier_with_proxy_retry(
|
||||
|
|
@ -356,29 +378,56 @@ class WebCrawlerConnector:
|
|||
status=getattr(page, "status", None),
|
||||
)
|
||||
|
||||
async def _crawl_with_stealthy(self, url: str) -> dict[str, Any] | None:
|
||||
async def _crawl_with_stealthy(
|
||||
self, url: str, captcha_state: dict[str, Any] | None = None
|
||||
) -> dict[str, Any] | None:
|
||||
"""
|
||||
Crawl URL using Scrapling's StealthyFetcher (patchright-Chromium) + Trafilatura.
|
||||
|
||||
Last-resort tier with anti-bot features. Runs the sync fetch in a worker
|
||||
thread for the same event-loop-safety reasons as DynamicFetcher. Falls
|
||||
back to the raw HTML when Trafilatura extraction is empty.
|
||||
"""
|
||||
return await asyncio.to_thread(self._crawl_with_stealthy_sync, url)
|
||||
|
||||
def _crawl_with_stealthy_sync(self, url: str) -> dict[str, Any] | None:
|
||||
``captcha_state`` (03d) is mutated in place by the captcha page_action
|
||||
(attempts/solved) so ``crawl_url`` can surface it on the outcome.
|
||||
"""
|
||||
return await asyncio.to_thread(
|
||||
self._crawl_with_stealthy_sync, url, captcha_state
|
||||
)
|
||||
|
||||
def _crawl_with_stealthy_sync(
|
||||
self, url: str, captcha_state: dict[str, Any] | None = None
|
||||
) -> dict[str, Any] | None:
|
||||
"""Synchronous StealthyFetcher crawl executed in a worker thread."""
|
||||
fetch_start = time.perf_counter()
|
||||
# Capture the proxy endpoint ONCE so the captcha solver egresses from the
|
||||
# SAME IP as this fetch (tokens are IP-bound). Re-calling get_proxy_url()
|
||||
# inside the page_action would rotate a pool-backed provider to a
|
||||
# different IP and invalidate the token (03d proxy-coherence caveat).
|
||||
proxy = get_proxy_url()
|
||||
|
||||
# Build the captcha page_action only when solving is enabled (and not
|
||||
# process-latched). ``None`` => stealth tier behaves exactly as before.
|
||||
page_action = None
|
||||
if captcha_state is not None and captcha_enabled():
|
||||
page_action = build_captcha_page_action(
|
||||
captcha_state, proxy, get_captcha_config()
|
||||
)
|
||||
|
||||
# ``solve_cloudflare=True`` runs the full Turnstile/Interstitial challenge
|
||||
# loop; scoped to this last-resort tier only (it spins up the browser).
|
||||
page = StealthyFetcher.fetch(
|
||||
url,
|
||||
headless=True,
|
||||
network_idle=True,
|
||||
block_ads=True,
|
||||
solve_cloudflare=True,
|
||||
proxy=get_proxy_url(),
|
||||
)
|
||||
# Scrapling runs solve_cloudflare BEFORE page_action, so Cloudflare is
|
||||
# cleared first, then our captcha injector runs.
|
||||
fetch_kwargs: dict[str, Any] = {
|
||||
"headless": True,
|
||||
"network_idle": True,
|
||||
"block_ads": True,
|
||||
"solve_cloudflare": True,
|
||||
"proxy": proxy,
|
||||
}
|
||||
if page_action is not None:
|
||||
fetch_kwargs["page_action"] = page_action
|
||||
page = StealthyFetcher.fetch(url, **fetch_kwargs)
|
||||
fetch_ms = (time.perf_counter() - fetch_start) * 1000
|
||||
return self._build_result(
|
||||
page.html_content,
|
||||
|
|
|
|||
|
|
@ -51,15 +51,31 @@ class WebCrawlCreditService:
|
|||
"""
|
||||
return int(successes) * config.WEB_CRAWL_MICROS_PER_SUCCESS
|
||||
|
||||
async def get_available_micros(self, user_id: str | UUID) -> int | None:
|
||||
"""Return spendable credit in micro-USD (``balance - reserved``).
|
||||
@staticmethod
|
||||
def captcha_billing_enabled() -> bool:
|
||||
"""Phase 3d: whether captcha *solves* are metered.
|
||||
|
||||
Returns ``None`` when crawl billing is disabled, which callers treat as
|
||||
"unlimited" (no blocking, no charge).
|
||||
Independent of crawl billing: a deployment may bill solves without
|
||||
billing crawls (or vice-versa). Off by default.
|
||||
"""
|
||||
if not config.WEB_CRAWL_CREDIT_BILLING_ENABLED:
|
||||
return None
|
||||
return config.WEB_CRAWL_CAPTCHA_BILLING_ENABLED
|
||||
|
||||
@staticmethod
|
||||
def captcha_solves_to_micros(attempts: int) -> int:
|
||||
"""Convert a captcha *attempt* count to USD micro-credits.
|
||||
|
||||
Reads ``config.WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE`` (single source of
|
||||
truth). Charged per attempt — not per success — because the solver
|
||||
vendor bills every attempt regardless of crawl outcome.
|
||||
"""
|
||||
return int(attempts) * config.WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE
|
||||
|
||||
async def _spendable_micros(self, user_id: str | UUID) -> int:
|
||||
"""Raw ``balance - reserved`` read, **ungated** by any billing flag.
|
||||
|
||||
Used by :meth:`check_balance` for combined (crawl + captcha) pre-flight,
|
||||
where the relevant gate is decided by the caller, not by a single flag.
|
||||
"""
|
||||
from app.db import User
|
||||
|
||||
result = await self.session.execute(
|
||||
|
|
@ -74,6 +90,39 @@ class WebCrawlCreditService:
|
|||
balance, reserved = row
|
||||
return balance - reserved
|
||||
|
||||
async def get_available_micros(self, user_id: str | UUID) -> int | None:
|
||||
"""Return spendable credit in micro-USD (``balance - reserved``).
|
||||
|
||||
Returns ``None`` when crawl billing is disabled, which callers treat as
|
||||
"unlimited" (no blocking, no charge).
|
||||
"""
|
||||
if not config.WEB_CRAWL_CREDIT_BILLING_ENABLED:
|
||||
return None
|
||||
return await self._spendable_micros(user_id)
|
||||
|
||||
async def check_balance(self, user_id: str | UUID, required_micros: int) -> None:
|
||||
"""Raise :class:`InsufficientCreditsError` if the wallet can't cover
|
||||
``required_micros`` (a combined crawl + worst-case captcha estimate).
|
||||
|
||||
Generic and **ungated** — the caller computes ``required_micros`` from
|
||||
whichever billers are enabled and only calls this when at least one is.
|
||||
No-op for a non-positive requirement.
|
||||
"""
|
||||
if required_micros <= 0:
|
||||
return
|
||||
available = await self._spendable_micros(user_id)
|
||||
if required_micros > available:
|
||||
raise InsufficientCreditsError(
|
||||
message=(
|
||||
"This run would exceed your available credit. "
|
||||
f"Available: ${available / 1_000_000:.2f}, "
|
||||
f"estimated need: ${required_micros / 1_000_000:.2f}. "
|
||||
"Add more credits to continue."
|
||||
),
|
||||
balance_micros=available,
|
||||
required_micros=required_micros,
|
||||
)
|
||||
|
||||
async def check_credits(
|
||||
self, user_id: str | UUID, estimated_successes: int = 1
|
||||
) -> None:
|
||||
|
|
@ -104,20 +153,14 @@ class WebCrawlCreditService:
|
|||
required_micros=required,
|
||||
)
|
||||
|
||||
async def charge_credits(
|
||||
self, user_id: str | UUID, successes: int
|
||||
) -> int | None:
|
||||
"""Debit the wallet for ``successes`` successful crawls.
|
||||
async def _apply_debit(self, user_id: str | UUID, cost_micros: int) -> int | None:
|
||||
"""Debit ``cost_micros`` from the wallet and commit (shared by all
|
||||
charge paths). Flushes any audit row the caller staged before this.
|
||||
|
||||
Commits the session (mirroring ``EtlCreditService.charge_credits``),
|
||||
which also flushes any audit row staged by the caller before this call.
|
||||
No-op when billing is disabled or ``successes <= 0``.
|
||||
|
||||
Returns the new balance in micros, or ``None`` when nothing was charged.
|
||||
Mirrors ``EtlCreditService.charge_credits``' commit-then-refresh +
|
||||
best-effort auto-reload. No-op for a non-positive cost.
|
||||
"""
|
||||
if not config.WEB_CRAWL_CREDIT_BILLING_ENABLED:
|
||||
return None
|
||||
if successes <= 0:
|
||||
if cost_micros <= 0:
|
||||
return None
|
||||
|
||||
from app.db import User
|
||||
|
|
@ -127,8 +170,7 @@ class WebCrawlCreditService:
|
|||
if not user:
|
||||
raise ValueError(f"User with ID {user_id} not found")
|
||||
|
||||
cost = self.successes_to_micros(successes)
|
||||
user.credit_micros_balance -= cost
|
||||
user.credit_micros_balance -= cost_micros
|
||||
await self.session.commit()
|
||||
await self.session.refresh(user)
|
||||
|
||||
|
|
@ -141,3 +183,32 @@ class WebCrawlCreditService:
|
|||
pass
|
||||
|
||||
return user.credit_micros_balance
|
||||
|
||||
async def charge_credits(
|
||||
self, user_id: str | UUID, successes: int
|
||||
) -> int | None:
|
||||
"""Debit the wallet for ``successes`` successful crawls.
|
||||
|
||||
No-op when crawl billing is disabled or ``successes <= 0``. Returns the
|
||||
new balance in micros, or ``None`` when nothing was charged.
|
||||
"""
|
||||
if not config.WEB_CRAWL_CREDIT_BILLING_ENABLED:
|
||||
return None
|
||||
if successes <= 0:
|
||||
return None
|
||||
return await self._apply_debit(user_id, self.successes_to_micros(successes))
|
||||
|
||||
async def charge_captcha(
|
||||
self, user_id: str | UUID, attempts: int
|
||||
) -> int | None:
|
||||
"""Debit the wallet for ``attempts`` captcha solves (Phase 3d).
|
||||
|
||||
Per-attempt (not per-success): the solver charges for every attempt even
|
||||
when the crawl ultimately fails. No-op when captcha billing is disabled
|
||||
or ``attempts <= 0``.
|
||||
"""
|
||||
if not config.WEB_CRAWL_CAPTCHA_BILLING_ENABLED:
|
||||
return None
|
||||
if attempts <= 0:
|
||||
return None
|
||||
return await self._apply_debit(user_id, self.captcha_solves_to_micros(attempts))
|
||||
|
|
|
|||
|
|
@ -17,11 +17,13 @@ from app.proprietary.web_crawler import (
|
|||
CrawlOutcomeStatus,
|
||||
WebCrawlerConnector,
|
||||
)
|
||||
from app.config import config
|
||||
from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType
|
||||
from app.services.etl_credit_service import InsufficientCreditsError
|
||||
from app.services.task_logging_service import TaskLoggingService
|
||||
from app.services.token_tracking_service import record_token_usage
|
||||
from app.services.web_crawl_credit_service import WebCrawlCreditService
|
||||
from app.utils.captcha import captcha_enabled
|
||||
from app.utils.document_converters import (
|
||||
create_document_chunks,
|
||||
embed_text,
|
||||
|
|
@ -175,19 +177,31 @@ async def index_crawled_urls(
|
|||
return 0, "No URLs provided for indexing"
|
||||
|
||||
# =======================================================================
|
||||
# Phase 3c: crawl-credit pre-flight. Bill the workspace OWNER. ``len(urls)``
|
||||
# is a safe upper bound on successes, so a passing check guarantees the
|
||||
# wallet can't go negative from this run. No-op when billing is disabled
|
||||
# (self-hosted/OSS) — we also skip the owner lookup in that case.
|
||||
# Phase 3c/3d: crawl-credit pre-flight. Bill the workspace OWNER. The
|
||||
# estimate is a safe UPPER BOUND so the wallet can't strand mid-batch:
|
||||
# crawl = len(urls) successes (3c; actual successes are <=)
|
||||
# captcha = len(urls) * MAX_ATTEMPTS (3d; per-attempt, worst case)
|
||||
# No-op (and no owner lookup) when both billers are disabled.
|
||||
# =======================================================================
|
||||
credit_service = WebCrawlCreditService(session)
|
||||
owner_user_id = user_id
|
||||
if credit_service.billing_enabled():
|
||||
if credit_service.billing_enabled() or credit_service.captcha_billing_enabled():
|
||||
owner_user_id = (
|
||||
await _resolve_workspace_owner(session, workspace_id) or user_id
|
||||
)
|
||||
required_micros = 0
|
||||
if credit_service.billing_enabled():
|
||||
required_micros += credit_service.successes_to_micros(len(urls))
|
||||
# Only reserve captcha budget when solving is actually enabled;
|
||||
# with solving off, attempts can never happen, so reserving would
|
||||
# wrongly block runs for captcha that will never be attempted.
|
||||
if credit_service.captcha_billing_enabled() and captcha_enabled():
|
||||
worst_case_attempts = len(urls) * config.CAPTCHA_MAX_ATTEMPTS_PER_URL
|
||||
required_micros += credit_service.captcha_solves_to_micros(
|
||||
worst_case_attempts
|
||||
)
|
||||
try:
|
||||
await credit_service.check_credits(owner_user_id, len(urls))
|
||||
await credit_service.check_balance(owner_user_id, required_micros)
|
||||
except InsufficientCreditsError as credit_error:
|
||||
await task_logger.log_task_failure(
|
||||
log_entry,
|
||||
|
|
@ -218,6 +232,11 @@ async def index_crawled_urls(
|
|||
# extracted content, independent of downstream KB dedupe/unchanged
|
||||
# bucketing. This is the billable unit Phase 3c meters on.
|
||||
crawls_succeeded = 0
|
||||
# Phase 3d: per-attempt captcha telemetry summed across the batch. Only
|
||||
# accumulated when captcha billing is on (avoids touching the outcome
|
||||
# field otherwise — and keeps MagicMock outcomes in tests arithmetic-safe).
|
||||
total_captcha_attempts = 0
|
||||
total_captcha_solved = 0
|
||||
|
||||
# Heartbeat tracking - update notification periodically to prevent appearing stuck
|
||||
last_heartbeat_time = time.time()
|
||||
|
|
@ -345,6 +364,13 @@ async def index_crawled_urls(
|
|||
# Crawl the URL
|
||||
outcome = await crawler.crawl_url(url)
|
||||
|
||||
# Phase 3d: count captcha attempts BEFORE the success branch — a
|
||||
# failed solve still cost real solver money and must bill.
|
||||
if credit_service.captcha_billing_enabled():
|
||||
total_captcha_attempts += outcome.captcha_attempts
|
||||
if outcome.captcha_solved:
|
||||
total_captcha_solved += 1
|
||||
|
||||
if (
|
||||
outcome.status is not CrawlOutcomeStatus.SUCCESS
|
||||
or not outcome.result
|
||||
|
|
@ -526,6 +552,29 @@ async def index_crawled_urls(
|
|||
)
|
||||
await credit_service.charge_credits(owner_user_id, crawls_succeeded)
|
||||
|
||||
# =======================================================================
|
||||
# Phase 3d: charge the workspace owner per captcha *attempt* (separate
|
||||
# unit; charged even when the crawl ultimately failed). Same stage +
|
||||
# owner as the crawl charge above.
|
||||
# =======================================================================
|
||||
if credit_service.captcha_billing_enabled() and total_captcha_attempts > 0:
|
||||
await record_token_usage(
|
||||
session,
|
||||
usage_type="web_crawl_captcha",
|
||||
workspace_id=workspace_id,
|
||||
user_id=owner_user_id,
|
||||
cost_micros=credit_service.captcha_solves_to_micros(
|
||||
total_captcha_attempts
|
||||
),
|
||||
call_details={
|
||||
"attempts": total_captcha_attempts,
|
||||
"solved": total_captcha_solved,
|
||||
"connector_id": connector_id,
|
||||
},
|
||||
message_id=None,
|
||||
)
|
||||
await credit_service.charge_captcha(owner_user_id, total_captcha_attempts)
|
||||
|
||||
# Build warning message if there were issues
|
||||
warning_parts = []
|
||||
if duplicate_content_count > 0:
|
||||
|
|
|
|||
22
surfsense_backend/app/utils/captcha/__init__.py
Normal file
22
surfsense_backend/app/utils/captcha/__init__.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
"""App-wide captcha-solver configuration (Apache-2.0, generic glue).
|
||||
|
||||
This package holds only the **generic, vendor-agnostic** config resolution for
|
||||
captcha solving — mirroring ``app/utils/proxy/`` (which stays Apache-2.0 even
|
||||
though the crawler that consumes it is proprietary). The actual bypass logic
|
||||
(challenge detection + token injection ``page_action``) lives in the separately
|
||||
licensed ``app/proprietary/web_crawler/captcha.py``.
|
||||
|
||||
``captchatools`` is itself the multi-vendor registry (capmonster / 2captcha /
|
||||
anticaptcha / capsolver / captchaai), so there is no provider hierarchy here —
|
||||
just one env-driven :class:`CaptchaConfig` and a cheap ``captcha_enabled()``
|
||||
gate callers check before constructing anything (mirrors
|
||||
``WebCrawlCreditService.billing_enabled()``).
|
||||
"""
|
||||
|
||||
from app.utils.captcha.config import (
|
||||
CaptchaConfig,
|
||||
captcha_enabled,
|
||||
get_captcha_config,
|
||||
)
|
||||
|
||||
__all__ = ["CaptchaConfig", "captcha_enabled", "get_captcha_config"]
|
||||
54
surfsense_backend/app/utils/captcha/config.py
Normal file
54
surfsense_backend/app/utils/captcha/config.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
"""Env-resolved captcha-solver config (one app-wide config; no per-connector).
|
||||
|
||||
Resolved from :data:`app.config.config` only, mirroring ``03b``'s single
|
||||
``PROXY_PROVIDER`` model. Off by default: ``captcha_enabled()`` is ``False``
|
||||
unless both ``CAPTCHA_SOLVING_ENABLED=TRUE`` and an API key are present, so a
|
||||
misconfigured deployment (flag on, key missing) never attempts a paid solve.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.config import config
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CaptchaConfig:
|
||||
"""Immutable snapshot of the captcha-solver settings for one crawl."""
|
||||
|
||||
enabled: bool
|
||||
solving_site: str
|
||||
api_key: str | None
|
||||
max_attempts_per_url: int
|
||||
timeout_s: int
|
||||
captcha_type_default: str
|
||||
v3_min_score: float
|
||||
v3_action: str
|
||||
|
||||
|
||||
def get_captcha_config() -> CaptchaConfig:
|
||||
"""Build a :class:`CaptchaConfig` from the current process config/env."""
|
||||
api_key = config.CAPTCHA_SOLVER_API_KEY
|
||||
flag_on = config.CAPTCHA_SOLVING_ENABLED
|
||||
return CaptchaConfig(
|
||||
# Effective-enabled requires a key too: a flag with no key would only
|
||||
# produce ``ErrWrongAPIKey`` per attempt (and still risk an IP ban from
|
||||
# the solver), so treat key-less config as disabled.
|
||||
enabled=bool(flag_on and api_key),
|
||||
solving_site=config.CAPTCHA_SOLVER_PROVIDER,
|
||||
api_key=api_key,
|
||||
max_attempts_per_url=config.CAPTCHA_MAX_ATTEMPTS_PER_URL,
|
||||
timeout_s=config.CAPTCHA_SOLVE_TIMEOUT_S,
|
||||
captcha_type_default=config.CAPTCHA_TYPE_DEFAULT,
|
||||
v3_min_score=config.CAPTCHA_V3_MIN_SCORE,
|
||||
v3_action=config.CAPTCHA_V3_ACTION,
|
||||
)
|
||||
|
||||
|
||||
def captcha_enabled() -> bool:
|
||||
"""Cheap gate: is captcha solving effectively configured (flag + key)?
|
||||
|
||||
Callers check this before building the ``page_action`` or capturing the
|
||||
crawl's proxy endpoint for the solver, so the stealth tier stays unchanged
|
||||
when solving is off.
|
||||
"""
|
||||
return bool(config.CAPTCHA_SOLVING_ENABLED and config.CAPTCHA_SOLVER_API_KEY)
|
||||
|
|
@ -46,6 +46,7 @@ dependencies = [
|
|||
"azure-storage-blob>=12.23.0",
|
||||
"fake-useragent>=2.2.0",
|
||||
"trafilatura>=2.0.0",
|
||||
"captchatools>=1.5.0",
|
||||
"fastapi-users[oauth,sqlalchemy]>=15.0.3",
|
||||
"chonkie[all]>=1.5.0",
|
||||
"langgraph-checkpoint-postgres>=3.0.2",
|
||||
|
|
|
|||
|
|
@ -53,7 +53,12 @@ def _make_session(owner_id, balance_micros, reserved_micros=0):
|
|||
return session, fake_user
|
||||
|
||||
|
||||
def _outcome(success: bool, content: str = "Hello content"):
|
||||
def _outcome(
|
||||
success: bool,
|
||||
content: str = "Hello content",
|
||||
captcha_attempts: int = 0,
|
||||
captcha_solved: bool = False,
|
||||
):
|
||||
o = MagicMock()
|
||||
if success:
|
||||
o.status = CrawlOutcomeStatus.SUCCESS
|
||||
|
|
@ -67,6 +72,9 @@ def _outcome(success: bool, content: str = "Hello content"):
|
|||
o.status = CrawlOutcomeStatus.FAILED
|
||||
o.result = None
|
||||
o.error = "blocked"
|
||||
# Real ints/bools (not MagicMock) so 03d accumulation arithmetic is safe.
|
||||
o.captcha_attempts = captcha_attempts
|
||||
o.captcha_solved = captcha_solved
|
||||
return o
|
||||
|
||||
|
||||
|
|
@ -188,3 +196,111 @@ async def test_disabled_skips_billing_but_still_crawls(indexer_env, monkeypatch)
|
|||
indexer_env["crawler"].crawl_url.assert_awaited_once()
|
||||
indexer_env["record_usage"].assert_not_awaited()
|
||||
assert user.credit_micros_balance == 100_000
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Phase 3d — captcha per-attempt billing in the indexer
|
||||
# ===================================================================
|
||||
|
||||
|
||||
async def test_captcha_attempts_charged_to_owner(indexer_env, monkeypatch):
|
||||
"""Captcha billed per attempt to the OWNER as a separate web_crawl_captcha
|
||||
unit. Crawl billing OFF here to isolate the captcha charge."""
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", False)
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", True)
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE", 3000)
|
||||
session, user = _make_session(_OWNER_USER, balance_micros=100_000)
|
||||
indexer_env["crawler"].crawl_url.side_effect = [
|
||||
_outcome(True, captcha_attempts=1, captcha_solved=True),
|
||||
_outcome(True, captcha_attempts=1, captcha_solved=True),
|
||||
]
|
||||
|
||||
total, _warning = await _run(
|
||||
indexer_env, session, ["https://a.com", "https://b.com"]
|
||||
)
|
||||
|
||||
assert total == 2
|
||||
# Owner debited 2 attempts * 3000 (no crawl charge — crawl billing off).
|
||||
assert user.credit_micros_balance == 100_000 - 6000
|
||||
indexer_env["record_usage"].assert_awaited_once()
|
||||
kwargs = indexer_env["record_usage"].await_args.kwargs
|
||||
assert kwargs["usage_type"] == "web_crawl_captcha"
|
||||
assert kwargs["user_id"] == _OWNER_USER
|
||||
assert kwargs["cost_micros"] == 6000
|
||||
assert kwargs["call_details"]["attempts"] == 2
|
||||
assert kwargs["call_details"]["solved"] == 2
|
||||
|
||||
|
||||
async def test_captcha_attempt_billed_even_when_crawl_fails(indexer_env, monkeypatch):
|
||||
"""A failed solve still cost solver money → bill the attempt anyway."""
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", False)
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", True)
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE", 3000)
|
||||
session, user = _make_session(_OWNER_USER, balance_micros=100_000)
|
||||
indexer_env["crawler"].crawl_url.side_effect = [
|
||||
_outcome(False, captcha_attempts=1, captcha_solved=False)
|
||||
]
|
||||
|
||||
total, _warning = await _run(indexer_env, session, ["https://a.com"])
|
||||
|
||||
assert total == 0 # crawl failed → not indexed
|
||||
assert user.credit_micros_balance == 100_000 - 3000 # but solve attempt billed
|
||||
kwargs = indexer_env["record_usage"].await_args.kwargs
|
||||
assert kwargs["usage_type"] == "web_crawl_captcha"
|
||||
assert kwargs["call_details"]["attempts"] == 1
|
||||
assert kwargs["call_details"]["solved"] == 0
|
||||
|
||||
|
||||
async def test_captcha_billing_on_but_solving_off_does_not_overreserve(
|
||||
indexer_env, monkeypatch
|
||||
):
|
||||
"""Regression: captcha billing ON but solving OFF must NOT reserve the
|
||||
worst-case captcha budget in pre-flight.
|
||||
|
||||
With solving off, attempts can never happen, so a balance below the
|
||||
(would-be) worst-case reservation must still let the run proceed rather than
|
||||
being wrongly blocked for captcha that never runs.
|
||||
"""
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", False)
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", True)
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE", 3000)
|
||||
monkeypatch.setattr(config, "CAPTCHA_MAX_ATTEMPTS_PER_URL", 3)
|
||||
# Solving OFF → captcha_enabled() is False (no flag + no key).
|
||||
monkeypatch.setattr(config, "CAPTCHA_SOLVING_ENABLED", False)
|
||||
monkeypatch.setattr(config, "CAPTCHA_SOLVER_API_KEY", "")
|
||||
|
||||
# Balance (5000) is below the buggy worst-case reservation
|
||||
# (2 URLs * 3 attempts * 3000 = 18000) but the run must still proceed.
|
||||
session, user = _make_session(_OWNER_USER, balance_micros=5000)
|
||||
indexer_env["crawler"].crawl_url.side_effect = [
|
||||
_outcome(True, captcha_attempts=0),
|
||||
_outcome(True, captcha_attempts=0),
|
||||
]
|
||||
|
||||
total, warning = await _run(
|
||||
indexer_env, session, ["https://a.com", "https://b.com"]
|
||||
)
|
||||
|
||||
assert total == 2
|
||||
assert warning is None
|
||||
assert indexer_env["crawler"].crawl_url.await_count == 2
|
||||
# Nothing billed: crawl billing off, no captcha attempts.
|
||||
indexer_env["record_usage"].assert_not_awaited()
|
||||
assert user.credit_micros_balance == 5000
|
||||
|
||||
|
||||
async def test_captcha_disabled_skips_captcha_billing(indexer_env, monkeypatch):
|
||||
"""Captcha billing OFF (default) → attempts on the outcome are ignored."""
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", False)
|
||||
session, user = _make_session(_OWNER_USER, balance_micros=100_000)
|
||||
indexer_env["crawler"].crawl_url.side_effect = [
|
||||
_outcome(True, captcha_attempts=5, captcha_solved=True)
|
||||
]
|
||||
|
||||
total, _warning = await _run(indexer_env, session, ["https://a.com"])
|
||||
|
||||
assert total == 1
|
||||
# Only the crawl unit billed (1 * 1000); captcha ignored entirely.
|
||||
assert user.credit_micros_balance == 100_000 - 1000
|
||||
kwargs = indexer_env["record_usage"].await_args.kwargs
|
||||
assert kwargs["usage_type"] == "web_crawl"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,237 @@
|
|||
"""Unit tests for the proprietary captcha page_action factory (Phase 3d).
|
||||
|
||||
The browser/solver boundary is mocked: a fake Playwright ``page`` and a
|
||||
monkeypatched ``_harvest_token`` / injection. We assert the glue logic —
|
||||
detection, proxy reformatting, attempt counting + state surfacing, the per-URL
|
||||
cap, and the no-balance process latch.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.proprietary.web_crawler import captcha as cap
|
||||
from app.utils.captcha import CaptchaConfig
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _cfg(**overrides) -> CaptchaConfig:
|
||||
base = dict(
|
||||
enabled=True,
|
||||
solving_site="capsolver",
|
||||
api_key="key-123",
|
||||
max_attempts_per_url=1,
|
||||
timeout_s=30,
|
||||
captcha_type_default="v2",
|
||||
v3_min_score=0.7,
|
||||
v3_action="verify",
|
||||
)
|
||||
base.update(overrides)
|
||||
return CaptchaConfig(**base)
|
||||
|
||||
|
||||
class _FakeEl:
|
||||
def __init__(self, attrs: dict[str, str]):
|
||||
self._attrs = attrs
|
||||
|
||||
def get_attribute(self, name: str) -> str | None:
|
||||
return self._attrs.get(name)
|
||||
|
||||
|
||||
class _FakePage:
|
||||
"""Minimal sync-Playwright-ish page for detection/injection tests."""
|
||||
|
||||
def __init__(self, widgets=None, iframes=None, scripts=None, url="https://t.test/p"):
|
||||
self._widgets = widgets or {} # selector -> _FakeEl
|
||||
self._iframes = iframes or [] # list[_FakeEl]
|
||||
self._scripts = scripts or [] # list[_FakeEl]
|
||||
self.url = url
|
||||
self.evaluated: list = []
|
||||
|
||||
def query_selector(self, selector: str):
|
||||
return self._widgets.get(selector)
|
||||
|
||||
def query_selector_all(self, selector: str):
|
||||
if selector == "iframe[src]":
|
||||
return self._iframes
|
||||
if selector == "script[src]":
|
||||
return self._scripts
|
||||
return []
|
||||
|
||||
def evaluate(self, js, arg=None):
|
||||
self.evaluated.append((js, arg))
|
||||
if "navigator.userAgent" in js:
|
||||
return "UA/1.0"
|
||||
return None
|
||||
|
||||
def wait_for_load_state(self, *_a, **_k):
|
||||
return None
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_latch():
|
||||
cap.reset_solver_latch()
|
||||
yield
|
||||
cap.reset_solver_latch()
|
||||
|
||||
|
||||
# --- proxy reformat --------------------------------------------------------
|
||||
|
||||
|
||||
class TestProxyReformat:
|
||||
def test_with_auth(self):
|
||||
assert (
|
||||
cap.proxy_url_to_captchatools("http://user:pass@1.2.3.4:8080")
|
||||
== "1.2.3.4:8080:user:pass"
|
||||
)
|
||||
|
||||
def test_without_auth(self):
|
||||
assert cap.proxy_url_to_captchatools("http://1.2.3.4:8080") == "1.2.3.4:8080"
|
||||
|
||||
def test_none_and_garbage(self):
|
||||
assert cap.proxy_url_to_captchatools(None) is None
|
||||
assert cap.proxy_url_to_captchatools("not-a-url") is None
|
||||
|
||||
|
||||
# --- detection -------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDetect:
|
||||
def test_recaptcha_v2_widget(self):
|
||||
page = _FakePage(
|
||||
widgets={".g-recaptcha[data-sitekey]": _FakeEl({"data-sitekey": "SK_V2"})}
|
||||
)
|
||||
assert cap.detect_challenge(page, _cfg()) == ("v2", "SK_V2")
|
||||
|
||||
def test_hcaptcha_widget(self):
|
||||
page = _FakePage(
|
||||
widgets={".h-captcha[data-sitekey]": _FakeEl({"data-sitekey": "SK_H"})}
|
||||
)
|
||||
assert cap.detect_challenge(page, _cfg()) == ("hcaptcha", "SK_H")
|
||||
|
||||
def test_iframe_fallback(self):
|
||||
page = _FakePage(
|
||||
iframes=[_FakeEl({"src": "https://www.google.com/recaptcha/api2?k=SK_IF"})]
|
||||
)
|
||||
assert cap.detect_challenge(page, _cfg()) == ("v2", "SK_IF")
|
||||
|
||||
def test_v3_render_param_when_default_v3(self):
|
||||
page = _FakePage(
|
||||
scripts=[_FakeEl({"src": "https://www.google.com/recaptcha/api.js?render=SK_V3"})]
|
||||
)
|
||||
assert cap.detect_challenge(page, _cfg(captcha_type_default="v3")) == (
|
||||
"v3",
|
||||
"SK_V3",
|
||||
)
|
||||
|
||||
def test_no_challenge(self):
|
||||
assert cap.detect_challenge(_FakePage(), _cfg()) is None
|
||||
|
||||
|
||||
# --- factory / page_action -------------------------------------------------
|
||||
|
||||
|
||||
class TestFactoryGating:
|
||||
def test_returns_none_when_disabled(self):
|
||||
state = {"attempts": 0, "solved": False}
|
||||
assert build_action(state, _cfg(enabled=False)) is None
|
||||
|
||||
def test_returns_none_when_latched(self):
|
||||
cap._latch_solver("test")
|
||||
state = {"attempts": 0, "solved": False}
|
||||
assert build_action(state, _cfg()) is None
|
||||
|
||||
|
||||
def build_action(state, cfg, proxy="http://u:p@1.2.3.4:9000"):
|
||||
return cap.build_captcha_page_action(state, proxy, cfg)
|
||||
|
||||
|
||||
class TestPageAction:
|
||||
def test_solves_and_records(self, monkeypatch):
|
||||
captured = {}
|
||||
|
||||
def _fake_harvest(cfg, ctype, sitekey, page_url, proxy, ua):
|
||||
captured.update(
|
||||
ctype=ctype, sitekey=sitekey, page_url=page_url, proxy=proxy, ua=ua
|
||||
)
|
||||
return "TOKEN"
|
||||
|
||||
injected = {}
|
||||
monkeypatch.setattr(cap, "_harvest_token", _fake_harvest)
|
||||
monkeypatch.setattr(
|
||||
cap,
|
||||
"_inject_and_submit",
|
||||
lambda page, ctype, token: injected.update(ctype=ctype, token=token),
|
||||
)
|
||||
|
||||
state = {"attempts": 0, "solved": False}
|
||||
action = build_action(state, _cfg())
|
||||
page = _FakePage(
|
||||
widgets={".g-recaptcha[data-sitekey]": _FakeEl({"data-sitekey": "SK"})}
|
||||
)
|
||||
action(page)
|
||||
|
||||
assert state == {"attempts": 1, "solved": True}
|
||||
# Solver egressed from the crawl's proxy, reformatted, with the page UA.
|
||||
assert captured["proxy"] == "1.2.3.4:9000:u:p"
|
||||
assert captured["ua"] == "UA/1.0"
|
||||
assert captured["ctype"] == "v2"
|
||||
assert injected == {"ctype": "v2", "token": "TOKEN"}
|
||||
|
||||
def test_no_challenge_no_attempt(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
cap, "_harvest_token", lambda *a, **k: pytest.fail("should not solve")
|
||||
)
|
||||
state = {"attempts": 0, "solved": False}
|
||||
build_action(state, _cfg())(_FakePage())
|
||||
assert state == {"attempts": 0, "solved": False}
|
||||
|
||||
def test_per_url_attempt_cap(self, monkeypatch):
|
||||
calls = {"n": 0}
|
||||
|
||||
def _h(*_a, **_k):
|
||||
calls["n"] += 1
|
||||
return "TOKEN"
|
||||
|
||||
monkeypatch.setattr(cap, "_harvest_token", _h)
|
||||
monkeypatch.setattr(cap, "_inject_and_submit", lambda *a, **k: None)
|
||||
|
||||
# Already at the cap → no further solve.
|
||||
state = {"attempts": 1, "solved": False}
|
||||
page = _FakePage(
|
||||
widgets={".g-recaptcha[data-sitekey]": _FakeEl({"data-sitekey": "SK"})}
|
||||
)
|
||||
build_action(state, _cfg(max_attempts_per_url=1))(page)
|
||||
assert calls["n"] == 0
|
||||
assert state["attempts"] == 1
|
||||
|
||||
def test_empty_token_counts_attempt_but_not_solved(self, monkeypatch):
|
||||
monkeypatch.setattr(cap, "_harvest_token", lambda *a, **k: None)
|
||||
monkeypatch.setattr(
|
||||
cap, "_inject_and_submit", lambda *a, **k: pytest.fail("no inject")
|
||||
)
|
||||
state = {"attempts": 0, "solved": False}
|
||||
page = _FakePage(
|
||||
widgets={".g-recaptcha[data-sitekey]": _FakeEl({"data-sitekey": "SK"})}
|
||||
)
|
||||
build_action(state, _cfg())(page)
|
||||
assert state == {"attempts": 1, "solved": False}
|
||||
|
||||
def test_no_balance_latches_and_stops(self, monkeypatch):
|
||||
class ErrNoBalance(Exception):
|
||||
pass
|
||||
|
||||
def _boom(*_a, **_k):
|
||||
raise ErrNoBalance("out of balance")
|
||||
|
||||
monkeypatch.setattr(cap, "_harvest_token", _boom)
|
||||
state = {"attempts": 0, "solved": False}
|
||||
page = _FakePage(
|
||||
widgets={".g-recaptcha[data-sitekey]": _FakeEl({"data-sitekey": "SK"})}
|
||||
)
|
||||
build_action(state, _cfg(max_attempts_per_url=3))(page)
|
||||
|
||||
# Attempt counted, not solved, and the process latch is tripped so the
|
||||
# next factory build refuses to wire solving at all (no retry loop).
|
||||
assert state == {"attempts": 1, "solved": False}
|
||||
assert cap.solver_latched() is True
|
||||
assert build_action({"attempts": 0, "solved": False}, _cfg()) is None
|
||||
|
|
@ -47,7 +47,7 @@ async def test_static_tier_success_short_circuits(
|
|||
later_calls.append("dynamic")
|
||||
return None
|
||||
|
||||
async def _record_stealthy(_url: str) -> None:
|
||||
async def _record_stealthy(_url: str, *_args) -> None:
|
||||
later_calls.append("stealthy")
|
||||
return None
|
||||
|
||||
|
|
@ -89,7 +89,7 @@ async def test_all_tiers_empty_is_empty(monkeypatch: pytest.MonkeyPatch) -> None
|
|||
"""Every tier fetched but extracted nothing -> EMPTY (not billable)."""
|
||||
crawler = WebCrawlerConnector()
|
||||
|
||||
async def _empty(_url: str) -> None:
|
||||
async def _empty(_url: str, *_args) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty)
|
||||
|
|
@ -106,7 +106,7 @@ async def test_all_tiers_raise_is_failed(monkeypatch: pytest.MonkeyPatch) -> Non
|
|||
"""Every tier raising (none reachable) -> FAILED with aggregated errors."""
|
||||
crawler = WebCrawlerConnector()
|
||||
|
||||
async def _boom(_url: str) -> None:
|
||||
async def _boom(_url: str, *_args) -> None:
|
||||
raise RuntimeError("fetch exploded")
|
||||
|
||||
monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _boom)
|
||||
|
|
@ -153,7 +153,7 @@ async def test_proxy_error_no_retry_when_single_endpoint(
|
|||
static_calls["n"] += 1
|
||||
raise RuntimeError("connection refused by upstream proxy")
|
||||
|
||||
async def _empty(_url: str) -> None:
|
||||
async def _empty(_url: str, *_args) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _proxy_err)
|
||||
|
|
@ -167,6 +167,77 @@ async def test_proxy_error_no_retry_when_single_endpoint(
|
|||
assert outcome.status is CrawlOutcomeStatus.EMPTY
|
||||
|
||||
|
||||
async def test_captcha_defaults_zero_on_non_stealthy_success(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""03d: a static success never touches captcha → fields stay 0/False."""
|
||||
crawler = WebCrawlerConnector()
|
||||
|
||||
async def _static(_url: str) -> dict:
|
||||
return _result("scrapling-static")
|
||||
|
||||
monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _static)
|
||||
outcome = await crawler.crawl_url("https://example.com")
|
||||
|
||||
assert outcome.status is CrawlOutcomeStatus.SUCCESS
|
||||
assert outcome.captcha_attempts == 0
|
||||
assert outcome.captcha_solved is False
|
||||
|
||||
|
||||
async def test_captcha_state_surfaced_onto_outcome(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""03d: the stealthy tier's captcha_state is stamped onto the outcome,
|
||||
even when the lower tiers missed and stealthy itself succeeds."""
|
||||
crawler = WebCrawlerConnector()
|
||||
|
||||
async def _empty(_url: str, *_args) -> None:
|
||||
return None
|
||||
|
||||
async def _stealthy(_url: str, captcha_state: dict) -> dict:
|
||||
# Simulate the page_action having solved a captcha mid-fetch.
|
||||
captcha_state["attempts"] = 2
|
||||
captcha_state["solved"] = True
|
||||
return _result("scrapling-stealthy")
|
||||
|
||||
monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty)
|
||||
monkeypatch.setattr(crawler, "_crawl_with_dynamic", _empty)
|
||||
monkeypatch.setattr(crawler, "_crawl_with_stealthy", _stealthy)
|
||||
|
||||
outcome = await crawler.crawl_url("https://example.com")
|
||||
|
||||
assert outcome.status is CrawlOutcomeStatus.SUCCESS
|
||||
assert outcome.tier == "scrapling-stealthy"
|
||||
assert outcome.captcha_attempts == 2
|
||||
assert outcome.captcha_solved is True
|
||||
|
||||
|
||||
async def test_captcha_attempts_surface_even_when_crawl_fails(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""03d: attempts are billed per-attempt → must surface on a FAILED outcome."""
|
||||
crawler = WebCrawlerConnector()
|
||||
|
||||
async def _empty(_url: str, *_args) -> None:
|
||||
return None
|
||||
|
||||
async def _stealthy_attempt_then_empty(_url: str, captcha_state: dict) -> None:
|
||||
captcha_state["attempts"] = 1 # solve attempted but crawl still empty
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty)
|
||||
monkeypatch.setattr(crawler, "_crawl_with_dynamic", _empty)
|
||||
monkeypatch.setattr(
|
||||
crawler, "_crawl_with_stealthy", _stealthy_attempt_then_empty
|
||||
)
|
||||
|
||||
outcome = await crawler.crawl_url("https://example.com")
|
||||
|
||||
assert outcome.status is CrawlOutcomeStatus.EMPTY
|
||||
assert outcome.captcha_attempts == 1
|
||||
assert outcome.captcha_solved is False
|
||||
|
||||
|
||||
async def test_non_proxy_error_never_retries(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
|
@ -178,7 +249,7 @@ async def test_non_proxy_error_never_retries(
|
|||
calls["n"] += 1
|
||||
raise RuntimeError("totally unrelated failure")
|
||||
|
||||
async def _empty(_url: str) -> None:
|
||||
async def _empty(_url: str, *_args) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _boom)
|
||||
|
|
|
|||
|
|
@ -213,3 +213,112 @@ class TestChatScrapeFold:
|
|||
# Run in a fresh context so the turn ContextVar resolves to its default
|
||||
# (None) regardless of other tests — must not raise.
|
||||
contextvars.Context().run(self._import_helper())
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# F) Captcha per-attempt unit (Phase 3d)
|
||||
# ===================================================================
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _enable_captcha_billing(monkeypatch):
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", True)
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE", 3000)
|
||||
|
||||
|
||||
class TestCaptchaStatics:
|
||||
def test_billing_enabled_reads_flag(self, monkeypatch):
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", True)
|
||||
assert WebCrawlCreditService.captcha_billing_enabled() is True
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", False)
|
||||
assert WebCrawlCreditService.captcha_billing_enabled() is False
|
||||
|
||||
def test_solves_to_micros_is_config_driven(self, monkeypatch):
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE", 3000)
|
||||
assert WebCrawlCreditService.captcha_solves_to_micros(1) == 3000
|
||||
assert WebCrawlCreditService.captcha_solves_to_micros(1000) == 3_000_000
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE", 5000)
|
||||
assert WebCrawlCreditService.captcha_solves_to_micros(2) == 10_000
|
||||
|
||||
|
||||
class TestChargeCaptcha:
|
||||
async def test_debits_per_attempt(self, _enable_captcha_billing):
|
||||
session, user = _make_session(balance_micros=100_000)
|
||||
svc = WebCrawlCreditService(session)
|
||||
new_balance = await svc.charge_captcha(_USER_ID, attempts=2)
|
||||
assert user.credit_micros_balance == 100_000 - 2 * 3000
|
||||
assert new_balance == 94_000
|
||||
session.commit.assert_awaited()
|
||||
|
||||
async def test_zero_attempts_is_noop(self, _enable_captcha_billing):
|
||||
session, user = _make_session(balance_micros=100_000)
|
||||
svc = WebCrawlCreditService(session)
|
||||
assert await svc.charge_captcha(_USER_ID, attempts=0) is None
|
||||
assert user.credit_micros_balance == 100_000
|
||||
session.commit.assert_not_awaited()
|
||||
|
||||
async def test_disabled_is_noop(self, monkeypatch):
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", False)
|
||||
session, user = _make_session(balance_micros=100_000)
|
||||
svc = WebCrawlCreditService(session)
|
||||
assert await svc.charge_captcha(_USER_ID, attempts=5) is None
|
||||
assert user.credit_micros_balance == 100_000
|
||||
session.execute.assert_not_called()
|
||||
|
||||
|
||||
class TestCheckBalance:
|
||||
"""Combined (crawl + captcha) pre-flight primitive — ungated."""
|
||||
|
||||
async def test_passes_when_affordable(self):
|
||||
session, _user = _make_session(balance_micros=10_000)
|
||||
await WebCrawlCreditService(session).check_balance(_USER_ID, 5_000)
|
||||
|
||||
async def test_raises_when_short(self):
|
||||
session, _user = _make_session(balance_micros=3_000)
|
||||
with pytest.raises(InsufficientCreditsError):
|
||||
await WebCrawlCreditService(session).check_balance(_USER_ID, 5_000)
|
||||
|
||||
async def test_zero_required_is_noop(self):
|
||||
session, _user = _make_session(balance_micros=0)
|
||||
await WebCrawlCreditService(session).check_balance(_USER_ID, 0)
|
||||
session.execute.assert_not_called()
|
||||
|
||||
|
||||
class TestCaptchaChatFold:
|
||||
"""The scrape tools fold captcha attempts into the live turn accumulator."""
|
||||
|
||||
def _import_helper(self):
|
||||
from app.agents.chat.multi_agent_chat.main_agent.tools.scrape_webpage import (
|
||||
_bill_captcha_attempts,
|
||||
)
|
||||
|
||||
return _bill_captcha_attempts
|
||||
|
||||
def _outcome(self, attempts):
|
||||
o = MagicMock()
|
||||
o.captcha_attempts = attempts
|
||||
return o
|
||||
|
||||
def test_folds_attempts_when_enabled(self, _enable_captcha_billing):
|
||||
from app.services.token_tracking_service import start_turn
|
||||
|
||||
acc = start_turn()
|
||||
self._import_helper()(self._outcome(2))
|
||||
assert acc.total_cost_micros == 6000
|
||||
assert len(acc.calls) == 1
|
||||
assert acc.calls[0].call_kind == "web_crawl_captcha"
|
||||
|
||||
def test_noop_when_zero_attempts(self, _enable_captcha_billing):
|
||||
from app.services.token_tracking_service import start_turn
|
||||
|
||||
acc = start_turn()
|
||||
self._import_helper()(self._outcome(0))
|
||||
assert acc.calls == []
|
||||
|
||||
def test_noop_when_billing_disabled(self, monkeypatch):
|
||||
from app.services.token_tracking_service import start_turn
|
||||
|
||||
monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", False)
|
||||
acc = start_turn()
|
||||
self._import_helper()(self._outcome(3))
|
||||
assert acc.calls == []
|
||||
|
|
|
|||
47
surfsense_backend/tests/unit/utils/test_captcha_config.py
Normal file
47
surfsense_backend/tests/unit/utils/test_captcha_config.py
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
"""Unit tests for the generic captcha config gate (Phase 3d, Apache-2 layer)."""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import config
|
||||
from app.utils.captcha import captcha_enabled, get_captcha_config
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class TestCaptchaEnabled:
|
||||
def test_off_by_default(self, monkeypatch):
|
||||
monkeypatch.setattr(config, "CAPTCHA_SOLVING_ENABLED", False)
|
||||
monkeypatch.setattr(config, "CAPTCHA_SOLVER_API_KEY", "key")
|
||||
assert captcha_enabled() is False
|
||||
|
||||
def test_flag_on_but_no_key_is_disabled(self, monkeypatch):
|
||||
monkeypatch.setattr(config, "CAPTCHA_SOLVING_ENABLED", True)
|
||||
monkeypatch.setattr(config, "CAPTCHA_SOLVER_API_KEY", None)
|
||||
assert captcha_enabled() is False
|
||||
|
||||
def test_flag_on_with_key_is_enabled(self, monkeypatch):
|
||||
monkeypatch.setattr(config, "CAPTCHA_SOLVING_ENABLED", True)
|
||||
monkeypatch.setattr(config, "CAPTCHA_SOLVER_API_KEY", "key")
|
||||
assert captcha_enabled() is True
|
||||
|
||||
|
||||
class TestGetCaptchaConfig:
|
||||
def test_snapshot_reflects_config(self, monkeypatch):
|
||||
monkeypatch.setattr(config, "CAPTCHA_SOLVING_ENABLED", True)
|
||||
monkeypatch.setattr(config, "CAPTCHA_SOLVER_API_KEY", "key")
|
||||
monkeypatch.setattr(config, "CAPTCHA_SOLVER_PROVIDER", "2captcha")
|
||||
monkeypatch.setattr(config, "CAPTCHA_MAX_ATTEMPTS_PER_URL", 3)
|
||||
monkeypatch.setattr(config, "CAPTCHA_SOLVE_TIMEOUT_S", 90)
|
||||
monkeypatch.setattr(config, "CAPTCHA_TYPE_DEFAULT", "hcaptcha")
|
||||
|
||||
cfg = get_captcha_config()
|
||||
assert cfg.enabled is True
|
||||
assert cfg.solving_site == "2captcha"
|
||||
assert cfg.max_attempts_per_url == 3
|
||||
assert cfg.timeout_s == 90
|
||||
assert cfg.captcha_type_default == "hcaptcha"
|
||||
|
||||
def test_keyless_config_is_not_enabled(self, monkeypatch):
|
||||
monkeypatch.setattr(config, "CAPTCHA_SOLVING_ENABLED", True)
|
||||
monkeypatch.setattr(config, "CAPTCHA_SOLVER_API_KEY", None)
|
||||
assert get_captcha_config().enabled is False
|
||||
146
surfsense_backend/uv.lock
generated
146
surfsense_backend/uv.lock
generated
|
|
@ -27,6 +27,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -52,6 +55,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -77,6 +83,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -102,6 +111,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -127,6 +139,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -152,6 +167,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -177,6 +195,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -202,6 +223,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -227,6 +251,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -252,6 +279,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -285,6 +315,10 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -310,6 +344,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -335,6 +372,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -360,6 +400,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -385,6 +428,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -410,6 +456,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -435,6 +484,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -460,6 +512,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -485,6 +540,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -510,6 +568,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -535,6 +596,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -568,6 +632,10 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -593,6 +661,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -618,6 +689,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -643,6 +717,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -668,6 +745,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -693,6 +773,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -718,6 +801,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -743,6 +829,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -768,6 +857,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -793,6 +885,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -818,6 +913,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -851,6 +949,10 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -876,6 +978,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -925,6 +1030,12 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -950,6 +1061,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -975,6 +1089,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1024,6 +1141,12 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1073,6 +1196,12 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1098,6 +1227,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
]
|
||||
conflicts = [[
|
||||
|
|
@ -1843,6 +1975,18 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/06/f3/39cf3367b8107baa44f861dc802cbf16263c945b62d8265d36034fc07bea/cachetools-7.0.5-py3-none-any.whl", hash = "sha256:46bc8ebefbe485407621d0a4264b23c080cedd913921bad7ac3ed2f26c183114", size = 13918 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "captchatools"
|
||||
version = "1.5.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "requests" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6b/38/01200f5b8169219002728ccddbad8cc50759e523c0884b7e811092d27352/captchatools-1.5.0.tar.gz", hash = "sha256:755ed685a93f6139608b5277ef0410f8682b3522e98f52d9cc3edff9df38ca12", size = 15282 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ea/f1/7faaa20ffd68314215da2e3c57076c2c89846ab05464f1b7167dc8a9a80b/captchatools-1.5.0-py3-none-any.whl", hash = "sha256:43b651f7603be5725df94e4d715e0ff770efec44b62bb6076eb9a63516c74d94", size = 16394 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "catalogue"
|
||||
version = "2.0.10"
|
||||
|
|
@ -9967,6 +10111,7 @@ dependencies = [
|
|||
{ name = "azure-ai-documentintelligence" },
|
||||
{ name = "azure-storage-blob" },
|
||||
{ name = "boto3" },
|
||||
{ name = "captchatools" },
|
||||
{ name = "celery", extra = ["redis"] },
|
||||
{ name = "chonkie", extra = ["all"] },
|
||||
{ name = "composio" },
|
||||
|
|
@ -10084,6 +10229,7 @@ requires-dist = [
|
|||
{ name = "azure-ai-documentintelligence", specifier = ">=1.0.2" },
|
||||
{ name = "azure-storage-blob", specifier = ">=12.23.0" },
|
||||
{ name = "boto3", specifier = ">=1.35.0" },
|
||||
{ name = "captchatools", specifier = ">=1.5.0" },
|
||||
{ name = "celery", extras = ["redis"], specifier = ">=5.5.3" },
|
||||
{ name = "chonkie", extras = ["all"], specifier = ">=1.5.0" },
|
||||
{ name = "composio", specifier = ">=0.10.9" },
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue