From 68d1832f1045327112ab1c84b3c098b6f896207f Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Fri, 17 Jul 2026 03:27:43 -0700 Subject: [PATCH] refactor: enhance Google search scraping capabilities with warm sticky-IP pool and improved captcha handling --- surfsense_backend/.env.example | 35 +- surfsense_backend/app/config/__init__.py | 9 +- .../platforms/google_search/README.md | 83 ++++- .../platforms/google_search/captcha.py | 178 ++++++++++ .../platforms/google_search/fetch.py | 279 ++++++++++++++-- .../platforms/google_search/pool_store.py | 121 +++++++ .../app/proprietary/web_crawler/captcha.py | 70 ++-- .../app/utils/captcha/__init__.py | 19 +- .../app/utils/captcha/solvers.py | 311 ++++++++++++++++++ surfsense_backend/pyproject.toml | 3 +- .../scripts/e2e_google_search.py | 12 +- .../scripts/scale_google_search.py | 229 +++++++++++++ .../scripts/stress_google_search.py | 207 ++++++++++++ .../platforms/google_search/test_captcha.py | 37 +++ .../google_search/test_fetch_concurrency.py | 2 +- .../google_search/test_fetch_pool.py | 99 ++++++ .../google_search/test_pool_store.py | 73 ++++ .../proprietary/web_crawler/test_captcha.py | 28 +- .../tests/unit/utils/captcha/test_solvers.py | 84 +++++ surfsense_backend/uv.lock | 196 ++++++++--- 20 files changed, 1900 insertions(+), 175 deletions(-) create mode 100644 surfsense_backend/app/proprietary/platforms/google_search/captcha.py create mode 100644 surfsense_backend/app/proprietary/platforms/google_search/pool_store.py create mode 100644 surfsense_backend/app/utils/captcha/solvers.py create mode 100644 surfsense_backend/scripts/scale_google_search.py create mode 100644 surfsense_backend/scripts/stress_google_search.py create mode 100644 surfsense_backend/tests/unit/platforms/google_search/test_captcha.py create mode 100644 surfsense_backend/tests/unit/platforms/google_search/test_fetch_pool.py create mode 100644 surfsense_backend/tests/unit/platforms/google_search/test_pool_store.py create mode 100644 surfsense_backend/tests/unit/utils/captcha/test_solvers.py diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 58e579e05..94508431c 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -374,15 +374,38 @@ TURNSTILE_SECRET_KEY= # PROXY_URL=http://user:pass@host:port # PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port +# --- Google Search scraper: scale / warm sticky-IP pool ----------------------- +# Per-process render concurrency AND the throughput lever: ceiling = +# MAX_CONCURRENT_PAGES / warm-render-secs (~4/14s ≈ 17 SERP/min). Renders are +# resource-light (disable_resources), so one Chromium holds many text contexts — +# raise this to trade RAM/CPU for throughput before adding processes +# (16 → ~69/min/process ⇒ ~7 processes cover 500/min). Validate with +# scripts/scale_google_search.py. +# GOOGLE_SEARCH_MAX_CONCURRENT_PAGES=4 +# Warm sticky-IP pool: concurrent renders spread across this many solved IPs so +# no single IP is hammered (Google re-walls a hot IP). Steady-state paid solves +# ≈ pool size, not request count. Size for the fleet: fleet per-IP load = +# IP_MAX_CONCURRENCY × processes must stay gentle. Solved IPs are shared across +# workers via Redis (REDIS_APP_URL) so the fleet solves each IP once. +# GOOGLE_SEARCH_WARM_POOL_TARGET=8 +# GOOGLE_SEARCH_IP_MAX_CONCURRENCY=2 +# How long a shared exemption stays reusable before a re-solve (Google's cookie +# outlives this; conservative TTL just re-solves idle IPs occasionally). +# GOOGLE_SEARCH_EXEMPTION_TTL_S=3600 +# Per-fetch budget before giving up under saturation (a cold solve is ~45s). +# GOOGLE_SEARCH_FETCH_DEADLINE_S=180 + # ===================================================================== -# Captcha solving (Phase 3d) — LAST-resort bypass tier via captchatools. -# Only fires on the stealth browser tier when a sitekey is detected AND -# CAPTCHA_SOLVING_ENABLED=TRUE. Cloudflare Turnstile is already solved free -# in-framework (no config needed). Off by default => zero attempts, zero cost. +# Captcha solving (Phase 3d) — LAST-resort bypass tier (in-house solver seam, +# app/utils/captcha/solvers.py). Only fires on the stealth browser tier when a +# sitekey is detected AND CAPTCHA_SOLVING_ENABLED=TRUE. Also unlocks Google's +# reCAPTCHA-Enterprise /sorry wall for the SERP scraper. Cloudflare Turnstile is +# already solved free in-framework. Off by default => zero attempts, zero cost. # NOTE: automated solving may violate a target site's ToS — opt-in, public -# data only (no logged-in bypass). captchatools is itself the vendor registry. +# data only (no logged-in bypass). # CAPTCHA_SOLVING_ENABLED=FALSE -# solving_site: capmonster | 2captcha | anticaptcha | capsolver | captchaai +# Provider: "capsolver" (AI-native, fastest on reCAPTCHA-Enterprise) or +# "2captcha" have in-house clients today (more added over time). # CAPTCHA_SOLVER_PROVIDER=capsolver # CAPTCHA_SOLVER_API_KEY= # Per-URL solve cap (bounds solver spend on a hostile page). diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 41b94a7e1..20497bb15 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -1102,7 +1102,8 @@ class Config: PROXY_URLS = os.getenv("PROXY_URLS") # ===================================================================== - # Phase 3d — Captcha solving (reCAPTCHA v2/v3, hCaptcha) via captchatools. + # Phase 3d — Captcha solving (reCAPTCHA v2/v3, hCaptcha, v2-Enterprise) via + # the in-house solver seam (app/utils/captcha/solvers.py). # The LAST-resort bypass tier: only fires on the StealthyFetcher browser # tier, only when a sitekey is detected, and only when explicitly enabled. # Cloudflare Turnstile is already handled free in-framework (03a), NOT here. @@ -1114,9 +1115,9 @@ class Config: CAPTCHA_SOLVING_ENABLED = ( os.getenv("CAPTCHA_SOLVING_ENABLED", "FALSE").upper() == "TRUE" ) - # captchatools "solving_site": capmonster | 2captcha | anticaptcha | - # capsolver | captchaai. captchatools is itself the provider registry, so we - # do not rebuild a vendor hierarchy. + # Solver vendor. "capsolver" (AI-native, fastest on reCAPTCHA-Enterprise) and + # "2captcha" have in-house clients today; anticaptcha / capmonster are added + # progressively in solvers._PROVIDERS. CAPTCHA_SOLVER_PROVIDER = os.getenv("CAPTCHA_SOLVER_PROVIDER", "capsolver") CAPTCHA_SOLVER_API_KEY = os.getenv("CAPTCHA_SOLVER_API_KEY") # Per-URL solve cap so one hostile page can't burn unbounded solver credit. diff --git a/surfsense_backend/app/proprietary/platforms/google_search/README.md b/surfsense_backend/app/proprietary/platforms/google_search/README.md index 726413194..dd7ee8f0f 100644 --- a/surfsense_backend/app/proprietary/platforms/google_search/README.md +++ b/surfsense_backend/app/proprietary/platforms/google_search/README.md @@ -27,22 +27,75 @@ traffic" wall, and the IPs that pass serve a JavaScript shell whose organic results only materialize after the page's JS runs. So `fetch.py` needs both a **non-blocked IP** and a **real browser render**, on the same IP: -1. Reuse the last known-good sticky IP if it still passes a cheap re-precheck; - otherwise race prechecks on several fresh sticky IPs at once (DataImpulse - maps ports → sessions) and take the first that passes, +1. Take a **warm sticky IP from the pool** (`_pool`): reuse an already-solved IP + under its soft per-IP concurrency cap — concurrent fetches fan out across the + pool's IPs (least-loaded first) instead of funneling onto one hot IP (which + Google re-walls). If the pool is below target, grow it: first try to **adopt** + an IP another process already solved (see the shared store below), else race + prechecks on several fresh sticky IPs at once (DataImpulse maps ports → + sessions) and solve on the first that passes. If the pool is full and every + IP is busy, wait briefly for a slot rather than burn a fresh solve, 2. render on that IP using a **shared long-lived browser** (launched once per process, per layout; each fetch only opens a fresh context carrying its - vetted proxy) — during which the render clicks the AI Overview's "Show - more" clamp and the initially-served People-Also-Ask questions open (all - clicks fired first, then one shared wait) so their content lands in the DOM; -3. retry on fresh IPs until one returns the results container. + vetted proxy). Renders drop image/font/media/stylesheet requests + (`disable_resources`; the parsed DOM is text/attributes only) — except AI + Mode, whose answer streams in. During the render it clicks the AI Overview's + "Show more" clamp and the initially-served People-Also-Ask questions open + (all clicks fired first, then one shared wait) so their content lands in the + DOM; +3. on success (re)admit the IP to the pool; a walled render evicts it (and drops + its exemption fleet-wide). Retry until a render returns the results container + or the per-fetch deadline / IP budget runs out. -A warm fetch (browser up, IP cached) runs ~8 s; the first fetch of a process -also pays the ~5 s Chromium launch and a vetting round. Requires the browser -tier (patchright Chromium via Scrapling's `AsyncStealthySession`) and a -residential proxy — set `PROXY_PROVIDER` + `PROXY_URL` (see -`.env`). Long-running callers can `await fetch.close_sessions()` on shutdown; -scripts that exit anyway can skip it. +**The reCAPTCHA wall.** A curl-vetted IP is only *half* the battle: the initial +`/search` GET returns a 200 JS shell, but the shell's JS reloads `/search` with +a `sei` token, which Google 302s into a reCAPTCHA-**Enterprise** `/sorry` page. +This trips *every* real browser (the block is on the browser access pattern, not +the IP — curl "passes" only because it never runs the reload, and so never gets +results either). When a solver is configured (`CAPTCHA_SOLVING_ENABLED=TRUE` + +`CAPTCHA_SOLVER_API_KEY`, see `app/utils/captcha/`), `captcha.py` runs inside the +render's `page_action`: on landing at `/sorry` it harvests an Enterprise token +(with the page's dynamic `data-s`) from the solver **egressing through the same +sticky proxy**, injects it, and submits. Google then issues a +`GOOGLE_ABUSE_EXEMPTION` cookie, which the fetch seam caches per proxy +(`_exemption_jar`) and replays via `page_setup` so **one solve unlocks the +sticky IP for all subsequent queries** on it. That solve is also **published to +Redis** (`pool_store.py`), so any *other* worker process can adopt the same +warm IP instead of re-solving — cost then tracks the shared pool size, not the +worker count (best-effort: no Redis ⇒ each process just keeps its own pool). +Without a solver configured the stealth tier is unchanged (and +brand/navigational queries will 429-wall). + +Timings (measured): a fetch that must solve runs **~40–110 s and is entirely +dominated by the solver** — the reCAPTCHA-Enterprise harvest alone measured +**27–39 s on capsolver** (the configured default; AI-native, tighter spread) and +37–100 s on 2captcha (pure vendor worker latency, not our overhead). NB the +sub-10 s figures both vendors advertise are for the plain reCAPTCHA *demo* +sitekey; Google's Enterprise `/sorry` wall is genuinely harder and slower for +everyone. Every *subsequent* fetch on that sticky IP reuses the cached exemption +and runs **~12–16 s** (two proxy navigations for Google's `sei` reload + ~3–4 s +of block expansion; the vet precheck is skipped, `render_ms` is the whole +render). The first fetch of a process also pays the ~5 s Chromium launch. So the +one lever that actually moves the needle is **solving less often** (the pool + +Redis-shared exemptions already amortize it) — or a faster solver. + +**Scale.** Per-process throughput ceiling = `GOOGLE_SEARCH_MAX_CONCURRENT_PAGES` +÷ warm-render-secs ≈ **4 / 14 s ≈ 17 SERP/min** at the default. Because renders +drop images/fonts/media, one Chromium holds well more than 4 text contexts, so +raise `GOOGLE_SEARCH_MAX_CONCURRENT_PAGES` to trade RAM/CPU for throughput +before adding processes (e.g. 16 → ~69/min/process ⇒ ~7 processes cover 500/min; +the sim in `scripts/scale_google_search.py` reproduces these numbers offline). +`GOOGLE_SEARCH_WARM_POOL_TARGET` sizes the shared warm-IP pool: steady-state +paid solves ≈ pool size (not request count), and it must be large enough that +fleet-wide per-IP load (local `GOOGLE_SEARCH_IP_MAX_CONCURRENCY` × processes) +stays gentle enough that Google doesn't re-wall a pool IP. Requires the +browser tier (patchright Chromium via +Scrapling's `AsyncStealthySession`) and a residential proxy — set +`PROXY_PROVIDER` + `PROXY_URL` (see `.env`; **do not** pin a `__cr.` +suffix — DataImpulse's per-country sub-pools are 429-walled even for the curl +precheck, which starves vetting; `gl=` in the URL sets result locale instead). +Long-running callers can `await fetch.close_sessions()` on shutdown; scripts +that exit anyway can skip it. ## Scope @@ -98,7 +151,9 @@ the progressive rollout.) | `schemas.py` | Pydantic input/output models mirroring the Apify camelCase spec. `extra="allow"` on outputs keeps the contract open. | | `scraper.py` | Orchestrator. `iter_serps` dispatches each `queries` line to `_term_flow` / `_url_flow` (+ `_ai_mode_flow` per term). | | `query_builder.py` | Pure: classify `queries` lines, fold advanced filters into search operators, resolve relative dates, build the URL. | -| `fetch.py` | Proxy-vetted two-phase fetch: cheap precheck GET + headless render on a shared sticky IP, retrying across IPs. | +| `fetch.py` | Proxy-vetted two-phase fetch: cheap precheck GET + headless render on a **warm sticky-IP pool** (spread across IPs, per-IP soft cap, grow-to-target), retrying across IPs. Caches per-IP reCAPTCHA exemptions. | +| `pool_store.py` | Best-effort Redis cache of solved-IP exemptions so a solve on one worker warms that IP for the whole fleet (adopt/publish/evict); silent no-op without Redis. | +| `captcha.py` | Async reCAPTCHA-Enterprise solve for the `/sorry` wall (via the `app.utils.captcha` solver seam — capsolver/2captcha, `enterprise`+`data-s`), run inside the render's `page_action`. | | `parsers.py` | Rendered SERP HTML → organic / text ads / product ads / related / People-Also-Ask / `resultsTotal` (degrades per-field). | ## Input semantics (matching Apify) diff --git a/surfsense_backend/app/proprietary/platforms/google_search/captcha.py b/surfsense_backend/app/proprietary/platforms/google_search/captcha.py new file mode 100644 index 000000000..2de1e9a0b --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_search/captcha.py @@ -0,0 +1,178 @@ +"""reCAPTCHA-Enterprise solve for Google's ``/sorry`` interstitial. + +Google walls a full browser's JS-driven ``sei`` reload by 302-ing it to +``/sorry/index`` — a reCAPTCHA **Enterprise** challenge. A curl-vetted IP still +hits this because the wall is on the *browser access pattern*, not the IP (curl +never runs the JS reload, so it only ever sees the useless no-JS shell). The +only way to keep rendering real SERPs on our own browser is to solve that +challenge. + +This runs inside the render's async ``page_action``: when the page lands on +``/sorry``, it harvests a token from the configured solver (egressing from the +**same sticky proxy** as the browser, so the token is IP-bound), injects it, and +submits the form. Google then issues a ``GOOGLE_ABUSE_EXEMPTION`` cookie that +unlocks real results on that IP — which the fetch seam caches per proxy so one +solve amortizes across subsequent fetches (see ``fetch._exemption_jar``). + +The actual vendor call goes through the shared, in-house solver seam +(:mod:`app.utils.captcha.solvers`) with ``enterprise=True`` + the page's +``data-s`` — this module just detects the wall, harvests via that seam egressing +through the render's own sticky proxy, injects the token, and submits. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import re +import time +from typing import Any + +from app.utils.captcha import CaptchaConfig +from app.utils.captcha import solvers + +logger = logging.getLogger(__name__) + +_LOG = "[google_search][captcha]" + +# The /sorry page embeds the widget sitekey and a dynamic `data-s` token that +# the Enterprise challenge binds to; both are mandatory in the solve request. +_SITEKEY_RE = re.compile(r'data-sitekey="([\w-]+)"') +_DATA_S_RE = re.compile(r'data-s="([^"]+)"') + +# After submitting the solved /sorry form, Google 302s back to the SERP. We wait +# only for the results container to attach — NOT for full network idle. A +# rendered SERP fires telemetry/XHRs that keep the network busy long past the +# point results are present, so a `networkidle` wait here burned up to its full +# timeout on every solve for no benefit; the container's presence is the real +# "we're through the wall" signal. +_POST_SOLVE_WAIT_MS = 15000 + +# Process-wide kill switch tripped on unrecoverable solver errors (no balance / +# bad key). Persists across the per-fetch page_action instances of one run; +# nothing short of a restart fixes balance/key, so keep hammering off. +_latched = False + + +def solver_latched() -> bool: + return _latched + + +def reset_solver_latch() -> None: + """Clear the process latch (test seam / explicit re-enable).""" + global _latched + _latched = False + + +def _latch(reason: str) -> None: + global _latched + _latched = True + logger.warning("%s solver latched OFF for this process: %s", _LOG, reason) + + +def on_sorry(page: Any) -> bool: + """True when the render landed on Google's ``/sorry`` reCAPTCHA wall.""" + return "/sorry/" in (getattr(page, "url", "") or "") + + +# Set the token into the response textarea and submit the /sorry form. Google's +# /sorry form posts to /sorry/index and, on a valid token, 302s back to the +# `continue` URL (the SERP) while setting GOOGLE_ABUSE_EXEMPTION. +_INJECT_JS = r""" +(token) => { + let ta = document.getElementById('g-recaptcha-response'); + if (!ta) { + ta = document.createElement('textarea'); + ta.id = 'g-recaptcha-response'; + ta.name = 'g-recaptcha-response'; + ta.style.display = 'none'; + (document.forms[0] || document.body).appendChild(ta); + } + ta.value = token; ta.innerHTML = token; + const form = ta.closest('form') || document.forms[0]; + if (form) { try { form.requestSubmit ? form.requestSubmit() : form.submit(); } catch (e) { form.submit(); } } +} +""" + +# Organic-results containers; presence means the exemption unlocked the SERP. +_RESULTS_SEL = "#search, #rso, div.tF2Cxc" + + +async def solve_sorry(page: Any, proxy_url: str | None, cfg: CaptchaConfig) -> bool: + """Solve the ``/sorry`` challenge on ``page`` and submit; return solved. + + Runs inside the async render. On success the page ends on the real SERP and + the context holds ``GOOGLE_ABUSE_EXEMPTION`` (the caller caches it). On any + failure returns ``False`` and leaves the page on ``/sorry`` so the fetch + loop moves to the next IP. + """ + if solver_latched(): + return False + try: + html = await page.content() + except Exception: + return False + sk = _SITEKEY_RE.search(html) + ds = _DATA_S_RE.search(html) + if not sk or not ds: + logger.warning("%s /sorry page missing sitekey/data-s; cannot solve", _LOG) + return False + + page_url = getattr(page, "url", "") or "" + try: + user_agent = await page.evaluate("() => navigator.userAgent") + except Exception: + user_agent = None + + logger.info("%s solving Enterprise reCAPTCHA site=%s", _LOG, sk.group(1)[:12]) + _t0 = time.perf_counter() + try: + token = await asyncio.to_thread( + solvers.solve, + cfg, + challenge_type="v2", + sitekey=sk.group(1), + page_url=page_url, + proxy_url=proxy_url, + user_agent=user_agent, + enterprise=True, + data_s=ds.group(1), + ) + except solvers.SolverError as e: + # Bad key / no balance / unsupported provider won't fix without a + # restart — latch so we stop hammering (and stop paying) this process. + _latch(repr(e)) + return False + except Exception as e: # never let a solver hiccup break the render + logger.warning("%s solve error: %s", _LOG, e) + return False + if not token: + return False + + harvest_ms = (time.perf_counter() - _t0) * 1000 + try: + await page.evaluate(_INJECT_JS, token) + except Exception as e: + logger.warning("%s token injection failed: %s", _LOG, e) + return False + with contextlib.suppress(Exception): + await page.wait_for_selector(_RESULTS_SEL, timeout=_POST_SOLVE_WAIT_MS) + + solved = not on_sorry(page) + logger.info( + "%s solve %s harvest_ms=%.0f total_ms=%.0f", + _LOG, + "OK" if solved else "did not clear wall", + harvest_ms, + (time.perf_counter() - _t0) * 1000, + ) + return solved + + +async def exemption_cookies(page: Any) -> list[dict]: + """Snapshot the context's google.com cookies (for the per-proxy jar).""" + try: + return await page.context.cookies("https://www.google.com") + except Exception: + return [] diff --git a/surfsense_backend/app/proprietary/platforms/google_search/fetch.py b/surfsense_backend/app/proprietary/platforms/google_search/fetch.py index 0e290fb4b..f11820047 100644 --- a/surfsense_backend/app/proprietary/platforms/google_search/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/google_search/fetch.py @@ -33,6 +33,7 @@ from __future__ import annotations import asyncio import contextlib import logging +import os import random import sys import threading @@ -41,6 +42,9 @@ from urllib.parse import urlsplit, urlunsplit from scrapling.fetchers import AsyncFetcher +from app.proprietary.platforms.google_search import captcha as _captcha +from app.proprietary.platforms.google_search import pool_store as _store +from app.utils.captcha import captcha_enabled, get_captcha_config from app.utils.proxy import get_proxy_url try: # browser tier is optional (needs `scrapling[fetchers]` browsers installed) @@ -55,6 +59,23 @@ logger = logging.getLogger(__name__) CONSENT_COOKIES = {"CONSENT": "PENDING+987", "SOCS": "CAESHAgBEhIaAB"} _HEADERS = {"Accept-Language": "en-US,en;q=0.9"} +# Context-cookie form of the consent cookies, injected before a render's +# navigation (page_setup) when captcha solving is on, so the browser skips the +# EU interstitial exactly like the curl precheck does. +_CONSENT_CTX = [ + {"name": k, "value": v, "domain": ".google.com", "path": "/"} + for k, v in CONSENT_COOKIES.items() +] + +# GOOGLE_ABUSE_EXEMPTION cookies harvested per sticky proxy after a solve. A hit +# lets a later render on the same IP skip straight to results (no re-solve), +# amortizing the one paid solve across the sticky session. +# ponytail: keyed by the full proxy URL (incl. sticky port). When that port's +# exit IP rotates the cached cookie goes stale -> the next render re-hits +# /sorry, re-solves, and overwrites the entry. Self-correcting; the sticky-port +# space is small so the dict never grows unbounded in practice. +_exemption_jar: dict[str, list[dict]] = {} + # Gateways whose sticky sessions are selected by destination port. A random port # in this range pins one residential IP for the duration of a browser session. _STICKY_HOSTS = {"gw.dataimpulse.com"} @@ -73,11 +94,37 @@ _VET_CONCURRENCY = 4 # the budget in a couple of seconds. _WALLED_ROUND_BACKOFF_S = 3.0 -# The sticky IP that most recently served a real SERP; the strongest hint for -# the next fetch. Re-vetted (cheap) before reuse, dropped on failure. -# ponytail: a single slot, not a pool — concurrent fetches share (and race) -# it; worst case a loser re-vets a fresh IP, which is the normal path anyway. -_last_good_proxy: str | None = None +# Warm sticky-IP pool. Concurrent fetches spread renders across the pool's IPs +# instead of funneling every render onto one sticky IP (a single hot residential +# IP is exactly what Google re-walls), and each pool IP is solved **once** — so +# steady-state paid solves track the pool size, not the request count. Replaces +# the old single "_last_good_proxy" slot, which both funneled load and forced a +# fresh solve whenever it was raced to None. +# +# State is touched from two loops (fetch_serp_html on the request loop; the +# exemption write in page_action on the browser loop), so a small lock guards +# every mutation. No awaits are held under the lock — every critical section is +# a handful of dict ops. +_pool: dict[str, float] = {} # warm proxy url -> last-used monotonic ts +_pool_inflight: dict[str, int] = {} # proxy url -> renders currently pinned to it +_pool_pending = 0 # fresh vets/solves in flight (growing the pool toward target) +_pool_lock = threading.Lock() + +# Target number of warm IPs to keep. Sized for throughput: it must cover +# (target SERP/min) / (safe SERP/min per IP), so a fleet serving 500/min across +# a handful of processes wants this in the low tens. Env-tunable without a code +# change; per-IP concurrency is a soft cap (exceeded only when the pool is full +# and every IP is busy, because reusing a warm IP always beats a fresh solve). +_WARM_POOL_TARGET = int(os.getenv("GOOGLE_SEARCH_WARM_POOL_TARGET", "8")) +_WARM_IP_MAX_CONCURRENCY = int(os.getenv("GOOGLE_SEARCH_IP_MAX_CONCURRENCY", "2")) +# When the pool is full and every IP is at its cap, wait this long for a slot to +# free instead of vetting+solving a fresh IP (this backpressure is what bounds +# solves to ~pool size under a burst). +_POOL_WAIT_S = 0.25 +# Absolute per-fetch budget. A cold solve is ~45 s and we may retry a couple of +# walled IPs, but a fetch that can't get results within this window fails rather +# than hanging a request forever under saturation. +_FETCH_DEADLINE_S = float(os.getenv("GOOGLE_SEARCH_FETCH_DEADLINE_S", "180")) # A usable precheck responds in <1 s; anything slower is a dead/slow sticky IP # (seen hanging ~60 s). Abandon it on this deadline so a slow IP costs no more @@ -172,6 +219,83 @@ async def _vet_fresh_ip(url: str, base: str) -> str | None: return winner +def _pool_take() -> tuple[str, str | None]: + """Decide how the next render gets an IP. Returns one of: + + * ``("reuse", url)`` — an existing warm IP under its soft concurrency cap + (its inflight count is already incremented; caller must ``_pool_settle``), + * ``("grow", None)`` — pool is below target, caller should vet a fresh IP + and solve on it (a pending slot is reserved), + * ``("wait", None)`` — pool is full and every IP is busy; caller should wait + briefly and retry rather than burn a fresh solve. + + Reuse prefers the least-loaded, least-recently-used warm IP so concurrent + renders fan out across the pool instead of piling onto one IP. + """ + global _pool_pending + with _pool_lock: + best: tuple[str, int, float] | None = None + for url, last in _pool.items(): + n = _pool_inflight.get(url, 0) + if n < _WARM_IP_MAX_CONCURRENCY and ( + best is None or (n, last) < (best[1], best[2]) + ): + best = (url, n, last) + if best is not None: + _pool_inflight[best[0]] = _pool_inflight.get(best[0], 0) + 1 + return ("reuse", best[0]) + if len(_pool) + _pool_pending < _WARM_POOL_TARGET: + _pool_pending += 1 + return ("grow", None) + return ("wait", None) + + +def _pool_bind_fresh(url: str) -> None: + """Pin the first render onto a just-vetted fresh IP (grow path).""" + with _pool_lock: + _pool_inflight[url] = _pool_inflight.get(url, 0) + 1 + + +def _pool_adopt(url: str) -> None: + """Turn a reserved grow slot into an IP adopted from the shared store: it + joins the warm pool without a local solve, so release the pending reservation + and pin this render onto it.""" + global _pool_pending + with _pool_lock: + _pool_pending = max(0, _pool_pending - 1) + _pool[url] = time.monotonic() + _pool_inflight[url] = _pool_inflight.get(url, 0) + 1 + + +def _pool_abort_grow() -> None: + """Release a reserved grow slot when the fresh vet found no usable IP.""" + global _pool_pending + with _pool_lock: + _pool_pending = max(0, _pool_pending - 1) + + +def _pool_settle(url: str, *, good: bool, grew: bool) -> None: + """Finish a render: drop its inflight hold, then admit or evict the IP. + + A good render (re)admits the IP to the warm pool (refreshing its LRU stamp); + a walled render evicts it and drops its now-worthless cached exemption. + """ + global _pool_pending + with _pool_lock: + n = _pool_inflight.get(url, 0) - 1 + if n <= 0: + _pool_inflight.pop(url, None) + else: + _pool_inflight[url] = n + if grew: + _pool_pending = max(0, _pool_pending - 1) + if good: + _pool[url] = time.monotonic() + else: + _pool.pop(url, None) + _exemption_jar.pop(url, None) + + # People-also-ask answers only load when a question is expanded (clicked). # We expand just the initially-served questions (~4); each expansion appends # more questions we deliberately leave collapsed, or the loop never ends. @@ -197,6 +321,7 @@ async def _expand_blocks(page): Both are free on pages without the block, and best-effort: a failed click just leaves that section collapsed rather than failing the render. """ + _t0 = time.perf_counter() clicked = 0 try: more = await page.query_selector(_AIO_SHOW_MORE_SEL) @@ -218,9 +343,45 @@ async def _expand_blocks(page): if clicked: # One shared wait while all the answer XHRs land in parallel. await page.wait_for_timeout(_PAA_ANSWER_WAIT_MS) + logger.info( + "[google_search][perf] expand_ms=%.0f clicked=%d", + (time.perf_counter() - _t0) * 1000, + clicked, + ) return page +def _make_page_setup(proxy: str | None): + """Per-fetch ``page_setup``: seed consent + any cached exemption for this IP.""" + + async def page_setup(page): + jar = list(_CONSENT_CTX) + cached = _exemption_jar.get(proxy or "") + if cached: + jar += cached + with contextlib.suppress(Exception): + await page.context.add_cookies(jar) + + return page_setup + + +def _make_page_action(proxy: str | None, cfg): + """Per-fetch ``page_action``: solve the /sorry wall if hit, then expand blocks. + + Overrides the session default (:func:`_expand_blocks`) so it must still call + it afterwards. On a successful solve the resulting ``GOOGLE_ABUSE_EXEMPTION`` + cookie is cached against this sticky proxy for reuse. + """ + + async def page_action(page): + if _captcha.on_sorry(page): + if await _captcha.solve_sorry(page, proxy, cfg): + _exemption_jar[proxy or ""] = await _captcha.exemption_cookies(page) + return await _expand_blocks(page) + + return page_action + + # Firefox-on-Android UA to make Google serve its mobile lightweight layout. # ponytail: the engine underneath is patchright's *Chromium*, so this UA lies # about the engine — but empirically it's what gets the mobile layout served @@ -279,9 +440,16 @@ _session_lock = asyncio.Lock() # second concurrent render raised RuntimeError('Maximum page limit (1) # reached'); the failure handler then closed the shared browser under the # sibling render (the TargetClosedError cascade seen in production when -# several scrape runs overlap). The pool and this gate are sized together: -# the gate queues excess renders instead of tripping the pool. -_MAX_CONCURRENT_PAGES = 4 +# several scrape runs overlap). The pool and this gate are sized together +# (the session's max_pages is set from this): the gate queues excess renders +# instead of tripping the pool. +# +# This is THE throughput lever: per-process ceiling = this / warm-render-secs +# (≈ 4/14 s ≈ 17 SERP/min at the default). Renders drop images/fonts/media +# (`disable_resources`), so a single Chromium can hold well more than 4 text +# contexts — raise this (env) to trade RAM/CPU for throughput before adding +# processes. e.g. 16 → ~68/min/process, so ~8 processes cover 500/min. +_MAX_CONCURRENT_PAGES = int(os.getenv("GOOGLE_SEARCH_MAX_CONCURRENT_PAGES", "4")) # Only ever awaited from coroutines running on the browser loop. _render_gate = asyncio.Semaphore(_MAX_CONCURRENT_PAGES) @@ -350,7 +518,23 @@ async def _render_on_loop(url: str, proxy: str | None, mobile: bool): session = await _get_session(mobile) _inflight[session] = _inflight.get(session, 0) + 1 try: - return await session.fetch(url, proxy=proxy) + kwargs: dict = {} + # Drop image/font/media/stylesheet requests: the SERP DOM we parse is + # text/attribute-only, so these are dead weight (a warm render moved + # ~2-7 MB per page) that also keeps `network_idle` from settling. + # AI Mode streams its answer into the DOM, so leave its resources on + # (dropping websockets/media can starve the stream) — detected by the + # udm=50 marker its URL always carries. + if "udm=50" not in url: + kwargs["disable_resources"] = True + # Only wire the solver when it's configured AND still viable this + # process; otherwise the session's default page_action (expand-only) + # runs and the stealth tier is unchanged. + if proxy and captcha_enabled() and not _captcha.solver_latched(): + cfg = get_captcha_config() + kwargs["page_setup"] = _make_page_setup(proxy) + kwargs["page_action"] = _make_page_action(proxy, cfg) + return await session.fetch(url, proxy=proxy, **kwargs) finally: _inflight[session] -= 1 if not _inflight[session]: @@ -374,35 +558,61 @@ async def _render(url: str, proxy: str | None, mobile: bool = False): async def fetch_serp_html(url: str, *, mobile: bool = False) -> str | None: """Return fully-rendered SERP HTML for ``url``, or ``None`` if unobtainable. - Reuses the last known-good sticky IP when it still passes the cheap - precheck; otherwise races prechecks on fresh sticky IPs and renders on the - first that passes. Retries until a render returns real results or the IP - budget runs out. Requires the browser tier — without it we cannot get - JS-built results. ``mobile`` renders with a phone UA/viewport (the - ``mobileResults`` input). + Renders on a **warm sticky-IP pool**: it reuses an already-solved IP under + its soft concurrency cap (spreading concurrent renders across the pool), + grows the pool by vetting+solving a fresh IP when it's below target, and + applies backpressure (a short wait) when the pool is full and busy rather + than burning an extra solve. Retries walled IPs until real results land or + the per-fetch deadline / IP budget runs out. Requires the browser tier — + without it we cannot get JS-built results. ``mobile`` renders with a phone + UA/viewport (the ``mobileResults`` input). """ - global _last_good_proxy if AsyncStealthySession is None: logger.error("[google_search] browser tier unavailable; cannot render SERPs") return None base = get_proxy_url() + if not base: + # No proxy configured: a single direct render, no pool bookkeeping. + with contextlib.suppress(Exception): + page = await _render(url, None, mobile=mobile) + html = page.html_content or "" + return html if (page.status == 200 and _has_results(html)) else None + return None + + deadline = time.monotonic() + _FETCH_DEADLINE_S ips_tried = 0 - while ips_tried < _MAX_IP_ATTEMPTS: - if base: - if _last_good_proxy and await _precheck(url, _last_good_proxy): - proxy = _last_good_proxy + while time.monotonic() < deadline and ips_tried < _MAX_IP_ATTEMPTS: + action, proxy = _pool_take() + if action == "wait": + # Pool full and every IP busy: wait for a warm slot instead of + # solving a fresh IP. The render gate drains continuously, so a slot + # frees shortly; this is the queue that bounds solves to ~pool size. + await asyncio.sleep(_POOL_WAIT_S) + continue + solved_fresh = False + vet_ms = 0.0 + if action == "grow": + # Before paying for a solve, try to adopt an IP another process + # already solved (fleet-shared exemption). Only if none is available + # do we vet a fresh IP and solve on it ourselves. + adopted = await _store.adopt(set(_pool)) + if adopted is not None: + proxy, cookies = adopted + _exemption_jar[proxy] = cookies + _pool_adopt(proxy) else: - _last_good_proxy = None + vt = time.perf_counter() proxy = await _vet_fresh_ip(url, base) + vet_ms = (time.perf_counter() - vt) * 1000 ips_tried += _VET_CONCURRENCY if proxy is None: + _pool_abort_grow() logger.debug("[google_search] vetting round: all IPs walled") await asyncio.sleep(_WALLED_ROUND_BACKOFF_S) continue - else: - proxy = None - ips_tried += 1 + _pool_bind_fresh(proxy) + solved_fresh = True started = time.perf_counter() try: page = await _render(url, proxy, mobile=mobile) @@ -411,23 +621,32 @@ async def fetch_serp_html(url: str, *, mobile: bool = False) -> str | None: # browser side is broken, so relaunch it rather than limp along. # repr(), not str(): e.g. NotImplementedError stringifies to "". logger.warning("[google_search] render failed: %r", e) - _last_good_proxy = None + _pool_settle(proxy, good=False, grew=solved_fresh) + await _store.evict(proxy) await _drop_session(mobile) continue fetch_ms = (time.perf_counter() - started) * 1000 html = page.html_content or "" good = page.status == 200 and _has_results(html) + _pool_settle(proxy, good=good, grew=solved_fresh) logger.info( - "[google_search][perf] status=%s bytes=%d has_results=%s fetch_ms=%.0f reused_ip=%s", + "[google_search][perf] status=%s bytes=%d has_results=%s vet_ms=%.0f render_ms=%.0f from_pool=%s pool=%d", page.status, len(html), good, + vet_ms, fetch_ms, - proxy == _last_good_proxy, + not solved_fresh, + len(_pool), ) if good: - _last_good_proxy = proxy + # Share a freshly-solved IP with the fleet; a walled IP is poison, + # so make sure no process keeps its stale exemption. + if solved_fresh and _exemption_jar.get(proxy): + await _store.publish(proxy, _exemption_jar[proxy]) return html - _last_good_proxy = None - logger.warning("[google_search] exhausted %d IPs for %s", _MAX_IP_ATTEMPTS, url) + await _store.evict(proxy) + logger.warning( + "[google_search] gave up on %s (deadline/%d-IP budget)", url, _MAX_IP_ATTEMPTS + ) return None diff --git a/surfsense_backend/app/proprietary/platforms/google_search/pool_store.py b/surfsense_backend/app/proprietary/platforms/google_search/pool_store.py new file mode 100644 index 000000000..f7271a54c --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_search/pool_store.py @@ -0,0 +1,121 @@ +"""Cross-process warm-IP exemption cache (Redis-backed, best-effort). + +The expensive unit is the reCAPTCHA solve. Within a process the warm pool +(:mod:`fetch`) already solves each sticky IP once; this shares those solves +across the whole worker fleet: a fresh solve on any process **publishes** the +IP's ``GOOGLE_ABUSE_EXEMPTION`` cookies to Redis, and every other process can +**adopt** that IP instead of solving it again. Without this, cost scales with +the number of worker processes; with it, cost scales with the (shared) pool +size. + +Best-effort by design: any Redis hiccup disables the layer and every process +falls back to its own local pool — correctness is unchanged, we just pay more +solves. Redis calls run in a worker thread (:func:`asyncio.to_thread`) so they +never block the request loop and don't care which loop the caller is on. + +``ponytail:`` the shared store holds only exemptions (the costly artifact), not +per-process render concurrency — global per-IP load is still governed by each +process's local per-IP cap × the number of processes, so size the pool for the +fleet (see ``GOOGLE_SEARCH_WARM_POOL_TARGET``). Full distributed inflight +accounting is the upgrade path if a single shared IP ever gets overloaded. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import logging +import os + +from app.config import config + +logger = logging.getLogger(__name__) + +_LOG = "[google_search][poolstore]" +_PREFIX = "gsearch:warm:" +# Google's GOOGLE_ABUSE_EXEMPTION outlives this comfortably; a conservative TTL +# just means the fleet re-solves an idle IP occasionally (cheap) and never +# serves a long-dead exemption. Env-tunable. +_TTL_S = int(os.getenv("GOOGLE_SEARCH_EXEMPTION_TTL_S", "3600")) + +_client = None +_disabled = False + + +def _key(proxy: str) -> str: + return _PREFIX + hashlib.sha1(proxy.encode()).hexdigest() + + +def _get_client(): + """Lazily build a short-timeout Redis client, disabling on first failure.""" + global _client, _disabled + if _disabled: + return None + if _client is not None: + return _client + try: + import redis + + _client = redis.Redis.from_url( + config.REDIS_APP_URL, socket_timeout=1, socket_connect_timeout=1 + ) + _client.ping() + except Exception as e: + logger.info("%s Redis unavailable; cross-process sharing off: %s", _LOG, e) + _disabled = True + _client = None + return _client + + +def _publish_sync(proxy: str, cookies: list[dict]) -> None: + c = _get_client() + if c is None: + return + try: + c.set(_key(proxy), json.dumps({"proxy": proxy, "cookies": cookies}), ex=_TTL_S) + except Exception as e: + logger.debug("%s publish failed: %s", _LOG, e) + + +def _adopt_sync(exclude: set[str]) -> tuple[str, list[dict]] | None: + c = _get_client() + if c is None: + return None + try: + for key in c.scan_iter(match=_PREFIX + "*", count=100): + raw = c.get(key) + if not raw: + continue + d = json.loads(raw) + proxy = d.get("proxy") + if proxy and proxy not in exclude: + return proxy, d.get("cookies") or [] + except Exception as e: + logger.debug("%s adopt failed: %s", _LOG, e) + return None + + +def _evict_sync(proxy: str) -> None: + c = _get_client() + if c is None: + return + try: + c.delete(_key(proxy)) + except Exception as e: + logger.debug("%s evict failed: %s", _LOG, e) + + +async def publish(proxy: str, cookies: list[dict]) -> None: + """Share a freshly-solved IP's exemption with the fleet.""" + await asyncio.to_thread(_publish_sync, proxy, cookies) + + +async def adopt(exclude: set[str]) -> tuple[str, list[dict]] | None: + """Return a fleet-warm ``(proxy, cookies)`` not in ``exclude``, or ``None``.""" + return await asyncio.to_thread(_adopt_sync, exclude) + + +async def evict(proxy: str) -> None: + """Drop a poisoned (walled) IP so no process adopts its stale exemption.""" + await asyncio.to_thread(_evict_sync, proxy) diff --git a/surfsense_backend/app/proprietary/web_crawler/captcha.py b/surfsense_backend/app/proprietary/web_crawler/captcha.py index 151eac4ce..b585a98fc 100644 --- a/surfsense_backend/app/proprietary/web_crawler/captcha.py +++ b/surfsense_backend/app/proprietary/web_crawler/captcha.py @@ -6,10 +6,10 @@ """Captcha detection + token-injection ``page_action`` (Phase 3d). This is the **bypass logic** (hence proprietary); the generic, vendor-agnostic -config lives in the Apache-2.0 ``app/utils/captcha/`` package. ``captchatools`` -is the multi-vendor solver registry — we only detect the challenge, harvest a -token egressing from **the crawl's own proxy IP** (token IP-binding), inject it, -and submit. +config and the vendor API clients live in the Apache-2.0 ``app/utils/captcha/`` +package (:mod:`app.utils.captcha.solvers`). We only detect the challenge, +harvest a token through that seam egressing from **the crawl's own proxy IP** +(token IP-binding), inject it, and submit. Why a closure cell: Scrapling runs ``page_action`` after navigation but **swallows its exceptions and discards its return value** (see @@ -29,7 +29,6 @@ import re from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout from typing import Any -from urllib.parse import urlsplit from app.utils.captcha import CaptchaConfig @@ -118,26 +117,6 @@ def detect_challenge(page: Any, cfg: CaptchaConfig) -> tuple[str, str] | None: # --- proxy / harvest ------------------------------------------------------- -def proxy_url_to_captchatools(proxy_url: str | None) -> str | None: - """Reformat ``http://user:pass@host:port`` -> ``host:port:user:pass``. - - captchatools wants the colon-delimited form. Returns ``None`` for a missing - or unparseable proxy so the solver harvests proxyless (the token may then be - IP-mismatched, but that's better than crashing the fetch). - """ - if not proxy_url: - return None - try: - p = urlsplit(proxy_url) - if not p.hostname or not p.port: - return None - if p.username and p.password: - return f"{p.hostname}:{p.port}:{p.username}:{p.password}" - return f"{p.hostname}:{p.port}" - except Exception: - return None - - def _harvest_token( cfg: CaptchaConfig, challenge_type: str, @@ -146,26 +125,21 @@ def _harvest_token( proxy: str | None, user_agent: str | None, ) -> str | None: - """Harvest a token from the solver. Raises on solver errors so the caller - can latch on the unrecoverable ones; returns ``None`` on an empty token. + """Harvest a token from the in-house solver seam. Raises on solver errors so + the caller can latch on the unrecoverable ones; returns ``None`` on an empty + token. ``proxy`` is a raw ``http://user:pass@host:port`` URL (the seam + reformats it per vendor). """ - import captchatools + from app.utils.captcha import solvers - harvester = captchatools.new_harvester( - api_key=cfg.api_key, - solving_site=cfg.solving_site, + return solvers.solve( + cfg, + challenge_type=challenge_type, sitekey=sitekey, - captcha_url=page_url, - captcha_type=challenge_type, - min_score=cfg.v3_min_score, - action=cfg.v3_action, - ) - token = harvester.get_token( - proxy=proxy, - proxy_type="HTTP", + page_url=page_url, + proxy_url=proxy, user_agent=user_agent, ) - return token or None # --- injection ------------------------------------------------------------- @@ -247,7 +221,9 @@ def build_captcha_page_action( if not cfg.enabled or solver_latched(): return None - captcha_proxy = proxy_url_to_captchatools(proxy_url) + # Raw URL; the solver seam reformats it per vendor and binds the token to + # this exit IP. + captcha_proxy = proxy_url def page_action(page: Any) -> Any: # Hard cap: never exceed the per-URL budget even across proxy-retry @@ -320,10 +296,14 @@ def build_captcha_page_action( def _is_unrecoverable(exc: Exception) -> bool: - """True for solver errors that must latch solving off (no balance / bad key). + """True for solver errors that must latch solving off. - Matched by class name to avoid a hard import of ``captchatools.exceptions`` - at module load (the dep is optional / lazily imported). + Covers the in-house seam's typed errors (``SolverBalanceError`` / + ``SolverAuthError`` / ``SolverUnsupported``) plus legacy/no-balance shapes. + Matched by class name so no solver module must be imported here. """ name = type(exc).__name__.lower() - return "nobalance" in name or "wrongapikey" in name or "apikey" in name + return any( + k in name + for k in ("balance", "apikey", "auth", "unsupported", "wronguser") + ) diff --git a/surfsense_backend/app/utils/captcha/__init__.py b/surfsense_backend/app/utils/captcha/__init__.py index 499eaa876..e446141e1 100644 --- a/surfsense_backend/app/utils/captcha/__init__.py +++ b/surfsense_backend/app/utils/captcha/__init__.py @@ -1,15 +1,16 @@ """App-wide captcha-solver configuration (Apache-2.0, generic glue). -This package holds only the **generic, vendor-agnostic** config resolution for -captcha solving — mirroring ``app/utils/proxy/`` (which stays Apache-2.0 even -though the crawler that consumes it is proprietary). The actual bypass logic -(challenge detection + token injection ``page_action``) lives in the separately -licensed ``app/proprietary/web_crawler/captcha.py``. +This package holds the **generic, vendor-agnostic** config resolution for +captcha solving (:class:`CaptchaConfig`) plus the in-house vendor API clients +(:mod:`app.utils.captcha.solvers`) — mirroring ``app/utils/proxy/`` (which stays +Apache-2.0 even though the crawler that consumes it is proprietary). The actual +bypass logic (challenge detection + token injection ``page_action``) lives in +the separately licensed ``app/proprietary/web_crawler/captcha.py`` and +``app/proprietary/platforms/google_search/captcha.py``. -``captchatools`` is itself the multi-vendor registry (capmonster / 2captcha / -anticaptcha / capsolver / captchaai), so there is no provider hierarchy here — -just one env-driven :class:`CaptchaConfig` and a cheap ``captcha_enabled()`` -gate callers check before constructing anything (mirrors +``solvers.solve()`` dispatches on ``CAPTCHA_SOLVER_PROVIDER``; only 2captcha is +wired today, more vendors are added progressively. ``captcha_enabled()`` is the +cheap gate callers check before constructing anything (mirrors ``WebCrawlCreditService.billing_enabled()``). """ diff --git a/surfsense_backend/app/utils/captcha/solvers.py b/surfsense_backend/app/utils/captcha/solvers.py new file mode 100644 index 000000000..db157f873 --- /dev/null +++ b/surfsense_backend/app/utils/captcha/solvers.py @@ -0,0 +1,311 @@ +"""In-house captcha-solver clients (vendor API glue, Apache-2.0). + +Replaces the unmaintained ``captchatools`` registry. Each provider is a small +function that submits a challenge to the vendor and polls for the token; +:func:`solve` dispatches on the app-wide ``CAPTCHA_SOLVER_PROVIDER``. + +Two providers are wired: **2captcha** (legacy ``in.php``/``res.php``) and +**capsolver** (AI-native ``createTask``/``getTaskResult``, materially faster on +the reCAPTCHA-*Enterprise* ``/sorry`` wall). Both express the Enterprise pieces +Google needs — the widget sitekey plus the page's dynamic ``data-s`` token +(something ``captchatools`` could not). More vendors (anticaptcha / capmonster) +are added progressively as new entries in :data:`_PROVIDERS`; until then an +unconfigured provider raises :class:`SolverUnsupported` so callers latch off +cleanly instead of leaking the API key to the wrong service. +""" + +from __future__ import annotations + +import logging +import time +from urllib.parse import urlsplit + +import requests + +from app.utils.captcha.config import CaptchaConfig + +logger = logging.getLogger(__name__) + +_LOG = "[captcha][solver]" + + +class SolverError(Exception): + """Base class for solver failures the caller may want to latch on.""" + + +class SolverAuthError(SolverError): + """Bad / unknown API key — unrecoverable without a config change.""" + + +class SolverBalanceError(SolverError): + """Solver account is out of balance — unrecoverable this process.""" + + +class SolverUnsupported(SolverError): + """Configured provider has no in-house client yet — unrecoverable.""" + + +# --- 2captcha (legacy in.php/res.php) -------------------------------------- + +_2CAP_IN = "http://2captcha.com/in.php" +_2CAP_RES = "http://2captcha.com/res.php" +# Historical captchatools soft_id, kept so solves stay attributed on 2captcha. +_2CAP_SOFT_ID = 4782723 + + +def proxy_login_form(proxy_url: str | None) -> str | None: + """``http://user:pass@host:port`` -> ``user:pass@host:port`` (no scheme). + + The vendor APIs want the proxy without a scheme prefix (2captcha rejects a + scheme with ``ERROR_PROXY_FORMAT``). Returns ``None`` for a missing or + unparseable proxy so the solve goes proxyless rather than crashing the fetch + (the token may then be IP-mismatched, but that fails cleanly). + """ + if not proxy_url: + return None + try: + p = urlsplit(proxy_url) + if not p.hostname or not p.port: + return None + if p.username and p.password: + return f"{p.username}:{p.password}@{p.hostname}:{p.port}" + return f"{p.hostname}:{p.port}" + except Exception: + return None + + +def _raise_2cap(err: str) -> None: + """Map a 2captcha error string to a typed exception (or just log a soft one).""" + up = err.upper() + if "ZERO_BALANCE" in up: + raise SolverBalanceError(err) + if "WRONG_USER_KEY" in up or "KEY_DOES_NOT_EXIST" in up: + raise SolverAuthError(err) + logger.warning("%s 2captcha error: %s", _LOG, err) + + +def _twocaptcha( + cfg: CaptchaConfig, + *, + challenge_type: str, + sitekey: str, + page_url: str, + proxy_url: str | None, + user_agent: str | None, + enterprise: bool, + data_s: str | None, +) -> str | None: + payload: dict = {"key": cfg.api_key, "json": 1, "soft_id": _2CAP_SOFT_ID} + if challenge_type in ("hcaptcha", "hcap"): + payload |= {"method": "hcaptcha", "sitekey": sitekey, "pageurl": page_url} + elif challenge_type == "v3": + payload |= { + "method": "userrecaptcha", + "version": "v3", + "googlekey": sitekey, + "pageurl": page_url, + "action": cfg.v3_action, + "min_score": cfg.v3_min_score, + } + else: # v2, optionally the Enterprise variant (adds enterprise=1 + data-s) + payload |= {"method": "userrecaptcha", "googlekey": sitekey, "pageurl": page_url} + if enterprise: + payload["enterprise"] = 1 + if data_s: + payload["data-s"] = data_s + + proxy = proxy_login_form(proxy_url) + if proxy: + payload["proxy"] = proxy + payload["proxytype"] = "HTTP" + if user_agent: + payload["userAgent"] = user_agent + + try: + submit = requests.post(_2CAP_IN, data=payload, timeout=30).json() + except requests.RequestException as e: + logger.warning("%s 2captcha submit request failed: %s", _LOG, e) + return None + if submit.get("status") != 1: + _raise_2cap(str(submit.get("request", ""))) + return None + + task_id = submit["request"] + deadline = time.monotonic() + cfg.timeout_s + while time.monotonic() < deadline: + time.sleep(5) + try: + got = requests.get( + f"{_2CAP_RES}?key={cfg.api_key}&action=get&id={task_id}&json=1", + timeout=30, + ).json() + except requests.RequestException: + continue + if got.get("status") == 1: + return got["request"] or None + req = str(got.get("request", "")) + if req != "CAPCHA_NOT_READY": + _raise_2cap(req) + return None + logger.warning("%s 2captcha solve timed out after %ss", _LOG, cfg.timeout_s) + return None + + +# --- capsolver (createTask / getTaskResult) -------------------------------- + +_CAPSOLVER_CREATE = "https://api.capsolver.com/createTask" +_CAPSOLVER_RESULT = "https://api.capsolver.com/getTaskResult" + + +def capsolver_proxy(proxy_url: str | None) -> str | None: + """``http://user:pass@host:port`` -> ``http:host:port:user:pass``. + + CapSolver's ``proxy`` field is a single colon-delimited + ``scheme:host:port[:user:pass]`` string (NOT a URL). Returns ``None`` for a + missing/unparseable proxy so the caller falls back to a proxyless task. + """ + if not proxy_url: + return None + try: + p = urlsplit(proxy_url) + if not p.hostname or not p.port: + return None + scheme = p.scheme or "http" + if p.username and p.password: + return f"{scheme}:{p.hostname}:{p.port}:{p.username}:{p.password}" + return f"{scheme}:{p.hostname}:{p.port}" + except Exception: + return None + + +def _raise_capsolver(code: str, desc: str) -> None: + """Map a CapSolver errorCode to a typed exception (or log a soft one).""" + up = (code or "").upper() + if "ZERO_BALANCE" in up or "INSUFFICIENT" in up: + raise SolverBalanceError(f"{code}: {desc}") + if "KEY" in up: # ERROR_KEY_DENIED_ACCESS / ERROR_KEY_DOES_NOT_EXIST + raise SolverAuthError(f"{code}: {desc}") + logger.warning("%s capsolver error: %s %s", _LOG, code, desc) + + +def _capsolver( + cfg: CaptchaConfig, + *, + challenge_type: str, + sitekey: str, + page_url: str, + proxy_url: str | None, + user_agent: str | None, + enterprise: bool, + data_s: str | None, +) -> str | None: + # We always egress through our own sticky proxy, so the proxied task types + # (no "ProxyLess" casing ambiguity) are the hot path; the proxyless variants + # are only the fallback when no proxy was threaded through. + proxy = capsolver_proxy(proxy_url) + has_proxy = proxy is not None + task: dict = {"websiteURL": page_url, "websiteKey": sitekey} + if challenge_type in ("hcaptcha", "hcap"): + task["type"] = "HCaptchaTask" if has_proxy else "HCaptchaTaskProxyLess" + elif challenge_type == "v3": + base = "ReCaptchaV3EnterpriseTask" if enterprise else "ReCaptchaV3Task" + task["type"] = base if has_proxy else base + "ProxyLess" + task["pageAction"] = cfg.v3_action + task["minScore"] = cfg.v3_min_score + else: # v2, optionally the Enterprise variant (Google /sorry) + base = "ReCaptchaV2EnterpriseTask" if enterprise else "ReCaptchaV2Task" + task["type"] = base if has_proxy else base + "ProxyLess" + if data_s: + # Enterprise carries the /anchor `s` under enterprisePayload; the + # normal-v2 field is recaptchaDataSValue. + task["enterprisePayload" if enterprise else "recaptchaDataSValue"] = ( + {"s": data_s} if enterprise else data_s + ) + if has_proxy: + task["proxy"] = proxy + if user_agent: + task["userAgent"] = user_agent + + try: + created = requests.post( + _CAPSOLVER_CREATE, + json={"clientKey": cfg.api_key, "task": task}, + timeout=30, + ).json() + except requests.RequestException as e: + logger.warning("%s capsolver createTask failed: %s", _LOG, e) + return None + if created.get("errorId"): + _raise_capsolver( + str(created.get("errorCode", "")), str(created.get("errorDescription", "")) + ) + return None + task_id = created.get("taskId") + if not task_id: + return None + + deadline = time.monotonic() + cfg.timeout_s + while time.monotonic() < deadline: + time.sleep(2) # AI solver is fast; poll tighter than 2captcha's 5 s + try: + got = requests.post( + _CAPSOLVER_RESULT, + json={"clientKey": cfg.api_key, "taskId": task_id}, + timeout=30, + ).json() + except requests.RequestException: + continue + if got.get("errorId"): + _raise_capsolver( + str(got.get("errorCode", "")), str(got.get("errorDescription", "")) + ) + return None + if got.get("status") == "ready": + return (got.get("solution") or {}).get("gRecaptchaResponse") or None + logger.warning("%s capsolver solve timed out after %ss", _LOG, cfg.timeout_s) + return None + + +# provider name (CAPTCHA_SOLVER_PROVIDER) -> client. Add vendors here. +_PROVIDERS = {"2captcha": _twocaptcha, "capsolver": _capsolver} + + +def supported_providers() -> list[str]: + return sorted(_PROVIDERS) + + +def solve( + cfg: CaptchaConfig, + *, + challenge_type: str, + sitekey: str, + page_url: str, + proxy_url: str | None = None, + user_agent: str | None = None, + enterprise: bool = False, + data_s: str | None = None, +) -> str | None: + """Harvest a token from the configured solver, or ``None`` on soft failure. + + ``challenge_type`` is ``v2`` / ``v3`` / ``hcaptcha``; set ``enterprise`` + + ``data_s`` for reCAPTCHA-v2-Enterprise pages (e.g. Google ``/sorry``). The + solve egresses through ``proxy_url`` so the token is bound to the crawl's + own exit IP. Raises a :class:`SolverError` subclass on unrecoverable errors + (bad key / no balance / unsupported provider) so callers can latch. + """ + client = _PROVIDERS.get((cfg.solving_site or "").lower()) + if client is None: + raise SolverUnsupported( + f"captcha provider {cfg.solving_site!r} has no in-house client " + f"(supported: {supported_providers()})" + ) + return client( + cfg, + challenge_type=challenge_type, + sitekey=sitekey, + page_url=page_url, + proxy_url=proxy_url, + user_agent=user_agent, + enterprise=enterprise, + data_s=data_s, + ) diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 30bb7ea5c..024c1b444 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -44,7 +44,6 @@ dependencies = [ "azure-storage-blob>=12.23.0", "fake-useragent>=2.2.0", "trafilatura>=2.0.0", - "captchatools>=1.5.0", "fastapi-users[oauth,sqlalchemy]>=15.0.3", "chonkie[all]>=1.5.0", "langgraph-checkpoint-postgres>=3.0.2", @@ -86,7 +85,7 @@ dependencies = [ "opentelemetry-instrumentation-logging>=0.61b0", "python-telegram-bot>=22.7", "croniter>=2.0.0", - "scrapling[fetchers]>=0.4.9", + "scrapling[fetchers]>=0.4.11", ] [project.optional-dependencies] diff --git a/surfsense_backend/scripts/e2e_google_search.py b/surfsense_backend/scripts/e2e_google_search.py index 9c3c02fcf..3742f3f22 100644 --- a/surfsense_backend/scripts/e2e_google_search.py +++ b/surfsense_backend/scripts/e2e_google_search.py @@ -28,9 +28,12 @@ sys.path.insert(0, str(_ROOT)) load_dotenv(_ROOT / ".env") logging.basicConfig(level=logging.WARNING) -logging.getLogger("app.proprietary.platforms.google_search.scraper").setLevel( - logging.INFO -) +for _name in ( + "app.proprietary.platforms.google_search.scraper", + "app.proprietary.platforms.google_search.fetch", + "app.proprietary.platforms.google_search.captcha", +): + logging.getLogger(_name).setLevel(logging.INFO) from app.proprietary.platforms.google_search import ( # noqa: E402 GoogleSearchScrapeInput, @@ -146,6 +149,9 @@ async def run( _CASES = { "plain": lambda: run("plain query", queries="python asyncio tutorial"), + # Prod incident 2026-07-17: single-word brand/navigational queries were + # exhausting all 24 IPs (precheck 200 but the browser render 429-walled). + "brand": lambda: run("brand query (notebooklm, prod repro)", queries="notebooklm"), "site": lambda: run("site: filter", queries="machine learning", site="arxiv.org"), "ads": lambda: run("text ads", queries="car insurance quotes", expect_ads=True), "products": lambda: run( diff --git a/surfsense_backend/scripts/scale_google_search.py b/surfsense_backend/scripts/scale_google_search.py new file mode 100644 index 000000000..5e4084b3e --- /dev/null +++ b/surfsense_backend/scripts/scale_google_search.py @@ -0,0 +1,229 @@ +"""Zero-cost concurrency simulation for the Google Search fetch seam. + +Drives many concurrent ``fetch_serp_html`` calls with the network I/O stubbed +out (no live Google, no proxy spend, no captcha solves) so we can measure the +*structure* of the scraper under load without paying for it: + +* **throughput** — completions/sec at steady state (the per-process ceiling is + the render gate divided by warm-render latency; nothing above it is real), +* **latency** — p50/p95/p99 wait as the gate queues excess renders, +* **solves** — how many paid solves the run would cost (the number we most want + the sticky-IP pool to shrink), and +* **per-IP peak concurrency** — the funneling metric: how many renders pile onto + a single sticky IP at once (a hot IP is what Google re-walls). + +The stub keeps the *real* ``fetch_serp_html`` loop intact (sticky reuse, +exemption skip-precheck, the render gate, inflight accounting) and only replaces +``_precheck`` / ``_get_session`` / ``get_proxy_url``. Timings are compressed +(warm≈0.1 s vs ~14 s live) and extrapolated to real seconds via the warm-render +ratio, since the system is gate-bounded sleeps + a semaphore (delays scale +linearly). + + .venv/Scripts/python.exe scripts/scale_google_search.py --count 400 --rate 60 +""" + +from __future__ import annotations + +import argparse +import asyncio +import statistics +import sys +import threading +import time +from pathlib import Path + +if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + +_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_ROOT)) + +from app.proprietary.platforms.google_search import fetch # noqa: E402 + +# Live-measured render latencies (README "Timings"); used only to translate the +# compressed sim clock back to real seconds for the human-readable report. +_REAL_WARM_S = 14.0 +_REAL_SOLVE_S = 45.0 + +_RESULTS_HTML = '
ok
' # trips fetch._has_results + + +class _Metrics: + """Browser-loop-side counters (fetch runs on two loops → guard with a lock).""" + + def __init__(self) -> None: + self.lock = threading.Lock() + self.solves = 0 + self.renders = 0 + self.ip_hits: dict[str, int] = {} + self.ip_now: dict[str, int] = {} + self.ip_peak: dict[str, int] = {} + + def enter(self, proxy: str, solved: bool) -> None: + with self.lock: + self.renders += 1 + if solved: + self.solves += 1 + self.ip_hits[proxy] = self.ip_hits.get(proxy, 0) + 1 + n = self.ip_now.get(proxy, 0) + 1 + self.ip_now[proxy] = n + self.ip_peak[proxy] = max(self.ip_peak.get(proxy, 0), n) + + def exit(self, proxy: str) -> None: + with self.lock: + self.ip_now[proxy] = self.ip_now.get(proxy, 1) - 1 + + +class _FakePage: + def __init__(self) -> None: + self.status = 200 + self.html_content = _RESULTS_HTML + + +class _FakeSession: + """Stands in for AsyncStealthySession: sleeps like a render, simulates the + solve on a cold IP by seeding fetch._exemption_jar (what page_action does).""" + + def __init__(self, metrics: _Metrics, warm_s: float, solve_s: float) -> None: + self.m = metrics + self.warm_s = warm_s + self.solve_s = solve_s + + async def fetch(self, url, proxy=None, **kwargs): + key = proxy or "" + # Cold IP (no cached exemption) pays the solve and warms the jar; a warm + # IP just pays the render. Mirrors _make_page_action's cache write. + cold = key not in fetch._exemption_jar + self.m.enter(key, solved=cold) + try: + if cold: + await asyncio.sleep(self.solve_s) + fetch._exemption_jar[key] = [{"name": "GOOGLE_ABUSE_EXEMPTION"}] + await asyncio.sleep(self.warm_s) + return _FakePage() + finally: + self.m.exit(key) + + +def _install_stubs(metrics: _Metrics, warm_s: float, solve_s: float, precheck_s: float): + base = "http://u:p@gw.dataimpulse.com:823" # hostname in fetch._STICKY_HOSTS + session = _FakeSession(metrics, warm_s, solve_s) + + async def fake_precheck(url, proxy): + await asyncio.sleep(precheck_s) + return True + + async def fake_get_session(mobile): + return session + + fetch.get_proxy_url = lambda: base + fetch._precheck = fake_precheck + fetch._get_session = fake_get_session + fetch.captcha_enabled = lambda: True + fetch.get_captcha_config = lambda: object() + fetch._captcha.solver_latched = lambda: False + # The pool's backpressure poll is negligible vs a real 14 s render; keep it + # proportional under the compressed sim clock so waiters don't idle. + fetch._POOL_WAIT_S = max(warm_s * 0.05, 0.001) + + # Isolate the sim to the LOCAL pool: the cross-process store (Redis) has its + # own unit test and would otherwise add ping latency / cross-run bleed here. + async def _no_adopt(exclude): + return None + + async def _noop(*a, **k): + return None + + fetch._store.adopt = _no_adopt + fetch._store.publish = _noop + fetch._store.evict = _noop + + +def _reset_state() -> None: + fetch._exemption_jar.clear() + for name in ("_pool", "_pool_inflight"): + obj = getattr(fetch, name, None) + if isinstance(obj, dict): + obj.clear() + if hasattr(fetch, "_pool_pending"): + fetch._pool_pending = 0 + if hasattr(fetch, "_last_good_proxy"): # pre-pool revision + fetch._last_good_proxy = None + + +async def _drive(count: int, rate: float) -> list[float]: + """Fire ``count`` fetches at ``rate``/sec (steady arrival), return latencies.""" + lat: list[float] = [] + interval = 1.0 / rate if rate > 0 else 0.0 + + async def one() -> None: + t0 = time.perf_counter() + await fetch.fetch_serp_html("https://www.google.com/search?q=notebooklm") + lat.append(time.perf_counter() - t0) + + tasks: list[asyncio.Task] = [] + for _ in range(count): + tasks.append(asyncio.create_task(one())) + if interval: + await asyncio.sleep(interval) + await asyncio.gather(*tasks) + return lat + + +def _pct(xs: list[float], p: float) -> float: + if not xs: + return 0.0 + xs = sorted(xs) + return xs[min(len(xs) - 1, int(p / 100 * len(xs)))] + + +async def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--count", type=int, default=400) + ap.add_argument("--rate", type=float, default=60.0, help="arrivals/sec (sim clock)") + ap.add_argument("--warm", type=float, default=0.10, help="warm render seconds (sim)") + ap.add_argument("--solve", type=float, default=0.45, help="solve seconds (sim)") + ap.add_argument("--precheck", type=float, default=0.01) + args = ap.parse_args() + + metrics = _Metrics() + _install_stubs(metrics, args.warm, args.solve, args.precheck) + _reset_state() + + scale = _REAL_WARM_S / args.warm # sim seconds → real seconds + gate = fetch._MAX_CONCURRENT_PAGES + pool = getattr(fetch, "_WARM_POOL_TARGET", None) + + wall0 = time.perf_counter() + lat = await _drive(args.count, args.rate) + wall = time.perf_counter() - wall0 + await fetch._in_browser_loop(asyncio.sleep(0)) # let last exit() settle + + hot_peak = max(metrics.ip_peak.values()) if metrics.ip_peak else 0 + top_hits = max(metrics.ip_hits.values()) if metrics.ip_hits else 0 + top_share = 100.0 * top_hits / metrics.renders if metrics.renders else 0.0 + thru_sim = args.count / wall if wall else 0.0 + thru_real = thru_sim / scale + + print("\n=== Google Search scale simulation ===") + print(f" gate (_MAX_CONCURRENT_PAGES) = {gate}" + + (f", warm-pool target = {pool}" if pool else " (single-slot sticky IP)")) + print(f" requests={args.count} arrival_rate={args.rate}/s (sim)") + print(f" --- structure (scale-free) ---") + print(f" paid solves = {metrics.solves} (want ~pool size, not ~requests)") + print(f" distinct sticky IPs = {len(metrics.ip_hits)}") + print(f" busiest IP carried = {top_hits}/{metrics.renders} renders " + f"({top_share:.0f}%) (funneling; want ~even spread)") + print(f" peak renders on 1 IP = {hot_peak} (concurrency; capped by gate)") + print(f" --- throughput / latency (extrapolated to live @ warm={_REAL_WARM_S}s) ---") + print(f" throughput = {thru_real*60:.0f} SERP/min ({thru_real:.2f}/s)") + print(f" latency p50 = {_pct(lat,50)*scale:6.1f}s p95 = {_pct(lat,95)*scale:6.1f}s " + f"p99 = {_pct(lat,99)*scale:6.1f}s") + print(f" ceiling (gate/warm) = {gate/_REAL_WARM_S*60:.0f} SERP/min per process") + need = 500 / (gate / _REAL_WARM_S * 60) + print(f" -> to sustain 500 SERP/min you need ~{need:.0f} such processes " + f"(or a larger gate)\n") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/surfsense_backend/scripts/stress_google_search.py b/surfsense_backend/scripts/stress_google_search.py new file mode 100644 index 000000000..e080a16d5 --- /dev/null +++ b/surfsense_backend/scripts/stress_google_search.py @@ -0,0 +1,207 @@ +"""Live stress test for the Google Search scraper (REAL solves + proxy spend). + +Simulates a burst of concurrent "users" hitting the real ``scrape_serps`` +pipeline with a mix of single-query and multi-query requests. Unlike the offline +sim (``scale_google_search.py``), this exercises the actual DataImpulse IPs, +CapSolver solves, and the shared browser — so it surfaces what the sim can't: +real warm-vs-cold latency, whether the warm-IP pool amortizes solves across real +IPs without Google re-walling them, and browser stability under concurrency. + + .venv/Scripts/python.exe scripts/stress_google_search.py --users 30 --gate 8 + +It COSTS money (each cold IP = one CapSolver solve, bounded by the pool target) +and proxy bandwidth. Metrics are tallied from the live perf/captcha logs. +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import os +import random +import re +import sys +import time +from pathlib import Path + +if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + +_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_ROOT)) + +from dotenv import load_dotenv # noqa: E402 + +load_dotenv(_ROOT / ".env") + +# Brand/navigational terms reliably trip the reCAPTCHA wall (a solve/pool grow); +# informational terms are the lighter, often-warm path. A realistic mix. +_BRAND = ["notebooklm", "figma", "notion", "linear app", "vercel", "perplexity ai"] +_INFO = [ + "python asyncio tutorial", + "best mechanical keyboard 2026", + "kubernetes vs docker", + "typescript generics explained", + "how to make sourdough bread", + "rust ownership model", +] + + +class _LogTally(logging.Handler): + """Counts solves / renders / walls / pool-reuse from the live log stream.""" + + _RENDER = re.compile( + r"has_results=(\w+).*from_pool=(\w+) pool=(\d+)" + ) + _SOLVE = re.compile(r"\[captcha\] solve (OK|did not)") + + def __init__(self) -> None: + super().__init__(level=logging.INFO) + self.solves_ok = 0 + self.solves_fail = 0 + self.renders_ok = 0 + self.renders_walled = 0 + self.reuse = 0 + self.grow = 0 + self.max_pool = 0 + + def emit(self, record: logging.LogRecord) -> None: + msg = record.getMessage() + m = self._RENDER.search(msg) + if m: + ok, from_pool, pool = m.group(1), m.group(2), int(m.group(3)) + if ok == "True": + self.renders_ok += 1 + else: + self.renders_walled += 1 + if from_pool == "True": + self.reuse += 1 + else: + self.grow += 1 + self.max_pool = max(self.max_pool, pool) + return + s = self._SOLVE.search(msg) + if s: + if s.group(1) == "OK": + self.solves_ok += 1 + else: + self.solves_fail += 1 + + +def _make_request(rng: random.Random, multi_ratio: float) -> tuple[str, int]: + """Build one user's request: a single term or a 2-3 term multi-query. + Returns (newline-joined queries, expected SERP count).""" + pool = _BRAND + _INFO + if rng.random() < multi_ratio: + n = rng.randint(2, 3) + terms = rng.sample(pool, n) + return "\n".join(terms), n + return rng.choice(pool), 1 + + +async def _user(uid: int, queries: str, want: int, results: list[dict]) -> None: + from app.proprietary.platforms.google_search import ( + GoogleSearchScrapeInput, + scrape_serps, + ) + + t0 = time.perf_counter() + inp = GoogleSearchScrapeInput(queries=queries, countryCode="us", languageCode="en") + try: + items = await scrape_serps(inp, limit=want) + got = len(items) + err = None + except Exception as e: # a user request should never crash the whole run + got, err = 0, repr(e) + results.append( + { + "uid": uid, + "want": want, + "got": got, + "secs": time.perf_counter() - t0, + "err": err, + } + ) + + +async def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--users", type=int, default=30, help="concurrent requests") + ap.add_argument("--gate", type=int, default=8, help="MAX_CONCURRENT_PAGES for run") + ap.add_argument("--multi-ratio", type=float, default=0.5) + ap.add_argument("--ramp", type=float, default=0.3, help="secs between user starts") + ap.add_argument("--seed", type=int, default=7) + args = ap.parse_args() + + # Must be set BEFORE fetch.py is imported (the gate is read at import). + os.environ["GOOGLE_SEARCH_MAX_CONCURRENT_PAGES"] = str(args.gate) + + logging.basicConfig(level=logging.WARNING) + tally = _LogTally() + for name in ( + "app.proprietary.platforms.google_search.fetch", + "app.proprietary.platforms.google_search.captcha", + ): + lg = logging.getLogger(name) + lg.setLevel(logging.INFO) + lg.addHandler(tally) + + from app.proprietary.platforms.google_search.fetch import ( # noqa: E402 + _WARM_POOL_TARGET, + close_sessions, + ) + + rng = random.Random(args.seed) + plans = [_make_request(rng, args.multi_ratio) for _ in range(args.users)] + total_serps = sum(w for _, w in plans) + n_multi = sum(1 for _, w in plans if w > 1) + + print(f"\n=== LIVE stress: {args.users} users " + f"({args.users - n_multi} single / {n_multi} multi), " + f"~{total_serps} SERPs, gate={args.gate}, pool_target={_WARM_POOL_TARGET} ===") + print(" (real solves + proxy spend; warming up...)\n") + + results: list[dict] = [] + wall0 = time.perf_counter() + tasks = [] + for uid, (queries, want) in enumerate(plans): + tasks.append(asyncio.create_task(_user(uid, queries, want, results))) + await asyncio.sleep(args.ramp) # gentle ramp, not a thundering herd + await asyncio.gather(*tasks) + wall = time.perf_counter() - wall0 + + await close_sessions() + + lat = sorted(r["secs"] for r in results) + got_serps = sum(r["got"] for r in results) + fails = [r for r in results if r["err"]] + short = [r for r in results if not r["err"] and r["got"] < r["want"]] + + def pct(p: float) -> float: + return lat[min(len(lat) - 1, int(p / 100 * len(lat)))] if lat else 0.0 + + print("=== results ===") + print(f" wall time = {wall:.0f}s") + print(f" requests ok/failed = {len(results) - len(fails)}/{len(fails)}") + print(f" SERPs got/expected = {got_serps}/{total_serps}" + + (f" ({len(short)} short)" if short else "")) + print(f" throughput = {got_serps / wall * 60:.0f} SERP/min") + print(f" request latency p50={pct(50):.0f}s p95={pct(95):.0f}s " + f"max={lat[-1] if lat else 0:.0f}s") + print(" --- pipeline (from live logs) ---") + print(f" paid solves ok/fail = {tally.solves_ok}/{tally.solves_fail} " + f"(bounded by pool target {_WARM_POOL_TARGET})") + print(f" renders ok/walled = {tally.renders_ok}/{tally.renders_walled}") + print(f" pool reuse/grow = {tally.reuse}/{tally.grow} " + f"(reuse share {100 * tally.reuse / max(1, tally.reuse + tally.grow):.0f}%)") + print(f" peak pool size = {tally.max_pool}") + if fails: + print(" --- failures ---") + for r in fails[:5]: + print(f" user{r['uid']}: {r['err']}") + print() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/surfsense_backend/tests/unit/platforms/google_search/test_captcha.py b/surfsense_backend/tests/unit/platforms/google_search/test_captcha.py new file mode 100644 index 000000000..03857a5dd --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/google_search/test_captcha.py @@ -0,0 +1,37 @@ +"""Offline checks for the /sorry reCAPTCHA solver's pure helpers. + +The network solve can't be unit-tested, but the parsing around it can: a missed +sitekey/data-s silently burns paid solves, so these guard the boundary logic. +(Proxy reformatting now lives in ``app.utils.captcha.solvers``.) +""" + +from app.proprietary.platforms.google_search import captcha + + +class _Page: + def __init__(self, url: str) -> None: + self.url = url + + +def test_on_sorry_detects_wall_only(): + assert captcha.on_sorry(_Page("https://www.google.com/sorry/index?continue=x")) + assert not captcha.on_sorry(_Page("https://www.google.com/search?q=notebooklm")) + assert not captcha.on_sorry(_Page("")) + + +def test_sitekey_and_data_s_extraction(): + html = ( + '
' + ) + assert captcha._SITEKEY_RE.search(html).group(1) == "6LdLLIMbAAAAAIl-KLj9p1ePhM" + assert captcha._DATA_S_RE.search(html).group(1) == "zBB1ixry9YzY_tok-en" + + +def test_latch_roundtrip(): + captcha.reset_solver_latch() + assert not captcha.solver_latched() + captcha._latch("no balance") + assert captcha.solver_latched() + captcha.reset_solver_latch() + assert not captcha.solver_latched() diff --git a/surfsense_backend/tests/unit/platforms/google_search/test_fetch_concurrency.py b/surfsense_backend/tests/unit/platforms/google_search/test_fetch_concurrency.py index c129f7c5f..9b719a153 100644 --- a/surfsense_backend/tests/unit/platforms/google_search/test_fetch_concurrency.py +++ b/surfsense_backend/tests/unit/platforms/google_search/test_fetch_concurrency.py @@ -16,7 +16,7 @@ class _FakeSession: self.release = asyncio.Event() self.closed = 0 - async def fetch(self, url, proxy=None): + async def fetch(self, url, proxy=None, **kwargs): await self.release.wait() return "page" diff --git a/surfsense_backend/tests/unit/platforms/google_search/test_fetch_pool.py b/surfsense_backend/tests/unit/platforms/google_search/test_fetch_pool.py new file mode 100644 index 000000000..1008783a8 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/google_search/test_fetch_pool.py @@ -0,0 +1,99 @@ +"""Warm sticky-IP pool bookkeeping (the scale-critical logic in fetch.py). + +These pure, lock-guarded helpers decide whether a render reuses a warm IP, +grows the pool (a paid solve), or waits — and how a finished render admits or +evicts its IP. Get the pending/inflight accounting wrong and the pool either +over-solves (cost) or funnels every render onto one IP (re-wall), so the +transitions are worth pinning down offline. +""" + +import pytest + +from app.proprietary.platforms.google_search import fetch + +pytestmark = pytest.mark.unit + + +@pytest.fixture(autouse=True) +def _clean_pool(): + def reset(): + fetch._pool.clear() + fetch._pool_inflight.clear() + fetch._pool_pending = 0 + fetch._exemption_jar.clear() + + reset() + yield + reset() + + +def test_take_grows_when_pool_empty(): + action, proxy = fetch._pool_take() + assert action == "grow" and proxy is None + assert fetch._pool_pending == 1 + + +def test_take_reuses_warm_under_cap(): + fetch._pool["p1"] = 1.0 + action, proxy = fetch._pool_take() + assert action == "reuse" and proxy == "p1" + assert fetch._pool_inflight["p1"] == 1 + + +def test_take_spreads_across_least_loaded_ip(): + fetch._pool.update({"p1": 1.0, "p2": 1.0}) + fetch._pool_inflight["p1"] = 1 # p2 is idle + action, proxy = fetch._pool_take() + assert action == "reuse" and proxy == "p2" + + +def test_take_waits_when_full_and_every_ip_capped(monkeypatch): + monkeypatch.setattr(fetch, "_WARM_POOL_TARGET", 2) + monkeypatch.setattr(fetch, "_WARM_IP_MAX_CONCURRENCY", 1) + fetch._pool.update({"p1": 1.0, "p2": 1.0}) + fetch._pool_inflight.update({"p1": 1, "p2": 1}) + action, proxy = fetch._pool_take() + assert action == "wait" and proxy is None + assert fetch._pool_pending == 0 # a wait must NOT reserve a solve + + +def test_take_grows_only_up_to_target_counting_pending(monkeypatch): + monkeypatch.setattr(fetch, "_WARM_POOL_TARGET", 2) + monkeypatch.setattr(fetch, "_WARM_IP_MAX_CONCURRENCY", 1) + fetch._pool["p1"] = 1.0 + fetch._pool_inflight["p1"] = 1 # warm but capped + assert fetch._pool_take()[0] == "grow" # pool(1)+pending(0) < 2 + assert fetch._pool_pending == 1 + assert fetch._pool_take()[0] == "wait" # pool(1)+pending(1) == 2 → no more solves + + +def test_settle_good_admits_and_releases(): + fetch._pool_pending = 1 + fetch._pool_inflight["p1"] = 1 + fetch._pool_settle("p1", good=True, grew=True) + assert "p1" in fetch._pool + assert "p1" not in fetch._pool_inflight + assert fetch._pool_pending == 0 + + +def test_settle_walled_evicts_ip_and_drops_exemption(): + fetch._pool["p1"] = 1.0 + fetch._pool_inflight["p1"] = 1 + fetch._exemption_jar["p1"] = [{"name": "GOOGLE_ABUSE_EXEMPTION"}] + fetch._pool_settle("p1", good=False, grew=False) + assert "p1" not in fetch._pool + assert "p1" not in fetch._exemption_jar + + +def test_adopt_releases_pending_and_pins_ip(): + fetch._pool_pending = 1 # reserved by a grow that we satisfy via the store + fetch._pool_adopt("shared") + assert fetch._pool_pending == 0 + assert "shared" in fetch._pool + assert fetch._pool_inflight["shared"] == 1 + + +def test_abort_grow_releases_the_reserved_slot(): + fetch._pool_pending = 2 + fetch._pool_abort_grow() + assert fetch._pool_pending == 1 diff --git a/surfsense_backend/tests/unit/platforms/google_search/test_pool_store.py b/surfsense_backend/tests/unit/platforms/google_search/test_pool_store.py new file mode 100644 index 000000000..6ad4c5143 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/google_search/test_pool_store.py @@ -0,0 +1,73 @@ +"""Cross-process warm-IP exemption store (best-effort Redis cache). + +Redis itself isn't unit-testable here, but the boundary is: adopt must skip IPs +this process already holds (or the fleet re-uses nothing new), and every path +must degrade to a silent no-op when Redis is down (or a hiccup would break the +fetch instead of just costing a solve). +""" + +import pytest + +from app.proprietary.platforms.google_search import pool_store as ps + +pytestmark = pytest.mark.unit + + +class _FakeRedis: + def __init__(self): + self.store: dict[str, str] = {} + + def ping(self): + return True + + def set(self, k, v, ex=None): + self.store[k] = v + + def get(self, k): + return self.store.get(k) + + def delete(self, k): + self.store.pop(k, None) + + def scan_iter(self, match=None, count=None): + return list(self.store.keys()) + + +@pytest.fixture +def fake(monkeypatch): + r = _FakeRedis() + monkeypatch.setattr(ps, "_client", r) + monkeypatch.setattr(ps, "_disabled", False) + return r + + +def test_key_is_stable_and_prefixed(): + assert ps._key("http://a").startswith(ps._PREFIX) + assert ps._key("http://a") == ps._key("http://a") + assert ps._key("http://a") != ps._key("http://b") + + +def test_publish_then_adopt_excludes_held_ips(fake): + ps._publish_sync("http://a", [{"n": 1}]) + ps._publish_sync("http://b", [{"n": 2}]) + proxy, cookies = ps._adopt_sync(exclude={"http://a"}) + assert proxy == "http://b" and cookies == [{"n": 2}] + + +def test_adopt_none_when_all_held(fake): + ps._publish_sync("http://a", []) + assert ps._adopt_sync(exclude={"http://a"}) is None + + +def test_evict_removes_entry(fake): + ps._publish_sync("http://a", [{"n": 1}]) + ps._evict_sync("http://a") + assert ps._adopt_sync(exclude=set()) is None + + +def test_disabled_is_silent_noop(monkeypatch): + monkeypatch.setattr(ps, "_disabled", True) + monkeypatch.setattr(ps, "_client", None) + assert ps._adopt_sync(set()) is None + ps._publish_sync("x", [{"n": 1}]) # must not raise + ps._evict_sync("x") # must not raise diff --git a/surfsense_backend/tests/unit/proprietary/web_crawler/test_captcha.py b/surfsense_backend/tests/unit/proprietary/web_crawler/test_captcha.py index de7228a76..6e7c67df4 100644 --- a/surfsense_backend/tests/unit/proprietary/web_crawler/test_captcha.py +++ b/surfsense_backend/tests/unit/proprietary/web_crawler/test_captcha.py @@ -2,8 +2,9 @@ The browser/solver boundary is mocked: a fake Playwright ``page`` and a monkeypatched ``_harvest_token`` / injection. We assert the glue logic — -detection, proxy reformatting, attempt counting + state surfacing, the per-URL -cap, and the no-balance process latch. +detection, proxy pass-through, attempt counting + state surfacing, the per-URL +cap, and the no-balance process latch. (Vendor proxy reformatting now lives in +``app.utils.captcha.solvers`` and is tested there.) """ import pytest @@ -76,24 +77,6 @@ def _clear_latch(): cap.reset_solver_latch() -# --- proxy reformat -------------------------------------------------------- - - -class TestProxyReformat: - def test_with_auth(self): - assert ( - cap.proxy_url_to_captchatools("http://user:pass@1.2.3.4:8080") - == "1.2.3.4:8080:user:pass" - ) - - def test_without_auth(self): - assert cap.proxy_url_to_captchatools("http://1.2.3.4:8080") == "1.2.3.4:8080" - - def test_none_and_garbage(self): - assert cap.proxy_url_to_captchatools(None) is None - assert cap.proxy_url_to_captchatools("not-a-url") is None - - # --- detection ------------------------------------------------------------- @@ -175,8 +158,9 @@ class TestPageAction: action(page) assert state == {"attempts": 1, "solved": True} - # Solver egressed from the crawl's proxy, reformatted, with the page UA. - assert captured["proxy"] == "1.2.3.4:9000:u:p" + # Solver egressed from the crawl's proxy (raw URL; the seam reformats + # per vendor), with the page UA. + assert captured["proxy"] == "http://u:p@1.2.3.4:9000" assert captured["ua"] == "UA/1.0" assert captured["ctype"] == "v2" assert injected == {"ctype": "v2", "token": "TOKEN"} diff --git a/surfsense_backend/tests/unit/utils/captcha/test_solvers.py b/surfsense_backend/tests/unit/utils/captcha/test_solvers.py new file mode 100644 index 000000000..6af4597c2 --- /dev/null +++ b/surfsense_backend/tests/unit/utils/captcha/test_solvers.py @@ -0,0 +1,84 @@ +"""Offline checks for the in-house captcha-solver seam. + +The network solve can't be unit-tested, but the boundary logic can: a wrong +proxy reformat (2captcha's ``ERROR_PROXY_FORMAT``) or dispatching to a provider +we haven't wired silently burns paid solves / leaks the key to the wrong vendor. +""" + +import pytest + +from app.utils.captcha import solvers +from app.utils.captcha.config import CaptchaConfig + +pytestmark = pytest.mark.unit + + +def _cfg(**overrides) -> CaptchaConfig: + base = { + "enabled": True, + "solving_site": "2captcha", + "api_key": "key-123", + "max_attempts_per_url": 1, + "timeout_s": 30, + "captcha_type_default": "v2", + "v3_min_score": 0.7, + "v3_action": "verify", + } + base.update(overrides) + return CaptchaConfig(**base) + + +# --- proxy reformat (2captcha wants login:pass@host:port, NO scheme) -------- + + +def test_proxy_login_form_strips_scheme(): + got = solvers.proxy_login_form("http://user:pass@gw.dataimpulse.com:15673") + assert got == "user:pass@gw.dataimpulse.com:15673" + + +def test_proxy_login_form_without_credentials(): + assert solvers.proxy_login_form("http://gw.dataimpulse.com:823") == "gw.dataimpulse.com:823" + + +def test_proxy_login_form_none_on_missing_or_bad(): + assert solvers.proxy_login_form(None) is None + assert solvers.proxy_login_form("not a url") is None + + +# --- capsolver proxy reformat (colon-delimited scheme:host:port:user:pass) -- + + +def test_capsolver_proxy_colon_delimited_with_creds(): + got = solvers.capsolver_proxy("http://user:pass@gw.dataimpulse.com:823") + assert got == "http:gw.dataimpulse.com:823:user:pass" + + +def test_capsolver_proxy_without_credentials(): + assert solvers.capsolver_proxy("http://gw.dataimpulse.com:823") == "http:gw.dataimpulse.com:823" + + +def test_capsolver_proxy_none_on_missing_or_bad(): + assert solvers.capsolver_proxy(None) is None + assert solvers.capsolver_proxy("not a url") is None + + +# --- dispatch -------------------------------------------------------------- + + +def test_unsupported_provider_raises_solvererror(): + # An unconfigured provider must fail loudly (latch) rather than POST the key + # to a wired vendor's endpoint under the wrong account. + with pytest.raises(solvers.SolverUnsupported): + solvers.solve( + _cfg(solving_site="anticaptcha"), + challenge_type="v2", + sitekey="SK", + page_url="https://t.test", + ) + assert issubclass(solvers.SolverUnsupported, solvers.SolverError) + + +def test_wired_providers_are_registered(): + supported = solvers.supported_providers() + assert "2captcha" in supported + assert "capsolver" in supported diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index 2b1272363..4c7c0305c 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -39,6 +39,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -76,6 +79,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -113,6 +119,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -150,6 +159,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -187,6 +199,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -224,6 +239,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -261,6 +279,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -298,6 +319,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -335,6 +359,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -372,6 +399,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -421,6 +451,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -458,6 +492,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -495,6 +532,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -532,6 +572,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -569,6 +612,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -606,6 +652,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -643,6 +692,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -680,6 +732,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -717,6 +772,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -754,6 +812,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -791,6 +852,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -840,6 +904,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -877,6 +945,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -914,6 +985,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -951,6 +1025,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -988,6 +1065,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1025,6 +1105,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1062,6 +1145,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1099,6 +1185,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1136,6 +1225,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1173,6 +1265,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1210,6 +1305,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1259,6 +1357,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1296,6 +1398,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1369,6 +1474,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1406,6 +1517,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1443,6 +1557,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1516,6 +1633,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1589,6 +1712,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1626,6 +1755,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", ] conflicts = [[ @@ -2371,18 +2503,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/06/f3/39cf3367b8107baa44f861dc802cbf16263c945b62d8265d36034fc07bea/cachetools-7.0.5-py3-none-any.whl", hash = "sha256:46bc8ebefbe485407621d0a4264b23c080cedd913921bad7ac3ed2f26c183114", size = 13918 }, ] -[[package]] -name = "captchatools" -version = "1.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6b/38/01200f5b8169219002728ccddbad8cc50759e523c0884b7e811092d27352/captchatools-1.5.0.tar.gz", hash = "sha256:755ed685a93f6139608b5277ef0410f8682b3522e98f52d9cc3edff9df38ca12", size = 15282 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/f1/7faaa20ffd68314215da2e3c57076c2c89846ab05464f1b7167dc8a9a80b/captchatools-1.5.0-py3-none-any.whl", hash = "sha256:43b651f7603be5725df94e4d715e0ff770efec44b62bb6076eb9a63516c74d94", size = 16394 }, -] - [[package]] name = "catalogue" version = "2.0.10" @@ -7942,21 +8062,21 @@ wheels = [ [[package]] name = "patchright" -version = "1.60.1" +version = "1.61.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "greenlet" }, { name = "pyee" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/71/26/c1e858fd1acc63e410b3d33243955f36d2a0814487b97a7aa604ad2baffd/patchright-1.60.1-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:e9492100d4e2a85ff92fc3a668dd16dee03f21df6e559c7b9f7c71e86ff48c6b", size = 43458936 }, - { url = "https://files.pythonhosted.org/packages/55/dd/2dd8e4e02489ec8fd57ad93dec9ef444b6f42adcc4fe95df30237c92841d/patchright-1.60.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:20bd806df2469b451ccd2ea10f5f944ceb0e0d83c716f5752b0c956c1ee59476", size = 42245629 }, - { url = "https://files.pythonhosted.org/packages/54/cc/0fa0bedec61045fd9068682e3695557b622c483f829d341093879ec8dbd9/patchright-1.60.1-py3-none-macosx_11_0_universal2.whl", hash = "sha256:9fd15a64c0ca80740dc2a3f41cda336a06a2ed6068d0ab893172654290b06e6b", size = 43458935 }, - { url = "https://files.pythonhosted.org/packages/ce/c2/4b8f69de0a20d90792980c43c0e60b10b801e08cf0224ccc8a8266e1fffb/patchright-1.60.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:547e7bfb813102309789cc42933780e5fdf7c4727de59fb2791e64bd1298a7f3", size = 47451519 }, - { url = "https://files.pythonhosted.org/packages/db/fc/9fd6a70818cf0bc3b62574af483d762963cb65ca0a369826a0536faffe41/patchright-1.60.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:023945a2fd30219a284721ca36385bd44075ae7b53071dc1da38036b6dbe88ec", size = 47139153 }, - { url = "https://files.pythonhosted.org/packages/08/bc/81fb621e5ab4131e6f324c9ba6cb2a9f3b146c92f12484bec95efe8c8347/patchright-1.60.1-py3-none-win32.whl", hash = "sha256:05b98a6afdbe7e6645fe223009c47cc8e7859df55fd8ce9d8a9925b3389b0ee1", size = 37886460 }, - { url = "https://files.pythonhosted.org/packages/74/8e/fff80350ed2c2c1f62145d667070799c60ca9e9b28d54e4f751f0b4f8da6/patchright-1.60.1-py3-none-win_amd64.whl", hash = "sha256:51b306ed55cd58f1bca24641458f5c9f7e86a1f1727dcffdead669cfe4c0a485", size = 37886465 }, - { url = "https://files.pythonhosted.org/packages/ee/b8/b6d1bfe98a420c1ccfb2f23a7bb2b4cdb40170907bae638e7ae92abd287c/patchright-1.60.1-py3-none-win_arm64.whl", hash = "sha256:f795728c1e27fc226dbe203c1aec713a537f01be963474fe0f3691f5e6457f9f", size = 34022283 }, + { url = "https://files.pythonhosted.org/packages/79/42/a76e2ea17dd4feccbf5ef6df017fb2e67f4b72a6f559f59d9aad1197f83a/patchright-1.61.2-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:43ae10cd2a3e46b222179a120ab39ba88f801f3a42aa41cab45fac9928f2103d", size = 43727572 }, + { url = "https://files.pythonhosted.org/packages/57/e7/1ff89853c9aeaf26cd20559f401375ec71a40e4041cf7ace03d41a741f06/patchright-1.61.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a3da24d7890631255c7bef467b94ffe7f7bfa36f96fc72c10d5787d3cb396873", size = 42512921 }, + { url = "https://files.pythonhosted.org/packages/e4/d0/d730536a7fc43c3aaf2781518595ca71ffa1fc2f24e580c0d0cf04e00eca/patchright-1.61.2-py3-none-macosx_11_0_universal2.whl", hash = "sha256:edef852aa7d28791fa2d3f7ce135edfea2ce07a2bc5e0a74fb2dfe269af62223", size = 43727573 }, + { url = "https://files.pythonhosted.org/packages/a2/b5/c76dcda275cb0d9651321e343f3268609ae24cef196bff56751754ffba16/patchright-1.61.2-py3-none-manylinux1_x86_64.whl", hash = "sha256:545bdf2c0bb5f9ad78ab315e91c8c2990cc6a702e7f18650b71fe1071049b5c1", size = 47729338 }, + { url = "https://files.pythonhosted.org/packages/19/ef/5de6feaedf47a6ba8efb9e8c4f8a42bad29cc59c35bbd088b1729a36b271/patchright-1.61.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1824da30cf6560749bc1bb4a7f002fec325d5b709090dd7c8c1b643899331cee", size = 47426969 }, + { url = "https://files.pythonhosted.org/packages/6c/6e/ce2c5d03a4c785ac8f3e0180bbf83ef85149cf41fdfa41c09bd8cbcfd945/patchright-1.61.2-py3-none-win32.whl", hash = "sha256:a5aaf4cec2bd38714f2447f84bcdfe7dcb592a16e9616bf6d2d872066d1c78af", size = 38155048 }, + { url = "https://files.pythonhosted.org/packages/46/c8/fceb57b9d32ed078f53fc6a757c6e4b54cb117f59412938252a3359105e5/patchright-1.61.2-py3-none-win_amd64.whl", hash = "sha256:506bca9b72e609fd8fee35e2a3ad26e5d92e25337002f5c2cdad53d5cacb0b67", size = 38155052 }, + { url = "https://files.pythonhosted.org/packages/33/96/67a791c6b7cd76424b44d356beaf414ab97c3aef2d42fad4641e7905b41d/patchright-1.61.2-py3-none-win_arm64.whl", hash = "sha256:3dee1e2daada9f98653763831f688d39c924e0cf0eeb988e8047b703766cee73", size = 34268624 }, ] [[package]] @@ -8215,21 +8335,21 @@ wheels = [ [[package]] name = "playwright" -version = "1.60.0" +version = "1.61.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "greenlet" }, { name = "pyee" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/21/f0/832bd9677194908da118064eef20082f2791e3d18215cc6d9391ee2c5a67/playwright-1.60.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:6a8cd0fec171fb3089e95e898c8bc8a6f35dea0b78b399e12fcc19427e91b1d7", size = 43474635 }, - { url = "https://files.pythonhosted.org/packages/59/7b/e1d32ae8a3ed937ec2be3721c5f728b13d731a0b7c6442e0b3bec5094ac0/playwright-1.60.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:39b5420ba6145045b69ced4c5c47d4d9fe5bddfc8ff816c518913afcb25ec7a5", size = 42261327 }, - { url = "https://files.pythonhosted.org/packages/d7/bc/23de499ded6411c188a20c5a0dea6f0cd4ed5d2b3cc6042a5dbd3ed609aa/playwright-1.60.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:2581d0e6a3392c71f91b27460c7fd093356818dc430f48153896c8aeeaef7705", size = 43474636 }, - { url = "https://files.pythonhosted.org/packages/22/7b/1d679f4fced4ea94efadd17103856d8c565384f68382a1681264e46f5925/playwright-1.60.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:1c2bfae7884fb3fb05b853290eab8f343d524e5016f2f1def702acbbdf14c93e", size = 47467220 }, - { url = "https://files.pythonhosted.org/packages/84/c2/1528d267d4442bd2c6b8eaeab819dd52c2030bf80e89293f0ba1f687473b/playwright-1.60.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43e66564125ee31b07a58cefb21e256d62d67d8d1713e6858df7a3019d8ed353", size = 47154856 }, - { url = "https://files.pythonhosted.org/packages/bb/4e/b008b6440a7a1624378041da94829956d4b8f7ab9ef5aad22d0dc3f2e26d/playwright-1.60.0-py3-none-win32.whl", hash = "sha256:ec94e416ea320711e0ad4bf185dcbf41833672961e90773e1885255d7db7b7e7", size = 37902157 }, - { url = "https://files.pythonhosted.org/packages/55/f0/0541524133104f9cc20bf900870ff4a736b76a23483f3a55295ddfa58409/playwright-1.60.0-py3-none-win_amd64.whl", hash = "sha256:9566821ce6030a1f9e7146a24e19355ab0d98805fd0f9be50bb3d8fef1750c02", size = 37902159 }, - { url = "https://files.pythonhosted.org/packages/80/c8/210f282d278e4709cdd71b12a31af45a30a22ab3207b387e29b37e478713/playwright-1.60.0-py3-none-win_arm64.whl", hash = "sha256:6e4f6700a4c2250efff8e690a81d66e3855754fb587b6b87cf5c784014f91537", size = 34037981 }, + { url = "https://files.pythonhosted.org/packages/44/ee/31e4e0db36588b817a10b299a0285082545fde7d36543c2abe498bb3d61a/playwright-1.61.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:ff138c3a604f69911e9d42fd036e55c2a171e5616edf04c1e7f60a2a285540b0", size = 43421877 }, + { url = "https://files.pythonhosted.org/packages/42/35/71395dd3ecc798965be4a3ef8c443217d4abca168e7cb34536304f9489e6/playwright-1.61.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:009588c2a7e499bc5a8b425b61fa65490968bbda9cd69e0cf2cff10f8304659a", size = 42205016 }, + { url = "https://files.pythonhosted.org/packages/f4/44/323164cf5cd1647bdefce76ffce27651aadb959d089b48f53ea40918276e/playwright-1.61.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:9f7de4536088d12037c13a52b7ea34b59270b78926bb56935070597ffac6b1af", size = 43421884 }, + { url = "https://files.pythonhosted.org/packages/ab/f8/a35bf179e4ba2522c1893635094a64e407572547bd61528820fc0abc87fe/playwright-1.61.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:54f3b39f6eab832e33458c1dd7da0b5682aedab3b09ae731b5c59fa12fd2024e", size = 47421381 }, + { url = "https://files.pythonhosted.org/packages/b7/eb/e3f922348ec17c315f98c463f72faa1181a1c3de0bfe31a8d2edf6561723/playwright-1.61.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93454322ade8c11d5d6c211bfd91bdfb9ffb4810e3e026371bcbc4bec1b7ee4c", size = 47120545 }, + { url = "https://files.pythonhosted.org/packages/c2/a6/5be4e52b40a9c0c8a073e7c5b0785c05cf5a9ea8f8a7b5b260e32d970342/playwright-1.61.0-py3-none-win32.whl", hash = "sha256:372d55a6f1248fa1dd47599686980cb8fb5bbe6fcda59eab793eb657c11d8a9b", size = 37844841 }, + { url = "https://files.pythonhosted.org/packages/6c/fd/2b78036e5fbe9d5f5645bbe08a1eac7160c51243c0093963edbcf67c35d9/playwright-1.61.0-py3-none-win_amd64.whl", hash = "sha256:35c6cc4589a5d00964a59d7b3e59641e0aac0c02f15479a7af77d20f6bc79597", size = 37844846 }, + { url = "https://files.pythonhosted.org/packages/27/0d/1b0f3c4ee4eb0514bc805b5c2f9a223e5b6de4f11a926f5235d51d0fc81b/playwright-1.61.0-py3-none-win_arm64.whl", hash = "sha256:e9fcbffcf557a8620fdedd92491eb59a32d18e23d6f3b4f6214b952be324fe51", size = 33955127 }, ] [[package]] @@ -8417,11 +8537,11 @@ wheels = [ [[package]] name = "protego" -version = "0.6.0" +version = "0.6.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/07/a7/955c422611d00a6e4a06d30b367ea9bb4fb09d48552e92aef1ba312493c7/protego-0.6.0.tar.gz", hash = "sha256:3466f41438421cf90008e98534d5fde47dc16a17482571d021143ac18b70ace9", size = 3137423 } +sdist = { url = "https://files.pythonhosted.org/packages/7d/1b/6b0ee60bb1561843bfc62f5c7c3cb1ef6147b47e829a5fd6b7fcd2752471/protego-0.6.2.tar.gz", hash = "sha256:88ff004544ce44e61269cc6f8735f7837d12e09bb77619fc94fbb26b72e5d137", size = 3137854 } wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/8c/f4dd590f48addf31398f78a78962eaa99eb4c87ac09c1927497032644731/protego-0.6.0-py3-none-any.whl", hash = "sha256:7210e6e06a8db839502baf1bfbcb810689a58e394d31408ef1ef9e4e3d79fc44", size = 10313 }, + { url = "https://files.pythonhosted.org/packages/61/10/cbd3c06603bb8b3e0e871df24ee793a6e3b53d8c4cc8763f1b8b936649aa/protego-0.6.2-py3-none-any.whl", hash = "sha256:714de21d82527c9be900066c3211b266985dd6a19b6e70c57e033fc1a589f3ff", size = 10296 }, ] [[package]] @@ -10011,7 +10131,7 @@ wheels = [ [[package]] name = "scrapling" -version = "0.4.9" +version = "0.4.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cssselect" }, @@ -10021,9 +10141,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "w3lib" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d0/34/61b72964fdeed6bcb9528262870b9870c936eac55541f39254ef876985ef/scrapling-0.4.9.tar.gz", hash = "sha256:e08afab736e5bd3337173e524fee99aea3073476e86d44d942af4bef9496a499", size = 156809 } +sdist = { url = "https://files.pythonhosted.org/packages/a9/e9/f1874e835ac83043a323014a5ce3776f6415041566177366250358e7649e/scrapling-0.4.11.tar.gz", hash = "sha256:92500fe60b601a52279842fa2505fcddd3954f14f657bd38d7e6ba09f02da947", size = 159785 } wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/e6/03712d0b3240b3b6af6dc8aa19f0f24348fa54f4a4497b09c2d5c511552f/scrapling-0.4.9-py3-none-any.whl", hash = "sha256:00c7fae4641d948fb26486fd1da22143e5296dc86ef821b8426c0c2854c529ef", size = 158609 }, + { url = "https://files.pythonhosted.org/packages/2e/21/979d476508766f6a4c54cf126d9eadf12fd613c18ebc1e31f78a28fcfd06/scrapling-0.4.11-py3-none-any.whl", hash = "sha256:10b781f9214502f293a46048062c4d3c211e9a05a897559f674edcbea1876e56", size = 163481 }, ] [package.optional-dependencies] @@ -10494,7 +10614,6 @@ dependencies = [ { name = "azure-ai-documentintelligence" }, { name = "azure-storage-blob" }, { name = "boto3" }, - { name = "captchatools" }, { name = "celery", extra = ["redis"] }, { name = "chonkie", extra = ["all"] }, { name = "composio" }, @@ -10610,7 +10729,6 @@ requires-dist = [ { name = "azure-ai-documentintelligence", specifier = ">=1.0.2" }, { name = "azure-storage-blob", specifier = ">=12.23.0" }, { name = "boto3", specifier = ">=1.35.0" }, - { name = "captchatools", specifier = ">=1.5.0" }, { name = "celery", extras = ["redis"], specifier = ">=5.5.3" }, { name = "chonkie", extras = ["all"], specifier = ">=1.5.0" }, { name = "composio", specifier = ">=0.10.9" }, @@ -10669,7 +10787,7 @@ requires-dist = [ { name = "python-telegram-bot", specifier = ">=22.7" }, { name = "redis", specifier = ">=5.2.1" }, { name = "rerankers", extras = ["flashrank"], specifier = ">=0.7.1" }, - { name = "scrapling", extras = ["fetchers"], specifier = ">=0.4.9" }, + { name = "scrapling", extras = ["fetchers"], specifier = ">=0.4.11" }, { name = "sentence-transformers", specifier = ">=3.4.1" }, { name = "slack-sdk", specifier = ">=3.34.0" }, { name = "slowapi", specifier = ">=0.1.9" },