mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-20 23:21:06 +02:00
refactor: enhance Google search scraping capabilities with warm sticky-IP pool and improved captcha handling
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
4b97e9a54e
commit
68d1832f10
20 changed files with 1900 additions and 175 deletions
|
|
@ -374,15 +374,38 @@ TURNSTILE_SECRET_KEY=
|
|||
# PROXY_URL=http://user:pass@host:port
|
||||
# PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port
|
||||
|
||||
# --- Google Search scraper: scale / warm sticky-IP pool -----------------------
|
||||
# Per-process render concurrency AND the throughput lever: ceiling =
|
||||
# MAX_CONCURRENT_PAGES / warm-render-secs (~4/14s ≈ 17 SERP/min). Renders are
|
||||
# resource-light (disable_resources), so one Chromium holds many text contexts —
|
||||
# raise this to trade RAM/CPU for throughput before adding processes
|
||||
# (16 → ~69/min/process ⇒ ~7 processes cover 500/min). Validate with
|
||||
# scripts/scale_google_search.py.
|
||||
# GOOGLE_SEARCH_MAX_CONCURRENT_PAGES=4
|
||||
# Warm sticky-IP pool: concurrent renders spread across this many solved IPs so
|
||||
# no single IP is hammered (Google re-walls a hot IP). Steady-state paid solves
|
||||
# ≈ pool size, not request count. Size for the fleet: fleet per-IP load =
|
||||
# IP_MAX_CONCURRENCY × processes must stay gentle. Solved IPs are shared across
|
||||
# workers via Redis (REDIS_APP_URL) so the fleet solves each IP once.
|
||||
# GOOGLE_SEARCH_WARM_POOL_TARGET=8
|
||||
# GOOGLE_SEARCH_IP_MAX_CONCURRENCY=2
|
||||
# How long a shared exemption stays reusable before a re-solve (Google's cookie
|
||||
# outlives this; conservative TTL just re-solves idle IPs occasionally).
|
||||
# GOOGLE_SEARCH_EXEMPTION_TTL_S=3600
|
||||
# Per-fetch budget before giving up under saturation (a cold solve is ~45s).
|
||||
# GOOGLE_SEARCH_FETCH_DEADLINE_S=180
|
||||
|
||||
# =====================================================================
|
||||
# Captcha solving (Phase 3d) — LAST-resort bypass tier via captchatools.
|
||||
# Only fires on the stealth browser tier when a sitekey is detected AND
|
||||
# CAPTCHA_SOLVING_ENABLED=TRUE. Cloudflare Turnstile is already solved free
|
||||
# in-framework (no config needed). Off by default => zero attempts, zero cost.
|
||||
# Captcha solving (Phase 3d) — LAST-resort bypass tier (in-house solver seam,
|
||||
# app/utils/captcha/solvers.py). Only fires on the stealth browser tier when a
|
||||
# sitekey is detected AND CAPTCHA_SOLVING_ENABLED=TRUE. Also unlocks Google's
|
||||
# reCAPTCHA-Enterprise /sorry wall for the SERP scraper. Cloudflare Turnstile is
|
||||
# already solved free in-framework. Off by default => zero attempts, zero cost.
|
||||
# NOTE: automated solving may violate a target site's ToS — opt-in, public
|
||||
# data only (no logged-in bypass). captchatools is itself the vendor registry.
|
||||
# data only (no logged-in bypass).
|
||||
# CAPTCHA_SOLVING_ENABLED=FALSE
|
||||
# solving_site: capmonster | 2captcha | anticaptcha | capsolver | captchaai
|
||||
# Provider: "capsolver" (AI-native, fastest on reCAPTCHA-Enterprise) or
|
||||
# "2captcha" have in-house clients today (more added over time).
|
||||
# CAPTCHA_SOLVER_PROVIDER=capsolver
|
||||
# CAPTCHA_SOLVER_API_KEY=
|
||||
# Per-URL solve cap (bounds solver spend on a hostile page).
|
||||
|
|
|
|||
|
|
@ -1102,7 +1102,8 @@ class Config:
|
|||
PROXY_URLS = os.getenv("PROXY_URLS")
|
||||
|
||||
# =====================================================================
|
||||
# Phase 3d — Captcha solving (reCAPTCHA v2/v3, hCaptcha) via captchatools.
|
||||
# Phase 3d — Captcha solving (reCAPTCHA v2/v3, hCaptcha, v2-Enterprise) via
|
||||
# the in-house solver seam (app/utils/captcha/solvers.py).
|
||||
# The LAST-resort bypass tier: only fires on the StealthyFetcher browser
|
||||
# tier, only when a sitekey is detected, and only when explicitly enabled.
|
||||
# Cloudflare Turnstile is already handled free in-framework (03a), NOT here.
|
||||
|
|
@ -1114,9 +1115,9 @@ class Config:
|
|||
CAPTCHA_SOLVING_ENABLED = (
|
||||
os.getenv("CAPTCHA_SOLVING_ENABLED", "FALSE").upper() == "TRUE"
|
||||
)
|
||||
# captchatools "solving_site": capmonster | 2captcha | anticaptcha |
|
||||
# capsolver | captchaai. captchatools is itself the provider registry, so we
|
||||
# do not rebuild a vendor hierarchy.
|
||||
# Solver vendor. "capsolver" (AI-native, fastest on reCAPTCHA-Enterprise) and
|
||||
# "2captcha" have in-house clients today; anticaptcha / capmonster are added
|
||||
# progressively in solvers._PROVIDERS.
|
||||
CAPTCHA_SOLVER_PROVIDER = os.getenv("CAPTCHA_SOLVER_PROVIDER", "capsolver")
|
||||
CAPTCHA_SOLVER_API_KEY = os.getenv("CAPTCHA_SOLVER_API_KEY")
|
||||
# Per-URL solve cap so one hostile page can't burn unbounded solver credit.
|
||||
|
|
|
|||
|
|
@ -27,22 +27,75 @@ traffic" wall, and the IPs that pass serve a JavaScript shell whose organic
|
|||
results only materialize after the page's JS runs. So `fetch.py` needs both a
|
||||
**non-blocked IP** and a **real browser render**, on the same IP:
|
||||
|
||||
1. Reuse the last known-good sticky IP if it still passes a cheap re-precheck;
|
||||
otherwise race prechecks on several fresh sticky IPs at once (DataImpulse
|
||||
maps ports → sessions) and take the first that passes,
|
||||
1. Take a **warm sticky IP from the pool** (`_pool`): reuse an already-solved IP
|
||||
under its soft per-IP concurrency cap — concurrent fetches fan out across the
|
||||
pool's IPs (least-loaded first) instead of funneling onto one hot IP (which
|
||||
Google re-walls). If the pool is below target, grow it: first try to **adopt**
|
||||
an IP another process already solved (see the shared store below), else race
|
||||
prechecks on several fresh sticky IPs at once (DataImpulse maps ports →
|
||||
sessions) and solve on the first that passes. If the pool is full and every
|
||||
IP is busy, wait briefly for a slot rather than burn a fresh solve,
|
||||
2. render on that IP using a **shared long-lived browser** (launched once per
|
||||
process, per layout; each fetch only opens a fresh context carrying its
|
||||
vetted proxy) — during which the render clicks the AI Overview's "Show
|
||||
more" clamp and the initially-served People-Also-Ask questions open (all
|
||||
clicks fired first, then one shared wait) so their content lands in the DOM;
|
||||
3. retry on fresh IPs until one returns the results container.
|
||||
vetted proxy). Renders drop image/font/media/stylesheet requests
|
||||
(`disable_resources`; the parsed DOM is text/attributes only) — except AI
|
||||
Mode, whose answer streams in. During the render it clicks the AI Overview's
|
||||
"Show more" clamp and the initially-served People-Also-Ask questions open
|
||||
(all clicks fired first, then one shared wait) so their content lands in the
|
||||
DOM;
|
||||
3. on success (re)admit the IP to the pool; a walled render evicts it (and drops
|
||||
its exemption fleet-wide). Retry until a render returns the results container
|
||||
or the per-fetch deadline / IP budget runs out.
|
||||
|
||||
A warm fetch (browser up, IP cached) runs ~8 s; the first fetch of a process
|
||||
also pays the ~5 s Chromium launch and a vetting round. Requires the browser
|
||||
tier (patchright Chromium via Scrapling's `AsyncStealthySession`) and a
|
||||
residential proxy — set `PROXY_PROVIDER` + `PROXY_URL` (see
|
||||
`.env`). Long-running callers can `await fetch.close_sessions()` on shutdown;
|
||||
scripts that exit anyway can skip it.
|
||||
**The reCAPTCHA wall.** A curl-vetted IP is only *half* the battle: the initial
|
||||
`/search` GET returns a 200 JS shell, but the shell's JS reloads `/search` with
|
||||
a `sei` token, which Google 302s into a reCAPTCHA-**Enterprise** `/sorry` page.
|
||||
This trips *every* real browser (the block is on the browser access pattern, not
|
||||
the IP — curl "passes" only because it never runs the reload, and so never gets
|
||||
results either). When a solver is configured (`CAPTCHA_SOLVING_ENABLED=TRUE` +
|
||||
`CAPTCHA_SOLVER_API_KEY`, see `app/utils/captcha/`), `captcha.py` runs inside the
|
||||
render's `page_action`: on landing at `/sorry` it harvests an Enterprise token
|
||||
(with the page's dynamic `data-s`) from the solver **egressing through the same
|
||||
sticky proxy**, injects it, and submits. Google then issues a
|
||||
`GOOGLE_ABUSE_EXEMPTION` cookie, which the fetch seam caches per proxy
|
||||
(`_exemption_jar`) and replays via `page_setup` so **one solve unlocks the
|
||||
sticky IP for all subsequent queries** on it. That solve is also **published to
|
||||
Redis** (`pool_store.py`), so any *other* worker process can adopt the same
|
||||
warm IP instead of re-solving — cost then tracks the shared pool size, not the
|
||||
worker count (best-effort: no Redis ⇒ each process just keeps its own pool).
|
||||
Without a solver configured the stealth tier is unchanged (and
|
||||
brand/navigational queries will 429-wall).
|
||||
|
||||
Timings (measured): a fetch that must solve runs **~40–110 s and is entirely
|
||||
dominated by the solver** — the reCAPTCHA-Enterprise harvest alone measured
|
||||
**27–39 s on capsolver** (the configured default; AI-native, tighter spread) and
|
||||
37–100 s on 2captcha (pure vendor worker latency, not our overhead). NB the
|
||||
sub-10 s figures both vendors advertise are for the plain reCAPTCHA *demo*
|
||||
sitekey; Google's Enterprise `/sorry` wall is genuinely harder and slower for
|
||||
everyone. Every *subsequent* fetch on that sticky IP reuses the cached exemption
|
||||
and runs **~12–16 s** (two proxy navigations for Google's `sei` reload + ~3–4 s
|
||||
of block expansion; the vet precheck is skipped, `render_ms` is the whole
|
||||
render). The first fetch of a process also pays the ~5 s Chromium launch. So the
|
||||
one lever that actually moves the needle is **solving less often** (the pool +
|
||||
Redis-shared exemptions already amortize it) — or a faster solver.
|
||||
|
||||
**Scale.** Per-process throughput ceiling = `GOOGLE_SEARCH_MAX_CONCURRENT_PAGES`
|
||||
÷ warm-render-secs ≈ **4 / 14 s ≈ 17 SERP/min** at the default. Because renders
|
||||
drop images/fonts/media, one Chromium holds well more than 4 text contexts, so
|
||||
raise `GOOGLE_SEARCH_MAX_CONCURRENT_PAGES` to trade RAM/CPU for throughput
|
||||
before adding processes (e.g. 16 → ~69/min/process ⇒ ~7 processes cover 500/min;
|
||||
the sim in `scripts/scale_google_search.py` reproduces these numbers offline).
|
||||
`GOOGLE_SEARCH_WARM_POOL_TARGET` sizes the shared warm-IP pool: steady-state
|
||||
paid solves ≈ pool size (not request count), and it must be large enough that
|
||||
fleet-wide per-IP load (local `GOOGLE_SEARCH_IP_MAX_CONCURRENCY` × processes)
|
||||
stays gentle enough that Google doesn't re-wall a pool IP. Requires the
|
||||
browser tier (patchright Chromium via
|
||||
Scrapling's `AsyncStealthySession`) and a residential proxy — set
|
||||
`PROXY_PROVIDER` + `PROXY_URL` (see `.env`; **do not** pin a `__cr.<country>`
|
||||
suffix — DataImpulse's per-country sub-pools are 429-walled even for the curl
|
||||
precheck, which starves vetting; `gl=` in the URL sets result locale instead).
|
||||
Long-running callers can `await fetch.close_sessions()` on shutdown; scripts
|
||||
that exit anyway can skip it.
|
||||
|
||||
## Scope
|
||||
|
||||
|
|
@ -98,7 +151,9 @@ the progressive rollout.)
|
|||
| `schemas.py` | Pydantic input/output models mirroring the Apify camelCase spec. `extra="allow"` on outputs keeps the contract open. |
|
||||
| `scraper.py` | Orchestrator. `iter_serps` dispatches each `queries` line to `_term_flow` / `_url_flow` (+ `_ai_mode_flow` per term). |
|
||||
| `query_builder.py` | Pure: classify `queries` lines, fold advanced filters into search operators, resolve relative dates, build the URL. |
|
||||
| `fetch.py` | Proxy-vetted two-phase fetch: cheap precheck GET + headless render on a shared sticky IP, retrying across IPs. |
|
||||
| `fetch.py` | Proxy-vetted two-phase fetch: cheap precheck GET + headless render on a **warm sticky-IP pool** (spread across IPs, per-IP soft cap, grow-to-target), retrying across IPs. Caches per-IP reCAPTCHA exemptions. |
|
||||
| `pool_store.py` | Best-effort Redis cache of solved-IP exemptions so a solve on one worker warms that IP for the whole fleet (adopt/publish/evict); silent no-op without Redis. |
|
||||
| `captcha.py` | Async reCAPTCHA-Enterprise solve for the `/sorry` wall (via the `app.utils.captcha` solver seam — capsolver/2captcha, `enterprise`+`data-s`), run inside the render's `page_action`. |
|
||||
| `parsers.py` | Rendered SERP HTML → organic / text ads / product ads / related / People-Also-Ask / `resultsTotal` (degrades per-field). |
|
||||
|
||||
## Input semantics (matching Apify)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,178 @@
|
|||
"""reCAPTCHA-Enterprise solve for Google's ``/sorry`` interstitial.
|
||||
|
||||
Google walls a full browser's JS-driven ``sei`` reload by 302-ing it to
|
||||
``/sorry/index`` — a reCAPTCHA **Enterprise** challenge. A curl-vetted IP still
|
||||
hits this because the wall is on the *browser access pattern*, not the IP (curl
|
||||
never runs the JS reload, so it only ever sees the useless no-JS shell). The
|
||||
only way to keep rendering real SERPs on our own browser is to solve that
|
||||
challenge.
|
||||
|
||||
This runs inside the render's async ``page_action``: when the page lands on
|
||||
``/sorry``, it harvests a token from the configured solver (egressing from the
|
||||
**same sticky proxy** as the browser, so the token is IP-bound), injects it, and
|
||||
submits the form. Google then issues a ``GOOGLE_ABUSE_EXEMPTION`` cookie that
|
||||
unlocks real results on that IP — which the fetch seam caches per proxy so one
|
||||
solve amortizes across subsequent fetches (see ``fetch._exemption_jar``).
|
||||
|
||||
The actual vendor call goes through the shared, in-house solver seam
|
||||
(:mod:`app.utils.captcha.solvers`) with ``enterprise=True`` + the page's
|
||||
``data-s`` — this module just detects the wall, harvests via that seam egressing
|
||||
through the render's own sticky proxy, injects the token, and submits.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from app.utils.captcha import CaptchaConfig
|
||||
from app.utils.captcha import solvers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LOG = "[google_search][captcha]"
|
||||
|
||||
# The /sorry page embeds the widget sitekey and a dynamic `data-s` token that
|
||||
# the Enterprise challenge binds to; both are mandatory in the solve request.
|
||||
_SITEKEY_RE = re.compile(r'data-sitekey="([\w-]+)"')
|
||||
_DATA_S_RE = re.compile(r'data-s="([^"]+)"')
|
||||
|
||||
# After submitting the solved /sorry form, Google 302s back to the SERP. We wait
|
||||
# only for the results container to attach — NOT for full network idle. A
|
||||
# rendered SERP fires telemetry/XHRs that keep the network busy long past the
|
||||
# point results are present, so a `networkidle` wait here burned up to its full
|
||||
# timeout on every solve for no benefit; the container's presence is the real
|
||||
# "we're through the wall" signal.
|
||||
_POST_SOLVE_WAIT_MS = 15000
|
||||
|
||||
# Process-wide kill switch tripped on unrecoverable solver errors (no balance /
|
||||
# bad key). Persists across the per-fetch page_action instances of one run;
|
||||
# nothing short of a restart fixes balance/key, so keep hammering off.
|
||||
_latched = False
|
||||
|
||||
|
||||
def solver_latched() -> bool:
|
||||
return _latched
|
||||
|
||||
|
||||
def reset_solver_latch() -> None:
|
||||
"""Clear the process latch (test seam / explicit re-enable)."""
|
||||
global _latched
|
||||
_latched = False
|
||||
|
||||
|
||||
def _latch(reason: str) -> None:
|
||||
global _latched
|
||||
_latched = True
|
||||
logger.warning("%s solver latched OFF for this process: %s", _LOG, reason)
|
||||
|
||||
|
||||
def on_sorry(page: Any) -> bool:
|
||||
"""True when the render landed on Google's ``/sorry`` reCAPTCHA wall."""
|
||||
return "/sorry/" in (getattr(page, "url", "") or "")
|
||||
|
||||
|
||||
# Set the token into the response textarea and submit the /sorry form. Google's
|
||||
# /sorry form posts to /sorry/index and, on a valid token, 302s back to the
|
||||
# `continue` URL (the SERP) while setting GOOGLE_ABUSE_EXEMPTION.
|
||||
_INJECT_JS = r"""
|
||||
(token) => {
|
||||
let ta = document.getElementById('g-recaptcha-response');
|
||||
if (!ta) {
|
||||
ta = document.createElement('textarea');
|
||||
ta.id = 'g-recaptcha-response';
|
||||
ta.name = 'g-recaptcha-response';
|
||||
ta.style.display = 'none';
|
||||
(document.forms[0] || document.body).appendChild(ta);
|
||||
}
|
||||
ta.value = token; ta.innerHTML = token;
|
||||
const form = ta.closest('form') || document.forms[0];
|
||||
if (form) { try { form.requestSubmit ? form.requestSubmit() : form.submit(); } catch (e) { form.submit(); } }
|
||||
}
|
||||
"""
|
||||
|
||||
# Organic-results containers; presence means the exemption unlocked the SERP.
|
||||
_RESULTS_SEL = "#search, #rso, div.tF2Cxc"
|
||||
|
||||
|
||||
async def solve_sorry(page: Any, proxy_url: str | None, cfg: CaptchaConfig) -> bool:
|
||||
"""Solve the ``/sorry`` challenge on ``page`` and submit; return solved.
|
||||
|
||||
Runs inside the async render. On success the page ends on the real SERP and
|
||||
the context holds ``GOOGLE_ABUSE_EXEMPTION`` (the caller caches it). On any
|
||||
failure returns ``False`` and leaves the page on ``/sorry`` so the fetch
|
||||
loop moves to the next IP.
|
||||
"""
|
||||
if solver_latched():
|
||||
return False
|
||||
try:
|
||||
html = await page.content()
|
||||
except Exception:
|
||||
return False
|
||||
sk = _SITEKEY_RE.search(html)
|
||||
ds = _DATA_S_RE.search(html)
|
||||
if not sk or not ds:
|
||||
logger.warning("%s /sorry page missing sitekey/data-s; cannot solve", _LOG)
|
||||
return False
|
||||
|
||||
page_url = getattr(page, "url", "") or ""
|
||||
try:
|
||||
user_agent = await page.evaluate("() => navigator.userAgent")
|
||||
except Exception:
|
||||
user_agent = None
|
||||
|
||||
logger.info("%s solving Enterprise reCAPTCHA site=%s", _LOG, sk.group(1)[:12])
|
||||
_t0 = time.perf_counter()
|
||||
try:
|
||||
token = await asyncio.to_thread(
|
||||
solvers.solve,
|
||||
cfg,
|
||||
challenge_type="v2",
|
||||
sitekey=sk.group(1),
|
||||
page_url=page_url,
|
||||
proxy_url=proxy_url,
|
||||
user_agent=user_agent,
|
||||
enterprise=True,
|
||||
data_s=ds.group(1),
|
||||
)
|
||||
except solvers.SolverError as e:
|
||||
# Bad key / no balance / unsupported provider won't fix without a
|
||||
# restart — latch so we stop hammering (and stop paying) this process.
|
||||
_latch(repr(e))
|
||||
return False
|
||||
except Exception as e: # never let a solver hiccup break the render
|
||||
logger.warning("%s solve error: %s", _LOG, e)
|
||||
return False
|
||||
if not token:
|
||||
return False
|
||||
|
||||
harvest_ms = (time.perf_counter() - _t0) * 1000
|
||||
try:
|
||||
await page.evaluate(_INJECT_JS, token)
|
||||
except Exception as e:
|
||||
logger.warning("%s token injection failed: %s", _LOG, e)
|
||||
return False
|
||||
with contextlib.suppress(Exception):
|
||||
await page.wait_for_selector(_RESULTS_SEL, timeout=_POST_SOLVE_WAIT_MS)
|
||||
|
||||
solved = not on_sorry(page)
|
||||
logger.info(
|
||||
"%s solve %s harvest_ms=%.0f total_ms=%.0f",
|
||||
_LOG,
|
||||
"OK" if solved else "did not clear wall",
|
||||
harvest_ms,
|
||||
(time.perf_counter() - _t0) * 1000,
|
||||
)
|
||||
return solved
|
||||
|
||||
|
||||
async def exemption_cookies(page: Any) -> list[dict]:
|
||||
"""Snapshot the context's google.com cookies (for the per-proxy jar)."""
|
||||
try:
|
||||
return await page.context.cookies("https://www.google.com")
|
||||
except Exception:
|
||||
return []
|
||||
|
|
@ -33,6 +33,7 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import threading
|
||||
|
|
@ -41,6 +42,9 @@ from urllib.parse import urlsplit, urlunsplit
|
|||
|
||||
from scrapling.fetchers import AsyncFetcher
|
||||
|
||||
from app.proprietary.platforms.google_search import captcha as _captcha
|
||||
from app.proprietary.platforms.google_search import pool_store as _store
|
||||
from app.utils.captcha import captcha_enabled, get_captcha_config
|
||||
from app.utils.proxy import get_proxy_url
|
||||
|
||||
try: # browser tier is optional (needs `scrapling[fetchers]` browsers installed)
|
||||
|
|
@ -55,6 +59,23 @@ logger = logging.getLogger(__name__)
|
|||
CONSENT_COOKIES = {"CONSENT": "PENDING+987", "SOCS": "CAESHAgBEhIaAB"}
|
||||
_HEADERS = {"Accept-Language": "en-US,en;q=0.9"}
|
||||
|
||||
# Context-cookie form of the consent cookies, injected before a render's
|
||||
# navigation (page_setup) when captcha solving is on, so the browser skips the
|
||||
# EU interstitial exactly like the curl precheck does.
|
||||
_CONSENT_CTX = [
|
||||
{"name": k, "value": v, "domain": ".google.com", "path": "/"}
|
||||
for k, v in CONSENT_COOKIES.items()
|
||||
]
|
||||
|
||||
# GOOGLE_ABUSE_EXEMPTION cookies harvested per sticky proxy after a solve. A hit
|
||||
# lets a later render on the same IP skip straight to results (no re-solve),
|
||||
# amortizing the one paid solve across the sticky session.
|
||||
# ponytail: keyed by the full proxy URL (incl. sticky port). When that port's
|
||||
# exit IP rotates the cached cookie goes stale -> the next render re-hits
|
||||
# /sorry, re-solves, and overwrites the entry. Self-correcting; the sticky-port
|
||||
# space is small so the dict never grows unbounded in practice.
|
||||
_exemption_jar: dict[str, list[dict]] = {}
|
||||
|
||||
# Gateways whose sticky sessions are selected by destination port. A random port
|
||||
# in this range pins one residential IP for the duration of a browser session.
|
||||
_STICKY_HOSTS = {"gw.dataimpulse.com"}
|
||||
|
|
@ -73,11 +94,37 @@ _VET_CONCURRENCY = 4
|
|||
# the budget in a couple of seconds.
|
||||
_WALLED_ROUND_BACKOFF_S = 3.0
|
||||
|
||||
# The sticky IP that most recently served a real SERP; the strongest hint for
|
||||
# the next fetch. Re-vetted (cheap) before reuse, dropped on failure.
|
||||
# ponytail: a single slot, not a pool — concurrent fetches share (and race)
|
||||
# it; worst case a loser re-vets a fresh IP, which is the normal path anyway.
|
||||
_last_good_proxy: str | None = None
|
||||
# Warm sticky-IP pool. Concurrent fetches spread renders across the pool's IPs
|
||||
# instead of funneling every render onto one sticky IP (a single hot residential
|
||||
# IP is exactly what Google re-walls), and each pool IP is solved **once** — so
|
||||
# steady-state paid solves track the pool size, not the request count. Replaces
|
||||
# the old single "_last_good_proxy" slot, which both funneled load and forced a
|
||||
# fresh solve whenever it was raced to None.
|
||||
#
|
||||
# State is touched from two loops (fetch_serp_html on the request loop; the
|
||||
# exemption write in page_action on the browser loop), so a small lock guards
|
||||
# every mutation. No awaits are held under the lock — every critical section is
|
||||
# a handful of dict ops.
|
||||
_pool: dict[str, float] = {} # warm proxy url -> last-used monotonic ts
|
||||
_pool_inflight: dict[str, int] = {} # proxy url -> renders currently pinned to it
|
||||
_pool_pending = 0 # fresh vets/solves in flight (growing the pool toward target)
|
||||
_pool_lock = threading.Lock()
|
||||
|
||||
# Target number of warm IPs to keep. Sized for throughput: it must cover
|
||||
# (target SERP/min) / (safe SERP/min per IP), so a fleet serving 500/min across
|
||||
# a handful of processes wants this in the low tens. Env-tunable without a code
|
||||
# change; per-IP concurrency is a soft cap (exceeded only when the pool is full
|
||||
# and every IP is busy, because reusing a warm IP always beats a fresh solve).
|
||||
_WARM_POOL_TARGET = int(os.getenv("GOOGLE_SEARCH_WARM_POOL_TARGET", "8"))
|
||||
_WARM_IP_MAX_CONCURRENCY = int(os.getenv("GOOGLE_SEARCH_IP_MAX_CONCURRENCY", "2"))
|
||||
# When the pool is full and every IP is at its cap, wait this long for a slot to
|
||||
# free instead of vetting+solving a fresh IP (this backpressure is what bounds
|
||||
# solves to ~pool size under a burst).
|
||||
_POOL_WAIT_S = 0.25
|
||||
# Absolute per-fetch budget. A cold solve is ~45 s and we may retry a couple of
|
||||
# walled IPs, but a fetch that can't get results within this window fails rather
|
||||
# than hanging a request forever under saturation.
|
||||
_FETCH_DEADLINE_S = float(os.getenv("GOOGLE_SEARCH_FETCH_DEADLINE_S", "180"))
|
||||
|
||||
# A usable precheck responds in <1 s; anything slower is a dead/slow sticky IP
|
||||
# (seen hanging ~60 s). Abandon it on this deadline so a slow IP costs no more
|
||||
|
|
@ -172,6 +219,83 @@ async def _vet_fresh_ip(url: str, base: str) -> str | None:
|
|||
return winner
|
||||
|
||||
|
||||
def _pool_take() -> tuple[str, str | None]:
|
||||
"""Decide how the next render gets an IP. Returns one of:
|
||||
|
||||
* ``("reuse", url)`` — an existing warm IP under its soft concurrency cap
|
||||
(its inflight count is already incremented; caller must ``_pool_settle``),
|
||||
* ``("grow", None)`` — pool is below target, caller should vet a fresh IP
|
||||
and solve on it (a pending slot is reserved),
|
||||
* ``("wait", None)`` — pool is full and every IP is busy; caller should wait
|
||||
briefly and retry rather than burn a fresh solve.
|
||||
|
||||
Reuse prefers the least-loaded, least-recently-used warm IP so concurrent
|
||||
renders fan out across the pool instead of piling onto one IP.
|
||||
"""
|
||||
global _pool_pending
|
||||
with _pool_lock:
|
||||
best: tuple[str, int, float] | None = None
|
||||
for url, last in _pool.items():
|
||||
n = _pool_inflight.get(url, 0)
|
||||
if n < _WARM_IP_MAX_CONCURRENCY and (
|
||||
best is None or (n, last) < (best[1], best[2])
|
||||
):
|
||||
best = (url, n, last)
|
||||
if best is not None:
|
||||
_pool_inflight[best[0]] = _pool_inflight.get(best[0], 0) + 1
|
||||
return ("reuse", best[0])
|
||||
if len(_pool) + _pool_pending < _WARM_POOL_TARGET:
|
||||
_pool_pending += 1
|
||||
return ("grow", None)
|
||||
return ("wait", None)
|
||||
|
||||
|
||||
def _pool_bind_fresh(url: str) -> None:
|
||||
"""Pin the first render onto a just-vetted fresh IP (grow path)."""
|
||||
with _pool_lock:
|
||||
_pool_inflight[url] = _pool_inflight.get(url, 0) + 1
|
||||
|
||||
|
||||
def _pool_adopt(url: str) -> None:
|
||||
"""Turn a reserved grow slot into an IP adopted from the shared store: it
|
||||
joins the warm pool without a local solve, so release the pending reservation
|
||||
and pin this render onto it."""
|
||||
global _pool_pending
|
||||
with _pool_lock:
|
||||
_pool_pending = max(0, _pool_pending - 1)
|
||||
_pool[url] = time.monotonic()
|
||||
_pool_inflight[url] = _pool_inflight.get(url, 0) + 1
|
||||
|
||||
|
||||
def _pool_abort_grow() -> None:
|
||||
"""Release a reserved grow slot when the fresh vet found no usable IP."""
|
||||
global _pool_pending
|
||||
with _pool_lock:
|
||||
_pool_pending = max(0, _pool_pending - 1)
|
||||
|
||||
|
||||
def _pool_settle(url: str, *, good: bool, grew: bool) -> None:
|
||||
"""Finish a render: drop its inflight hold, then admit or evict the IP.
|
||||
|
||||
A good render (re)admits the IP to the warm pool (refreshing its LRU stamp);
|
||||
a walled render evicts it and drops its now-worthless cached exemption.
|
||||
"""
|
||||
global _pool_pending
|
||||
with _pool_lock:
|
||||
n = _pool_inflight.get(url, 0) - 1
|
||||
if n <= 0:
|
||||
_pool_inflight.pop(url, None)
|
||||
else:
|
||||
_pool_inflight[url] = n
|
||||
if grew:
|
||||
_pool_pending = max(0, _pool_pending - 1)
|
||||
if good:
|
||||
_pool[url] = time.monotonic()
|
||||
else:
|
||||
_pool.pop(url, None)
|
||||
_exemption_jar.pop(url, None)
|
||||
|
||||
|
||||
# People-also-ask answers only load when a question is expanded (clicked).
|
||||
# We expand just the initially-served questions (~4); each expansion appends
|
||||
# more questions we deliberately leave collapsed, or the loop never ends.
|
||||
|
|
@ -197,6 +321,7 @@ async def _expand_blocks(page):
|
|||
Both are free on pages without the block, and best-effort: a failed click
|
||||
just leaves that section collapsed rather than failing the render.
|
||||
"""
|
||||
_t0 = time.perf_counter()
|
||||
clicked = 0
|
||||
try:
|
||||
more = await page.query_selector(_AIO_SHOW_MORE_SEL)
|
||||
|
|
@ -218,9 +343,45 @@ async def _expand_blocks(page):
|
|||
if clicked:
|
||||
# One shared wait while all the answer XHRs land in parallel.
|
||||
await page.wait_for_timeout(_PAA_ANSWER_WAIT_MS)
|
||||
logger.info(
|
||||
"[google_search][perf] expand_ms=%.0f clicked=%d",
|
||||
(time.perf_counter() - _t0) * 1000,
|
||||
clicked,
|
||||
)
|
||||
return page
|
||||
|
||||
|
||||
def _make_page_setup(proxy: str | None):
|
||||
"""Per-fetch ``page_setup``: seed consent + any cached exemption for this IP."""
|
||||
|
||||
async def page_setup(page):
|
||||
jar = list(_CONSENT_CTX)
|
||||
cached = _exemption_jar.get(proxy or "")
|
||||
if cached:
|
||||
jar += cached
|
||||
with contextlib.suppress(Exception):
|
||||
await page.context.add_cookies(jar)
|
||||
|
||||
return page_setup
|
||||
|
||||
|
||||
def _make_page_action(proxy: str | None, cfg):
|
||||
"""Per-fetch ``page_action``: solve the /sorry wall if hit, then expand blocks.
|
||||
|
||||
Overrides the session default (:func:`_expand_blocks`) so it must still call
|
||||
it afterwards. On a successful solve the resulting ``GOOGLE_ABUSE_EXEMPTION``
|
||||
cookie is cached against this sticky proxy for reuse.
|
||||
"""
|
||||
|
||||
async def page_action(page):
|
||||
if _captcha.on_sorry(page):
|
||||
if await _captcha.solve_sorry(page, proxy, cfg):
|
||||
_exemption_jar[proxy or ""] = await _captcha.exemption_cookies(page)
|
||||
return await _expand_blocks(page)
|
||||
|
||||
return page_action
|
||||
|
||||
|
||||
# Firefox-on-Android UA to make Google serve its mobile lightweight layout.
|
||||
# ponytail: the engine underneath is patchright's *Chromium*, so this UA lies
|
||||
# about the engine — but empirically it's what gets the mobile layout served
|
||||
|
|
@ -279,9 +440,16 @@ _session_lock = asyncio.Lock()
|
|||
# second concurrent render raised RuntimeError('Maximum page limit (1)
|
||||
# reached'); the failure handler then closed the shared browser under the
|
||||
# sibling render (the TargetClosedError cascade seen in production when
|
||||
# several scrape runs overlap). The pool and this gate are sized together:
|
||||
# the gate queues excess renders instead of tripping the pool.
|
||||
_MAX_CONCURRENT_PAGES = 4
|
||||
# several scrape runs overlap). The pool and this gate are sized together
|
||||
# (the session's max_pages is set from this): the gate queues excess renders
|
||||
# instead of tripping the pool.
|
||||
#
|
||||
# This is THE throughput lever: per-process ceiling = this / warm-render-secs
|
||||
# (≈ 4/14 s ≈ 17 SERP/min at the default). Renders drop images/fonts/media
|
||||
# (`disable_resources`), so a single Chromium can hold well more than 4 text
|
||||
# contexts — raise this (env) to trade RAM/CPU for throughput before adding
|
||||
# processes. e.g. 16 → ~68/min/process, so ~8 processes cover 500/min.
|
||||
_MAX_CONCURRENT_PAGES = int(os.getenv("GOOGLE_SEARCH_MAX_CONCURRENT_PAGES", "4"))
|
||||
# Only ever awaited from coroutines running on the browser loop.
|
||||
_render_gate = asyncio.Semaphore(_MAX_CONCURRENT_PAGES)
|
||||
|
||||
|
|
@ -350,7 +518,23 @@ async def _render_on_loop(url: str, proxy: str | None, mobile: bool):
|
|||
session = await _get_session(mobile)
|
||||
_inflight[session] = _inflight.get(session, 0) + 1
|
||||
try:
|
||||
return await session.fetch(url, proxy=proxy)
|
||||
kwargs: dict = {}
|
||||
# Drop image/font/media/stylesheet requests: the SERP DOM we parse is
|
||||
# text/attribute-only, so these are dead weight (a warm render moved
|
||||
# ~2-7 MB per page) that also keeps `network_idle` from settling.
|
||||
# AI Mode streams its answer into the DOM, so leave its resources on
|
||||
# (dropping websockets/media can starve the stream) — detected by the
|
||||
# udm=50 marker its URL always carries.
|
||||
if "udm=50" not in url:
|
||||
kwargs["disable_resources"] = True
|
||||
# Only wire the solver when it's configured AND still viable this
|
||||
# process; otherwise the session's default page_action (expand-only)
|
||||
# runs and the stealth tier is unchanged.
|
||||
if proxy and captcha_enabled() and not _captcha.solver_latched():
|
||||
cfg = get_captcha_config()
|
||||
kwargs["page_setup"] = _make_page_setup(proxy)
|
||||
kwargs["page_action"] = _make_page_action(proxy, cfg)
|
||||
return await session.fetch(url, proxy=proxy, **kwargs)
|
||||
finally:
|
||||
_inflight[session] -= 1
|
||||
if not _inflight[session]:
|
||||
|
|
@ -374,35 +558,61 @@ async def _render(url: str, proxy: str | None, mobile: bool = False):
|
|||
async def fetch_serp_html(url: str, *, mobile: bool = False) -> str | None:
|
||||
"""Return fully-rendered SERP HTML for ``url``, or ``None`` if unobtainable.
|
||||
|
||||
Reuses the last known-good sticky IP when it still passes the cheap
|
||||
precheck; otherwise races prechecks on fresh sticky IPs and renders on the
|
||||
first that passes. Retries until a render returns real results or the IP
|
||||
budget runs out. Requires the browser tier — without it we cannot get
|
||||
JS-built results. ``mobile`` renders with a phone UA/viewport (the
|
||||
``mobileResults`` input).
|
||||
Renders on a **warm sticky-IP pool**: it reuses an already-solved IP under
|
||||
its soft concurrency cap (spreading concurrent renders across the pool),
|
||||
grows the pool by vetting+solving a fresh IP when it's below target, and
|
||||
applies backpressure (a short wait) when the pool is full and busy rather
|
||||
than burning an extra solve. Retries walled IPs until real results land or
|
||||
the per-fetch deadline / IP budget runs out. Requires the browser tier —
|
||||
without it we cannot get JS-built results. ``mobile`` renders with a phone
|
||||
UA/viewport (the ``mobileResults`` input).
|
||||
"""
|
||||
global _last_good_proxy
|
||||
if AsyncStealthySession is None:
|
||||
logger.error("[google_search] browser tier unavailable; cannot render SERPs")
|
||||
return None
|
||||
|
||||
base = get_proxy_url()
|
||||
if not base:
|
||||
# No proxy configured: a single direct render, no pool bookkeeping.
|
||||
with contextlib.suppress(Exception):
|
||||
page = await _render(url, None, mobile=mobile)
|
||||
html = page.html_content or ""
|
||||
return html if (page.status == 200 and _has_results(html)) else None
|
||||
return None
|
||||
|
||||
deadline = time.monotonic() + _FETCH_DEADLINE_S
|
||||
ips_tried = 0
|
||||
while ips_tried < _MAX_IP_ATTEMPTS:
|
||||
if base:
|
||||
if _last_good_proxy and await _precheck(url, _last_good_proxy):
|
||||
proxy = _last_good_proxy
|
||||
while time.monotonic() < deadline and ips_tried < _MAX_IP_ATTEMPTS:
|
||||
action, proxy = _pool_take()
|
||||
if action == "wait":
|
||||
# Pool full and every IP busy: wait for a warm slot instead of
|
||||
# solving a fresh IP. The render gate drains continuously, so a slot
|
||||
# frees shortly; this is the queue that bounds solves to ~pool size.
|
||||
await asyncio.sleep(_POOL_WAIT_S)
|
||||
continue
|
||||
solved_fresh = False
|
||||
vet_ms = 0.0
|
||||
if action == "grow":
|
||||
# Before paying for a solve, try to adopt an IP another process
|
||||
# already solved (fleet-shared exemption). Only if none is available
|
||||
# do we vet a fresh IP and solve on it ourselves.
|
||||
adopted = await _store.adopt(set(_pool))
|
||||
if adopted is not None:
|
||||
proxy, cookies = adopted
|
||||
_exemption_jar[proxy] = cookies
|
||||
_pool_adopt(proxy)
|
||||
else:
|
||||
_last_good_proxy = None
|
||||
vt = time.perf_counter()
|
||||
proxy = await _vet_fresh_ip(url, base)
|
||||
vet_ms = (time.perf_counter() - vt) * 1000
|
||||
ips_tried += _VET_CONCURRENCY
|
||||
if proxy is None:
|
||||
_pool_abort_grow()
|
||||
logger.debug("[google_search] vetting round: all IPs walled")
|
||||
await asyncio.sleep(_WALLED_ROUND_BACKOFF_S)
|
||||
continue
|
||||
else:
|
||||
proxy = None
|
||||
ips_tried += 1
|
||||
_pool_bind_fresh(proxy)
|
||||
solved_fresh = True
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
page = await _render(url, proxy, mobile=mobile)
|
||||
|
|
@ -411,23 +621,32 @@ async def fetch_serp_html(url: str, *, mobile: bool = False) -> str | None:
|
|||
# browser side is broken, so relaunch it rather than limp along.
|
||||
# repr(), not str(): e.g. NotImplementedError stringifies to "".
|
||||
logger.warning("[google_search] render failed: %r", e)
|
||||
_last_good_proxy = None
|
||||
_pool_settle(proxy, good=False, grew=solved_fresh)
|
||||
await _store.evict(proxy)
|
||||
await _drop_session(mobile)
|
||||
continue
|
||||
fetch_ms = (time.perf_counter() - started) * 1000
|
||||
html = page.html_content or ""
|
||||
good = page.status == 200 and _has_results(html)
|
||||
_pool_settle(proxy, good=good, grew=solved_fresh)
|
||||
logger.info(
|
||||
"[google_search][perf] status=%s bytes=%d has_results=%s fetch_ms=%.0f reused_ip=%s",
|
||||
"[google_search][perf] status=%s bytes=%d has_results=%s vet_ms=%.0f render_ms=%.0f from_pool=%s pool=%d",
|
||||
page.status,
|
||||
len(html),
|
||||
good,
|
||||
vet_ms,
|
||||
fetch_ms,
|
||||
proxy == _last_good_proxy,
|
||||
not solved_fresh,
|
||||
len(_pool),
|
||||
)
|
||||
if good:
|
||||
_last_good_proxy = proxy
|
||||
# Share a freshly-solved IP with the fleet; a walled IP is poison,
|
||||
# so make sure no process keeps its stale exemption.
|
||||
if solved_fresh and _exemption_jar.get(proxy):
|
||||
await _store.publish(proxy, _exemption_jar[proxy])
|
||||
return html
|
||||
_last_good_proxy = None
|
||||
logger.warning("[google_search] exhausted %d IPs for %s", _MAX_IP_ATTEMPTS, url)
|
||||
await _store.evict(proxy)
|
||||
logger.warning(
|
||||
"[google_search] gave up on %s (deadline/%d-IP budget)", url, _MAX_IP_ATTEMPTS
|
||||
)
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -0,0 +1,121 @@
|
|||
"""Cross-process warm-IP exemption cache (Redis-backed, best-effort).
|
||||
|
||||
The expensive unit is the reCAPTCHA solve. Within a process the warm pool
|
||||
(:mod:`fetch`) already solves each sticky IP once; this shares those solves
|
||||
across the whole worker fleet: a fresh solve on any process **publishes** the
|
||||
IP's ``GOOGLE_ABUSE_EXEMPTION`` cookies to Redis, and every other process can
|
||||
**adopt** that IP instead of solving it again. Without this, cost scales with
|
||||
the number of worker processes; with it, cost scales with the (shared) pool
|
||||
size.
|
||||
|
||||
Best-effort by design: any Redis hiccup disables the layer and every process
|
||||
falls back to its own local pool — correctness is unchanged, we just pay more
|
||||
solves. Redis calls run in a worker thread (:func:`asyncio.to_thread`) so they
|
||||
never block the request loop and don't care which loop the caller is on.
|
||||
|
||||
``ponytail:`` the shared store holds only exemptions (the costly artifact), not
|
||||
per-process render concurrency — global per-IP load is still governed by each
|
||||
process's local per-IP cap × the number of processes, so size the pool for the
|
||||
fleet (see ``GOOGLE_SEARCH_WARM_POOL_TARGET``). Full distributed inflight
|
||||
accounting is the upgrade path if a single shared IP ever gets overloaded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
from app.config import config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LOG = "[google_search][poolstore]"
|
||||
_PREFIX = "gsearch:warm:"
|
||||
# Google's GOOGLE_ABUSE_EXEMPTION outlives this comfortably; a conservative TTL
|
||||
# just means the fleet re-solves an idle IP occasionally (cheap) and never
|
||||
# serves a long-dead exemption. Env-tunable.
|
||||
_TTL_S = int(os.getenv("GOOGLE_SEARCH_EXEMPTION_TTL_S", "3600"))
|
||||
|
||||
_client = None
|
||||
_disabled = False
|
||||
|
||||
|
||||
def _key(proxy: str) -> str:
|
||||
return _PREFIX + hashlib.sha1(proxy.encode()).hexdigest()
|
||||
|
||||
|
||||
def _get_client():
|
||||
"""Lazily build a short-timeout Redis client, disabling on first failure."""
|
||||
global _client, _disabled
|
||||
if _disabled:
|
||||
return None
|
||||
if _client is not None:
|
||||
return _client
|
||||
try:
|
||||
import redis
|
||||
|
||||
_client = redis.Redis.from_url(
|
||||
config.REDIS_APP_URL, socket_timeout=1, socket_connect_timeout=1
|
||||
)
|
||||
_client.ping()
|
||||
except Exception as e:
|
||||
logger.info("%s Redis unavailable; cross-process sharing off: %s", _LOG, e)
|
||||
_disabled = True
|
||||
_client = None
|
||||
return _client
|
||||
|
||||
|
||||
def _publish_sync(proxy: str, cookies: list[dict]) -> None:
|
||||
c = _get_client()
|
||||
if c is None:
|
||||
return
|
||||
try:
|
||||
c.set(_key(proxy), json.dumps({"proxy": proxy, "cookies": cookies}), ex=_TTL_S)
|
||||
except Exception as e:
|
||||
logger.debug("%s publish failed: %s", _LOG, e)
|
||||
|
||||
|
||||
def _adopt_sync(exclude: set[str]) -> tuple[str, list[dict]] | None:
|
||||
c = _get_client()
|
||||
if c is None:
|
||||
return None
|
||||
try:
|
||||
for key in c.scan_iter(match=_PREFIX + "*", count=100):
|
||||
raw = c.get(key)
|
||||
if not raw:
|
||||
continue
|
||||
d = json.loads(raw)
|
||||
proxy = d.get("proxy")
|
||||
if proxy and proxy not in exclude:
|
||||
return proxy, d.get("cookies") or []
|
||||
except Exception as e:
|
||||
logger.debug("%s adopt failed: %s", _LOG, e)
|
||||
return None
|
||||
|
||||
|
||||
def _evict_sync(proxy: str) -> None:
|
||||
c = _get_client()
|
||||
if c is None:
|
||||
return
|
||||
try:
|
||||
c.delete(_key(proxy))
|
||||
except Exception as e:
|
||||
logger.debug("%s evict failed: %s", _LOG, e)
|
||||
|
||||
|
||||
async def publish(proxy: str, cookies: list[dict]) -> None:
|
||||
"""Share a freshly-solved IP's exemption with the fleet."""
|
||||
await asyncio.to_thread(_publish_sync, proxy, cookies)
|
||||
|
||||
|
||||
async def adopt(exclude: set[str]) -> tuple[str, list[dict]] | None:
|
||||
"""Return a fleet-warm ``(proxy, cookies)`` not in ``exclude``, or ``None``."""
|
||||
return await asyncio.to_thread(_adopt_sync, exclude)
|
||||
|
||||
|
||||
async def evict(proxy: str) -> None:
|
||||
"""Drop a poisoned (walled) IP so no process adopts its stale exemption."""
|
||||
await asyncio.to_thread(_evict_sync, proxy)
|
||||
|
|
@ -6,10 +6,10 @@
|
|||
"""Captcha detection + token-injection ``page_action`` (Phase 3d).
|
||||
|
||||
This is the **bypass logic** (hence proprietary); the generic, vendor-agnostic
|
||||
config lives in the Apache-2.0 ``app/utils/captcha/`` package. ``captchatools``
|
||||
is the multi-vendor solver registry — we only detect the challenge, harvest a
|
||||
token egressing from **the crawl's own proxy IP** (token IP-binding), inject it,
|
||||
and submit.
|
||||
config and the vendor API clients live in the Apache-2.0 ``app/utils/captcha/``
|
||||
package (:mod:`app.utils.captcha.solvers`). We only detect the challenge,
|
||||
harvest a token through that seam egressing from **the crawl's own proxy IP**
|
||||
(token IP-binding), inject it, and submit.
|
||||
|
||||
Why a closure cell: Scrapling runs ``page_action`` after navigation but
|
||||
**swallows its exceptions and discards its return value** (see
|
||||
|
|
@ -29,7 +29,6 @@ import re
|
|||
from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from app.utils.captcha import CaptchaConfig
|
||||
|
||||
|
|
@ -118,26 +117,6 @@ def detect_challenge(page: Any, cfg: CaptchaConfig) -> tuple[str, str] | None:
|
|||
# --- proxy / harvest -------------------------------------------------------
|
||||
|
||||
|
||||
def proxy_url_to_captchatools(proxy_url: str | None) -> str | None:
|
||||
"""Reformat ``http://user:pass@host:port`` -> ``host:port:user:pass``.
|
||||
|
||||
captchatools wants the colon-delimited form. Returns ``None`` for a missing
|
||||
or unparseable proxy so the solver harvests proxyless (the token may then be
|
||||
IP-mismatched, but that's better than crashing the fetch).
|
||||
"""
|
||||
if not proxy_url:
|
||||
return None
|
||||
try:
|
||||
p = urlsplit(proxy_url)
|
||||
if not p.hostname or not p.port:
|
||||
return None
|
||||
if p.username and p.password:
|
||||
return f"{p.hostname}:{p.port}:{p.username}:{p.password}"
|
||||
return f"{p.hostname}:{p.port}"
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _harvest_token(
|
||||
cfg: CaptchaConfig,
|
||||
challenge_type: str,
|
||||
|
|
@ -146,26 +125,21 @@ def _harvest_token(
|
|||
proxy: str | None,
|
||||
user_agent: str | None,
|
||||
) -> str | None:
|
||||
"""Harvest a token from the solver. Raises on solver errors so the caller
|
||||
can latch on the unrecoverable ones; returns ``None`` on an empty token.
|
||||
"""Harvest a token from the in-house solver seam. Raises on solver errors so
|
||||
the caller can latch on the unrecoverable ones; returns ``None`` on an empty
|
||||
token. ``proxy`` is a raw ``http://user:pass@host:port`` URL (the seam
|
||||
reformats it per vendor).
|
||||
"""
|
||||
import captchatools
|
||||
from app.utils.captcha import solvers
|
||||
|
||||
harvester = captchatools.new_harvester(
|
||||
api_key=cfg.api_key,
|
||||
solving_site=cfg.solving_site,
|
||||
return solvers.solve(
|
||||
cfg,
|
||||
challenge_type=challenge_type,
|
||||
sitekey=sitekey,
|
||||
captcha_url=page_url,
|
||||
captcha_type=challenge_type,
|
||||
min_score=cfg.v3_min_score,
|
||||
action=cfg.v3_action,
|
||||
)
|
||||
token = harvester.get_token(
|
||||
proxy=proxy,
|
||||
proxy_type="HTTP",
|
||||
page_url=page_url,
|
||||
proxy_url=proxy,
|
||||
user_agent=user_agent,
|
||||
)
|
||||
return token or None
|
||||
|
||||
|
||||
# --- injection -------------------------------------------------------------
|
||||
|
|
@ -247,7 +221,9 @@ def build_captcha_page_action(
|
|||
if not cfg.enabled or solver_latched():
|
||||
return None
|
||||
|
||||
captcha_proxy = proxy_url_to_captchatools(proxy_url)
|
||||
# Raw URL; the solver seam reformats it per vendor and binds the token to
|
||||
# this exit IP.
|
||||
captcha_proxy = proxy_url
|
||||
|
||||
def page_action(page: Any) -> Any:
|
||||
# Hard cap: never exceed the per-URL budget even across proxy-retry
|
||||
|
|
@ -320,10 +296,14 @@ def build_captcha_page_action(
|
|||
|
||||
|
||||
def _is_unrecoverable(exc: Exception) -> bool:
|
||||
"""True for solver errors that must latch solving off (no balance / bad key).
|
||||
"""True for solver errors that must latch solving off.
|
||||
|
||||
Matched by class name to avoid a hard import of ``captchatools.exceptions``
|
||||
at module load (the dep is optional / lazily imported).
|
||||
Covers the in-house seam's typed errors (``SolverBalanceError`` /
|
||||
``SolverAuthError`` / ``SolverUnsupported``) plus legacy/no-balance shapes.
|
||||
Matched by class name so no solver module must be imported here.
|
||||
"""
|
||||
name = type(exc).__name__.lower()
|
||||
return "nobalance" in name or "wrongapikey" in name or "apikey" in name
|
||||
return any(
|
||||
k in name
|
||||
for k in ("balance", "apikey", "auth", "unsupported", "wronguser")
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,15 +1,16 @@
|
|||
"""App-wide captcha-solver configuration (Apache-2.0, generic glue).
|
||||
|
||||
This package holds only the **generic, vendor-agnostic** config resolution for
|
||||
captcha solving — mirroring ``app/utils/proxy/`` (which stays Apache-2.0 even
|
||||
though the crawler that consumes it is proprietary). The actual bypass logic
|
||||
(challenge detection + token injection ``page_action``) lives in the separately
|
||||
licensed ``app/proprietary/web_crawler/captcha.py``.
|
||||
This package holds the **generic, vendor-agnostic** config resolution for
|
||||
captcha solving (:class:`CaptchaConfig`) plus the in-house vendor API clients
|
||||
(:mod:`app.utils.captcha.solvers`) — mirroring ``app/utils/proxy/`` (which stays
|
||||
Apache-2.0 even though the crawler that consumes it is proprietary). The actual
|
||||
bypass logic (challenge detection + token injection ``page_action``) lives in
|
||||
the separately licensed ``app/proprietary/web_crawler/captcha.py`` and
|
||||
``app/proprietary/platforms/google_search/captcha.py``.
|
||||
|
||||
``captchatools`` is itself the multi-vendor registry (capmonster / 2captcha /
|
||||
anticaptcha / capsolver / captchaai), so there is no provider hierarchy here —
|
||||
just one env-driven :class:`CaptchaConfig` and a cheap ``captcha_enabled()``
|
||||
gate callers check before constructing anything (mirrors
|
||||
``solvers.solve()`` dispatches on ``CAPTCHA_SOLVER_PROVIDER``; only 2captcha is
|
||||
wired today, more vendors are added progressively. ``captcha_enabled()`` is the
|
||||
cheap gate callers check before constructing anything (mirrors
|
||||
``WebCrawlCreditService.billing_enabled()``).
|
||||
"""
|
||||
|
||||
|
|
|
|||
311
surfsense_backend/app/utils/captcha/solvers.py
Normal file
311
surfsense_backend/app/utils/captcha/solvers.py
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
"""In-house captcha-solver clients (vendor API glue, Apache-2.0).
|
||||
|
||||
Replaces the unmaintained ``captchatools`` registry. Each provider is a small
|
||||
function that submits a challenge to the vendor and polls for the token;
|
||||
:func:`solve` dispatches on the app-wide ``CAPTCHA_SOLVER_PROVIDER``.
|
||||
|
||||
Two providers are wired: **2captcha** (legacy ``in.php``/``res.php``) and
|
||||
**capsolver** (AI-native ``createTask``/``getTaskResult``, materially faster on
|
||||
the reCAPTCHA-*Enterprise* ``/sorry`` wall). Both express the Enterprise pieces
|
||||
Google needs — the widget sitekey plus the page's dynamic ``data-s`` token
|
||||
(something ``captchatools`` could not). More vendors (anticaptcha / capmonster)
|
||||
are added progressively as new entries in :data:`_PROVIDERS`; until then an
|
||||
unconfigured provider raises :class:`SolverUnsupported` so callers latch off
|
||||
cleanly instead of leaking the API key to the wrong service.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
from app.utils.captcha.config import CaptchaConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LOG = "[captcha][solver]"
|
||||
|
||||
|
||||
class SolverError(Exception):
|
||||
"""Base class for solver failures the caller may want to latch on."""
|
||||
|
||||
|
||||
class SolverAuthError(SolverError):
|
||||
"""Bad / unknown API key — unrecoverable without a config change."""
|
||||
|
||||
|
||||
class SolverBalanceError(SolverError):
|
||||
"""Solver account is out of balance — unrecoverable this process."""
|
||||
|
||||
|
||||
class SolverUnsupported(SolverError):
|
||||
"""Configured provider has no in-house client yet — unrecoverable."""
|
||||
|
||||
|
||||
# --- 2captcha (legacy in.php/res.php) --------------------------------------
|
||||
|
||||
_2CAP_IN = "http://2captcha.com/in.php"
|
||||
_2CAP_RES = "http://2captcha.com/res.php"
|
||||
# Historical captchatools soft_id, kept so solves stay attributed on 2captcha.
|
||||
_2CAP_SOFT_ID = 4782723
|
||||
|
||||
|
||||
def proxy_login_form(proxy_url: str | None) -> str | None:
|
||||
"""``http://user:pass@host:port`` -> ``user:pass@host:port`` (no scheme).
|
||||
|
||||
The vendor APIs want the proxy without a scheme prefix (2captcha rejects a
|
||||
scheme with ``ERROR_PROXY_FORMAT``). Returns ``None`` for a missing or
|
||||
unparseable proxy so the solve goes proxyless rather than crashing the fetch
|
||||
(the token may then be IP-mismatched, but that fails cleanly).
|
||||
"""
|
||||
if not proxy_url:
|
||||
return None
|
||||
try:
|
||||
p = urlsplit(proxy_url)
|
||||
if not p.hostname or not p.port:
|
||||
return None
|
||||
if p.username and p.password:
|
||||
return f"{p.username}:{p.password}@{p.hostname}:{p.port}"
|
||||
return f"{p.hostname}:{p.port}"
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _raise_2cap(err: str) -> None:
|
||||
"""Map a 2captcha error string to a typed exception (or just log a soft one)."""
|
||||
up = err.upper()
|
||||
if "ZERO_BALANCE" in up:
|
||||
raise SolverBalanceError(err)
|
||||
if "WRONG_USER_KEY" in up or "KEY_DOES_NOT_EXIST" in up:
|
||||
raise SolverAuthError(err)
|
||||
logger.warning("%s 2captcha error: %s", _LOG, err)
|
||||
|
||||
|
||||
def _twocaptcha(
|
||||
cfg: CaptchaConfig,
|
||||
*,
|
||||
challenge_type: str,
|
||||
sitekey: str,
|
||||
page_url: str,
|
||||
proxy_url: str | None,
|
||||
user_agent: str | None,
|
||||
enterprise: bool,
|
||||
data_s: str | None,
|
||||
) -> str | None:
|
||||
payload: dict = {"key": cfg.api_key, "json": 1, "soft_id": _2CAP_SOFT_ID}
|
||||
if challenge_type in ("hcaptcha", "hcap"):
|
||||
payload |= {"method": "hcaptcha", "sitekey": sitekey, "pageurl": page_url}
|
||||
elif challenge_type == "v3":
|
||||
payload |= {
|
||||
"method": "userrecaptcha",
|
||||
"version": "v3",
|
||||
"googlekey": sitekey,
|
||||
"pageurl": page_url,
|
||||
"action": cfg.v3_action,
|
||||
"min_score": cfg.v3_min_score,
|
||||
}
|
||||
else: # v2, optionally the Enterprise variant (adds enterprise=1 + data-s)
|
||||
payload |= {"method": "userrecaptcha", "googlekey": sitekey, "pageurl": page_url}
|
||||
if enterprise:
|
||||
payload["enterprise"] = 1
|
||||
if data_s:
|
||||
payload["data-s"] = data_s
|
||||
|
||||
proxy = proxy_login_form(proxy_url)
|
||||
if proxy:
|
||||
payload["proxy"] = proxy
|
||||
payload["proxytype"] = "HTTP"
|
||||
if user_agent:
|
||||
payload["userAgent"] = user_agent
|
||||
|
||||
try:
|
||||
submit = requests.post(_2CAP_IN, data=payload, timeout=30).json()
|
||||
except requests.RequestException as e:
|
||||
logger.warning("%s 2captcha submit request failed: %s", _LOG, e)
|
||||
return None
|
||||
if submit.get("status") != 1:
|
||||
_raise_2cap(str(submit.get("request", "")))
|
||||
return None
|
||||
|
||||
task_id = submit["request"]
|
||||
deadline = time.monotonic() + cfg.timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
time.sleep(5)
|
||||
try:
|
||||
got = requests.get(
|
||||
f"{_2CAP_RES}?key={cfg.api_key}&action=get&id={task_id}&json=1",
|
||||
timeout=30,
|
||||
).json()
|
||||
except requests.RequestException:
|
||||
continue
|
||||
if got.get("status") == 1:
|
||||
return got["request"] or None
|
||||
req = str(got.get("request", ""))
|
||||
if req != "CAPCHA_NOT_READY":
|
||||
_raise_2cap(req)
|
||||
return None
|
||||
logger.warning("%s 2captcha solve timed out after %ss", _LOG, cfg.timeout_s)
|
||||
return None
|
||||
|
||||
|
||||
# --- capsolver (createTask / getTaskResult) --------------------------------
|
||||
|
||||
_CAPSOLVER_CREATE = "https://api.capsolver.com/createTask"
|
||||
_CAPSOLVER_RESULT = "https://api.capsolver.com/getTaskResult"
|
||||
|
||||
|
||||
def capsolver_proxy(proxy_url: str | None) -> str | None:
|
||||
"""``http://user:pass@host:port`` -> ``http:host:port:user:pass``.
|
||||
|
||||
CapSolver's ``proxy`` field is a single colon-delimited
|
||||
``scheme:host:port[:user:pass]`` string (NOT a URL). Returns ``None`` for a
|
||||
missing/unparseable proxy so the caller falls back to a proxyless task.
|
||||
"""
|
||||
if not proxy_url:
|
||||
return None
|
||||
try:
|
||||
p = urlsplit(proxy_url)
|
||||
if not p.hostname or not p.port:
|
||||
return None
|
||||
scheme = p.scheme or "http"
|
||||
if p.username and p.password:
|
||||
return f"{scheme}:{p.hostname}:{p.port}:{p.username}:{p.password}"
|
||||
return f"{scheme}:{p.hostname}:{p.port}"
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _raise_capsolver(code: str, desc: str) -> None:
|
||||
"""Map a CapSolver errorCode to a typed exception (or log a soft one)."""
|
||||
up = (code or "").upper()
|
||||
if "ZERO_BALANCE" in up or "INSUFFICIENT" in up:
|
||||
raise SolverBalanceError(f"{code}: {desc}")
|
||||
if "KEY" in up: # ERROR_KEY_DENIED_ACCESS / ERROR_KEY_DOES_NOT_EXIST
|
||||
raise SolverAuthError(f"{code}: {desc}")
|
||||
logger.warning("%s capsolver error: %s %s", _LOG, code, desc)
|
||||
|
||||
|
||||
def _capsolver(
|
||||
cfg: CaptchaConfig,
|
||||
*,
|
||||
challenge_type: str,
|
||||
sitekey: str,
|
||||
page_url: str,
|
||||
proxy_url: str | None,
|
||||
user_agent: str | None,
|
||||
enterprise: bool,
|
||||
data_s: str | None,
|
||||
) -> str | None:
|
||||
# We always egress through our own sticky proxy, so the proxied task types
|
||||
# (no "ProxyLess" casing ambiguity) are the hot path; the proxyless variants
|
||||
# are only the fallback when no proxy was threaded through.
|
||||
proxy = capsolver_proxy(proxy_url)
|
||||
has_proxy = proxy is not None
|
||||
task: dict = {"websiteURL": page_url, "websiteKey": sitekey}
|
||||
if challenge_type in ("hcaptcha", "hcap"):
|
||||
task["type"] = "HCaptchaTask" if has_proxy else "HCaptchaTaskProxyLess"
|
||||
elif challenge_type == "v3":
|
||||
base = "ReCaptchaV3EnterpriseTask" if enterprise else "ReCaptchaV3Task"
|
||||
task["type"] = base if has_proxy else base + "ProxyLess"
|
||||
task["pageAction"] = cfg.v3_action
|
||||
task["minScore"] = cfg.v3_min_score
|
||||
else: # v2, optionally the Enterprise variant (Google /sorry)
|
||||
base = "ReCaptchaV2EnterpriseTask" if enterprise else "ReCaptchaV2Task"
|
||||
task["type"] = base if has_proxy else base + "ProxyLess"
|
||||
if data_s:
|
||||
# Enterprise carries the /anchor `s` under enterprisePayload; the
|
||||
# normal-v2 field is recaptchaDataSValue.
|
||||
task["enterprisePayload" if enterprise else "recaptchaDataSValue"] = (
|
||||
{"s": data_s} if enterprise else data_s
|
||||
)
|
||||
if has_proxy:
|
||||
task["proxy"] = proxy
|
||||
if user_agent:
|
||||
task["userAgent"] = user_agent
|
||||
|
||||
try:
|
||||
created = requests.post(
|
||||
_CAPSOLVER_CREATE,
|
||||
json={"clientKey": cfg.api_key, "task": task},
|
||||
timeout=30,
|
||||
).json()
|
||||
except requests.RequestException as e:
|
||||
logger.warning("%s capsolver createTask failed: %s", _LOG, e)
|
||||
return None
|
||||
if created.get("errorId"):
|
||||
_raise_capsolver(
|
||||
str(created.get("errorCode", "")), str(created.get("errorDescription", ""))
|
||||
)
|
||||
return None
|
||||
task_id = created.get("taskId")
|
||||
if not task_id:
|
||||
return None
|
||||
|
||||
deadline = time.monotonic() + cfg.timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
time.sleep(2) # AI solver is fast; poll tighter than 2captcha's 5 s
|
||||
try:
|
||||
got = requests.post(
|
||||
_CAPSOLVER_RESULT,
|
||||
json={"clientKey": cfg.api_key, "taskId": task_id},
|
||||
timeout=30,
|
||||
).json()
|
||||
except requests.RequestException:
|
||||
continue
|
||||
if got.get("errorId"):
|
||||
_raise_capsolver(
|
||||
str(got.get("errorCode", "")), str(got.get("errorDescription", ""))
|
||||
)
|
||||
return None
|
||||
if got.get("status") == "ready":
|
||||
return (got.get("solution") or {}).get("gRecaptchaResponse") or None
|
||||
logger.warning("%s capsolver solve timed out after %ss", _LOG, cfg.timeout_s)
|
||||
return None
|
||||
|
||||
|
||||
# provider name (CAPTCHA_SOLVER_PROVIDER) -> client. Add vendors here.
|
||||
_PROVIDERS = {"2captcha": _twocaptcha, "capsolver": _capsolver}
|
||||
|
||||
|
||||
def supported_providers() -> list[str]:
|
||||
return sorted(_PROVIDERS)
|
||||
|
||||
|
||||
def solve(
|
||||
cfg: CaptchaConfig,
|
||||
*,
|
||||
challenge_type: str,
|
||||
sitekey: str,
|
||||
page_url: str,
|
||||
proxy_url: str | None = None,
|
||||
user_agent: str | None = None,
|
||||
enterprise: bool = False,
|
||||
data_s: str | None = None,
|
||||
) -> str | None:
|
||||
"""Harvest a token from the configured solver, or ``None`` on soft failure.
|
||||
|
||||
``challenge_type`` is ``v2`` / ``v3`` / ``hcaptcha``; set ``enterprise`` +
|
||||
``data_s`` for reCAPTCHA-v2-Enterprise pages (e.g. Google ``/sorry``). The
|
||||
solve egresses through ``proxy_url`` so the token is bound to the crawl's
|
||||
own exit IP. Raises a :class:`SolverError` subclass on unrecoverable errors
|
||||
(bad key / no balance / unsupported provider) so callers can latch.
|
||||
"""
|
||||
client = _PROVIDERS.get((cfg.solving_site or "").lower())
|
||||
if client is None:
|
||||
raise SolverUnsupported(
|
||||
f"captcha provider {cfg.solving_site!r} has no in-house client "
|
||||
f"(supported: {supported_providers()})"
|
||||
)
|
||||
return client(
|
||||
cfg,
|
||||
challenge_type=challenge_type,
|
||||
sitekey=sitekey,
|
||||
page_url=page_url,
|
||||
proxy_url=proxy_url,
|
||||
user_agent=user_agent,
|
||||
enterprise=enterprise,
|
||||
data_s=data_s,
|
||||
)
|
||||
|
|
@ -44,7 +44,6 @@ dependencies = [
|
|||
"azure-storage-blob>=12.23.0",
|
||||
"fake-useragent>=2.2.0",
|
||||
"trafilatura>=2.0.0",
|
||||
"captchatools>=1.5.0",
|
||||
"fastapi-users[oauth,sqlalchemy]>=15.0.3",
|
||||
"chonkie[all]>=1.5.0",
|
||||
"langgraph-checkpoint-postgres>=3.0.2",
|
||||
|
|
@ -86,7 +85,7 @@ dependencies = [
|
|||
"opentelemetry-instrumentation-logging>=0.61b0",
|
||||
"python-telegram-bot>=22.7",
|
||||
"croniter>=2.0.0",
|
||||
"scrapling[fetchers]>=0.4.9",
|
||||
"scrapling[fetchers]>=0.4.11",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
|
|
|||
|
|
@ -28,9 +28,12 @@ sys.path.insert(0, str(_ROOT))
|
|||
load_dotenv(_ROOT / ".env")
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logging.getLogger("app.proprietary.platforms.google_search.scraper").setLevel(
|
||||
logging.INFO
|
||||
)
|
||||
for _name in (
|
||||
"app.proprietary.platforms.google_search.scraper",
|
||||
"app.proprietary.platforms.google_search.fetch",
|
||||
"app.proprietary.platforms.google_search.captcha",
|
||||
):
|
||||
logging.getLogger(_name).setLevel(logging.INFO)
|
||||
|
||||
from app.proprietary.platforms.google_search import ( # noqa: E402
|
||||
GoogleSearchScrapeInput,
|
||||
|
|
@ -146,6 +149,9 @@ async def run(
|
|||
|
||||
_CASES = {
|
||||
"plain": lambda: run("plain query", queries="python asyncio tutorial"),
|
||||
# Prod incident 2026-07-17: single-word brand/navigational queries were
|
||||
# exhausting all 24 IPs (precheck 200 but the browser render 429-walled).
|
||||
"brand": lambda: run("brand query (notebooklm, prod repro)", queries="notebooklm"),
|
||||
"site": lambda: run("site: filter", queries="machine learning", site="arxiv.org"),
|
||||
"ads": lambda: run("text ads", queries="car insurance quotes", expect_ads=True),
|
||||
"products": lambda: run(
|
||||
|
|
|
|||
229
surfsense_backend/scripts/scale_google_search.py
Normal file
229
surfsense_backend/scripts/scale_google_search.py
Normal file
|
|
@ -0,0 +1,229 @@
|
|||
"""Zero-cost concurrency simulation for the Google Search fetch seam.
|
||||
|
||||
Drives many concurrent ``fetch_serp_html`` calls with the network I/O stubbed
|
||||
out (no live Google, no proxy spend, no captcha solves) so we can measure the
|
||||
*structure* of the scraper under load without paying for it:
|
||||
|
||||
* **throughput** — completions/sec at steady state (the per-process ceiling is
|
||||
the render gate divided by warm-render latency; nothing above it is real),
|
||||
* **latency** — p50/p95/p99 wait as the gate queues excess renders,
|
||||
* **solves** — how many paid solves the run would cost (the number we most want
|
||||
the sticky-IP pool to shrink), and
|
||||
* **per-IP peak concurrency** — the funneling metric: how many renders pile onto
|
||||
a single sticky IP at once (a hot IP is what Google re-walls).
|
||||
|
||||
The stub keeps the *real* ``fetch_serp_html`` loop intact (sticky reuse,
|
||||
exemption skip-precheck, the render gate, inflight accounting) and only replaces
|
||||
``_precheck`` / ``_get_session`` / ``get_proxy_url``. Timings are compressed
|
||||
(warm≈0.1 s vs ~14 s live) and extrapolated to real seconds via the warm-render
|
||||
ratio, since the system is gate-bounded sleeps + a semaphore (delays scale
|
||||
linearly).
|
||||
|
||||
.venv/Scripts/python.exe scripts/scale_google_search.py --count 400 --rate 60
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import statistics
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(_ROOT))
|
||||
|
||||
from app.proprietary.platforms.google_search import fetch # noqa: E402
|
||||
|
||||
# Live-measured render latencies (README "Timings"); used only to translate the
|
||||
# compressed sim clock back to real seconds for the human-readable report.
|
||||
_REAL_WARM_S = 14.0
|
||||
_REAL_SOLVE_S = 45.0
|
||||
|
||||
_RESULTS_HTML = '<div id="rso">ok</div>' # trips fetch._has_results
|
||||
|
||||
|
||||
class _Metrics:
|
||||
"""Browser-loop-side counters (fetch runs on two loops → guard with a lock)."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.lock = threading.Lock()
|
||||
self.solves = 0
|
||||
self.renders = 0
|
||||
self.ip_hits: dict[str, int] = {}
|
||||
self.ip_now: dict[str, int] = {}
|
||||
self.ip_peak: dict[str, int] = {}
|
||||
|
||||
def enter(self, proxy: str, solved: bool) -> None:
|
||||
with self.lock:
|
||||
self.renders += 1
|
||||
if solved:
|
||||
self.solves += 1
|
||||
self.ip_hits[proxy] = self.ip_hits.get(proxy, 0) + 1
|
||||
n = self.ip_now.get(proxy, 0) + 1
|
||||
self.ip_now[proxy] = n
|
||||
self.ip_peak[proxy] = max(self.ip_peak.get(proxy, 0), n)
|
||||
|
||||
def exit(self, proxy: str) -> None:
|
||||
with self.lock:
|
||||
self.ip_now[proxy] = self.ip_now.get(proxy, 1) - 1
|
||||
|
||||
|
||||
class _FakePage:
|
||||
def __init__(self) -> None:
|
||||
self.status = 200
|
||||
self.html_content = _RESULTS_HTML
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Stands in for AsyncStealthySession: sleeps like a render, simulates the
|
||||
solve on a cold IP by seeding fetch._exemption_jar (what page_action does)."""
|
||||
|
||||
def __init__(self, metrics: _Metrics, warm_s: float, solve_s: float) -> None:
|
||||
self.m = metrics
|
||||
self.warm_s = warm_s
|
||||
self.solve_s = solve_s
|
||||
|
||||
async def fetch(self, url, proxy=None, **kwargs):
|
||||
key = proxy or ""
|
||||
# Cold IP (no cached exemption) pays the solve and warms the jar; a warm
|
||||
# IP just pays the render. Mirrors _make_page_action's cache write.
|
||||
cold = key not in fetch._exemption_jar
|
||||
self.m.enter(key, solved=cold)
|
||||
try:
|
||||
if cold:
|
||||
await asyncio.sleep(self.solve_s)
|
||||
fetch._exemption_jar[key] = [{"name": "GOOGLE_ABUSE_EXEMPTION"}]
|
||||
await asyncio.sleep(self.warm_s)
|
||||
return _FakePage()
|
||||
finally:
|
||||
self.m.exit(key)
|
||||
|
||||
|
||||
def _install_stubs(metrics: _Metrics, warm_s: float, solve_s: float, precheck_s: float):
|
||||
base = "http://u:p@gw.dataimpulse.com:823" # hostname in fetch._STICKY_HOSTS
|
||||
session = _FakeSession(metrics, warm_s, solve_s)
|
||||
|
||||
async def fake_precheck(url, proxy):
|
||||
await asyncio.sleep(precheck_s)
|
||||
return True
|
||||
|
||||
async def fake_get_session(mobile):
|
||||
return session
|
||||
|
||||
fetch.get_proxy_url = lambda: base
|
||||
fetch._precheck = fake_precheck
|
||||
fetch._get_session = fake_get_session
|
||||
fetch.captcha_enabled = lambda: True
|
||||
fetch.get_captcha_config = lambda: object()
|
||||
fetch._captcha.solver_latched = lambda: False
|
||||
# The pool's backpressure poll is negligible vs a real 14 s render; keep it
|
||||
# proportional under the compressed sim clock so waiters don't idle.
|
||||
fetch._POOL_WAIT_S = max(warm_s * 0.05, 0.001)
|
||||
|
||||
# Isolate the sim to the LOCAL pool: the cross-process store (Redis) has its
|
||||
# own unit test and would otherwise add ping latency / cross-run bleed here.
|
||||
async def _no_adopt(exclude):
|
||||
return None
|
||||
|
||||
async def _noop(*a, **k):
|
||||
return None
|
||||
|
||||
fetch._store.adopt = _no_adopt
|
||||
fetch._store.publish = _noop
|
||||
fetch._store.evict = _noop
|
||||
|
||||
|
||||
def _reset_state() -> None:
|
||||
fetch._exemption_jar.clear()
|
||||
for name in ("_pool", "_pool_inflight"):
|
||||
obj = getattr(fetch, name, None)
|
||||
if isinstance(obj, dict):
|
||||
obj.clear()
|
||||
if hasattr(fetch, "_pool_pending"):
|
||||
fetch._pool_pending = 0
|
||||
if hasattr(fetch, "_last_good_proxy"): # pre-pool revision
|
||||
fetch._last_good_proxy = None
|
||||
|
||||
|
||||
async def _drive(count: int, rate: float) -> list[float]:
|
||||
"""Fire ``count`` fetches at ``rate``/sec (steady arrival), return latencies."""
|
||||
lat: list[float] = []
|
||||
interval = 1.0 / rate if rate > 0 else 0.0
|
||||
|
||||
async def one() -> None:
|
||||
t0 = time.perf_counter()
|
||||
await fetch.fetch_serp_html("https://www.google.com/search?q=notebooklm")
|
||||
lat.append(time.perf_counter() - t0)
|
||||
|
||||
tasks: list[asyncio.Task] = []
|
||||
for _ in range(count):
|
||||
tasks.append(asyncio.create_task(one()))
|
||||
if interval:
|
||||
await asyncio.sleep(interval)
|
||||
await asyncio.gather(*tasks)
|
||||
return lat
|
||||
|
||||
|
||||
def _pct(xs: list[float], p: float) -> float:
|
||||
if not xs:
|
||||
return 0.0
|
||||
xs = sorted(xs)
|
||||
return xs[min(len(xs) - 1, int(p / 100 * len(xs)))]
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--count", type=int, default=400)
|
||||
ap.add_argument("--rate", type=float, default=60.0, help="arrivals/sec (sim clock)")
|
||||
ap.add_argument("--warm", type=float, default=0.10, help="warm render seconds (sim)")
|
||||
ap.add_argument("--solve", type=float, default=0.45, help="solve seconds (sim)")
|
||||
ap.add_argument("--precheck", type=float, default=0.01)
|
||||
args = ap.parse_args()
|
||||
|
||||
metrics = _Metrics()
|
||||
_install_stubs(metrics, args.warm, args.solve, args.precheck)
|
||||
_reset_state()
|
||||
|
||||
scale = _REAL_WARM_S / args.warm # sim seconds → real seconds
|
||||
gate = fetch._MAX_CONCURRENT_PAGES
|
||||
pool = getattr(fetch, "_WARM_POOL_TARGET", None)
|
||||
|
||||
wall0 = time.perf_counter()
|
||||
lat = await _drive(args.count, args.rate)
|
||||
wall = time.perf_counter() - wall0
|
||||
await fetch._in_browser_loop(asyncio.sleep(0)) # let last exit() settle
|
||||
|
||||
hot_peak = max(metrics.ip_peak.values()) if metrics.ip_peak else 0
|
||||
top_hits = max(metrics.ip_hits.values()) if metrics.ip_hits else 0
|
||||
top_share = 100.0 * top_hits / metrics.renders if metrics.renders else 0.0
|
||||
thru_sim = args.count / wall if wall else 0.0
|
||||
thru_real = thru_sim / scale
|
||||
|
||||
print("\n=== Google Search scale simulation ===")
|
||||
print(f" gate (_MAX_CONCURRENT_PAGES) = {gate}"
|
||||
+ (f", warm-pool target = {pool}" if pool else " (single-slot sticky IP)"))
|
||||
print(f" requests={args.count} arrival_rate={args.rate}/s (sim)")
|
||||
print(f" --- structure (scale-free) ---")
|
||||
print(f" paid solves = {metrics.solves} (want ~pool size, not ~requests)")
|
||||
print(f" distinct sticky IPs = {len(metrics.ip_hits)}")
|
||||
print(f" busiest IP carried = {top_hits}/{metrics.renders} renders "
|
||||
f"({top_share:.0f}%) (funneling; want ~even spread)")
|
||||
print(f" peak renders on 1 IP = {hot_peak} (concurrency; capped by gate)")
|
||||
print(f" --- throughput / latency (extrapolated to live @ warm={_REAL_WARM_S}s) ---")
|
||||
print(f" throughput = {thru_real*60:.0f} SERP/min ({thru_real:.2f}/s)")
|
||||
print(f" latency p50 = {_pct(lat,50)*scale:6.1f}s p95 = {_pct(lat,95)*scale:6.1f}s "
|
||||
f"p99 = {_pct(lat,99)*scale:6.1f}s")
|
||||
print(f" ceiling (gate/warm) = {gate/_REAL_WARM_S*60:.0f} SERP/min per process")
|
||||
need = 500 / (gate / _REAL_WARM_S * 60)
|
||||
print(f" -> to sustain 500 SERP/min you need ~{need:.0f} such processes "
|
||||
f"(or a larger gate)\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
207
surfsense_backend/scripts/stress_google_search.py
Normal file
207
surfsense_backend/scripts/stress_google_search.py
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
"""Live stress test for the Google Search scraper (REAL solves + proxy spend).
|
||||
|
||||
Simulates a burst of concurrent "users" hitting the real ``scrape_serps``
|
||||
pipeline with a mix of single-query and multi-query requests. Unlike the offline
|
||||
sim (``scale_google_search.py``), this exercises the actual DataImpulse IPs,
|
||||
CapSolver solves, and the shared browser — so it surfaces what the sim can't:
|
||||
real warm-vs-cold latency, whether the warm-IP pool amortizes solves across real
|
||||
IPs without Google re-walling them, and browser stability under concurrency.
|
||||
|
||||
.venv/Scripts/python.exe scripts/stress_google_search.py --users 30 --gate 8
|
||||
|
||||
It COSTS money (each cold IP = one CapSolver solve, bounded by the pool target)
|
||||
and proxy bandwidth. Metrics are tallied from the live perf/captcha logs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(_ROOT))
|
||||
|
||||
from dotenv import load_dotenv # noqa: E402
|
||||
|
||||
load_dotenv(_ROOT / ".env")
|
||||
|
||||
# Brand/navigational terms reliably trip the reCAPTCHA wall (a solve/pool grow);
|
||||
# informational terms are the lighter, often-warm path. A realistic mix.
|
||||
_BRAND = ["notebooklm", "figma", "notion", "linear app", "vercel", "perplexity ai"]
|
||||
_INFO = [
|
||||
"python asyncio tutorial",
|
||||
"best mechanical keyboard 2026",
|
||||
"kubernetes vs docker",
|
||||
"typescript generics explained",
|
||||
"how to make sourdough bread",
|
||||
"rust ownership model",
|
||||
]
|
||||
|
||||
|
||||
class _LogTally(logging.Handler):
|
||||
"""Counts solves / renders / walls / pool-reuse from the live log stream."""
|
||||
|
||||
_RENDER = re.compile(
|
||||
r"has_results=(\w+).*from_pool=(\w+) pool=(\d+)"
|
||||
)
|
||||
_SOLVE = re.compile(r"\[captcha\] solve (OK|did not)")
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(level=logging.INFO)
|
||||
self.solves_ok = 0
|
||||
self.solves_fail = 0
|
||||
self.renders_ok = 0
|
||||
self.renders_walled = 0
|
||||
self.reuse = 0
|
||||
self.grow = 0
|
||||
self.max_pool = 0
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
msg = record.getMessage()
|
||||
m = self._RENDER.search(msg)
|
||||
if m:
|
||||
ok, from_pool, pool = m.group(1), m.group(2), int(m.group(3))
|
||||
if ok == "True":
|
||||
self.renders_ok += 1
|
||||
else:
|
||||
self.renders_walled += 1
|
||||
if from_pool == "True":
|
||||
self.reuse += 1
|
||||
else:
|
||||
self.grow += 1
|
||||
self.max_pool = max(self.max_pool, pool)
|
||||
return
|
||||
s = self._SOLVE.search(msg)
|
||||
if s:
|
||||
if s.group(1) == "OK":
|
||||
self.solves_ok += 1
|
||||
else:
|
||||
self.solves_fail += 1
|
||||
|
||||
|
||||
def _make_request(rng: random.Random, multi_ratio: float) -> tuple[str, int]:
|
||||
"""Build one user's request: a single term or a 2-3 term multi-query.
|
||||
Returns (newline-joined queries, expected SERP count)."""
|
||||
pool = _BRAND + _INFO
|
||||
if rng.random() < multi_ratio:
|
||||
n = rng.randint(2, 3)
|
||||
terms = rng.sample(pool, n)
|
||||
return "\n".join(terms), n
|
||||
return rng.choice(pool), 1
|
||||
|
||||
|
||||
async def _user(uid: int, queries: str, want: int, results: list[dict]) -> None:
|
||||
from app.proprietary.platforms.google_search import (
|
||||
GoogleSearchScrapeInput,
|
||||
scrape_serps,
|
||||
)
|
||||
|
||||
t0 = time.perf_counter()
|
||||
inp = GoogleSearchScrapeInput(queries=queries, countryCode="us", languageCode="en")
|
||||
try:
|
||||
items = await scrape_serps(inp, limit=want)
|
||||
got = len(items)
|
||||
err = None
|
||||
except Exception as e: # a user request should never crash the whole run
|
||||
got, err = 0, repr(e)
|
||||
results.append(
|
||||
{
|
||||
"uid": uid,
|
||||
"want": want,
|
||||
"got": got,
|
||||
"secs": time.perf_counter() - t0,
|
||||
"err": err,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--users", type=int, default=30, help="concurrent requests")
|
||||
ap.add_argument("--gate", type=int, default=8, help="MAX_CONCURRENT_PAGES for run")
|
||||
ap.add_argument("--multi-ratio", type=float, default=0.5)
|
||||
ap.add_argument("--ramp", type=float, default=0.3, help="secs between user starts")
|
||||
ap.add_argument("--seed", type=int, default=7)
|
||||
args = ap.parse_args()
|
||||
|
||||
# Must be set BEFORE fetch.py is imported (the gate is read at import).
|
||||
os.environ["GOOGLE_SEARCH_MAX_CONCURRENT_PAGES"] = str(args.gate)
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
tally = _LogTally()
|
||||
for name in (
|
||||
"app.proprietary.platforms.google_search.fetch",
|
||||
"app.proprietary.platforms.google_search.captcha",
|
||||
):
|
||||
lg = logging.getLogger(name)
|
||||
lg.setLevel(logging.INFO)
|
||||
lg.addHandler(tally)
|
||||
|
||||
from app.proprietary.platforms.google_search.fetch import ( # noqa: E402
|
||||
_WARM_POOL_TARGET,
|
||||
close_sessions,
|
||||
)
|
||||
|
||||
rng = random.Random(args.seed)
|
||||
plans = [_make_request(rng, args.multi_ratio) for _ in range(args.users)]
|
||||
total_serps = sum(w for _, w in plans)
|
||||
n_multi = sum(1 for _, w in plans if w > 1)
|
||||
|
||||
print(f"\n=== LIVE stress: {args.users} users "
|
||||
f"({args.users - n_multi} single / {n_multi} multi), "
|
||||
f"~{total_serps} SERPs, gate={args.gate}, pool_target={_WARM_POOL_TARGET} ===")
|
||||
print(" (real solves + proxy spend; warming up...)\n")
|
||||
|
||||
results: list[dict] = []
|
||||
wall0 = time.perf_counter()
|
||||
tasks = []
|
||||
for uid, (queries, want) in enumerate(plans):
|
||||
tasks.append(asyncio.create_task(_user(uid, queries, want, results)))
|
||||
await asyncio.sleep(args.ramp) # gentle ramp, not a thundering herd
|
||||
await asyncio.gather(*tasks)
|
||||
wall = time.perf_counter() - wall0
|
||||
|
||||
await close_sessions()
|
||||
|
||||
lat = sorted(r["secs"] for r in results)
|
||||
got_serps = sum(r["got"] for r in results)
|
||||
fails = [r for r in results if r["err"]]
|
||||
short = [r for r in results if not r["err"] and r["got"] < r["want"]]
|
||||
|
||||
def pct(p: float) -> float:
|
||||
return lat[min(len(lat) - 1, int(p / 100 * len(lat)))] if lat else 0.0
|
||||
|
||||
print("=== results ===")
|
||||
print(f" wall time = {wall:.0f}s")
|
||||
print(f" requests ok/failed = {len(results) - len(fails)}/{len(fails)}")
|
||||
print(f" SERPs got/expected = {got_serps}/{total_serps}"
|
||||
+ (f" ({len(short)} short)" if short else ""))
|
||||
print(f" throughput = {got_serps / wall * 60:.0f} SERP/min")
|
||||
print(f" request latency p50={pct(50):.0f}s p95={pct(95):.0f}s "
|
||||
f"max={lat[-1] if lat else 0:.0f}s")
|
||||
print(" --- pipeline (from live logs) ---")
|
||||
print(f" paid solves ok/fail = {tally.solves_ok}/{tally.solves_fail} "
|
||||
f"(bounded by pool target {_WARM_POOL_TARGET})")
|
||||
print(f" renders ok/walled = {tally.renders_ok}/{tally.renders_walled}")
|
||||
print(f" pool reuse/grow = {tally.reuse}/{tally.grow} "
|
||||
f"(reuse share {100 * tally.reuse / max(1, tally.reuse + tally.grow):.0f}%)")
|
||||
print(f" peak pool size = {tally.max_pool}")
|
||||
if fails:
|
||||
print(" --- failures ---")
|
||||
for r in fails[:5]:
|
||||
print(f" user{r['uid']}: {r['err']}")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
"""Offline checks for the /sorry reCAPTCHA solver's pure helpers.
|
||||
|
||||
The network solve can't be unit-tested, but the parsing around it can: a missed
|
||||
sitekey/data-s silently burns paid solves, so these guard the boundary logic.
|
||||
(Proxy reformatting now lives in ``app.utils.captcha.solvers``.)
|
||||
"""
|
||||
|
||||
from app.proprietary.platforms.google_search import captcha
|
||||
|
||||
|
||||
class _Page:
|
||||
def __init__(self, url: str) -> None:
|
||||
self.url = url
|
||||
|
||||
|
||||
def test_on_sorry_detects_wall_only():
|
||||
assert captcha.on_sorry(_Page("https://www.google.com/sorry/index?continue=x"))
|
||||
assert not captcha.on_sorry(_Page("https://www.google.com/search?q=notebooklm"))
|
||||
assert not captcha.on_sorry(_Page(""))
|
||||
|
||||
|
||||
def test_sitekey_and_data_s_extraction():
|
||||
html = (
|
||||
'<div class="g-recaptcha" data-sitekey="6LdLLIMbAAAAAIl-KLj9p1ePhM"'
|
||||
' data-s="zBB1ixry9YzY_tok-en"></div>'
|
||||
)
|
||||
assert captcha._SITEKEY_RE.search(html).group(1) == "6LdLLIMbAAAAAIl-KLj9p1ePhM"
|
||||
assert captcha._DATA_S_RE.search(html).group(1) == "zBB1ixry9YzY_tok-en"
|
||||
|
||||
|
||||
def test_latch_roundtrip():
|
||||
captcha.reset_solver_latch()
|
||||
assert not captcha.solver_latched()
|
||||
captcha._latch("no balance")
|
||||
assert captcha.solver_latched()
|
||||
captcha.reset_solver_latch()
|
||||
assert not captcha.solver_latched()
|
||||
|
|
@ -16,7 +16,7 @@ class _FakeSession:
|
|||
self.release = asyncio.Event()
|
||||
self.closed = 0
|
||||
|
||||
async def fetch(self, url, proxy=None):
|
||||
async def fetch(self, url, proxy=None, **kwargs):
|
||||
await self.release.wait()
|
||||
return "page"
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,99 @@
|
|||
"""Warm sticky-IP pool bookkeeping (the scale-critical logic in fetch.py).
|
||||
|
||||
These pure, lock-guarded helpers decide whether a render reuses a warm IP,
|
||||
grows the pool (a paid solve), or waits — and how a finished render admits or
|
||||
evicts its IP. Get the pending/inflight accounting wrong and the pool either
|
||||
over-solves (cost) or funnels every render onto one IP (re-wall), so the
|
||||
transitions are worth pinning down offline.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.proprietary.platforms.google_search import fetch
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_pool():
|
||||
def reset():
|
||||
fetch._pool.clear()
|
||||
fetch._pool_inflight.clear()
|
||||
fetch._pool_pending = 0
|
||||
fetch._exemption_jar.clear()
|
||||
|
||||
reset()
|
||||
yield
|
||||
reset()
|
||||
|
||||
|
||||
def test_take_grows_when_pool_empty():
|
||||
action, proxy = fetch._pool_take()
|
||||
assert action == "grow" and proxy is None
|
||||
assert fetch._pool_pending == 1
|
||||
|
||||
|
||||
def test_take_reuses_warm_under_cap():
|
||||
fetch._pool["p1"] = 1.0
|
||||
action, proxy = fetch._pool_take()
|
||||
assert action == "reuse" and proxy == "p1"
|
||||
assert fetch._pool_inflight["p1"] == 1
|
||||
|
||||
|
||||
def test_take_spreads_across_least_loaded_ip():
|
||||
fetch._pool.update({"p1": 1.0, "p2": 1.0})
|
||||
fetch._pool_inflight["p1"] = 1 # p2 is idle
|
||||
action, proxy = fetch._pool_take()
|
||||
assert action == "reuse" and proxy == "p2"
|
||||
|
||||
|
||||
def test_take_waits_when_full_and_every_ip_capped(monkeypatch):
|
||||
monkeypatch.setattr(fetch, "_WARM_POOL_TARGET", 2)
|
||||
monkeypatch.setattr(fetch, "_WARM_IP_MAX_CONCURRENCY", 1)
|
||||
fetch._pool.update({"p1": 1.0, "p2": 1.0})
|
||||
fetch._pool_inflight.update({"p1": 1, "p2": 1})
|
||||
action, proxy = fetch._pool_take()
|
||||
assert action == "wait" and proxy is None
|
||||
assert fetch._pool_pending == 0 # a wait must NOT reserve a solve
|
||||
|
||||
|
||||
def test_take_grows_only_up_to_target_counting_pending(monkeypatch):
|
||||
monkeypatch.setattr(fetch, "_WARM_POOL_TARGET", 2)
|
||||
monkeypatch.setattr(fetch, "_WARM_IP_MAX_CONCURRENCY", 1)
|
||||
fetch._pool["p1"] = 1.0
|
||||
fetch._pool_inflight["p1"] = 1 # warm but capped
|
||||
assert fetch._pool_take()[0] == "grow" # pool(1)+pending(0) < 2
|
||||
assert fetch._pool_pending == 1
|
||||
assert fetch._pool_take()[0] == "wait" # pool(1)+pending(1) == 2 → no more solves
|
||||
|
||||
|
||||
def test_settle_good_admits_and_releases():
|
||||
fetch._pool_pending = 1
|
||||
fetch._pool_inflight["p1"] = 1
|
||||
fetch._pool_settle("p1", good=True, grew=True)
|
||||
assert "p1" in fetch._pool
|
||||
assert "p1" not in fetch._pool_inflight
|
||||
assert fetch._pool_pending == 0
|
||||
|
||||
|
||||
def test_settle_walled_evicts_ip_and_drops_exemption():
|
||||
fetch._pool["p1"] = 1.0
|
||||
fetch._pool_inflight["p1"] = 1
|
||||
fetch._exemption_jar["p1"] = [{"name": "GOOGLE_ABUSE_EXEMPTION"}]
|
||||
fetch._pool_settle("p1", good=False, grew=False)
|
||||
assert "p1" not in fetch._pool
|
||||
assert "p1" not in fetch._exemption_jar
|
||||
|
||||
|
||||
def test_adopt_releases_pending_and_pins_ip():
|
||||
fetch._pool_pending = 1 # reserved by a grow that we satisfy via the store
|
||||
fetch._pool_adopt("shared")
|
||||
assert fetch._pool_pending == 0
|
||||
assert "shared" in fetch._pool
|
||||
assert fetch._pool_inflight["shared"] == 1
|
||||
|
||||
|
||||
def test_abort_grow_releases_the_reserved_slot():
|
||||
fetch._pool_pending = 2
|
||||
fetch._pool_abort_grow()
|
||||
assert fetch._pool_pending == 1
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
"""Cross-process warm-IP exemption store (best-effort Redis cache).
|
||||
|
||||
Redis itself isn't unit-testable here, but the boundary is: adopt must skip IPs
|
||||
this process already holds (or the fleet re-uses nothing new), and every path
|
||||
must degrade to a silent no-op when Redis is down (or a hiccup would break the
|
||||
fetch instead of just costing a solve).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.proprietary.platforms.google_search import pool_store as ps
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class _FakeRedis:
|
||||
def __init__(self):
|
||||
self.store: dict[str, str] = {}
|
||||
|
||||
def ping(self):
|
||||
return True
|
||||
|
||||
def set(self, k, v, ex=None):
|
||||
self.store[k] = v
|
||||
|
||||
def get(self, k):
|
||||
return self.store.get(k)
|
||||
|
||||
def delete(self, k):
|
||||
self.store.pop(k, None)
|
||||
|
||||
def scan_iter(self, match=None, count=None):
|
||||
return list(self.store.keys())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake(monkeypatch):
|
||||
r = _FakeRedis()
|
||||
monkeypatch.setattr(ps, "_client", r)
|
||||
monkeypatch.setattr(ps, "_disabled", False)
|
||||
return r
|
||||
|
||||
|
||||
def test_key_is_stable_and_prefixed():
|
||||
assert ps._key("http://a").startswith(ps._PREFIX)
|
||||
assert ps._key("http://a") == ps._key("http://a")
|
||||
assert ps._key("http://a") != ps._key("http://b")
|
||||
|
||||
|
||||
def test_publish_then_adopt_excludes_held_ips(fake):
|
||||
ps._publish_sync("http://a", [{"n": 1}])
|
||||
ps._publish_sync("http://b", [{"n": 2}])
|
||||
proxy, cookies = ps._adopt_sync(exclude={"http://a"})
|
||||
assert proxy == "http://b" and cookies == [{"n": 2}]
|
||||
|
||||
|
||||
def test_adopt_none_when_all_held(fake):
|
||||
ps._publish_sync("http://a", [])
|
||||
assert ps._adopt_sync(exclude={"http://a"}) is None
|
||||
|
||||
|
||||
def test_evict_removes_entry(fake):
|
||||
ps._publish_sync("http://a", [{"n": 1}])
|
||||
ps._evict_sync("http://a")
|
||||
assert ps._adopt_sync(exclude=set()) is None
|
||||
|
||||
|
||||
def test_disabled_is_silent_noop(monkeypatch):
|
||||
monkeypatch.setattr(ps, "_disabled", True)
|
||||
monkeypatch.setattr(ps, "_client", None)
|
||||
assert ps._adopt_sync(set()) is None
|
||||
ps._publish_sync("x", [{"n": 1}]) # must not raise
|
||||
ps._evict_sync("x") # must not raise
|
||||
|
|
@ -2,8 +2,9 @@
|
|||
|
||||
The browser/solver boundary is mocked: a fake Playwright ``page`` and a
|
||||
monkeypatched ``_harvest_token`` / injection. We assert the glue logic —
|
||||
detection, proxy reformatting, attempt counting + state surfacing, the per-URL
|
||||
cap, and the no-balance process latch.
|
||||
detection, proxy pass-through, attempt counting + state surfacing, the per-URL
|
||||
cap, and the no-balance process latch. (Vendor proxy reformatting now lives in
|
||||
``app.utils.captcha.solvers`` and is tested there.)
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
|
@ -76,24 +77,6 @@ def _clear_latch():
|
|||
cap.reset_solver_latch()
|
||||
|
||||
|
||||
# --- proxy reformat --------------------------------------------------------
|
||||
|
||||
|
||||
class TestProxyReformat:
|
||||
def test_with_auth(self):
|
||||
assert (
|
||||
cap.proxy_url_to_captchatools("http://user:pass@1.2.3.4:8080")
|
||||
== "1.2.3.4:8080:user:pass"
|
||||
)
|
||||
|
||||
def test_without_auth(self):
|
||||
assert cap.proxy_url_to_captchatools("http://1.2.3.4:8080") == "1.2.3.4:8080"
|
||||
|
||||
def test_none_and_garbage(self):
|
||||
assert cap.proxy_url_to_captchatools(None) is None
|
||||
assert cap.proxy_url_to_captchatools("not-a-url") is None
|
||||
|
||||
|
||||
# --- detection -------------------------------------------------------------
|
||||
|
||||
|
||||
|
|
@ -175,8 +158,9 @@ class TestPageAction:
|
|||
action(page)
|
||||
|
||||
assert state == {"attempts": 1, "solved": True}
|
||||
# Solver egressed from the crawl's proxy, reformatted, with the page UA.
|
||||
assert captured["proxy"] == "1.2.3.4:9000:u:p"
|
||||
# Solver egressed from the crawl's proxy (raw URL; the seam reformats
|
||||
# per vendor), with the page UA.
|
||||
assert captured["proxy"] == "http://u:p@1.2.3.4:9000"
|
||||
assert captured["ua"] == "UA/1.0"
|
||||
assert captured["ctype"] == "v2"
|
||||
assert injected == {"ctype": "v2", "token": "TOKEN"}
|
||||
|
|
|
|||
84
surfsense_backend/tests/unit/utils/captcha/test_solvers.py
Normal file
84
surfsense_backend/tests/unit/utils/captcha/test_solvers.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
"""Offline checks for the in-house captcha-solver seam.
|
||||
|
||||
The network solve can't be unit-tested, but the boundary logic can: a wrong
|
||||
proxy reformat (2captcha's ``ERROR_PROXY_FORMAT``) or dispatching to a provider
|
||||
we haven't wired silently burns paid solves / leaks the key to the wrong vendor.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.utils.captcha import solvers
|
||||
from app.utils.captcha.config import CaptchaConfig
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _cfg(**overrides) -> CaptchaConfig:
|
||||
base = {
|
||||
"enabled": True,
|
||||
"solving_site": "2captcha",
|
||||
"api_key": "key-123",
|
||||
"max_attempts_per_url": 1,
|
||||
"timeout_s": 30,
|
||||
"captcha_type_default": "v2",
|
||||
"v3_min_score": 0.7,
|
||||
"v3_action": "verify",
|
||||
}
|
||||
base.update(overrides)
|
||||
return CaptchaConfig(**base)
|
||||
|
||||
|
||||
# --- proxy reformat (2captcha wants login:pass@host:port, NO scheme) --------
|
||||
|
||||
|
||||
def test_proxy_login_form_strips_scheme():
|
||||
got = solvers.proxy_login_form("http://user:pass@gw.dataimpulse.com:15673")
|
||||
assert got == "user:pass@gw.dataimpulse.com:15673"
|
||||
|
||||
|
||||
def test_proxy_login_form_without_credentials():
|
||||
assert solvers.proxy_login_form("http://gw.dataimpulse.com:823") == "gw.dataimpulse.com:823"
|
||||
|
||||
|
||||
def test_proxy_login_form_none_on_missing_or_bad():
|
||||
assert solvers.proxy_login_form(None) is None
|
||||
assert solvers.proxy_login_form("not a url") is None
|
||||
|
||||
|
||||
# --- capsolver proxy reformat (colon-delimited scheme:host:port:user:pass) --
|
||||
|
||||
|
||||
def test_capsolver_proxy_colon_delimited_with_creds():
|
||||
got = solvers.capsolver_proxy("http://user:pass@gw.dataimpulse.com:823")
|
||||
assert got == "http:gw.dataimpulse.com:823:user:pass"
|
||||
|
||||
|
||||
def test_capsolver_proxy_without_credentials():
|
||||
assert solvers.capsolver_proxy("http://gw.dataimpulse.com:823") == "http:gw.dataimpulse.com:823"
|
||||
|
||||
|
||||
def test_capsolver_proxy_none_on_missing_or_bad():
|
||||
assert solvers.capsolver_proxy(None) is None
|
||||
assert solvers.capsolver_proxy("not a url") is None
|
||||
|
||||
|
||||
# --- dispatch --------------------------------------------------------------
|
||||
|
||||
|
||||
def test_unsupported_provider_raises_solvererror():
|
||||
# An unconfigured provider must fail loudly (latch) rather than POST the key
|
||||
# to a wired vendor's endpoint under the wrong account.
|
||||
with pytest.raises(solvers.SolverUnsupported):
|
||||
solvers.solve(
|
||||
_cfg(solving_site="anticaptcha"),
|
||||
challenge_type="v2",
|
||||
sitekey="SK",
|
||||
page_url="https://t.test",
|
||||
)
|
||||
assert issubclass(solvers.SolverUnsupported, solvers.SolverError)
|
||||
|
||||
|
||||
def test_wired_providers_are_registered():
|
||||
supported = solvers.supported_providers()
|
||||
assert "2captcha" in supported
|
||||
assert "capsolver" in supported
|
||||
196
surfsense_backend/uv.lock
generated
196
surfsense_backend/uv.lock
generated
|
|
@ -39,6 +39,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -76,6 +79,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -113,6 +119,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -150,6 +159,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -187,6 +199,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -224,6 +239,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -261,6 +279,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -298,6 +319,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -335,6 +359,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -372,6 +399,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -421,6 +451,10 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -458,6 +492,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -495,6 +532,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -532,6 +572,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -569,6 +612,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -606,6 +652,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -643,6 +692,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -680,6 +732,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -717,6 +772,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -754,6 +812,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -791,6 +852,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -840,6 +904,10 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -877,6 +945,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -914,6 +985,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -951,6 +1025,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -988,6 +1065,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1025,6 +1105,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1062,6 +1145,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1099,6 +1185,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1136,6 +1225,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1173,6 +1265,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1210,6 +1305,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1259,6 +1357,10 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1296,6 +1398,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1369,6 +1474,12 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1406,6 +1517,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1443,6 +1557,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1516,6 +1633,12 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1589,6 +1712,12 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1626,6 +1755,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
]
|
||||
conflicts = [[
|
||||
|
|
@ -2371,18 +2503,6 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/06/f3/39cf3367b8107baa44f861dc802cbf16263c945b62d8265d36034fc07bea/cachetools-7.0.5-py3-none-any.whl", hash = "sha256:46bc8ebefbe485407621d0a4264b23c080cedd913921bad7ac3ed2f26c183114", size = 13918 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "captchatools"
|
||||
version = "1.5.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "requests" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6b/38/01200f5b8169219002728ccddbad8cc50759e523c0884b7e811092d27352/captchatools-1.5.0.tar.gz", hash = "sha256:755ed685a93f6139608b5277ef0410f8682b3522e98f52d9cc3edff9df38ca12", size = 15282 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ea/f1/7faaa20ffd68314215da2e3c57076c2c89846ab05464f1b7167dc8a9a80b/captchatools-1.5.0-py3-none-any.whl", hash = "sha256:43b651f7603be5725df94e4d715e0ff770efec44b62bb6076eb9a63516c74d94", size = 16394 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "catalogue"
|
||||
version = "2.0.10"
|
||||
|
|
@ -7942,21 +8062,21 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "patchright"
|
||||
version = "1.60.1"
|
||||
version = "1.61.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "greenlet" },
|
||||
{ name = "pyee" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/71/26/c1e858fd1acc63e410b3d33243955f36d2a0814487b97a7aa604ad2baffd/patchright-1.60.1-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:e9492100d4e2a85ff92fc3a668dd16dee03f21df6e559c7b9f7c71e86ff48c6b", size = 43458936 },
|
||||
{ url = "https://files.pythonhosted.org/packages/55/dd/2dd8e4e02489ec8fd57ad93dec9ef444b6f42adcc4fe95df30237c92841d/patchright-1.60.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:20bd806df2469b451ccd2ea10f5f944ceb0e0d83c716f5752b0c956c1ee59476", size = 42245629 },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/cc/0fa0bedec61045fd9068682e3695557b622c483f829d341093879ec8dbd9/patchright-1.60.1-py3-none-macosx_11_0_universal2.whl", hash = "sha256:9fd15a64c0ca80740dc2a3f41cda336a06a2ed6068d0ab893172654290b06e6b", size = 43458935 },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/c2/4b8f69de0a20d90792980c43c0e60b10b801e08cf0224ccc8a8266e1fffb/patchright-1.60.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:547e7bfb813102309789cc42933780e5fdf7c4727de59fb2791e64bd1298a7f3", size = 47451519 },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/fc/9fd6a70818cf0bc3b62574af483d762963cb65ca0a369826a0536faffe41/patchright-1.60.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:023945a2fd30219a284721ca36385bd44075ae7b53071dc1da38036b6dbe88ec", size = 47139153 },
|
||||
{ url = "https://files.pythonhosted.org/packages/08/bc/81fb621e5ab4131e6f324c9ba6cb2a9f3b146c92f12484bec95efe8c8347/patchright-1.60.1-py3-none-win32.whl", hash = "sha256:05b98a6afdbe7e6645fe223009c47cc8e7859df55fd8ce9d8a9925b3389b0ee1", size = 37886460 },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/8e/fff80350ed2c2c1f62145d667070799c60ca9e9b28d54e4f751f0b4f8da6/patchright-1.60.1-py3-none-win_amd64.whl", hash = "sha256:51b306ed55cd58f1bca24641458f5c9f7e86a1f1727dcffdead669cfe4c0a485", size = 37886465 },
|
||||
{ url = "https://files.pythonhosted.org/packages/ee/b8/b6d1bfe98a420c1ccfb2f23a7bb2b4cdb40170907bae638e7ae92abd287c/patchright-1.60.1-py3-none-win_arm64.whl", hash = "sha256:f795728c1e27fc226dbe203c1aec713a537f01be963474fe0f3691f5e6457f9f", size = 34022283 },
|
||||
{ url = "https://files.pythonhosted.org/packages/79/42/a76e2ea17dd4feccbf5ef6df017fb2e67f4b72a6f559f59d9aad1197f83a/patchright-1.61.2-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:43ae10cd2a3e46b222179a120ab39ba88f801f3a42aa41cab45fac9928f2103d", size = 43727572 },
|
||||
{ url = "https://files.pythonhosted.org/packages/57/e7/1ff89853c9aeaf26cd20559f401375ec71a40e4041cf7ace03d41a741f06/patchright-1.61.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a3da24d7890631255c7bef467b94ffe7f7bfa36f96fc72c10d5787d3cb396873", size = 42512921 },
|
||||
{ url = "https://files.pythonhosted.org/packages/e4/d0/d730536a7fc43c3aaf2781518595ca71ffa1fc2f24e580c0d0cf04e00eca/patchright-1.61.2-py3-none-macosx_11_0_universal2.whl", hash = "sha256:edef852aa7d28791fa2d3f7ce135edfea2ce07a2bc5e0a74fb2dfe269af62223", size = 43727573 },
|
||||
{ url = "https://files.pythonhosted.org/packages/a2/b5/c76dcda275cb0d9651321e343f3268609ae24cef196bff56751754ffba16/patchright-1.61.2-py3-none-manylinux1_x86_64.whl", hash = "sha256:545bdf2c0bb5f9ad78ab315e91c8c2990cc6a702e7f18650b71fe1071049b5c1", size = 47729338 },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/ef/5de6feaedf47a6ba8efb9e8c4f8a42bad29cc59c35bbd088b1729a36b271/patchright-1.61.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1824da30cf6560749bc1bb4a7f002fec325d5b709090dd7c8c1b643899331cee", size = 47426969 },
|
||||
{ url = "https://files.pythonhosted.org/packages/6c/6e/ce2c5d03a4c785ac8f3e0180bbf83ef85149cf41fdfa41c09bd8cbcfd945/patchright-1.61.2-py3-none-win32.whl", hash = "sha256:a5aaf4cec2bd38714f2447f84bcdfe7dcb592a16e9616bf6d2d872066d1c78af", size = 38155048 },
|
||||
{ url = "https://files.pythonhosted.org/packages/46/c8/fceb57b9d32ed078f53fc6a757c6e4b54cb117f59412938252a3359105e5/patchright-1.61.2-py3-none-win_amd64.whl", hash = "sha256:506bca9b72e609fd8fee35e2a3ad26e5d92e25337002f5c2cdad53d5cacb0b67", size = 38155052 },
|
||||
{ url = "https://files.pythonhosted.org/packages/33/96/67a791c6b7cd76424b44d356beaf414ab97c3aef2d42fad4641e7905b41d/patchright-1.61.2-py3-none-win_arm64.whl", hash = "sha256:3dee1e2daada9f98653763831f688d39c924e0cf0eeb988e8047b703766cee73", size = 34268624 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -8215,21 +8335,21 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "playwright"
|
||||
version = "1.60.0"
|
||||
version = "1.61.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "greenlet" },
|
||||
{ name = "pyee" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/21/f0/832bd9677194908da118064eef20082f2791e3d18215cc6d9391ee2c5a67/playwright-1.60.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:6a8cd0fec171fb3089e95e898c8bc8a6f35dea0b78b399e12fcc19427e91b1d7", size = 43474635 },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/7b/e1d32ae8a3ed937ec2be3721c5f728b13d731a0b7c6442e0b3bec5094ac0/playwright-1.60.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:39b5420ba6145045b69ced4c5c47d4d9fe5bddfc8ff816c518913afcb25ec7a5", size = 42261327 },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/bc/23de499ded6411c188a20c5a0dea6f0cd4ed5d2b3cc6042a5dbd3ed609aa/playwright-1.60.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:2581d0e6a3392c71f91b27460c7fd093356818dc430f48153896c8aeeaef7705", size = 43474636 },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/7b/1d679f4fced4ea94efadd17103856d8c565384f68382a1681264e46f5925/playwright-1.60.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:1c2bfae7884fb3fb05b853290eab8f343d524e5016f2f1def702acbbdf14c93e", size = 47467220 },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/c2/1528d267d4442bd2c6b8eaeab819dd52c2030bf80e89293f0ba1f687473b/playwright-1.60.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43e66564125ee31b07a58cefb21e256d62d67d8d1713e6858df7a3019d8ed353", size = 47154856 },
|
||||
{ url = "https://files.pythonhosted.org/packages/bb/4e/b008b6440a7a1624378041da94829956d4b8f7ab9ef5aad22d0dc3f2e26d/playwright-1.60.0-py3-none-win32.whl", hash = "sha256:ec94e416ea320711e0ad4bf185dcbf41833672961e90773e1885255d7db7b7e7", size = 37902157 },
|
||||
{ url = "https://files.pythonhosted.org/packages/55/f0/0541524133104f9cc20bf900870ff4a736b76a23483f3a55295ddfa58409/playwright-1.60.0-py3-none-win_amd64.whl", hash = "sha256:9566821ce6030a1f9e7146a24e19355ab0d98805fd0f9be50bb3d8fef1750c02", size = 37902159 },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/c8/210f282d278e4709cdd71b12a31af45a30a22ab3207b387e29b37e478713/playwright-1.60.0-py3-none-win_arm64.whl", hash = "sha256:6e4f6700a4c2250efff8e690a81d66e3855754fb587b6b87cf5c784014f91537", size = 34037981 },
|
||||
{ url = "https://files.pythonhosted.org/packages/44/ee/31e4e0db36588b817a10b299a0285082545fde7d36543c2abe498bb3d61a/playwright-1.61.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:ff138c3a604f69911e9d42fd036e55c2a171e5616edf04c1e7f60a2a285540b0", size = 43421877 },
|
||||
{ url = "https://files.pythonhosted.org/packages/42/35/71395dd3ecc798965be4a3ef8c443217d4abca168e7cb34536304f9489e6/playwright-1.61.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:009588c2a7e499bc5a8b425b61fa65490968bbda9cd69e0cf2cff10f8304659a", size = 42205016 },
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/44/323164cf5cd1647bdefce76ffce27651aadb959d089b48f53ea40918276e/playwright-1.61.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:9f7de4536088d12037c13a52b7ea34b59270b78926bb56935070597ffac6b1af", size = 43421884 },
|
||||
{ url = "https://files.pythonhosted.org/packages/ab/f8/a35bf179e4ba2522c1893635094a64e407572547bd61528820fc0abc87fe/playwright-1.61.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:54f3b39f6eab832e33458c1dd7da0b5682aedab3b09ae731b5c59fa12fd2024e", size = 47421381 },
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/eb/e3f922348ec17c315f98c463f72faa1181a1c3de0bfe31a8d2edf6561723/playwright-1.61.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93454322ade8c11d5d6c211bfd91bdfb9ffb4810e3e026371bcbc4bec1b7ee4c", size = 47120545 },
|
||||
{ url = "https://files.pythonhosted.org/packages/c2/a6/5be4e52b40a9c0c8a073e7c5b0785c05cf5a9ea8f8a7b5b260e32d970342/playwright-1.61.0-py3-none-win32.whl", hash = "sha256:372d55a6f1248fa1dd47599686980cb8fb5bbe6fcda59eab793eb657c11d8a9b", size = 37844841 },
|
||||
{ url = "https://files.pythonhosted.org/packages/6c/fd/2b78036e5fbe9d5f5645bbe08a1eac7160c51243c0093963edbcf67c35d9/playwright-1.61.0-py3-none-win_amd64.whl", hash = "sha256:35c6cc4589a5d00964a59d7b3e59641e0aac0c02f15479a7af77d20f6bc79597", size = 37844846 },
|
||||
{ url = "https://files.pythonhosted.org/packages/27/0d/1b0f3c4ee4eb0514bc805b5c2f9a223e5b6de4f11a926f5235d51d0fc81b/playwright-1.61.0-py3-none-win_arm64.whl", hash = "sha256:e9fcbffcf557a8620fdedd92491eb59a32d18e23d6f3b4f6214b952be324fe51", size = 33955127 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -8417,11 +8537,11 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "protego"
|
||||
version = "0.6.0"
|
||||
version = "0.6.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/07/a7/955c422611d00a6e4a06d30b367ea9bb4fb09d48552e92aef1ba312493c7/protego-0.6.0.tar.gz", hash = "sha256:3466f41438421cf90008e98534d5fde47dc16a17482571d021143ac18b70ace9", size = 3137423 }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7d/1b/6b0ee60bb1561843bfc62f5c7c3cb1ef6147b47e829a5fd6b7fcd2752471/protego-0.6.2.tar.gz", hash = "sha256:88ff004544ce44e61269cc6f8735f7837d12e09bb77619fc94fbb26b72e5d137", size = 3137854 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d8/8c/f4dd590f48addf31398f78a78962eaa99eb4c87ac09c1927497032644731/protego-0.6.0-py3-none-any.whl", hash = "sha256:7210e6e06a8db839502baf1bfbcb810689a58e394d31408ef1ef9e4e3d79fc44", size = 10313 },
|
||||
{ url = "https://files.pythonhosted.org/packages/61/10/cbd3c06603bb8b3e0e871df24ee793a6e3b53d8c4cc8763f1b8b936649aa/protego-0.6.2-py3-none-any.whl", hash = "sha256:714de21d82527c9be900066c3211b266985dd6a19b6e70c57e033fc1a589f3ff", size = 10296 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -10011,7 +10131,7 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "scrapling"
|
||||
version = "0.4.9"
|
||||
version = "0.4.11"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cssselect" },
|
||||
|
|
@ -10021,9 +10141,9 @@ dependencies = [
|
|||
{ name = "typing-extensions" },
|
||||
{ name = "w3lib" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d0/34/61b72964fdeed6bcb9528262870b9870c936eac55541f39254ef876985ef/scrapling-0.4.9.tar.gz", hash = "sha256:e08afab736e5bd3337173e524fee99aea3073476e86d44d942af4bef9496a499", size = 156809 }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a9/e9/f1874e835ac83043a323014a5ce3776f6415041566177366250358e7649e/scrapling-0.4.11.tar.gz", hash = "sha256:92500fe60b601a52279842fa2505fcddd3954f14f657bd38d7e6ba09f02da947", size = 159785 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/e6/03712d0b3240b3b6af6dc8aa19f0f24348fa54f4a4497b09c2d5c511552f/scrapling-0.4.9-py3-none-any.whl", hash = "sha256:00c7fae4641d948fb26486fd1da22143e5296dc86ef821b8426c0c2854c529ef", size = 158609 },
|
||||
{ url = "https://files.pythonhosted.org/packages/2e/21/979d476508766f6a4c54cf126d9eadf12fd613c18ebc1e31f78a28fcfd06/scrapling-0.4.11-py3-none-any.whl", hash = "sha256:10b781f9214502f293a46048062c4d3c211e9a05a897559f674edcbea1876e56", size = 163481 },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
|
|
@ -10494,7 +10614,6 @@ dependencies = [
|
|||
{ name = "azure-ai-documentintelligence" },
|
||||
{ name = "azure-storage-blob" },
|
||||
{ name = "boto3" },
|
||||
{ name = "captchatools" },
|
||||
{ name = "celery", extra = ["redis"] },
|
||||
{ name = "chonkie", extra = ["all"] },
|
||||
{ name = "composio" },
|
||||
|
|
@ -10610,7 +10729,6 @@ requires-dist = [
|
|||
{ name = "azure-ai-documentintelligence", specifier = ">=1.0.2" },
|
||||
{ name = "azure-storage-blob", specifier = ">=12.23.0" },
|
||||
{ name = "boto3", specifier = ">=1.35.0" },
|
||||
{ name = "captchatools", specifier = ">=1.5.0" },
|
||||
{ name = "celery", extras = ["redis"], specifier = ">=5.5.3" },
|
||||
{ name = "chonkie", extras = ["all"], specifier = ">=1.5.0" },
|
||||
{ name = "composio", specifier = ">=0.10.9" },
|
||||
|
|
@ -10669,7 +10787,7 @@ requires-dist = [
|
|||
{ name = "python-telegram-bot", specifier = ">=22.7" },
|
||||
{ name = "redis", specifier = ">=5.2.1" },
|
||||
{ name = "rerankers", extras = ["flashrank"], specifier = ">=0.7.1" },
|
||||
{ name = "scrapling", extras = ["fetchers"], specifier = ">=0.4.9" },
|
||||
{ name = "scrapling", extras = ["fetchers"], specifier = ">=0.4.11" },
|
||||
{ name = "sentence-transformers", specifier = ">=3.4.1" },
|
||||
{ name = "slack-sdk", specifier = ">=3.34.0" },
|
||||
{ name = "slowapi", specifier = ">=0.1.9" },
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue