mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-10 22:32:16 +02:00
feat(proxy): refactor proxy configuration and add DataImpulse provider
- Updated proxy configuration in `.env.example` files to use `PROXY_URL` and `PROXY_URLS` instead of `CUSTOM_PROXY_URL` and `CUSTOM_PROXY_URLS`. - Introduced `DataImpulseProvider` for proxy management, replacing the deprecated `AnonymousProxiesProvider`. - Enhanced documentation to reflect changes in proxy setup and usage. - Adjusted related code in the proxy registry and configuration files to support the new provider structure.
This commit is contained in:
parent
203215adc0
commit
701f888b9e
19 changed files with 303 additions and 193 deletions
|
|
@ -15,6 +15,7 @@ registering it in ``registry.py``.
|
|||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
class ProxyProvider(ABC):
|
||||
|
|
@ -27,12 +28,33 @@ class ProxyProvider(ABC):
|
|||
def get_proxy_url(self) -> str | None:
|
||||
"""Return ``http://user:pass@host:port`` (no trailing slash), or ``None``.
|
||||
|
||||
This is the canonical form Scrapling/curl_cffi consume directly.
|
||||
This is the canonical form Scrapling/curl_cffi consume directly, and the
|
||||
single source every provider must supply — the ``requests`` and Playwright
|
||||
shapes below are derived from it.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_playwright_proxy(self) -> dict[str, str] | None:
|
||||
"""Return a Playwright proxy dict, or ``None`` when not configured."""
|
||||
"""Return a Playwright ``{"server","username","password"}`` dict, or ``None``.
|
||||
|
||||
Parsed from :meth:`get_proxy_url` (the canonical URL) by default, since
|
||||
every provider's credentials already live in that URL. Override only for a
|
||||
vendor whose Playwright form can't be expressed as a parse of the URL.
|
||||
"""
|
||||
proxy_url = self.get_proxy_url()
|
||||
if not proxy_url:
|
||||
return None
|
||||
parts = urlsplit(proxy_url)
|
||||
if not parts.hostname:
|
||||
return None
|
||||
server = f"{parts.scheme or 'http'}://{parts.hostname}"
|
||||
if parts.port:
|
||||
server = f"{server}:{parts.port}"
|
||||
proxy: dict[str, str] = {"server": server}
|
||||
if parts.username:
|
||||
proxy["username"] = parts.username
|
||||
if parts.password:
|
||||
proxy["password"] = parts.password
|
||||
return proxy
|
||||
|
||||
def get_requests_proxies(self) -> dict[str, str] | None:
|
||||
"""Return a ``requests``/``aiohttp`` proxies dict, or ``None``.
|
||||
|
|
@ -45,14 +67,25 @@ class ProxyProvider(ABC):
|
|||
return None
|
||||
return {"http": proxy_url, "https": proxy_url}
|
||||
|
||||
def get_location(self) -> str:
|
||||
"""Return the proxy's configured exit region (e.g. ``"us"``), or ``""``.
|
||||
|
||||
Vendor-agnostic hook the crawler's geoip-match (``03e``) uses to align the
|
||||
browser locale/timezone with the exit IP's country. Default ``""``
|
||||
(unknown) for providers that don't pin a region (e.g. BYO ``custom`` URLs,
|
||||
where the region is baked opaquely into the URL). Override in providers
|
||||
that hold the region as a discrete field.
|
||||
"""
|
||||
return ""
|
||||
|
||||
@property
|
||||
def is_pool_backed(self) -> bool:
|
||||
"""Whether this provider rotates across a *client-side* pool of endpoints.
|
||||
|
||||
``False`` for single-endpoint providers (including server-side rotating
|
||||
gateways like ``anonymous_proxies``, whose rotation happens upstream).
|
||||
The crawler performs its bounded proxy-error rotation-retry **only** when
|
||||
this is ``True`` — retrying a single static endpoint would just re-hit the
|
||||
same dead proxy.
|
||||
gateways like ``dataimpulse``, whose rotation happens upstream). The
|
||||
crawler performs its bounded proxy-error rotation-retry **only** when this
|
||||
is ``True`` — retrying a single static endpoint would just re-hit the same
|
||||
dead proxy.
|
||||
"""
|
||||
return False
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue