mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
701f888b9e
- Updated proxy configuration in `.env.example` files to use `PROXY_URL` and `PROXY_URLS` instead of `CUSTOM_PROXY_URL` and `CUSTOM_PROXY_URLS`. - Introduced `DataImpulseProvider` for proxy management, replacing the deprecated `AnonymousProxiesProvider`. - Enhanced documentation to reflect changes in proxy setup and usage. - Adjusted related code in the proxy registry and configuration files to support the new provider structure.
61 lines
2.1 KiB
Python
61 lines
2.1 KiB
Python
"""Bring-your-own (BYO) custom proxy provider.
|
|
|
|
Reads one or more proxy endpoints from the shared env (``PROXY_URL`` and/or the
|
|
comma-separated ``PROXY_URLS``). With a single endpoint it behaves like a static
|
|
proxy; with a pool (>1) it rotates client-side via Scrapling's thread-safe
|
|
``ProxyRotator`` (cyclic), transparently to every caller of the zero-arg getters.
|
|
|
|
No vendor-specific auth assumptions: a user who wants a specific vendor points
|
|
``PROXY_URLS`` at that vendor's ``http://user:pass@host:port`` endpoints.
|
|
"""
|
|
|
|
import logging
|
|
|
|
from scrapling.engines.toolbelt import ProxyRotator
|
|
|
|
from app.config import Config
|
|
from app.utils.proxy.base import ProxyProvider
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class CustomProxyProvider(ProxyProvider):
|
|
"""BYO provider for a single endpoint or a rotating pool of endpoints."""
|
|
|
|
name = "custom"
|
|
|
|
def __init__(self) -> None:
|
|
self._urls = self._load_urls()
|
|
# Only build a rotator for an actual pool; a single endpoint stays static.
|
|
self._rotator = ProxyRotator(self._urls) if len(self._urls) > 1 else None
|
|
if not self._urls:
|
|
logger.warning(
|
|
"PROXY_PROVIDER='custom' selected but neither PROXY_URL nor "
|
|
"PROXY_URLS is set; crawls will run without a proxy."
|
|
)
|
|
|
|
@staticmethod
|
|
def _load_urls() -> list[str]:
|
|
"""Collect proxy URLs from env (pool first, then single), de-duplicated."""
|
|
urls: list[str] = []
|
|
pool = Config.PROXY_URLS
|
|
if pool:
|
|
urls.extend(part.strip() for part in pool.split(",") if part.strip())
|
|
|
|
single = (Config.PROXY_URL or "").strip()
|
|
if single and single not in urls:
|
|
urls.append(single)
|
|
|
|
return urls
|
|
|
|
@property
|
|
def is_pool_backed(self) -> bool:
|
|
return self._rotator is not None
|
|
|
|
def get_proxy_url(self) -> str | None:
|
|
if not self._urls:
|
|
return None
|
|
if self._rotator is not None:
|
|
# Advances the cyclic index on every call (thread-safe).
|
|
return self._rotator.get_proxy()
|
|
return self._urls[0]
|