mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-06 22:12:12 +02:00
62260125f7
Add CustomProxyProvider (single endpoint or rotating pool via Scrapling ProxyRotator), registered as 'custom' alongside anonymous_proxies and selectable via PROXY_PROVIDER. Adds is_pool_backed to the ProxyProvider ABC + a zero-arg package helper. The web crawler does a bounded one-per-tier is_proxy_error rotation-retry gated on is_pool_backed() (single-endpoint providers no-op). Config/.env.example gain CUSTOM_PROXY_URL(S). Zero-arg getter contract unchanged for all consumers. Documents the proprietary boundary test (generic proxy infra stays Apache-2). Tests: provider, registry, crawler rotation (16). Co-authored-by: Cursor <cursoragent@cursor.com>
46 lines
1.5 KiB
Python
46 lines
1.5 KiB
Python
"""Proxy provider registry.
|
|
|
|
Maps the ``PROXY_PROVIDER`` config value to a :class:`ProxyProvider`
|
|
implementation. To add a new vendor: implement a provider in ``providers/`` and
|
|
add a single entry to ``_PROVIDERS`` below - no caller changes required.
|
|
"""
|
|
|
|
import logging
|
|
|
|
from app.config import Config
|
|
from app.utils.proxy.base import ProxyProvider
|
|
from app.utils.proxy.providers.anonymous_proxies import AnonymousProxiesProvider
|
|
from app.utils.proxy.providers.custom import CustomProxyProvider
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Registered proxy providers, keyed by their ``name``.
|
|
_PROVIDERS: dict[str, type[ProxyProvider]] = {
|
|
AnonymousProxiesProvider.name: AnonymousProxiesProvider,
|
|
CustomProxyProvider.name: CustomProxyProvider,
|
|
}
|
|
|
|
_DEFAULT_PROVIDER = AnonymousProxiesProvider.name
|
|
|
|
_active_provider: ProxyProvider | None = None
|
|
|
|
|
|
def get_active_provider() -> ProxyProvider:
|
|
"""Return the configured proxy provider instance (cached for the process)."""
|
|
global _active_provider
|
|
if _active_provider is not None:
|
|
return _active_provider
|
|
|
|
key = (Config.PROXY_PROVIDER or _DEFAULT_PROVIDER).strip()
|
|
provider_cls = _PROVIDERS.get(key)
|
|
if provider_cls is None:
|
|
logger.warning(
|
|
"Unknown PROXY_PROVIDER '%s'; falling back to '%s'. Available: %s",
|
|
key,
|
|
_DEFAULT_PROVIDER,
|
|
", ".join(sorted(_PROVIDERS)),
|
|
)
|
|
provider_cls = _PROVIDERS[_DEFAULT_PROVIDER]
|
|
|
|
_active_provider = provider_cls()
|
|
return _active_provider
|