feat(03b): add BYO custom proxy provider + bounded crawler rotation-retry

Add CustomProxyProvider (single endpoint or rotating pool via Scrapling ProxyRotator), registered as 'custom' alongside anonymous_proxies and selectable via PROXY_PROVIDER. Adds is_pool_backed to the ProxyProvider ABC + a zero-arg package helper. The web crawler does a bounded one-per-tier is_proxy_error rotation-retry gated on is_pool_backed() (single-endpoint providers no-op). Config/.env.example gain CUSTOM_PROXY_URL(S). Zero-arg getter contract unchanged for all consumers. Documents the proprietary boundary test (generic proxy infra stays Apache-2). Tests: provider, registry, crawler rotation (16).

Co-authored-by: Cursor <cursoragent@cursor.com>

surfsense_backend/app/utils/proxy/__init__.py

@@ -25,6 +25,14 @@ def get_requests_proxies() -> dict[str, str] | None:
     return get_active_provider().get_requests_proxies()
 
 
+def is_pool_backed() -> bool:
+    """Whether the active provider rotates across a client-side pool of endpoints.
+
+    The crawler gates its bounded proxy-error rotation-retry on this.
+    """
+    return get_active_provider().is_pool_backed
+
+
 def get_residential_proxy_url() -> str | None:
     """Backward-compatible alias for :func:`get_proxy_url`."""
     return get_proxy_url()
@@ -37,4 +45,5 @@ __all__ = [
     "get_proxy_url",
     "get_requests_proxies",
     "get_residential_proxy_url",
+    "is_pool_backed",
 ]

surfsense_backend/app/utils/proxy/base.py

@@ -44,3 +44,15 @@ class ProxyProvider(ABC):
         if proxy_url is None:
             return None
         return {"http": proxy_url, "https": proxy_url}
+
+    @property
+    def is_pool_backed(self) -> bool:
+        """Whether this provider rotates across a *client-side* pool of endpoints.
+
+        ``False`` for single-endpoint providers (including server-side rotating
+        gateways like ``anonymous_proxies``, whose rotation happens upstream).
+        The crawler performs its bounded proxy-error rotation-retry **only** when
+        this is ``True`` — retrying a single static endpoint would just re-hit the
+        same dead proxy.
+        """
+        return False

surfsense_backend/app/utils/proxy/providers/custom.py

@@ -0,0 +1,82 @@
+"""Bring-your-own (BYO) custom proxy provider.
+
+Reads one or more proxy endpoints from env (``CUSTOM_PROXY_URL`` and/or the
+comma-separated ``CUSTOM_PROXY_URLS``). With a single endpoint it behaves like a
+static proxy; with a pool (>1) it rotates client-side via Scrapling's thread-safe
+``ProxyRotator`` (cyclic), transparently to every caller of the zero-arg getters.
+
+No vendor-specific auth assumptions: a user who wants a specific vendor points
+``CUSTOM_PROXY_URLS`` at that vendor's ``http://user:pass@host:port`` endpoints.
+"""
+
+import logging
+from urllib.parse import urlsplit
+
+from scrapling.engines.toolbelt import ProxyRotator
+
+from app.config import Config
+from app.utils.proxy.base import ProxyProvider
+
+logger = logging.getLogger(__name__)
+
+
+class CustomProxyProvider(ProxyProvider):
+    """BYO provider for a single endpoint or a rotating pool of endpoints."""
+
+    name = "custom"
+
+    def __init__(self) -> None:
+        self._urls = self._load_urls()
+        # Only build a rotator for an actual pool; a single endpoint stays static.
+        self._rotator = ProxyRotator(self._urls) if len(self._urls) > 1 else None
+        if not self._urls:
+            logger.warning(
+                "PROXY_PROVIDER='custom' selected but neither CUSTOM_PROXY_URL nor "
+                "CUSTOM_PROXY_URLS is set; crawls will run without a proxy."
+            )
+
+    @staticmethod
+    def _load_urls() -> list[str]:
+        """Collect proxy URLs from env (pool first, then single), de-duplicated."""
+        urls: list[str] = []
+        pool = Config.CUSTOM_PROXY_URLS
+        if pool:
+            urls.extend(part.strip() for part in pool.split(",") if part.strip())
+
+        single = (Config.CUSTOM_PROXY_URL or "").strip()
+        if single and single not in urls:
+            urls.append(single)
+
+        return urls
+
+    @property
+    def is_pool_backed(self) -> bool:
+        return self._rotator is not None
+
+    def get_proxy_url(self) -> str | None:
+        if not self._urls:
+            return None
+        if self._rotator is not None:
+            # Advances the cyclic index on every call (thread-safe).
+            return self._rotator.get_proxy()
+        return self._urls[0]
+
+    def get_playwright_proxy(self) -> dict[str, str] | None:
+        proxy_url = self.get_proxy_url()
+        if not proxy_url:
+            return None
+
+        parts = urlsplit(proxy_url)
+        if not parts.hostname:
+            return None
+
+        server = f"{parts.scheme or 'http'}://{parts.hostname}"
+        if parts.port:
+            server = f"{server}:{parts.port}"
+
+        proxy: dict[str, str] = {"server": server}
+        if parts.username:
+            proxy["username"] = parts.username
+        if parts.password:
+            proxy["password"] = parts.password
+        return proxy

surfsense_backend/app/utils/proxy/registry.py

@@ -10,12 +10,14 @@ import logging
 from app.config import Config
 from app.utils.proxy.base import ProxyProvider
 from app.utils.proxy.providers.anonymous_proxies import AnonymousProxiesProvider
+from app.utils.proxy.providers.custom import CustomProxyProvider
 
 logger = logging.getLogger(__name__)
 
 # Registered proxy providers, keyed by their ``name``.
 _PROVIDERS: dict[str, type[ProxyProvider]] = {
     AnonymousProxiesProvider.name: AnonymousProxiesProvider,
+    CustomProxyProvider.name: CustomProxyProvider,
 }
 
 _DEFAULT_PROVIDER = AnonymousProxiesProvider.name