feat(03b): add BYO custom proxy provider + bounded crawler rotation-retry

Add CustomProxyProvider (single endpoint or rotating pool via Scrapling ProxyRotator), registered as 'custom' alongside anonymous_proxies and selectable via PROXY_PROVIDER. Adds is_pool_backed to the ProxyProvider ABC + a zero-arg package helper. The web crawler does a bounded one-per-tier is_proxy_error rotation-retry gated on is_pool_backed() (single-endpoint providers no-op). Config/.env.example gain CUSTOM_PROXY_URL(S). Zero-arg getter contract unchanged for all consumers. Documents the proprietary boundary test (generic proxy infra stays Apache-2). Tests: provider, registry, crawler rotation (16).

Co-authored-by: Cursor <cursoragent@cursor.com>

surfsense_backend/.env.example

@@ -325,6 +325,12 @@ TURNSTILE_SECRET_KEY=
 # RESIDENTIAL_PROXY_LOCATION=
 # RESIDENTIAL_PROXY_TYPE=1
 
+# Custom (BYO) proxy provider. Activate with PROXY_PROVIDER=custom.
+# Provide a single endpoint and/or a comma-separated pool; a pool of >1 rotates
+# client-side (cyclic). Each value is a full http://user:pass@host:port URL.
+# CUSTOM_PROXY_URL=http://user:pass@host:port
+# CUSTOM_PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port
+
 # File Parser Service
 ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING
 UNSTRUCTURED_API_KEY=Tpu3P0U8iy

surfsense_backend/app/config/__init__.py

@@ -1017,6 +1017,14 @@ class Config:
     RESIDENTIAL_PROXY_LOCATION = os.getenv("RESIDENTIAL_PROXY_LOCATION", "")
     RESIDENTIAL_PROXY_TYPE = int(os.getenv("RESIDENTIAL_PROXY_TYPE", "1"))
 
+    # Custom (BYO) proxy provider, selected via PROXY_PROVIDER="custom".
+    # Consumed by the "custom" provider (app/utils/proxy/providers/custom.py).
+    # Provide a single endpoint (CUSTOM_PROXY_URL) and/or a comma-separated pool
+    # (CUSTOM_PROXY_URLS); a pool of >1 rotates client-side via Scrapling's
+    # ProxyRotator. Each value is a full http://user:pass@host:port URL.
+    CUSTOM_PROXY_URL = os.getenv("CUSTOM_PROXY_URL")
+    CUSTOM_PROXY_URLS = os.getenv("CUSTOM_PROXY_URLS")
+
     # Litellm TTS Configuration
     TTS_SERVICE = os.getenv("TTS_SERVICE")
     TTS_SERVICE_API_BASE = os.getenv("TTS_SERVICE_API_BASE")

surfsense_backend/app/proprietary/README.md

@@ -30,3 +30,11 @@ Apache-2.0* — instead of per-file headers scattered across the tree.
   chat `scrape_webpage` tools do); that does not move them under this license.
 - Depend only on the public API exported from each subpackage's `__init__`, not
   on internal modules, so the boundary stays clean and swappable.
+- **Boundary test:** put code here only if it is used *exclusively* by the moat.
+  Generic infrastructure that Apache-2 features also depend on stays Apache-2
+  even when the crawler uses it too. Example: `app/utils/proxy/` (provider
+  abstraction, registry, `CustomProxyProvider` + rotation — a thin wrapper over
+  Scrapling's public `ProxyRotator`) is shared with the YouTube/transcript and
+  chat features, so it stays Apache-2; only the crawl-ladder-coupled
+  rotation-retry orchestration (`web_crawler/connector.py::_run_tier_with_proxy_retry`)
+  lives here.

surfsense_backend/app/proprietary/web_crawler/connector.py

@@ -23,15 +23,17 @@ which tier produced it.
 import asyncio
 import logging
 import time
+from collections.abc import Awaitable, Callable
 from dataclasses import dataclass
 from enum import Enum
 from typing import Any
 
 import trafilatura
 import validators
+from scrapling.engines.toolbelt import is_proxy_error
 from scrapling.fetchers import AsyncFetcher, DynamicFetcher, StealthyFetcher
 
-from app.utils.proxy import get_proxy_url
+from app.utils.proxy import get_proxy_url, is_pool_backed
 
 logger = logging.getLogger(__name__)
 
@@ -104,7 +106,9 @@ class WebCrawlerConnector:
             tier_start = time.perf_counter()
             try:
                 logger.info(f"[webcrawler] Using Scrapling AsyncFetcher for: {url}")
-                result = await self._crawl_with_async_fetcher(url)
+                result = await self._run_tier_with_proxy_retry(
+                    "scrapling-static", lambda: self._crawl_with_async_fetcher(url)
+                )
                 if result:
                     self._log_tier_outcome(
                         "scrapling-static", url, tier_start, "success"
@@ -128,7 +132,9 @@ class WebCrawlerConnector:
             tier_start = time.perf_counter()
             try:
                 logger.info(f"[webcrawler] Using Scrapling DynamicFetcher for: {url}")
-                result = await self._crawl_with_dynamic(url)
+                result = await self._run_tier_with_proxy_retry(
+                    "scrapling-dynamic", lambda: self._crawl_with_dynamic(url)
+                )
                 if result:
                     self._log_tier_outcome(
                         "scrapling-dynamic", url, tier_start, "success"
@@ -160,7 +166,9 @@ class WebCrawlerConnector:
             tier_start = time.perf_counter()
             try:
                 logger.info(f"[webcrawler] Using Scrapling StealthyFetcher for: {url}")
-                result = await self._crawl_with_stealthy(url)
+                result = await self._run_tier_with_proxy_retry(
+                    "scrapling-stealthy", lambda: self._crawl_with_stealthy(url)
+                )
                 if result:
                     self._log_tier_outcome(
                         "scrapling-stealthy", url, tier_start, "success"
@@ -206,6 +214,36 @@ class WebCrawlerConnector:
                 error=f"Error crawling URL {url}: {e!s}",
             )
 
+    async def _run_tier_with_proxy_retry(
+        self,
+        tier: str,
+        attempt: Callable[[], Awaitable[dict[str, Any] | None]],
+    ) -> dict[str, Any] | None:
+        """Run one fetch tier, retrying once on a proxy error when pool-backed.
+
+        ``03b`` rotation: a pool-backed ``CustomProxyProvider`` yields the *next*
+        endpoint on every ``get_proxy_url()`` call, so simply re-invoking the tier
+        rotates the proxy. Bounded to a single extra attempt per tier — no
+        unbounded fan-out on billable crawls. Single-endpoint providers
+        (including the server-side-rotating ``anonymous_proxies``) skip the retry
+        entirely (``is_pool_backed()`` is ``False``), since retrying the same
+        static endpoint would just re-hit the same dead proxy. Non-proxy errors
+        (and ``NotImplementedError`` from the browser tiers) propagate unchanged
+        for the caller's existing per-tier handling.
+        """
+        try:
+            return await attempt()
+        except Exception as exc:
+            if is_proxy_error(exc) and is_pool_backed():
+                logger.warning(
+                    "%s tier=%s proxy error; rotating endpoint, retrying once: %s",
+                    _PERF,
+                    tier,
+                    exc,
+                )
+                return await attempt()
+            raise
+
     @staticmethod
     def _log_tier_outcome(
         tier: str,

surfsense_backend/app/utils/proxy/__init__.py

@@ -25,6 +25,14 @@ def get_requests_proxies() -> dict[str, str] | None:
     return get_active_provider().get_requests_proxies()
 
 
+def is_pool_backed() -> bool:
+    """Whether the active provider rotates across a client-side pool of endpoints.
+
+    The crawler gates its bounded proxy-error rotation-retry on this.
+    """
+    return get_active_provider().is_pool_backed
+
+
 def get_residential_proxy_url() -> str | None:
     """Backward-compatible alias for :func:`get_proxy_url`."""
     return get_proxy_url()
@@ -37,4 +45,5 @@ __all__ = [
     "get_proxy_url",
     "get_requests_proxies",
     "get_residential_proxy_url",
+    "is_pool_backed",
 ]

surfsense_backend/app/utils/proxy/base.py

@@ -44,3 +44,15 @@ class ProxyProvider(ABC):
         if proxy_url is None:
             return None
         return {"http": proxy_url, "https": proxy_url}
+
+    @property
+    def is_pool_backed(self) -> bool:
+        """Whether this provider rotates across a *client-side* pool of endpoints.
+
+        ``False`` for single-endpoint providers (including server-side rotating
+        gateways like ``anonymous_proxies``, whose rotation happens upstream).
+        The crawler performs its bounded proxy-error rotation-retry **only** when
+        this is ``True`` — retrying a single static endpoint would just re-hit the
+        same dead proxy.
+        """
+        return False

surfsense_backend/app/utils/proxy/providers/custom.py

@@ -0,0 +1,82 @@
+"""Bring-your-own (BYO) custom proxy provider.
+
+Reads one or more proxy endpoints from env (``CUSTOM_PROXY_URL`` and/or the
+comma-separated ``CUSTOM_PROXY_URLS``). With a single endpoint it behaves like a
+static proxy; with a pool (>1) it rotates client-side via Scrapling's thread-safe
+``ProxyRotator`` (cyclic), transparently to every caller of the zero-arg getters.
+
+No vendor-specific auth assumptions: a user who wants a specific vendor points
+``CUSTOM_PROXY_URLS`` at that vendor's ``http://user:pass@host:port`` endpoints.
+"""
+
+import logging
+from urllib.parse import urlsplit
+
+from scrapling.engines.toolbelt import ProxyRotator
+
+from app.config import Config
+from app.utils.proxy.base import ProxyProvider
+
+logger = logging.getLogger(__name__)
+
+
+class CustomProxyProvider(ProxyProvider):
+    """BYO provider for a single endpoint or a rotating pool of endpoints."""
+
+    name = "custom"
+
+    def __init__(self) -> None:
+        self._urls = self._load_urls()
+        # Only build a rotator for an actual pool; a single endpoint stays static.
+        self._rotator = ProxyRotator(self._urls) if len(self._urls) > 1 else None
+        if not self._urls:
+            logger.warning(
+                "PROXY_PROVIDER='custom' selected but neither CUSTOM_PROXY_URL nor "
+                "CUSTOM_PROXY_URLS is set; crawls will run without a proxy."
+            )
+
+    @staticmethod
+    def _load_urls() -> list[str]:
+        """Collect proxy URLs from env (pool first, then single), de-duplicated."""
+        urls: list[str] = []
+        pool = Config.CUSTOM_PROXY_URLS
+        if pool:
+            urls.extend(part.strip() for part in pool.split(",") if part.strip())
+
+        single = (Config.CUSTOM_PROXY_URL or "").strip()
+        if single and single not in urls:
+            urls.append(single)
+
+        return urls
+
+    @property
+    def is_pool_backed(self) -> bool:
+        return self._rotator is not None
+
+    def get_proxy_url(self) -> str | None:
+        if not self._urls:
+            return None
+        if self._rotator is not None:
+            # Advances the cyclic index on every call (thread-safe).
+            return self._rotator.get_proxy()
+        return self._urls[0]
+
+    def get_playwright_proxy(self) -> dict[str, str] | None:
+        proxy_url = self.get_proxy_url()
+        if not proxy_url:
+            return None
+
+        parts = urlsplit(proxy_url)
+        if not parts.hostname:
+            return None
+
+        server = f"{parts.scheme or 'http'}://{parts.hostname}"
+        if parts.port:
+            server = f"{server}:{parts.port}"
+
+        proxy: dict[str, str] = {"server": server}
+        if parts.username:
+            proxy["username"] = parts.username
+        if parts.password:
+            proxy["password"] = parts.password
+        return proxy

surfsense_backend/app/utils/proxy/registry.py

@@ -10,12 +10,14 @@ import logging
 from app.config import Config
 from app.utils.proxy.base import ProxyProvider
 from app.utils.proxy.providers.anonymous_proxies import AnonymousProxiesProvider
+from app.utils.proxy.providers.custom import CustomProxyProvider
 
 logger = logging.getLogger(__name__)
 
 # Registered proxy providers, keyed by their ``name``.
 _PROVIDERS: dict[str, type[ProxyProvider]] = {
     AnonymousProxiesProvider.name: AnonymousProxiesProvider,
+    CustomProxyProvider.name: CustomProxyProvider,
 }
 
 _DEFAULT_PROVIDER = AnonymousProxiesProvider.name

surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py

@@ -11,6 +11,7 @@ from app.proprietary.web_crawler import (
     CrawlOutcomeStatus,
     WebCrawlerConnector,
 )
+from app.proprietary.web_crawler import connector as connector_module
 
 pytestmark = pytest.mark.unit
 
@@ -116,3 +117,76 @@ async def test_all_tiers_raise_is_failed(monkeypatch: pytest.MonkeyPatch) -> Non
 
     assert outcome.status is CrawlOutcomeStatus.FAILED
     assert "fetch exploded" in (outcome.error or "")
+
+
+async def test_proxy_error_rotates_once_when_pool_backed(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """03b: a pool-backed provider retries the tier once on a proxy error."""
+    crawler = WebCrawlerConnector()
+    calls = {"n": 0}
+
+    async def _flaky(_url: str) -> dict:
+        calls["n"] += 1
+        if calls["n"] == 1:
+            raise RuntimeError("connection refused by upstream proxy")
+        return _result("scrapling-static")
+
+    monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _flaky)
+    monkeypatch.setattr(connector_module, "is_pool_backed", lambda: True)
+
+    outcome = await crawler.crawl_url("https://example.com")
+
+    assert outcome.status is CrawlOutcomeStatus.SUCCESS
+    assert outcome.tier == "scrapling-static"
+    assert calls["n"] == 2  # original attempt + one rotation retry
+
+
+async def test_proxy_error_no_retry_when_single_endpoint(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Single-endpoint providers skip the retry (no re-hit of the dead proxy)."""
+    crawler = WebCrawlerConnector()
+    static_calls = {"n": 0}
+
+    async def _proxy_err(_url: str) -> None:
+        static_calls["n"] += 1
+        raise RuntimeError("connection refused by upstream proxy")
+
+    async def _empty(_url: str) -> None:
+        return None
+
+    monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _proxy_err)
+    monkeypatch.setattr(crawler, "_crawl_with_dynamic", _empty)
+    monkeypatch.setattr(crawler, "_crawl_with_stealthy", _empty)
+    monkeypatch.setattr(connector_module, "is_pool_backed", lambda: False)
+
+    outcome = await crawler.crawl_url("https://example.com")
+
+    assert static_calls["n"] == 1  # no retry
+    assert outcome.status is CrawlOutcomeStatus.EMPTY
+
+
+async def test_non_proxy_error_never_retries(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A non-proxy error is not retried even when pool-backed."""
+    crawler = WebCrawlerConnector()
+    calls = {"n": 0}
+
+    async def _boom(_url: str) -> None:
+        calls["n"] += 1
+        raise RuntimeError("totally unrelated failure")
+
+    async def _empty(_url: str) -> None:
+        return None
+
+    monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _boom)
+    monkeypatch.setattr(crawler, "_crawl_with_dynamic", _empty)
+    monkeypatch.setattr(crawler, "_crawl_with_stealthy", _empty)
+    monkeypatch.setattr(connector_module, "is_pool_backed", lambda: True)
+
+    outcome = await crawler.crawl_url("https://example.com")
+
+    assert calls["n"] == 1  # not retried (not a proxy error)
+    assert outcome.status is CrawlOutcomeStatus.EMPTY

surfsense_backend/tests/unit/utils/proxy/test_custom_provider.py

@@ -0,0 +1,79 @@
+"""Unit tests for the BYO ``CustomProxyProvider`` (Phase 3b).
+
+Covers single-endpoint vs pool behavior, cyclic rotation, env de-duplication,
+the empty/unconfigured case, and the playwright-dict parse.
+"""
+
+import pytest
+
+from app.config import Config
+from app.utils.proxy.providers.custom import CustomProxyProvider
+
+pytestmark = pytest.mark.unit
+
+
+@pytest.fixture(autouse=True)
+def _clear_proxy_env(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Default both knobs to unset; individual tests set what they need."""
+    monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", None)
+    monkeypatch.setattr(Config, "CUSTOM_PROXY_URLS", None)
+
+
+def test_single_url_is_static(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", "http://u:p@host:8080")
+    provider = CustomProxyProvider()
+
+    assert provider.is_pool_backed is False
+    assert provider.get_proxy_url() == "http://u:p@host:8080"
+    # Static endpoint returns the same value every call.
+    assert provider.get_proxy_url() == "http://u:p@host:8080"
+
+
+def test_pool_rotates_cyclically(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(
+        Config,
+        "CUSTOM_PROXY_URLS",
+        "http://a:1@h1:8080, http://b:2@h2:8080 ,http://c:3@h3:8080",
+    )
+    provider = CustomProxyProvider()
+
+    assert provider.is_pool_backed is True
+    seen = [provider.get_proxy_url() for _ in range(4)]
+    assert seen == [
+        "http://a:1@h1:8080",
+        "http://b:2@h2:8080",
+        "http://c:3@h3:8080",
+        "http://a:1@h1:8080",  # wraps around
+    ]
+
+
+def test_single_plus_pool_dedupes(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A URL present in both CUSTOM_PROXY_URL and the pool is not duplicated."""
+    monkeypatch.setattr(Config, "CUSTOM_PROXY_URLS", "http://a@h1:80,http://b@h2:80")
+    monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", "http://a@h1:80")
+    provider = CustomProxyProvider()
+
+    assert provider.is_pool_backed is True
+    seen = [provider.get_proxy_url() for _ in range(3)]
+    assert seen == ["http://a@h1:80", "http://b@h2:80", "http://a@h1:80"]
+
+
+def test_unconfigured_returns_none() -> None:
+    provider = CustomProxyProvider()
+
+    assert provider.is_pool_backed is False
+    assert provider.get_proxy_url() is None
+    assert provider.get_requests_proxies() is None
+
+
+def test_playwright_proxy_parses_credentials(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", "http://user:pass@host:8080")
+    provider = CustomProxyProvider()
+
+    assert provider.get_playwright_proxy() == {
+        "server": "http://host:8080",
+        "username": "user",
+        "password": "pass",
+    }

surfsense_backend/tests/unit/utils/proxy/test_registry.py

@@ -0,0 +1,36 @@
+"""Unit tests for proxy provider selection (Phase 3b).
+
+``PROXY_PROVIDER`` selects the single app-wide provider; ``custom`` is now
+registered alongside ``anonymous_proxies``, and unknown values still warn and
+fall back to the default.
+"""
+
+import pytest
+
+from app.config import Config
+from app.utils.proxy import registry
+from app.utils.proxy.providers.anonymous_proxies import AnonymousProxiesProvider
+from app.utils.proxy.providers.custom import CustomProxyProvider
+
+pytestmark = pytest.mark.unit
+
+
+@pytest.fixture(autouse=True)
+def _reset_active_provider(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Clear the process-wide provider cache so each test resolves fresh."""
+    monkeypatch.setattr(registry, "_active_provider", None)
+
+
+def test_resolves_custom(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(Config, "PROXY_PROVIDER", "custom")
+    assert isinstance(registry.get_active_provider(), CustomProxyProvider)
+
+
+def test_resolves_anonymous(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(Config, "PROXY_PROVIDER", "anonymous_proxies")
+    assert isinstance(registry.get_active_provider(), AnonymousProxiesProvider)
+
+
+def test_unknown_falls_back_to_default(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(Config, "PROXY_PROVIDER", "does_not_exist")
+    assert isinstance(registry.get_active_provider(), AnonymousProxiesProvider)