diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 6caa33c4e..22851dd6d 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -325,6 +325,12 @@ TURNSTILE_SECRET_KEY= # RESIDENTIAL_PROXY_LOCATION= # RESIDENTIAL_PROXY_TYPE=1 +# Custom (BYO) proxy provider. Activate with PROXY_PROVIDER=custom. +# Provide a single endpoint and/or a comma-separated pool; a pool of >1 rotates +# client-side (cyclic). Each value is a full http://user:pass@host:port URL. +# CUSTOM_PROXY_URL=http://user:pass@host:port +# CUSTOM_PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port + # File Parser Service ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING UNSTRUCTURED_API_KEY=Tpu3P0U8iy diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 9e69d6d53..8673ded7d 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -1017,6 +1017,14 @@ class Config: RESIDENTIAL_PROXY_LOCATION = os.getenv("RESIDENTIAL_PROXY_LOCATION", "") RESIDENTIAL_PROXY_TYPE = int(os.getenv("RESIDENTIAL_PROXY_TYPE", "1")) + # Custom (BYO) proxy provider, selected via PROXY_PROVIDER="custom". + # Consumed by the "custom" provider (app/utils/proxy/providers/custom.py). + # Provide a single endpoint (CUSTOM_PROXY_URL) and/or a comma-separated pool + # (CUSTOM_PROXY_URLS); a pool of >1 rotates client-side via Scrapling's + # ProxyRotator. Each value is a full http://user:pass@host:port URL. + CUSTOM_PROXY_URL = os.getenv("CUSTOM_PROXY_URL") + CUSTOM_PROXY_URLS = os.getenv("CUSTOM_PROXY_URLS") + # Litellm TTS Configuration TTS_SERVICE = os.getenv("TTS_SERVICE") TTS_SERVICE_API_BASE = os.getenv("TTS_SERVICE_API_BASE") diff --git a/surfsense_backend/app/proprietary/README.md b/surfsense_backend/app/proprietary/README.md index b1c29aa35..26f29436e 100644 --- a/surfsense_backend/app/proprietary/README.md +++ b/surfsense_backend/app/proprietary/README.md @@ -30,3 +30,11 @@ Apache-2.0* — instead of per-file headers scattered across the tree. chat `scrape_webpage` tools do); that does not move them under this license. - Depend only on the public API exported from each subpackage's `__init__`, not on internal modules, so the boundary stays clean and swappable. +- **Boundary test:** put code here only if it is used *exclusively* by the moat. + Generic infrastructure that Apache-2 features also depend on stays Apache-2 + even when the crawler uses it too. Example: `app/utils/proxy/` (provider + abstraction, registry, `CustomProxyProvider` + rotation — a thin wrapper over + Scrapling's public `ProxyRotator`) is shared with the YouTube/transcript and + chat features, so it stays Apache-2; only the crawl-ladder-coupled + rotation-retry orchestration (`web_crawler/connector.py::_run_tier_with_proxy_retry`) + lives here. diff --git a/surfsense_backend/app/proprietary/web_crawler/connector.py b/surfsense_backend/app/proprietary/web_crawler/connector.py index 64f0e4afd..259db41f7 100644 --- a/surfsense_backend/app/proprietary/web_crawler/connector.py +++ b/surfsense_backend/app/proprietary/web_crawler/connector.py @@ -23,15 +23,17 @@ which tier produced it. import asyncio import logging import time +from collections.abc import Awaitable, Callable from dataclasses import dataclass from enum import Enum from typing import Any import trafilatura import validators +from scrapling.engines.toolbelt import is_proxy_error from scrapling.fetchers import AsyncFetcher, DynamicFetcher, StealthyFetcher -from app.utils.proxy import get_proxy_url +from app.utils.proxy import get_proxy_url, is_pool_backed logger = logging.getLogger(__name__) @@ -104,7 +106,9 @@ class WebCrawlerConnector: tier_start = time.perf_counter() try: logger.info(f"[webcrawler] Using Scrapling AsyncFetcher for: {url}") - result = await self._crawl_with_async_fetcher(url) + result = await self._run_tier_with_proxy_retry( + "scrapling-static", lambda: self._crawl_with_async_fetcher(url) + ) if result: self._log_tier_outcome( "scrapling-static", url, tier_start, "success" @@ -128,7 +132,9 @@ class WebCrawlerConnector: tier_start = time.perf_counter() try: logger.info(f"[webcrawler] Using Scrapling DynamicFetcher for: {url}") - result = await self._crawl_with_dynamic(url) + result = await self._run_tier_with_proxy_retry( + "scrapling-dynamic", lambda: self._crawl_with_dynamic(url) + ) if result: self._log_tier_outcome( "scrapling-dynamic", url, tier_start, "success" @@ -160,7 +166,9 @@ class WebCrawlerConnector: tier_start = time.perf_counter() try: logger.info(f"[webcrawler] Using Scrapling StealthyFetcher for: {url}") - result = await self._crawl_with_stealthy(url) + result = await self._run_tier_with_proxy_retry( + "scrapling-stealthy", lambda: self._crawl_with_stealthy(url) + ) if result: self._log_tier_outcome( "scrapling-stealthy", url, tier_start, "success" @@ -206,6 +214,36 @@ class WebCrawlerConnector: error=f"Error crawling URL {url}: {e!s}", ) + async def _run_tier_with_proxy_retry( + self, + tier: str, + attempt: Callable[[], Awaitable[dict[str, Any] | None]], + ) -> dict[str, Any] | None: + """Run one fetch tier, retrying once on a proxy error when pool-backed. + + ``03b`` rotation: a pool-backed ``CustomProxyProvider`` yields the *next* + endpoint on every ``get_proxy_url()`` call, so simply re-invoking the tier + rotates the proxy. Bounded to a single extra attempt per tier — no + unbounded fan-out on billable crawls. Single-endpoint providers + (including the server-side-rotating ``anonymous_proxies``) skip the retry + entirely (``is_pool_backed()`` is ``False``), since retrying the same + static endpoint would just re-hit the same dead proxy. Non-proxy errors + (and ``NotImplementedError`` from the browser tiers) propagate unchanged + for the caller's existing per-tier handling. + """ + try: + return await attempt() + except Exception as exc: + if is_proxy_error(exc) and is_pool_backed(): + logger.warning( + "%s tier=%s proxy error; rotating endpoint, retrying once: %s", + _PERF, + tier, + exc, + ) + return await attempt() + raise + @staticmethod def _log_tier_outcome( tier: str, diff --git a/surfsense_backend/app/utils/proxy/__init__.py b/surfsense_backend/app/utils/proxy/__init__.py index 8ff489a41..3e4158587 100644 --- a/surfsense_backend/app/utils/proxy/__init__.py +++ b/surfsense_backend/app/utils/proxy/__init__.py @@ -25,6 +25,14 @@ def get_requests_proxies() -> dict[str, str] | None: return get_active_provider().get_requests_proxies() +def is_pool_backed() -> bool: + """Whether the active provider rotates across a client-side pool of endpoints. + + The crawler gates its bounded proxy-error rotation-retry on this. + """ + return get_active_provider().is_pool_backed + + def get_residential_proxy_url() -> str | None: """Backward-compatible alias for :func:`get_proxy_url`.""" return get_proxy_url() @@ -37,4 +45,5 @@ __all__ = [ "get_proxy_url", "get_requests_proxies", "get_residential_proxy_url", + "is_pool_backed", ] diff --git a/surfsense_backend/app/utils/proxy/base.py b/surfsense_backend/app/utils/proxy/base.py index a3e84faf0..bc6eb5041 100644 --- a/surfsense_backend/app/utils/proxy/base.py +++ b/surfsense_backend/app/utils/proxy/base.py @@ -44,3 +44,15 @@ class ProxyProvider(ABC): if proxy_url is None: return None return {"http": proxy_url, "https": proxy_url} + + @property + def is_pool_backed(self) -> bool: + """Whether this provider rotates across a *client-side* pool of endpoints. + + ``False`` for single-endpoint providers (including server-side rotating + gateways like ``anonymous_proxies``, whose rotation happens upstream). + The crawler performs its bounded proxy-error rotation-retry **only** when + this is ``True`` — retrying a single static endpoint would just re-hit the + same dead proxy. + """ + return False diff --git a/surfsense_backend/app/utils/proxy/providers/custom.py b/surfsense_backend/app/utils/proxy/providers/custom.py new file mode 100644 index 000000000..16d87a9a7 --- /dev/null +++ b/surfsense_backend/app/utils/proxy/providers/custom.py @@ -0,0 +1,82 @@ +"""Bring-your-own (BYO) custom proxy provider. + +Reads one or more proxy endpoints from env (``CUSTOM_PROXY_URL`` and/or the +comma-separated ``CUSTOM_PROXY_URLS``). With a single endpoint it behaves like a +static proxy; with a pool (>1) it rotates client-side via Scrapling's thread-safe +``ProxyRotator`` (cyclic), transparently to every caller of the zero-arg getters. + +No vendor-specific auth assumptions: a user who wants a specific vendor points +``CUSTOM_PROXY_URLS`` at that vendor's ``http://user:pass@host:port`` endpoints. +""" + +import logging +from urllib.parse import urlsplit + +from scrapling.engines.toolbelt import ProxyRotator + +from app.config import Config +from app.utils.proxy.base import ProxyProvider + +logger = logging.getLogger(__name__) + + +class CustomProxyProvider(ProxyProvider): + """BYO provider for a single endpoint or a rotating pool of endpoints.""" + + name = "custom" + + def __init__(self) -> None: + self._urls = self._load_urls() + # Only build a rotator for an actual pool; a single endpoint stays static. + self._rotator = ProxyRotator(self._urls) if len(self._urls) > 1 else None + if not self._urls: + logger.warning( + "PROXY_PROVIDER='custom' selected but neither CUSTOM_PROXY_URL nor " + "CUSTOM_PROXY_URLS is set; crawls will run without a proxy." + ) + + @staticmethod + def _load_urls() -> list[str]: + """Collect proxy URLs from env (pool first, then single), de-duplicated.""" + urls: list[str] = [] + pool = Config.CUSTOM_PROXY_URLS + if pool: + urls.extend(part.strip() for part in pool.split(",") if part.strip()) + + single = (Config.CUSTOM_PROXY_URL or "").strip() + if single and single not in urls: + urls.append(single) + + return urls + + @property + def is_pool_backed(self) -> bool: + return self._rotator is not None + + def get_proxy_url(self) -> str | None: + if not self._urls: + return None + if self._rotator is not None: + # Advances the cyclic index on every call (thread-safe). + return self._rotator.get_proxy() + return self._urls[0] + + def get_playwright_proxy(self) -> dict[str, str] | None: + proxy_url = self.get_proxy_url() + if not proxy_url: + return None + + parts = urlsplit(proxy_url) + if not parts.hostname: + return None + + server = f"{parts.scheme or 'http'}://{parts.hostname}" + if parts.port: + server = f"{server}:{parts.port}" + + proxy: dict[str, str] = {"server": server} + if parts.username: + proxy["username"] = parts.username + if parts.password: + proxy["password"] = parts.password + return proxy diff --git a/surfsense_backend/app/utils/proxy/registry.py b/surfsense_backend/app/utils/proxy/registry.py index 777dfc049..7f1feb70d 100644 --- a/surfsense_backend/app/utils/proxy/registry.py +++ b/surfsense_backend/app/utils/proxy/registry.py @@ -10,12 +10,14 @@ import logging from app.config import Config from app.utils.proxy.base import ProxyProvider from app.utils.proxy.providers.anonymous_proxies import AnonymousProxiesProvider +from app.utils.proxy.providers.custom import CustomProxyProvider logger = logging.getLogger(__name__) # Registered proxy providers, keyed by their ``name``. _PROVIDERS: dict[str, type[ProxyProvider]] = { AnonymousProxiesProvider.name: AnonymousProxiesProvider, + CustomProxyProvider.name: CustomProxyProvider, } _DEFAULT_PROVIDER = AnonymousProxiesProvider.name diff --git a/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py b/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py index 81759787f..18112fe7e 100644 --- a/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py +++ b/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py @@ -11,6 +11,7 @@ from app.proprietary.web_crawler import ( CrawlOutcomeStatus, WebCrawlerConnector, ) +from app.proprietary.web_crawler import connector as connector_module pytestmark = pytest.mark.unit @@ -116,3 +117,76 @@ async def test_all_tiers_raise_is_failed(monkeypatch: pytest.MonkeyPatch) -> Non assert outcome.status is CrawlOutcomeStatus.FAILED assert "fetch exploded" in (outcome.error or "") + + +async def test_proxy_error_rotates_once_when_pool_backed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """03b: a pool-backed provider retries the tier once on a proxy error.""" + crawler = WebCrawlerConnector() + calls = {"n": 0} + + async def _flaky(_url: str) -> dict: + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("connection refused by upstream proxy") + return _result("scrapling-static") + + monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _flaky) + monkeypatch.setattr(connector_module, "is_pool_backed", lambda: True) + + outcome = await crawler.crawl_url("https://example.com") + + assert outcome.status is CrawlOutcomeStatus.SUCCESS + assert outcome.tier == "scrapling-static" + assert calls["n"] == 2 # original attempt + one rotation retry + + +async def test_proxy_error_no_retry_when_single_endpoint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Single-endpoint providers skip the retry (no re-hit of the dead proxy).""" + crawler = WebCrawlerConnector() + static_calls = {"n": 0} + + async def _proxy_err(_url: str) -> None: + static_calls["n"] += 1 + raise RuntimeError("connection refused by upstream proxy") + + async def _empty(_url: str) -> None: + return None + + monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _proxy_err) + monkeypatch.setattr(crawler, "_crawl_with_dynamic", _empty) + monkeypatch.setattr(crawler, "_crawl_with_stealthy", _empty) + monkeypatch.setattr(connector_module, "is_pool_backed", lambda: False) + + outcome = await crawler.crawl_url("https://example.com") + + assert static_calls["n"] == 1 # no retry + assert outcome.status is CrawlOutcomeStatus.EMPTY + + +async def test_non_proxy_error_never_retries( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A non-proxy error is not retried even when pool-backed.""" + crawler = WebCrawlerConnector() + calls = {"n": 0} + + async def _boom(_url: str) -> None: + calls["n"] += 1 + raise RuntimeError("totally unrelated failure") + + async def _empty(_url: str) -> None: + return None + + monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _boom) + monkeypatch.setattr(crawler, "_crawl_with_dynamic", _empty) + monkeypatch.setattr(crawler, "_crawl_with_stealthy", _empty) + monkeypatch.setattr(connector_module, "is_pool_backed", lambda: True) + + outcome = await crawler.crawl_url("https://example.com") + + assert calls["n"] == 1 # not retried (not a proxy error) + assert outcome.status is CrawlOutcomeStatus.EMPTY diff --git a/surfsense_backend/tests/unit/utils/proxy/__init__.py b/surfsense_backend/tests/unit/utils/proxy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/utils/proxy/test_custom_provider.py b/surfsense_backend/tests/unit/utils/proxy/test_custom_provider.py new file mode 100644 index 000000000..3ab244376 --- /dev/null +++ b/surfsense_backend/tests/unit/utils/proxy/test_custom_provider.py @@ -0,0 +1,79 @@ +"""Unit tests for the BYO ``CustomProxyProvider`` (Phase 3b). + +Covers single-endpoint vs pool behavior, cyclic rotation, env de-duplication, +the empty/unconfigured case, and the playwright-dict parse. +""" + +import pytest + +from app.config import Config +from app.utils.proxy.providers.custom import CustomProxyProvider + +pytestmark = pytest.mark.unit + + +@pytest.fixture(autouse=True) +def _clear_proxy_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Default both knobs to unset; individual tests set what they need.""" + monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", None) + monkeypatch.setattr(Config, "CUSTOM_PROXY_URLS", None) + + +def test_single_url_is_static(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", "http://u:p@host:8080") + provider = CustomProxyProvider() + + assert provider.is_pool_backed is False + assert provider.get_proxy_url() == "http://u:p@host:8080" + # Static endpoint returns the same value every call. + assert provider.get_proxy_url() == "http://u:p@host:8080" + + +def test_pool_rotates_cyclically(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + Config, + "CUSTOM_PROXY_URLS", + "http://a:1@h1:8080, http://b:2@h2:8080 ,http://c:3@h3:8080", + ) + provider = CustomProxyProvider() + + assert provider.is_pool_backed is True + seen = [provider.get_proxy_url() for _ in range(4)] + assert seen == [ + "http://a:1@h1:8080", + "http://b:2@h2:8080", + "http://c:3@h3:8080", + "http://a:1@h1:8080", # wraps around + ] + + +def test_single_plus_pool_dedupes(monkeypatch: pytest.MonkeyPatch) -> None: + """A URL present in both CUSTOM_PROXY_URL and the pool is not duplicated.""" + monkeypatch.setattr(Config, "CUSTOM_PROXY_URLS", "http://a@h1:80,http://b@h2:80") + monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", "http://a@h1:80") + provider = CustomProxyProvider() + + assert provider.is_pool_backed is True + seen = [provider.get_proxy_url() for _ in range(3)] + assert seen == ["http://a@h1:80", "http://b@h2:80", "http://a@h1:80"] + + +def test_unconfigured_returns_none() -> None: + provider = CustomProxyProvider() + + assert provider.is_pool_backed is False + assert provider.get_proxy_url() is None + assert provider.get_requests_proxies() is None + + +def test_playwright_proxy_parses_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", "http://user:pass@host:8080") + provider = CustomProxyProvider() + + assert provider.get_playwright_proxy() == { + "server": "http://host:8080", + "username": "user", + "password": "pass", + } diff --git a/surfsense_backend/tests/unit/utils/proxy/test_registry.py b/surfsense_backend/tests/unit/utils/proxy/test_registry.py new file mode 100644 index 000000000..c67a286ba --- /dev/null +++ b/surfsense_backend/tests/unit/utils/proxy/test_registry.py @@ -0,0 +1,36 @@ +"""Unit tests for proxy provider selection (Phase 3b). + +``PROXY_PROVIDER`` selects the single app-wide provider; ``custom`` is now +registered alongside ``anonymous_proxies``, and unknown values still warn and +fall back to the default. +""" + +import pytest + +from app.config import Config +from app.utils.proxy import registry +from app.utils.proxy.providers.anonymous_proxies import AnonymousProxiesProvider +from app.utils.proxy.providers.custom import CustomProxyProvider + +pytestmark = pytest.mark.unit + + +@pytest.fixture(autouse=True) +def _reset_active_provider(monkeypatch: pytest.MonkeyPatch) -> None: + """Clear the process-wide provider cache so each test resolves fresh.""" + monkeypatch.setattr(registry, "_active_provider", None) + + +def test_resolves_custom(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Config, "PROXY_PROVIDER", "custom") + assert isinstance(registry.get_active_provider(), CustomProxyProvider) + + +def test_resolves_anonymous(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Config, "PROXY_PROVIDER", "anonymous_proxies") + assert isinstance(registry.get_active_provider(), AnonymousProxiesProvider) + + +def test_unknown_falls_back_to_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Config, "PROXY_PROVIDER", "does_not_exist") + assert isinstance(registry.get_active_provider(), AnonymousProxiesProvider)