feat(proxy): refactor proxy configuration and add DataImpulse provider

- Updated proxy configuration in `.env.example` files to use `PROXY_URL` and `PROXY_URLS` instead of `CUSTOM_PROXY_URL` and `CUSTOM_PROXY_URLS`.
- Introduced `DataImpulseProvider` for proxy management, replacing the deprecated `AnonymousProxiesProvider`.
- Enhanced documentation to reflect changes in proxy setup and usage.
- Adjusted related code in the proxy registry and configuration files to support the new provider structure.

surfsense_backend/app/utils/proxy/base.py

@@ -15,6 +15,7 @@ registering it in ``registry.py``.
 """
 
 from abc import ABC, abstractmethod
+from urllib.parse import urlsplit
 
 
 class ProxyProvider(ABC):
@@ -27,12 +28,33 @@ class ProxyProvider(ABC):
     def get_proxy_url(self) -> str | None:
         """Return ``http://user:pass@host:port`` (no trailing slash), or ``None``.
 
-        This is the canonical form Scrapling/curl_cffi consume directly.
+        This is the canonical form Scrapling/curl_cffi consume directly, and the
+        single source every provider must supply — the ``requests`` and Playwright
+        shapes below are derived from it.
         """
 
-    @abstractmethod
     def get_playwright_proxy(self) -> dict[str, str] | None:
-        """Return a Playwright proxy dict, or ``None`` when not configured."""
+        """Return a Playwright ``{"server","username","password"}`` dict, or ``None``.
+
+        Parsed from :meth:`get_proxy_url` (the canonical URL) by default, since
+        every provider's credentials already live in that URL. Override only for a
+        vendor whose Playwright form can't be expressed as a parse of the URL.
+        """
+        proxy_url = self.get_proxy_url()
+        if not proxy_url:
+            return None
+        parts = urlsplit(proxy_url)
+        if not parts.hostname:
+            return None
+        server = f"{parts.scheme or 'http'}://{parts.hostname}"
+        if parts.port:
+            server = f"{server}:{parts.port}"
+        proxy: dict[str, str] = {"server": server}
+        if parts.username:
+            proxy["username"] = parts.username
+        if parts.password:
+            proxy["password"] = parts.password
+        return proxy
 
     def get_requests_proxies(self) -> dict[str, str] | None:
         """Return a ``requests``/``aiohttp`` proxies dict, or ``None``.
@@ -45,14 +67,25 @@ class ProxyProvider(ABC):
             return None
         return {"http": proxy_url, "https": proxy_url}
 
+    def get_location(self) -> str:
+        """Return the proxy's configured exit region (e.g. ``"us"``), or ``""``.
+
+        Vendor-agnostic hook the crawler's geoip-match (``03e``) uses to align the
+        browser locale/timezone with the exit IP's country. Default ``""``
+        (unknown) for providers that don't pin a region (e.g. BYO ``custom`` URLs,
+        where the region is baked opaquely into the URL). Override in providers
+        that hold the region as a discrete field.
+        """
+        return ""
+
     @property
     def is_pool_backed(self) -> bool:
         """Whether this provider rotates across a *client-side* pool of endpoints.
 
         ``False`` for single-endpoint providers (including server-side rotating
-        gateways like ``anonymous_proxies``, whose rotation happens upstream).
-        The crawler performs its bounded proxy-error rotation-retry **only** when
-        this is ``True`` — retrying a single static endpoint would just re-hit the
-        same dead proxy.
+        gateways like ``dataimpulse``, whose rotation happens upstream). The
+        crawler performs its bounded proxy-error rotation-retry **only** when this
+        is ``True`` — retrying a single static endpoint would just re-hit the same
+        dead proxy.
         """
         return False

surfsense_backend/app/utils/proxy/providers/anonymous_proxies.py

@@ -1,65 +0,0 @@
-"""anonymous-proxies.net residential / rotating proxy provider.
-
-The vendor (``rotating.dnsproxifier.com``) encodes the location and rotation
-``type`` options inside a base64-encoded JSON "password". The hostname already
-includes the port (e.g. ``rotating.dnsproxifier.com:31230``).
-"""
-
-import base64
-import json
-import logging
-
-from app.config import Config
-from app.utils.proxy.base import ProxyProvider
-
-logger = logging.getLogger(__name__)
-
-
-class AnonymousProxiesProvider(ProxyProvider):
-    """Provider for anonymous-proxies.net credentials in ``RESIDENTIAL_PROXY_*``."""
-
-    name = "anonymous_proxies"
-
-    def _password_b64(self) -> str | None:
-        """Build the base64-encoded password dict required by the vendor.
-
-        Returns ``None`` when the password is not configured.
-        """
-        password = Config.RESIDENTIAL_PROXY_PASSWORD
-        if not password:
-            return None
-
-        password_dict = {
-            "p": password,
-            "l": Config.RESIDENTIAL_PROXY_LOCATION,
-            "t": Config.RESIDENTIAL_PROXY_TYPE,
-        }
-        return base64.b64encode(json.dumps(password_dict).encode("utf-8")).decode(
-            "utf-8"
-        )
-
-    def get_proxy_url(self) -> str | None:
-        username = Config.RESIDENTIAL_PROXY_USERNAME
-        hostname = Config.RESIDENTIAL_PROXY_HOSTNAME
-        password_b64 = self._password_b64()
-
-        if not all([username, hostname, password_b64]):
-            return None
-
-        # No trailing slash: curl_cffi (Scrapling static fetcher) expects a bare
-        # ``http://user:pass@host:port`` URL.
-        return f"http://{username}:{password_b64}@{hostname}"
-
-    def get_playwright_proxy(self) -> dict[str, str] | None:
-        username = Config.RESIDENTIAL_PROXY_USERNAME
-        hostname = Config.RESIDENTIAL_PROXY_HOSTNAME
-        password_b64 = self._password_b64()
-
-        if not all([username, hostname, password_b64]):
-            return None
-
-        return {
-            "server": f"http://{hostname}",
-            "username": username,
-            "password": password_b64,
-        }

surfsense_backend/app/utils/proxy/providers/custom.py

@@ -1,16 +1,15 @@
 """Bring-your-own (BYO) custom proxy provider.
 
-Reads one or more proxy endpoints from env (``CUSTOM_PROXY_URL`` and/or the
-comma-separated ``CUSTOM_PROXY_URLS``). With a single endpoint it behaves like a
-static proxy; with a pool (>1) it rotates client-side via Scrapling's thread-safe
+Reads one or more proxy endpoints from the shared env (``PROXY_URL`` and/or the
+comma-separated ``PROXY_URLS``). With a single endpoint it behaves like a static
+proxy; with a pool (>1) it rotates client-side via Scrapling's thread-safe
 ``ProxyRotator`` (cyclic), transparently to every caller of the zero-arg getters.
 
 No vendor-specific auth assumptions: a user who wants a specific vendor points
-``CUSTOM_PROXY_URLS`` at that vendor's ``http://user:pass@host:port`` endpoints.
+``PROXY_URLS`` at that vendor's ``http://user:pass@host:port`` endpoints.
 """
 
 import logging
-from urllib.parse import urlsplit
 
 from scrapling.engines.toolbelt import ProxyRotator
 
@@ -31,19 +30,19 @@ class CustomProxyProvider(ProxyProvider):
         self._rotator = ProxyRotator(self._urls) if len(self._urls) > 1 else None
         if not self._urls:
             logger.warning(
-                "PROXY_PROVIDER='custom' selected but neither CUSTOM_PROXY_URL nor "
-                "CUSTOM_PROXY_URLS is set; crawls will run without a proxy."
+                "PROXY_PROVIDER='custom' selected but neither PROXY_URL nor "
+                "PROXY_URLS is set; crawls will run without a proxy."
             )
 
     @staticmethod
     def _load_urls() -> list[str]:
         """Collect proxy URLs from env (pool first, then single), de-duplicated."""
         urls: list[str] = []
-        pool = Config.CUSTOM_PROXY_URLS
+        pool = Config.PROXY_URLS
         if pool:
             urls.extend(part.strip() for part in pool.split(",") if part.strip())
 
-        single = (Config.CUSTOM_PROXY_URL or "").strip()
+        single = (Config.PROXY_URL or "").strip()
         if single and single not in urls:
             urls.append(single)
 
@@ -60,23 +59,3 @@ class CustomProxyProvider(ProxyProvider):
             # Advances the cyclic index on every call (thread-safe).
             return self._rotator.get_proxy()
         return self._urls[0]
-
-    def get_playwright_proxy(self) -> dict[str, str] | None:
-        proxy_url = self.get_proxy_url()
-        if not proxy_url:
-            return None
-
-        parts = urlsplit(proxy_url)
-        if not parts.hostname:
-            return None
-
-        server = f"{parts.scheme or 'http'}://{parts.hostname}"
-        if parts.port:
-            server = f"{server}:{parts.port}"
-
-        proxy: dict[str, str] = {"server": server}
-        if parts.username:
-            proxy["username"] = parts.username
-        if parts.password:
-            proxy["password"] = parts.password
-        return proxy

surfsense_backend/app/utils/proxy/providers/dataimpulse.py

@@ -0,0 +1,56 @@
+"""DataImpulse residential / rotating proxy provider.
+
+Takes the shared ``PROXY_URL`` env, exactly like the BYO ``custom`` provider —
+the format is uniform, paste it straight from the vendor dashboard. What makes
+this a *named* provider rather than just ``custom`` is the vendor-specific
+knowledge it layers on top of that URL:
+
+* :meth:`get_location` reads DataImpulse's ``__cr.<country>`` username suffix so
+  the crawler's geoip-match can align the browser locale with the exit country
+  (``custom`` can't — it treats the URL as opaque).
+
+Rotation happens server-side (a fresh exit IP per request on the default pool
+port), so this is NOT :pyattr:`~ProxyProvider.is_pool_backed`.
+
+Example URL::
+
+    http://<token>__cr.us:<password>@gw.dataimpulse.com:823
+
+ponytail: sticky sessions (a stable exit IP across requests) are another
+username suffix (``__sid.<id>``) — the lever the Reddit scraper's README flags as
+a TODO for its ``loid``-per-IP flow. Not built yet: Reddit isn't wired to a
+route, so there's no caller to thread a session id through. Add a
+``get_sticky_proxy_url(session_id)`` here (rewriting the username) when it lands.
+"""
+
+import logging
+from urllib.parse import urlsplit
+
+from app.config import Config
+from app.utils.proxy.base import ProxyProvider
+
+logger = logging.getLogger(__name__)
+
+# DataImpulse encodes country routing as a "__cr.<country>" username suffix; the
+# country token runs until the next "__" param (e.g. "__sid") or the end.
+_COUNTRY_MARKER = "__cr."
+
+
+class DataImpulseProvider(ProxyProvider):
+    """Provider for a DataImpulse proxy URL in the shared ``PROXY_URL`` env."""
+
+    name = "dataimpulse"
+
+    def get_proxy_url(self) -> str | None:
+        url = (Config.PROXY_URL or "").strip()
+        return url or None
+
+    def get_location(self) -> str:
+        """Country parsed from the ``__cr.<country>`` username suffix, or ``""``."""
+        url = self.get_proxy_url()
+        if not url:
+            return ""
+        username = urlsplit(url).username or ""
+        if _COUNTRY_MARKER not in username:
+            return ""
+        return username.split(_COUNTRY_MARKER, 1)[1].split("__", 1)[0].lower()

surfsense_backend/app/utils/proxy/registry.py

@@ -9,18 +9,20 @@ import logging
 
 from app.config import Config
 from app.utils.proxy.base import ProxyProvider
-from app.utils.proxy.providers.anonymous_proxies import AnonymousProxiesProvider
 from app.utils.proxy.providers.custom import CustomProxyProvider
+from app.utils.proxy.providers.dataimpulse import DataImpulseProvider
 
 logger = logging.getLogger(__name__)
 
 # Registered proxy providers, keyed by their ``name``.
 _PROVIDERS: dict[str, type[ProxyProvider]] = {
-    AnonymousProxiesProvider.name: AnonymousProxiesProvider,
     CustomProxyProvider.name: CustomProxyProvider,
+    DataImpulseProvider.name: DataImpulseProvider,
 }
 
-_DEFAULT_PROVIDER = AnonymousProxiesProvider.name
+# BYO ``custom`` is the neutral default: it needs no vendor and returns no proxy
+# until PROXY_URL(S) is set, so an unconfigured install simply runs direct.
+_DEFAULT_PROVIDER = CustomProxyProvider.name
 
 _active_provider: ProxyProvider | None = None