feat(proxy): integrate Scrapling for enhanced web scraping capabilities

- Replaced Playwright with Scrapling's fetchers in the web crawling and YouTube processing modules for improved performance and flexibility.
- Updated proxy configuration to support dynamic proxy selection via environment variables.
- Enhanced logging to track performance metrics during web scraping operations.
- Refactored related modules to utilize the new proxy utilities and streamline the scraping process.

surfsense_backend/app/utils/proxy/__init__.py

@@ -0,0 +1,40 @@
+"""Modular residential / rotating proxy provider package.
+
+Selects a provider via the ``PROXY_PROVIDER`` env var (see ``registry.py``) and
+exposes proxy settings in the formats different HTTP libraries expect. Add new
+vendors by implementing :class:`ProxyProvider` in ``providers/`` and registering
+them in ``registry.py``.
+"""
+
+from app.utils.proxy.base import ProxyProvider
+from app.utils.proxy.registry import get_active_provider
+
+
+def get_proxy_url() -> str | None:
+    """Canonical ``http://user:pass@host:port`` URL for Scrapling/curl_cffi."""
+    return get_active_provider().get_proxy_url()
+
+
+def get_playwright_proxy() -> dict[str, str] | None:
+    """Playwright-style proxy dict, or ``None`` when not configured."""
+    return get_active_provider().get_playwright_proxy()
+
+
+def get_requests_proxies() -> dict[str, str] | None:
+    """``{"http": ..., "https": ...}`` dict for requests/aiohttp, or ``None``."""
+    return get_active_provider().get_requests_proxies()
+
+
+def get_residential_proxy_url() -> str | None:
+    """Backward-compatible alias for :func:`get_proxy_url`."""
+    return get_proxy_url()
+
+
+__all__ = [
+    "ProxyProvider",
+    "get_active_provider",
+    "get_playwright_proxy",
+    "get_proxy_url",
+    "get_requests_proxies",
+    "get_residential_proxy_url",
+]

surfsense_backend/app/utils/proxy/base.py

@@ -0,0 +1,46 @@
+"""Abstract base class for residential / rotating proxy providers.
+
+Each provider reads its own credentials from the application Config and exposes
+proxy settings in the formats the different HTTP stacks expect:
+
+* ``get_proxy_url`` -> canonical ``http://user:pass@host:port`` string consumed
+  by Scrapling's fetchers (curl_cffi / patchright / camoufox).
+* ``get_requests_proxies`` -> ``{"http": ..., "https": ...}`` dict for
+  ``requests`` / ``aiohttp``.
+* ``get_playwright_proxy`` -> Playwright-style ``{"server", "username",
+  "password"}`` dict.
+
+Add a new vendor by subclassing :class:`ProxyProvider` in ``providers/`` and
+registering it in ``registry.py``.
+"""
+
+from abc import ABC, abstractmethod
+
+
+class ProxyProvider(ABC):
+    """Interface every proxy provider must implement."""
+
+    #: Unique key used to select this provider via the ``PROXY_PROVIDER`` env var.
+    name: str = "base"
+
+    @abstractmethod
+    def get_proxy_url(self) -> str | None:
+        """Return ``http://user:pass@host:port`` (no trailing slash), or ``None``.
+
+        This is the canonical form Scrapling/curl_cffi consume directly.
+        """
+
+    @abstractmethod
+    def get_playwright_proxy(self) -> dict[str, str] | None:
+        """Return a Playwright proxy dict, or ``None`` when not configured."""
+
+    def get_requests_proxies(self) -> dict[str, str] | None:
+        """Return a ``requests``/``aiohttp`` proxies dict, or ``None``.
+
+        Built from :meth:`get_proxy_url` by default; override if a provider needs
+        different http vs https endpoints.
+        """
+        proxy_url = self.get_proxy_url()
+        if proxy_url is None:
+            return None
+        return {"http": proxy_url, "https": proxy_url}

surfsense_backend/app/utils/proxy/providers/__init__.py

@@ -0,0 +1 @@
+"""Concrete proxy provider implementations."""

surfsense_backend/app/utils/proxy/providers/anonymous_proxies.py

@@ -0,0 +1,65 @@
+"""anonymous-proxies.net residential / rotating proxy provider.
+
+The vendor (``rotating.dnsproxifier.com``) encodes the location and rotation
+``type`` options inside a base64-encoded JSON "password". The hostname already
+includes the port (e.g. ``rotating.dnsproxifier.com:31230``).
+"""
+
+import base64
+import json
+import logging
+
+from app.config import Config
+from app.utils.proxy.base import ProxyProvider
+
+logger = logging.getLogger(__name__)
+
+
+class AnonymousProxiesProvider(ProxyProvider):
+    """Provider for anonymous-proxies.net credentials in ``RESIDENTIAL_PROXY_*``."""
+
+    name = "anonymous_proxies"
+
+    def _password_b64(self) -> str | None:
+        """Build the base64-encoded password dict required by the vendor.
+
+        Returns ``None`` when the password is not configured.
+        """
+        password = Config.RESIDENTIAL_PROXY_PASSWORD
+        if not password:
+            return None
+
+        password_dict = {
+            "p": password,
+            "l": Config.RESIDENTIAL_PROXY_LOCATION,
+            "t": Config.RESIDENTIAL_PROXY_TYPE,
+        }
+        return base64.b64encode(
+            json.dumps(password_dict).encode("utf-8")
+        ).decode("utf-8")
+
+    def get_proxy_url(self) -> str | None:
+        username = Config.RESIDENTIAL_PROXY_USERNAME
+        hostname = Config.RESIDENTIAL_PROXY_HOSTNAME
+        password_b64 = self._password_b64()
+
+        if not all([username, hostname, password_b64]):
+            return None
+
+        # No trailing slash: curl_cffi (Scrapling static fetcher) expects a bare
+        # ``http://user:pass@host:port`` URL.
+        return f"http://{username}:{password_b64}@{hostname}"
+
+    def get_playwright_proxy(self) -> dict[str, str] | None:
+        username = Config.RESIDENTIAL_PROXY_USERNAME
+        hostname = Config.RESIDENTIAL_PROXY_HOSTNAME
+        password_b64 = self._password_b64()
+
+        if not all([username, hostname, password_b64]):
+            return None
+
+        return {
+            "server": f"http://{hostname}",
+            "username": username,
+            "password": password_b64,
+        }

surfsense_backend/app/utils/proxy/registry.py

@@ -0,0 +1,44 @@
+"""Proxy provider registry.
+
+Maps the ``PROXY_PROVIDER`` config value to a :class:`ProxyProvider`
+implementation. To add a new vendor: implement a provider in ``providers/`` and
+add a single entry to ``_PROVIDERS`` below - no caller changes required.
+"""
+
+import logging
+
+from app.config import Config
+from app.utils.proxy.base import ProxyProvider
+from app.utils.proxy.providers.anonymous_proxies import AnonymousProxiesProvider
+
+logger = logging.getLogger(__name__)
+
+# Registered proxy providers, keyed by their ``name``.
+_PROVIDERS: dict[str, type[ProxyProvider]] = {
+    AnonymousProxiesProvider.name: AnonymousProxiesProvider,
+}
+
+_DEFAULT_PROVIDER = AnonymousProxiesProvider.name
+
+_active_provider: ProxyProvider | None = None
+
+
+def get_active_provider() -> ProxyProvider:
+    """Return the configured proxy provider instance (cached for the process)."""
+    global _active_provider
+    if _active_provider is not None:
+        return _active_provider
+
+    key = (Config.PROXY_PROVIDER or _DEFAULT_PROVIDER).strip()
+    provider_cls = _PROVIDERS.get(key)
+    if provider_cls is None:
+        logger.warning(
+            "Unknown PROXY_PROVIDER '%s'; falling back to '%s'. Available: %s",
+            key,
+            _DEFAULT_PROVIDER,
+            ", ".join(sorted(_PROVIDERS)),
+        )
+        provider_cls = _PROVIDERS[_DEFAULT_PROVIDER]
+
+    _active_provider = provider_cls()
+    return _active_provider

surfsense_backend/app/utils/proxy_config.py

@@ -1,86 +1,20 @@
-"""
-Residential proxy configuration utility.
+"""Backward-compatible shim for the proxy helpers.
 
-Reads proxy credentials from the application Config and provides helper
-functions that return proxy configs in the format expected by different
-HTTP libraries (requests, httpx, aiohttp, Playwright).
+The implementation now lives in the modular :mod:`app.utils.proxy` package.
+Existing imports of ``app.utils.proxy_config`` keep working via these re-exports.
+Prefer importing from ``app.utils.proxy`` (and ``get_proxy_url``) in new code.
 """
 
-import base64
-import json
-import logging
+from app.utils.proxy import (
+    get_playwright_proxy,
+    get_proxy_url,
+    get_requests_proxies,
+    get_residential_proxy_url,
+)
 
-from app.config import Config
-
-logger = logging.getLogger(__name__)
-
-
-def _build_password_b64() -> str | None:
-    """
-    Build the base64-encoded password dict required by anonymous-proxies.net.
-
-    Returns ``None`` when the required config values are not set.
-    """
-    password = Config.RESIDENTIAL_PROXY_PASSWORD
-    if not password:
-        return None
-
-    password_dict = {
-        "p": password,
-        "l": Config.RESIDENTIAL_PROXY_LOCATION,
-        "t": Config.RESIDENTIAL_PROXY_TYPE,
-    }
-    return base64.b64encode(json.dumps(password_dict).encode("utf-8")).decode("utf-8")
-
-
-def get_residential_proxy_url() -> str | None:
-    """
-    Return the fully-formed residential proxy URL, or ``None`` when not
-    configured.
-
-    The URL format is::
-
-        http://<username>:<base64_password>@<hostname>/
-    """
-    username = Config.RESIDENTIAL_PROXY_USERNAME
-    hostname = Config.RESIDENTIAL_PROXY_HOSTNAME
-    password_b64 = _build_password_b64()
-
-    if not all([username, hostname, password_b64]):
-        return None
-
-    return f"http://{username}:{password_b64}@{hostname}/"
-
-
-def get_requests_proxies() -> dict[str, str] | None:
-    """
-    Return a ``{"http": …, "https": …}`` dict suitable for
-    ``requests.Session.proxies`` and ``aiohttp`` ``proxy=`` kwarg,
-    or ``None`` when not configured.
-    """
-    proxy_url = get_residential_proxy_url()
-    if proxy_url is None:
-        return None
-    return {"http": proxy_url, "https": proxy_url}
-
-
-def get_playwright_proxy() -> dict[str, str] | None:
-    """
-    Return a Playwright-compatible proxy dict::
-
-        {"server": "http://host:port", "username": "…", "password": "…"}
-
-    or ``None`` when not configured.
-    """
-    username = Config.RESIDENTIAL_PROXY_USERNAME
-    hostname = Config.RESIDENTIAL_PROXY_HOSTNAME
-    password_b64 = _build_password_b64()
-
-    if not all([username, hostname, password_b64]):
-        return None
-
-    return {
-        "server": f"http://{hostname}",
-        "username": username,
-        "password": password_b64,
-    }
+__all__ = [
+    "get_playwright_proxy",
+    "get_proxy_url",
+    "get_requests_proxies",
+    "get_residential_proxy_url",
+]