From 701f888b9e2a9181d136c368a3024fa8641555a8 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sat, 4 Jul 2026 17:18:40 -0700 Subject: [PATCH] feat(proxy): refactor proxy configuration and add DataImpulse provider - Updated proxy configuration in `.env.example` files to use `PROXY_URL` and `PROXY_URLS` instead of `CUSTOM_PROXY_URL` and `CUSTOM_PROXY_URLS`. - Introduced `DataImpulseProvider` for proxy management, replacing the deprecated `AnonymousProxiesProvider`. - Enhanced documentation to reflect changes in proxy setup and usage. - Adjusted related code in the proxy registry and configuration files to support the new provider structure. --- docker/.env.example | 15 ++-- surfsense_backend/.env.example | 32 ++++----- surfsense_backend/app/config/__init__.py | 33 ++++----- .../platforms/google_search/README.md | 2 +- .../app/proprietary/scrapers/reddit/README.md | 5 +- .../app/proprietary/scrapers/reddit/fetch.py | 15 ++-- .../proprietary/scrapers/reddit/scraper.py | 29 ++++++-- .../app/proprietary/web_crawler/connector.py | 2 +- .../app/proprietary/web_crawler/stealth.py | 8 ++- surfsense_backend/app/utils/proxy/base.py | 47 ++++++++++-- .../proxy/providers/anonymous_proxies.py | 65 ----------------- .../app/utils/proxy/providers/custom.py | 37 +++------- .../app/utils/proxy/providers/dataimpulse.py | 56 +++++++++++++++ surfsense_backend/app/utils/proxy/registry.py | 8 ++- .../proprietary/web_crawler/test_stealth.py | 19 +++-- .../unit/utils/proxy/test_custom_provider.py | 16 ++--- .../utils/proxy/test_dataimpulse_provider.py | 72 +++++++++++++++++++ .../tests/unit/utils/proxy/test_registry.py | 18 ++--- .../content/docs/how-to/web-search.mdx | 17 +++-- 19 files changed, 303 insertions(+), 193 deletions(-) delete mode 100644 surfsense_backend/app/utils/proxy/providers/anonymous_proxies.py create mode 100644 surfsense_backend/app/utils/proxy/providers/dataimpulse.py create mode 100644 surfsense_backend/tests/unit/utils/proxy/test_dataimpulse_provider.py diff --git a/docker/.env.example b/docker/.env.example index 18142c614..fe57698ab 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -467,12 +467,15 @@ NOLOGIN_MODE_ENABLED=FALSE # Connector indexing lock TTL in seconds (default: 28800 = 8 hours) # CONNECTOR_INDEXING_LOCK_TTL_SECONDS=28800 -# Residential proxy for web crawling -# RESIDENTIAL_PROXY_USERNAME= -# RESIDENTIAL_PROXY_PASSWORD= -# RESIDENTIAL_PROXY_HOSTNAME= -# RESIDENTIAL_PROXY_LOCATION= -# RESIDENTIAL_PROXY_TYPE=1 +# Proxy provider selection: "custom" (default) or "dataimpulse". +# PROXY_PROVIDER=custom + +# Proxy endpoint(s), shared across providers. PROXY_URL is a single full URL used +# by every provider (for "dataimpulse", country is a "__cr." username +# suffix the provider parses for geoip). PROXY_URLS is a comma-separated pool the +# "custom" provider rotates client-side. Leave unset to disable proxying. +# PROXY_URL=http://user:pass@host:port +# PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port # ============================================================================== # DEV / DEPS-ONLY COMPOSE OVERRIDES diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 435782159..f2b3f73b1 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -338,23 +338,21 @@ TURNSTILE_SECRET_KEY= # Proxy provider selection. Selects a ProxyProvider implementation registered in -# app/utils/proxy/registry.py. Default: "anonymous_proxies". Add new vendors there. -# PROXY_PROVIDER=anonymous_proxies +# app/utils/proxy/registry.py. Default: "custom". Options: "custom", "dataimpulse". +# PROXY_PROVIDER=custom -# Residential Proxy Configuration (anonymous-proxies.net) -# Used for web crawling, link previews, and YouTube transcript fetching to avoid IP bans. -# Consumed by the "anonymous_proxies" provider. Leave commented out to disable proxying. -# RESIDENTIAL_PROXY_USERNAME=your_proxy_username -# RESIDENTIAL_PROXY_PASSWORD=your_proxy_password -# RESIDENTIAL_PROXY_HOSTNAME=rotating.dnsproxifier.com:31230 -# RESIDENTIAL_PROXY_LOCATION= -# RESIDENTIAL_PROXY_TYPE=1 - -# Custom (BYO) proxy provider. Activate with PROXY_PROVIDER=custom. -# Provide a single endpoint and/or a comma-separated pool; a pool of >1 rotates -# client-side (cyclic). Each value is a full http://user:pass@host:port URL. -# CUSTOM_PROXY_URL=http://user:pass@host:port -# CUSTOM_PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port +# Proxy endpoint(s), shared across providers — PROXY_PROVIDER picks the behavior, +# not a different env name. Used for web crawling, link previews, and YouTube +# transcript fetching to avoid IP bans. Leave unset to disable proxying. +# +# PROXY_URL: a single full http://user:pass@host:port endpoint (used by every +# provider). For "dataimpulse", country is a "__cr." username suffix the +# provider parses for geoip-match, e.g.: +# PROXY_URL=http://your_token__cr.us:your_password@gw.dataimpulse.com:823 +# PROXY_URLS: a comma-separated pool the "custom" provider rotates client-side +# (cyclic); server-side-rotating gateways ignore it. +# PROXY_URL=http://user:pass@host:port +# PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port # ===================================================================== # Captcha solving (Phase 3d) — LAST-resort bypass tier via captchatools. @@ -380,7 +378,7 @@ TURNSTILE_SECRET_KEY= # Stealth hardening (Phase 3e, Slice A) — runtime/config-level levers on the # stealth browser tier. Consumed by app/proprietary/web_crawler/stealth.py. # Defaults add no crawl-speed regression and preserve today's behavior. -# Match browser locale/timezone to RESIDENTIAL_PROXY_LOCATION (no exit-IP lookup). +# Match browser locale/timezone to the proxy provider's exit region (no exit-IP lookup). # CRAWL_GEOIP_MATCH_ENABLED=FALSE # WebRTC respects the proxy (anti local-IP leak). Cheap + safe. # CRAWL_BLOCK_WEBRTC=TRUE diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index b0d37ea39..89790c952 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -1041,24 +1041,17 @@ class Config: # Proxy provider selection. Maps to a ProxyProvider implementation registered # in app/utils/proxy/registry.py. Add new vendors there and switch via this var. - PROXY_PROVIDER = os.getenv("PROXY_PROVIDER", "anonymous_proxies") + PROXY_PROVIDER = os.getenv("PROXY_PROVIDER", "custom") - # Residential Proxy Configuration (anonymous-proxies.net) - # Used for web crawling and YouTube transcript fetching to avoid IP bans. - # Consumed by the "anonymous_proxies" proxy provider. - RESIDENTIAL_PROXY_USERNAME = os.getenv("RESIDENTIAL_PROXY_USERNAME") - RESIDENTIAL_PROXY_PASSWORD = os.getenv("RESIDENTIAL_PROXY_PASSWORD") - RESIDENTIAL_PROXY_HOSTNAME = os.getenv("RESIDENTIAL_PROXY_HOSTNAME") - RESIDENTIAL_PROXY_LOCATION = os.getenv("RESIDENTIAL_PROXY_LOCATION", "") - RESIDENTIAL_PROXY_TYPE = int(os.getenv("RESIDENTIAL_PROXY_TYPE", "1")) - - # Custom (BYO) proxy provider, selected via PROXY_PROVIDER="custom". - # Consumed by the "custom" provider (app/utils/proxy/providers/custom.py). - # Provide a single endpoint (CUSTOM_PROXY_URL) and/or a comma-separated pool - # (CUSTOM_PROXY_URLS); a pool of >1 rotates client-side via Scrapling's - # ProxyRotator. Each value is a full http://user:pass@host:port URL. - CUSTOM_PROXY_URL = os.getenv("CUSTOM_PROXY_URL") - CUSTOM_PROXY_URLS = os.getenv("CUSTOM_PROXY_URLS") + # Proxy endpoint(s), shared across all providers — PROXY_PROVIDER selects how + # they're interpreted, not a different env name. PROXY_URL is a single full + # http://user:pass@host:port endpoint (used by every provider); e.g. DataImpulse + # encodes country as a "__cr." username suffix that its provider parses + # for geoip-match. PROXY_URLS is a comma-separated pool that the "custom" provider + # rotates client-side (server-side-rotating gateways ignore it). Leave unset to + # disable proxying. + PROXY_URL = os.getenv("PROXY_URL") + PROXY_URLS = os.getenv("PROXY_URLS") # ===================================================================== # Phase 3d — Captcha solving (reCAPTCHA v2/v3, hCaptcha) via captchatools. @@ -1097,9 +1090,9 @@ class Config: # (no test-vs-prod drift). Defaults preserve today's behavior / # introduce no crawl-speed regression. See plans/backend/03e-stealth-hardening.md. # ===================================================================== - # Map the configured proxy region (RESIDENTIAL_PROXY_LOCATION) -> browser - # locale/timezone so the fingerprint coheres with the proxy exit geo. No - # exit-IP lookup (zero added latency); unknown/empty region => skip. + # Map the active proxy provider's exit region (ProxyProvider.get_location()) + # -> browser locale/timezone so the fingerprint coheres with the proxy exit + # geo. No exit-IP lookup (zero added latency); unknown/empty region => skip. CRAWL_GEOIP_MATCH_ENABLED = ( os.getenv("CRAWL_GEOIP_MATCH_ENABLED", "FALSE").upper() == "TRUE" ) diff --git a/surfsense_backend/app/proprietary/platforms/google_search/README.md b/surfsense_backend/app/proprietary/platforms/google_search/README.md index e1c1861bf..726413194 100644 --- a/surfsense_backend/app/proprietary/platforms/google_search/README.md +++ b/surfsense_backend/app/proprietary/platforms/google_search/README.md @@ -40,7 +40,7 @@ results only materialize after the page's JS runs. So `fetch.py` needs both a A warm fetch (browser up, IP cached) runs ~8 s; the first fetch of a process also pays the ~5 s Chromium launch and a vetting round. Requires the browser tier (patchright Chromium via Scrapling's `AsyncStealthySession`) and a -residential proxy — set `PROXY_PROVIDER=custom` + `CUSTOM_PROXY_URL` (see +residential proxy — set `PROXY_PROVIDER` + `PROXY_URL` (see `.env`). Long-running callers can `await fetch.close_sessions()` on shutdown; scripts that exit anyway can skip it. diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/README.md b/surfsense_backend/app/proprietary/scrapers/reddit/README.md index 58c601cf5..4ed352393 100644 --- a/surfsense_backend/app/proprietary/scrapers/reddit/README.md +++ b/surfsense_backend/app/proprietary/scrapers/reddit/README.md @@ -77,8 +77,9 @@ cd surfsense_backend ## TODO / out of scope (v1) - Sticky-IP provider support: the fetch layer assumes a sticky exit IP per - session (the `loid` binds to it). `anonymous_proxies` does not yet set the - `"si"` sticky key — add it (proxy layer) before high-volume production use. + session (the `loid` binds to it). The `dataimpulse` provider does not yet emit + a sticky `__sid.` username suffix — add it (proxy layer) before high-volume + production use. - `/api/morechildren` deep-comment expansion — `more` stubs terminate the tree walk today. - Routes / `connector_service.py` / ingestion / Celery wiring. diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py b/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py index ce45795e7..41193a2c9 100644 --- a/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py +++ b/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py @@ -72,10 +72,17 @@ _MAX_ROTATIONS = 3 _MAX_BACKOFFS = 4 _BACKOFF_BASE_S = 5.0 -# Reddit 429s aggressively (~60-100 req/min/IP). Pace each sticky session to -# ~1 req/s + jitter to stay under the per-IP threshold. -_MIN_INTERVAL_S = 1.0 -_PACE_JITTER_S = 0.5 +# Reddit 429s aggressively (~60-100 req/min/IP). Pace each sticky session so a +# fast IP can't burst past the per-IP threshold. A live probe (12 rapid fetches +# on one sticky IP => 0x429; scripts/_bench_reddit2.py) showed the natural +# ~0.85s request latency already holds a session near ~1 req/s, so the old 1.0s +# floor just ADDED ~0.4s of dead sleep to every fetch. A 0.5s floor is a no-op +# for typical fetches yet still caps a fast IP at ~2 req/s; the 429 backoff below +# is the real safety net if an IP's limit is tighter. +# ponytail: 0.5s is tuned to dataimpulse residential exits. A pool with a +# stricter per-IP cap may need it raised — watch for 429 log spam and bump it. +_MIN_INTERVAL_S = 0.5 +_PACE_JITTER_S = 0.25 _HEADERS = {"Accept-Language": "en-US,en;q=0.9"} diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py b/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py index e97cda024..6fe97877a 100644 --- a/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py +++ b/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py @@ -56,6 +56,14 @@ _LISTING_LIMIT = 100 _MAX_PAGES = 10 _EMPTY_STREAK_ABORT = 2 +# A subreddit's per-post comment fetches are independent (each is a separate +# .json), so after paging the listing on one sticky IP we fan them across their +# own warm sessions instead of walking them sequentially — the dominant cost of +# a subreddit+comments scrape (~3.6x on the comment phase; scripts/_bench_reddit2). +# Kept below the top-level fan-out width: with N concurrent subreddit targets the +# worst case is N x this many proxy IPs, so this bounds that multiplication. +_COMMENT_CONCURRENCY = 8 + # Search sorts differ from listing sorts; fall back to "new" for a listing path # when the input carries a search-only sort. _LISTING_SORTS = frozenset({"hot", "new", "top", "rising", "controversial", "best"}) @@ -227,6 +235,7 @@ async def _subreddit_flow( if sort == "top" and input_model.time: params["t"] = input_model.time + post_ids: list[str] = [] async for data in _paginate_listing( f"r/{subreddit}/{sort}", params, @@ -238,14 +247,24 @@ async def _subreddit_flow( item = _emit(parse_post(data), include_nsfw=input_model.includeNSFW) if item is not None: yield item - if not input_model.skipComments and isinstance(data.get("id"), str): - async for comment in _post_flow( - data["id"], + # Collect ids now; fetch the comment trees in parallel below. Walking + # them here would serialize one .json per post on this single sticky IP. + parsed_id = data.get("id") + if not input_model.skipComments and isinstance(parsed_id, str): + post_ids.append(parsed_id) + + if post_ids: + comment_jobs = [ + _post_flow( + pid, input_model=input_model, subreddit=subreddit, include_post=False, - ): - yield comment + ) + for pid in post_ids + ] + async for comment in fan_out(comment_jobs, concurrency=_COMMENT_CONCURRENCY): + yield comment async def _user_flow( diff --git a/surfsense_backend/app/proprietary/web_crawler/connector.py b/surfsense_backend/app/proprietary/web_crawler/connector.py index 061cc1b00..e393ff5b4 100644 --- a/surfsense_backend/app/proprietary/web_crawler/connector.py +++ b/surfsense_backend/app/proprietary/web_crawler/connector.py @@ -273,7 +273,7 @@ class WebCrawlerConnector: endpoint on every ``get_proxy_url()`` call, so simply re-invoking the tier rotates the proxy. Bounded to a single extra attempt per tier — no unbounded fan-out on billable crawls. Single-endpoint providers - (including the server-side-rotating ``anonymous_proxies``) skip the retry + (including the server-side-rotating ``dataimpulse``) skip the retry entirely (``is_pool_backed()`` is ``False``), since retrying the same static endpoint would just re-hit the same dead proxy. Non-proxy errors (and ``NotImplementedError`` from the browser tiers) propagate unchanged diff --git a/surfsense_backend/app/proprietary/web_crawler/stealth.py b/surfsense_backend/app/proprietary/web_crawler/stealth.py index 25b48d69b..5b0f469ff 100644 --- a/surfsense_backend/app/proprietary/web_crawler/stealth.py +++ b/surfsense_backend/app/proprietary/web_crawler/stealth.py @@ -27,6 +27,7 @@ from dataclasses import dataclass from typing import Any from app.config import config +from app.utils.proxy import get_active_provider # Coarse region -> (locale, IANA timezone) map. Country granularity only: per the # 03e "Geoip accuracy" risk, wrong-but-coherent beats a default mismatch and @@ -131,8 +132,9 @@ def location_to_locale_timezone( ) -> tuple[str | None, str | None]: """Map a free-form proxy region string -> (locale, timezone_id). - ``location`` is the vendor-specific ``RESIDENTIAL_PROXY_LOCATION`` value - (``03b``). Best-effort: matches an ISO-3166 alpha-2 code (e.g. ``"us"``) or a + ``location`` is the active proxy provider's exit region + (``ProxyProvider.get_location()``). Best-effort: matches an ISO-3166 alpha-2 + code (e.g. ``"us"``) or a common country name (e.g. ``"Germany"``); only the leading token is honored, so vendor strings like ``"us:nyc"`` or ``"de region"`` still resolve. Returns ``(None, None)`` for empty/unknown input (caller then leaves the browser @@ -181,7 +183,7 @@ def get_stealth_config() -> StealthConfig: """Build a :class:`StealthConfig` from the current process config/env.""" return StealthConfig( geoip_match_enabled=config.CRAWL_GEOIP_MATCH_ENABLED, - proxy_location=config.RESIDENTIAL_PROXY_LOCATION or "", + proxy_location=get_active_provider().get_location(), block_webrtc=config.CRAWL_BLOCK_WEBRTC, hide_canvas=config.CRAWL_HIDE_CANVAS, google_search=config.CRAWL_GOOGLE_SEARCH_REFERER, diff --git a/surfsense_backend/app/utils/proxy/base.py b/surfsense_backend/app/utils/proxy/base.py index bc6eb5041..ed67cd384 100644 --- a/surfsense_backend/app/utils/proxy/base.py +++ b/surfsense_backend/app/utils/proxy/base.py @@ -15,6 +15,7 @@ registering it in ``registry.py``. """ from abc import ABC, abstractmethod +from urllib.parse import urlsplit class ProxyProvider(ABC): @@ -27,12 +28,33 @@ class ProxyProvider(ABC): def get_proxy_url(self) -> str | None: """Return ``http://user:pass@host:port`` (no trailing slash), or ``None``. - This is the canonical form Scrapling/curl_cffi consume directly. + This is the canonical form Scrapling/curl_cffi consume directly, and the + single source every provider must supply — the ``requests`` and Playwright + shapes below are derived from it. """ - @abstractmethod def get_playwright_proxy(self) -> dict[str, str] | None: - """Return a Playwright proxy dict, or ``None`` when not configured.""" + """Return a Playwright ``{"server","username","password"}`` dict, or ``None``. + + Parsed from :meth:`get_proxy_url` (the canonical URL) by default, since + every provider's credentials already live in that URL. Override only for a + vendor whose Playwright form can't be expressed as a parse of the URL. + """ + proxy_url = self.get_proxy_url() + if not proxy_url: + return None + parts = urlsplit(proxy_url) + if not parts.hostname: + return None + server = f"{parts.scheme or 'http'}://{parts.hostname}" + if parts.port: + server = f"{server}:{parts.port}" + proxy: dict[str, str] = {"server": server} + if parts.username: + proxy["username"] = parts.username + if parts.password: + proxy["password"] = parts.password + return proxy def get_requests_proxies(self) -> dict[str, str] | None: """Return a ``requests``/``aiohttp`` proxies dict, or ``None``. @@ -45,14 +67,25 @@ class ProxyProvider(ABC): return None return {"http": proxy_url, "https": proxy_url} + def get_location(self) -> str: + """Return the proxy's configured exit region (e.g. ``"us"``), or ``""``. + + Vendor-agnostic hook the crawler's geoip-match (``03e``) uses to align the + browser locale/timezone with the exit IP's country. Default ``""`` + (unknown) for providers that don't pin a region (e.g. BYO ``custom`` URLs, + where the region is baked opaquely into the URL). Override in providers + that hold the region as a discrete field. + """ + return "" + @property def is_pool_backed(self) -> bool: """Whether this provider rotates across a *client-side* pool of endpoints. ``False`` for single-endpoint providers (including server-side rotating - gateways like ``anonymous_proxies``, whose rotation happens upstream). - The crawler performs its bounded proxy-error rotation-retry **only** when - this is ``True`` — retrying a single static endpoint would just re-hit the - same dead proxy. + gateways like ``dataimpulse``, whose rotation happens upstream). The + crawler performs its bounded proxy-error rotation-retry **only** when this + is ``True`` — retrying a single static endpoint would just re-hit the same + dead proxy. """ return False diff --git a/surfsense_backend/app/utils/proxy/providers/anonymous_proxies.py b/surfsense_backend/app/utils/proxy/providers/anonymous_proxies.py deleted file mode 100644 index a005a9e72..000000000 --- a/surfsense_backend/app/utils/proxy/providers/anonymous_proxies.py +++ /dev/null @@ -1,65 +0,0 @@ -"""anonymous-proxies.net residential / rotating proxy provider. - -The vendor (``rotating.dnsproxifier.com``) encodes the location and rotation -``type`` options inside a base64-encoded JSON "password". The hostname already -includes the port (e.g. ``rotating.dnsproxifier.com:31230``). -""" - -import base64 -import json -import logging - -from app.config import Config -from app.utils.proxy.base import ProxyProvider - -logger = logging.getLogger(__name__) - - -class AnonymousProxiesProvider(ProxyProvider): - """Provider for anonymous-proxies.net credentials in ``RESIDENTIAL_PROXY_*``.""" - - name = "anonymous_proxies" - - def _password_b64(self) -> str | None: - """Build the base64-encoded password dict required by the vendor. - - Returns ``None`` when the password is not configured. - """ - password = Config.RESIDENTIAL_PROXY_PASSWORD - if not password: - return None - - password_dict = { - "p": password, - "l": Config.RESIDENTIAL_PROXY_LOCATION, - "t": Config.RESIDENTIAL_PROXY_TYPE, - } - return base64.b64encode(json.dumps(password_dict).encode("utf-8")).decode( - "utf-8" - ) - - def get_proxy_url(self) -> str | None: - username = Config.RESIDENTIAL_PROXY_USERNAME - hostname = Config.RESIDENTIAL_PROXY_HOSTNAME - password_b64 = self._password_b64() - - if not all([username, hostname, password_b64]): - return None - - # No trailing slash: curl_cffi (Scrapling static fetcher) expects a bare - # ``http://user:pass@host:port`` URL. - return f"http://{username}:{password_b64}@{hostname}" - - def get_playwright_proxy(self) -> dict[str, str] | None: - username = Config.RESIDENTIAL_PROXY_USERNAME - hostname = Config.RESIDENTIAL_PROXY_HOSTNAME - password_b64 = self._password_b64() - - if not all([username, hostname, password_b64]): - return None - - return { - "server": f"http://{hostname}", - "username": username, - "password": password_b64, - } diff --git a/surfsense_backend/app/utils/proxy/providers/custom.py b/surfsense_backend/app/utils/proxy/providers/custom.py index 16d87a9a7..3dc55128e 100644 --- a/surfsense_backend/app/utils/proxy/providers/custom.py +++ b/surfsense_backend/app/utils/proxy/providers/custom.py @@ -1,16 +1,15 @@ """Bring-your-own (BYO) custom proxy provider. -Reads one or more proxy endpoints from env (``CUSTOM_PROXY_URL`` and/or the -comma-separated ``CUSTOM_PROXY_URLS``). With a single endpoint it behaves like a -static proxy; with a pool (>1) it rotates client-side via Scrapling's thread-safe +Reads one or more proxy endpoints from the shared env (``PROXY_URL`` and/or the +comma-separated ``PROXY_URLS``). With a single endpoint it behaves like a static +proxy; with a pool (>1) it rotates client-side via Scrapling's thread-safe ``ProxyRotator`` (cyclic), transparently to every caller of the zero-arg getters. No vendor-specific auth assumptions: a user who wants a specific vendor points -``CUSTOM_PROXY_URLS`` at that vendor's ``http://user:pass@host:port`` endpoints. +``PROXY_URLS`` at that vendor's ``http://user:pass@host:port`` endpoints. """ import logging -from urllib.parse import urlsplit from scrapling.engines.toolbelt import ProxyRotator @@ -31,19 +30,19 @@ class CustomProxyProvider(ProxyProvider): self._rotator = ProxyRotator(self._urls) if len(self._urls) > 1 else None if not self._urls: logger.warning( - "PROXY_PROVIDER='custom' selected but neither CUSTOM_PROXY_URL nor " - "CUSTOM_PROXY_URLS is set; crawls will run without a proxy." + "PROXY_PROVIDER='custom' selected but neither PROXY_URL nor " + "PROXY_URLS is set; crawls will run without a proxy." ) @staticmethod def _load_urls() -> list[str]: """Collect proxy URLs from env (pool first, then single), de-duplicated.""" urls: list[str] = [] - pool = Config.CUSTOM_PROXY_URLS + pool = Config.PROXY_URLS if pool: urls.extend(part.strip() for part in pool.split(",") if part.strip()) - single = (Config.CUSTOM_PROXY_URL or "").strip() + single = (Config.PROXY_URL or "").strip() if single and single not in urls: urls.append(single) @@ -60,23 +59,3 @@ class CustomProxyProvider(ProxyProvider): # Advances the cyclic index on every call (thread-safe). return self._rotator.get_proxy() return self._urls[0] - - def get_playwright_proxy(self) -> dict[str, str] | None: - proxy_url = self.get_proxy_url() - if not proxy_url: - return None - - parts = urlsplit(proxy_url) - if not parts.hostname: - return None - - server = f"{parts.scheme or 'http'}://{parts.hostname}" - if parts.port: - server = f"{server}:{parts.port}" - - proxy: dict[str, str] = {"server": server} - if parts.username: - proxy["username"] = parts.username - if parts.password: - proxy["password"] = parts.password - return proxy diff --git a/surfsense_backend/app/utils/proxy/providers/dataimpulse.py b/surfsense_backend/app/utils/proxy/providers/dataimpulse.py new file mode 100644 index 000000000..17d09325f --- /dev/null +++ b/surfsense_backend/app/utils/proxy/providers/dataimpulse.py @@ -0,0 +1,56 @@ +"""DataImpulse residential / rotating proxy provider. + +Takes the shared ``PROXY_URL`` env, exactly like the BYO ``custom`` provider — +the format is uniform, paste it straight from the vendor dashboard. What makes +this a *named* provider rather than just ``custom`` is the vendor-specific +knowledge it layers on top of that URL: + +* :meth:`get_location` reads DataImpulse's ``__cr.`` username suffix so + the crawler's geoip-match can align the browser locale with the exit country + (``custom`` can't — it treats the URL as opaque). + +Rotation happens server-side (a fresh exit IP per request on the default pool +port), so this is NOT :pyattr:`~ProxyProvider.is_pool_backed`. + +Example URL:: + + http://__cr.us:@gw.dataimpulse.com:823 + +ponytail: sticky sessions (a stable exit IP across requests) are another +username suffix (``__sid.``) — the lever the Reddit scraper's README flags as +a TODO for its ``loid``-per-IP flow. Not built yet: Reddit isn't wired to a +route, so there's no caller to thread a session id through. Add a +``get_sticky_proxy_url(session_id)`` here (rewriting the username) when it lands. +""" + +import logging +from urllib.parse import urlsplit + +from app.config import Config +from app.utils.proxy.base import ProxyProvider + +logger = logging.getLogger(__name__) + +# DataImpulse encodes country routing as a "__cr." username suffix; the +# country token runs until the next "__" param (e.g. "__sid") or the end. +_COUNTRY_MARKER = "__cr." + + +class DataImpulseProvider(ProxyProvider): + """Provider for a DataImpulse proxy URL in the shared ``PROXY_URL`` env.""" + + name = "dataimpulse" + + def get_proxy_url(self) -> str | None: + url = (Config.PROXY_URL or "").strip() + return url or None + + def get_location(self) -> str: + """Country parsed from the ``__cr.`` username suffix, or ``""``.""" + url = self.get_proxy_url() + if not url: + return "" + username = urlsplit(url).username or "" + if _COUNTRY_MARKER not in username: + return "" + return username.split(_COUNTRY_MARKER, 1)[1].split("__", 1)[0].lower() diff --git a/surfsense_backend/app/utils/proxy/registry.py b/surfsense_backend/app/utils/proxy/registry.py index 7f1feb70d..b222af83c 100644 --- a/surfsense_backend/app/utils/proxy/registry.py +++ b/surfsense_backend/app/utils/proxy/registry.py @@ -9,18 +9,20 @@ import logging from app.config import Config from app.utils.proxy.base import ProxyProvider -from app.utils.proxy.providers.anonymous_proxies import AnonymousProxiesProvider from app.utils.proxy.providers.custom import CustomProxyProvider +from app.utils.proxy.providers.dataimpulse import DataImpulseProvider logger = logging.getLogger(__name__) # Registered proxy providers, keyed by their ``name``. _PROVIDERS: dict[str, type[ProxyProvider]] = { - AnonymousProxiesProvider.name: AnonymousProxiesProvider, CustomProxyProvider.name: CustomProxyProvider, + DataImpulseProvider.name: DataImpulseProvider, } -_DEFAULT_PROVIDER = AnonymousProxiesProvider.name +# BYO ``custom`` is the neutral default: it needs no vendor and returns no proxy +# until PROXY_URL(S) is set, so an unconfigured install simply runs direct. +_DEFAULT_PROVIDER = CustomProxyProvider.name _active_provider: ProxyProvider | None = None diff --git a/surfsense_backend/tests/unit/proprietary/web_crawler/test_stealth.py b/surfsense_backend/tests/unit/proprietary/web_crawler/test_stealth.py index 951fcd614..0aa17ef3a 100644 --- a/surfsense_backend/tests/unit/proprietary/web_crawler/test_stealth.py +++ b/surfsense_backend/tests/unit/proprietary/web_crawler/test_stealth.py @@ -3,6 +3,7 @@ import pytest from app.config import config +from app.proprietary.web_crawler import stealth from app.proprietary.web_crawler.stealth import ( build_stealthy_kwargs, get_stealth_config, @@ -12,6 +13,16 @@ from app.proprietary.web_crawler.stealth import ( pytestmark = pytest.mark.unit +def _set_proxy_location(monkeypatch: pytest.MonkeyPatch, location: str) -> None: + """Point get_stealth_config at a stub provider with the given exit region.""" + + class _StubProvider: + def get_location(self) -> str: + return location + + monkeypatch.setattr(stealth, "get_active_provider", lambda: _StubProvider()) + + class TestLocationToLocaleTimezone: def test_alpha2_code(self): assert location_to_locale_timezone("us") == ("en-US", "America/New_York") @@ -58,7 +69,7 @@ class TestBuildStealthyKwargs: monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", False) monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", True) monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", False) - monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "us") + _set_proxy_location(monkeypatch, "us") kwargs = build_stealthy_kwargs(get_stealth_config()) @@ -77,7 +88,7 @@ class TestBuildStealthyKwargs: monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", True) monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", False) monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", True) - monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "") + _set_proxy_location(monkeypatch, "") kwargs = build_stealthy_kwargs(get_stealth_config()) @@ -92,7 +103,7 @@ class TestBuildStealthyKwargs: monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", False) monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", True) monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", False) - monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "de") + _set_proxy_location(monkeypatch, "de") kwargs = build_stealthy_kwargs(get_stealth_config()) @@ -105,7 +116,7 @@ class TestBuildStealthyKwargs: monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", False) monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", True) monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", False) - monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "atlantis") + _set_proxy_location(monkeypatch, "atlantis") kwargs = build_stealthy_kwargs(get_stealth_config()) diff --git a/surfsense_backend/tests/unit/utils/proxy/test_custom_provider.py b/surfsense_backend/tests/unit/utils/proxy/test_custom_provider.py index 3ab244376..c5563290d 100644 --- a/surfsense_backend/tests/unit/utils/proxy/test_custom_provider.py +++ b/surfsense_backend/tests/unit/utils/proxy/test_custom_provider.py @@ -15,12 +15,12 @@ pytestmark = pytest.mark.unit @pytest.fixture(autouse=True) def _clear_proxy_env(monkeypatch: pytest.MonkeyPatch) -> None: """Default both knobs to unset; individual tests set what they need.""" - monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", None) - monkeypatch.setattr(Config, "CUSTOM_PROXY_URLS", None) + monkeypatch.setattr(Config, "PROXY_URL", None) + monkeypatch.setattr(Config, "PROXY_URLS", None) def test_single_url_is_static(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", "http://u:p@host:8080") + monkeypatch.setattr(Config, "PROXY_URL", "http://u:p@host:8080") provider = CustomProxyProvider() assert provider.is_pool_backed is False @@ -32,7 +32,7 @@ def test_single_url_is_static(monkeypatch: pytest.MonkeyPatch) -> None: def test_pool_rotates_cyclically(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( Config, - "CUSTOM_PROXY_URLS", + "PROXY_URLS", "http://a:1@h1:8080, http://b:2@h2:8080 ,http://c:3@h3:8080", ) provider = CustomProxyProvider() @@ -48,9 +48,9 @@ def test_pool_rotates_cyclically(monkeypatch: pytest.MonkeyPatch) -> None: def test_single_plus_pool_dedupes(monkeypatch: pytest.MonkeyPatch) -> None: - """A URL present in both CUSTOM_PROXY_URL and the pool is not duplicated.""" - monkeypatch.setattr(Config, "CUSTOM_PROXY_URLS", "http://a@h1:80,http://b@h2:80") - monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", "http://a@h1:80") + """A URL present in both PROXY_URL and the pool is not duplicated.""" + monkeypatch.setattr(Config, "PROXY_URLS", "http://a@h1:80,http://b@h2:80") + monkeypatch.setattr(Config, "PROXY_URL", "http://a@h1:80") provider = CustomProxyProvider() assert provider.is_pool_backed is True @@ -69,7 +69,7 @@ def test_unconfigured_returns_none() -> None: def test_playwright_proxy_parses_credentials( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", "http://user:pass@host:8080") + monkeypatch.setattr(Config, "PROXY_URL", "http://user:pass@host:8080") provider = CustomProxyProvider() assert provider.get_playwright_proxy() == { diff --git a/surfsense_backend/tests/unit/utils/proxy/test_dataimpulse_provider.py b/surfsense_backend/tests/unit/utils/proxy/test_dataimpulse_provider.py new file mode 100644 index 000000000..e2d96e0e1 --- /dev/null +++ b/surfsense_backend/tests/unit/utils/proxy/test_dataimpulse_provider.py @@ -0,0 +1,72 @@ +"""Unit tests for the DataImpulseProvider. + +Takes a single full URL (like ``custom``); the vendor-specific bit is parsing the +``__cr.`` username suffix for :meth:`get_location`. Playwright/requests +shapes come from the shared base parse, so a couple of checks cover the wiring. +""" + +import pytest + +from app.config import Config +from app.utils.proxy.providers.dataimpulse import DataImpulseProvider + +pytestmark = pytest.mark.unit + +_URL = "http://tok123__cr.us:secret@gw.dataimpulse.com:823" + + +@pytest.fixture(autouse=True) +def _clear_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Config, "PROXY_URL", None) + + +def test_returns_configured_url(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Config, "PROXY_URL", _URL) + provider = DataImpulseProvider() + + assert provider.is_pool_backed is False + assert provider.get_proxy_url() == _URL + assert provider.get_requests_proxies() == {"http": _URL, "https": _URL} + + +def test_location_parsed_from_country_suffix(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Config, "PROXY_URL", _URL) + assert DataImpulseProvider().get_location() == "us" + + +def test_location_stops_at_next_param(monkeypatch: pytest.MonkeyPatch) -> None: + # A sticky/session suffix after the country must not bleed into the location. + monkeypatch.setattr( + Config, + "PROXY_URL", + "http://tok__cr.de__sid.abc123:secret@gw.dataimpulse.com:823", + ) + assert DataImpulseProvider().get_location() == "de" + + +def test_no_country_suffix_yields_empty_location( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + Config, "PROXY_URL", "http://tok:secret@gw.dataimpulse.com:823" + ) + assert DataImpulseProvider().get_location() == "" + + +def test_unconfigured_returns_none() -> None: + provider = DataImpulseProvider() + + assert provider.get_proxy_url() is None + assert provider.get_requests_proxies() is None + assert provider.get_playwright_proxy() is None + assert provider.get_location() == "" + + +def test_playwright_proxy_from_base_parse(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Config, "PROXY_URL", _URL) + + assert DataImpulseProvider().get_playwright_proxy() == { + "server": "http://gw.dataimpulse.com:823", + "username": "tok123__cr.us", + "password": "secret", + } diff --git a/surfsense_backend/tests/unit/utils/proxy/test_registry.py b/surfsense_backend/tests/unit/utils/proxy/test_registry.py index c67a286ba..5a6bde403 100644 --- a/surfsense_backend/tests/unit/utils/proxy/test_registry.py +++ b/surfsense_backend/tests/unit/utils/proxy/test_registry.py @@ -1,16 +1,16 @@ -"""Unit tests for proxy provider selection (Phase 3b). +"""Unit tests for proxy provider selection. -``PROXY_PROVIDER`` selects the single app-wide provider; ``custom`` is now -registered alongside ``anonymous_proxies``, and unknown values still warn and -fall back to the default. +``PROXY_PROVIDER`` selects the single app-wide provider; ``custom`` (the default) +and ``dataimpulse`` are registered, and unknown values still warn and fall back +to the default. """ import pytest from app.config import Config from app.utils.proxy import registry -from app.utils.proxy.providers.anonymous_proxies import AnonymousProxiesProvider from app.utils.proxy.providers.custom import CustomProxyProvider +from app.utils.proxy.providers.dataimpulse import DataImpulseProvider pytestmark = pytest.mark.unit @@ -26,11 +26,11 @@ def test_resolves_custom(monkeypatch: pytest.MonkeyPatch) -> None: assert isinstance(registry.get_active_provider(), CustomProxyProvider) -def test_resolves_anonymous(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(Config, "PROXY_PROVIDER", "anonymous_proxies") - assert isinstance(registry.get_active_provider(), AnonymousProxiesProvider) +def test_resolves_dataimpulse(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Config, "PROXY_PROVIDER", "dataimpulse") + assert isinstance(registry.get_active_provider(), DataImpulseProvider) def test_unknown_falls_back_to_default(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(Config, "PROXY_PROVIDER", "does_not_exist") - assert isinstance(registry.get_active_provider(), AnonymousProxiesProvider) + assert isinstance(registry.get_active_provider(), CustomProxyProvider) diff --git a/surfsense_web/content/docs/how-to/web-search.mdx b/surfsense_web/content/docs/how-to/web-search.mdx index cbea11d36..cb8007f52 100644 --- a/surfsense_web/content/docs/how-to/web-search.mdx +++ b/surfsense_web/content/docs/how-to/web-search.mdx @@ -110,17 +110,16 @@ In production, search engines may rate-limit or block your server's IP. To avoid ### Step 1: Build the Proxy URL -SurfSense uses [anonymous-proxies.net](https://anonymous-proxies.net/) style residential proxies where the password is a base64-encoded JSON object. Build the URL using your proxy credentials: - -```bash -# Encode the password (replace with your actual values) -echo -n '{"p": "YOUR_PASSWORD", "l": "LOCATION", "t": PROXY_TYPE}' | base64 -``` - -The full proxy URL format is: +Use any residential proxy that exposes a standard `http://user:pass@host:port` endpoint. For example, [DataImpulse](https://dataimpulse.com/) encodes country routing as a username suffix (`__cr.`): ``` -http://:@:/ +http://__cr.us:@gw.dataimpulse.com:823 +``` + +The general proxy URL format is: + +``` +http://:@:/ ``` ### Step 2: Add to SearXNG Settings