mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
feat(proxy): refactor proxy configuration and add DataImpulse provider
- Updated proxy configuration in `.env.example` files to use `PROXY_URL` and `PROXY_URLS` instead of `CUSTOM_PROXY_URL` and `CUSTOM_PROXY_URLS`. - Introduced `DataImpulseProvider` for proxy management, replacing the deprecated `AnonymousProxiesProvider`. - Enhanced documentation to reflect changes in proxy setup and usage. - Adjusted related code in the proxy registry and configuration files to support the new provider structure.
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
203215adc0
commit
701f888b9e
19 changed files with 303 additions and 193 deletions
|
|
@ -467,12 +467,15 @@ NOLOGIN_MODE_ENABLED=FALSE
|
|||
# Connector indexing lock TTL in seconds (default: 28800 = 8 hours)
|
||||
# CONNECTOR_INDEXING_LOCK_TTL_SECONDS=28800
|
||||
|
||||
# Residential proxy for web crawling
|
||||
# RESIDENTIAL_PROXY_USERNAME=
|
||||
# RESIDENTIAL_PROXY_PASSWORD=
|
||||
# RESIDENTIAL_PROXY_HOSTNAME=
|
||||
# RESIDENTIAL_PROXY_LOCATION=
|
||||
# RESIDENTIAL_PROXY_TYPE=1
|
||||
# Proxy provider selection: "custom" (default) or "dataimpulse".
|
||||
# PROXY_PROVIDER=custom
|
||||
|
||||
# Proxy endpoint(s), shared across providers. PROXY_URL is a single full URL used
|
||||
# by every provider (for "dataimpulse", country is a "__cr.<country>" username
|
||||
# suffix the provider parses for geoip). PROXY_URLS is a comma-separated pool the
|
||||
# "custom" provider rotates client-side. Leave unset to disable proxying.
|
||||
# PROXY_URL=http://user:pass@host:port
|
||||
# PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port
|
||||
|
||||
# ==============================================================================
|
||||
# DEV / DEPS-ONLY COMPOSE OVERRIDES
|
||||
|
|
|
|||
|
|
@ -338,23 +338,21 @@ TURNSTILE_SECRET_KEY=
|
|||
|
||||
|
||||
# Proxy provider selection. Selects a ProxyProvider implementation registered in
|
||||
# app/utils/proxy/registry.py. Default: "anonymous_proxies". Add new vendors there.
|
||||
# PROXY_PROVIDER=anonymous_proxies
|
||||
# app/utils/proxy/registry.py. Default: "custom". Options: "custom", "dataimpulse".
|
||||
# PROXY_PROVIDER=custom
|
||||
|
||||
# Residential Proxy Configuration (anonymous-proxies.net)
|
||||
# Used for web crawling, link previews, and YouTube transcript fetching to avoid IP bans.
|
||||
# Consumed by the "anonymous_proxies" provider. Leave commented out to disable proxying.
|
||||
# RESIDENTIAL_PROXY_USERNAME=your_proxy_username
|
||||
# RESIDENTIAL_PROXY_PASSWORD=your_proxy_password
|
||||
# RESIDENTIAL_PROXY_HOSTNAME=rotating.dnsproxifier.com:31230
|
||||
# RESIDENTIAL_PROXY_LOCATION=
|
||||
# RESIDENTIAL_PROXY_TYPE=1
|
||||
|
||||
# Custom (BYO) proxy provider. Activate with PROXY_PROVIDER=custom.
|
||||
# Provide a single endpoint and/or a comma-separated pool; a pool of >1 rotates
|
||||
# client-side (cyclic). Each value is a full http://user:pass@host:port URL.
|
||||
# CUSTOM_PROXY_URL=http://user:pass@host:port
|
||||
# CUSTOM_PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port
|
||||
# Proxy endpoint(s), shared across providers — PROXY_PROVIDER picks the behavior,
|
||||
# not a different env name. Used for web crawling, link previews, and YouTube
|
||||
# transcript fetching to avoid IP bans. Leave unset to disable proxying.
|
||||
#
|
||||
# PROXY_URL: a single full http://user:pass@host:port endpoint (used by every
|
||||
# provider). For "dataimpulse", country is a "__cr.<country>" username suffix the
|
||||
# provider parses for geoip-match, e.g.:
|
||||
# PROXY_URL=http://your_token__cr.us:your_password@gw.dataimpulse.com:823
|
||||
# PROXY_URLS: a comma-separated pool the "custom" provider rotates client-side
|
||||
# (cyclic); server-side-rotating gateways ignore it.
|
||||
# PROXY_URL=http://user:pass@host:port
|
||||
# PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port
|
||||
|
||||
# =====================================================================
|
||||
# Captcha solving (Phase 3d) — LAST-resort bypass tier via captchatools.
|
||||
|
|
@ -380,7 +378,7 @@ TURNSTILE_SECRET_KEY=
|
|||
# Stealth hardening (Phase 3e, Slice A) — runtime/config-level levers on the
|
||||
# stealth browser tier. Consumed by app/proprietary/web_crawler/stealth.py.
|
||||
# Defaults add no crawl-speed regression and preserve today's behavior.
|
||||
# Match browser locale/timezone to RESIDENTIAL_PROXY_LOCATION (no exit-IP lookup).
|
||||
# Match browser locale/timezone to the proxy provider's exit region (no exit-IP lookup).
|
||||
# CRAWL_GEOIP_MATCH_ENABLED=FALSE
|
||||
# WebRTC respects the proxy (anti local-IP leak). Cheap + safe.
|
||||
# CRAWL_BLOCK_WEBRTC=TRUE
|
||||
|
|
|
|||
|
|
@ -1041,24 +1041,17 @@ class Config:
|
|||
|
||||
# Proxy provider selection. Maps to a ProxyProvider implementation registered
|
||||
# in app/utils/proxy/registry.py. Add new vendors there and switch via this var.
|
||||
PROXY_PROVIDER = os.getenv("PROXY_PROVIDER", "anonymous_proxies")
|
||||
PROXY_PROVIDER = os.getenv("PROXY_PROVIDER", "custom")
|
||||
|
||||
# Residential Proxy Configuration (anonymous-proxies.net)
|
||||
# Used for web crawling and YouTube transcript fetching to avoid IP bans.
|
||||
# Consumed by the "anonymous_proxies" proxy provider.
|
||||
RESIDENTIAL_PROXY_USERNAME = os.getenv("RESIDENTIAL_PROXY_USERNAME")
|
||||
RESIDENTIAL_PROXY_PASSWORD = os.getenv("RESIDENTIAL_PROXY_PASSWORD")
|
||||
RESIDENTIAL_PROXY_HOSTNAME = os.getenv("RESIDENTIAL_PROXY_HOSTNAME")
|
||||
RESIDENTIAL_PROXY_LOCATION = os.getenv("RESIDENTIAL_PROXY_LOCATION", "")
|
||||
RESIDENTIAL_PROXY_TYPE = int(os.getenv("RESIDENTIAL_PROXY_TYPE", "1"))
|
||||
|
||||
# Custom (BYO) proxy provider, selected via PROXY_PROVIDER="custom".
|
||||
# Consumed by the "custom" provider (app/utils/proxy/providers/custom.py).
|
||||
# Provide a single endpoint (CUSTOM_PROXY_URL) and/or a comma-separated pool
|
||||
# (CUSTOM_PROXY_URLS); a pool of >1 rotates client-side via Scrapling's
|
||||
# ProxyRotator. Each value is a full http://user:pass@host:port URL.
|
||||
CUSTOM_PROXY_URL = os.getenv("CUSTOM_PROXY_URL")
|
||||
CUSTOM_PROXY_URLS = os.getenv("CUSTOM_PROXY_URLS")
|
||||
# Proxy endpoint(s), shared across all providers — PROXY_PROVIDER selects how
|
||||
# they're interpreted, not a different env name. PROXY_URL is a single full
|
||||
# http://user:pass@host:port endpoint (used by every provider); e.g. DataImpulse
|
||||
# encodes country as a "__cr.<country>" username suffix that its provider parses
|
||||
# for geoip-match. PROXY_URLS is a comma-separated pool that the "custom" provider
|
||||
# rotates client-side (server-side-rotating gateways ignore it). Leave unset to
|
||||
# disable proxying.
|
||||
PROXY_URL = os.getenv("PROXY_URL")
|
||||
PROXY_URLS = os.getenv("PROXY_URLS")
|
||||
|
||||
# =====================================================================
|
||||
# Phase 3d — Captcha solving (reCAPTCHA v2/v3, hCaptcha) via captchatools.
|
||||
|
|
@ -1097,9 +1090,9 @@ class Config:
|
|||
# (no test-vs-prod drift). Defaults preserve today's behavior /
|
||||
# introduce no crawl-speed regression. See plans/backend/03e-stealth-hardening.md.
|
||||
# =====================================================================
|
||||
# Map the configured proxy region (RESIDENTIAL_PROXY_LOCATION) -> browser
|
||||
# locale/timezone so the fingerprint coheres with the proxy exit geo. No
|
||||
# exit-IP lookup (zero added latency); unknown/empty region => skip.
|
||||
# Map the active proxy provider's exit region (ProxyProvider.get_location())
|
||||
# -> browser locale/timezone so the fingerprint coheres with the proxy exit
|
||||
# geo. No exit-IP lookup (zero added latency); unknown/empty region => skip.
|
||||
CRAWL_GEOIP_MATCH_ENABLED = (
|
||||
os.getenv("CRAWL_GEOIP_MATCH_ENABLED", "FALSE").upper() == "TRUE"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ results only materialize after the page's JS runs. So `fetch.py` needs both a
|
|||
A warm fetch (browser up, IP cached) runs ~8 s; the first fetch of a process
|
||||
also pays the ~5 s Chromium launch and a vetting round. Requires the browser
|
||||
tier (patchright Chromium via Scrapling's `AsyncStealthySession`) and a
|
||||
residential proxy — set `PROXY_PROVIDER=custom` + `CUSTOM_PROXY_URL` (see
|
||||
residential proxy — set `PROXY_PROVIDER` + `PROXY_URL` (see
|
||||
`.env`). Long-running callers can `await fetch.close_sessions()` on shutdown;
|
||||
scripts that exit anyway can skip it.
|
||||
|
||||
|
|
|
|||
|
|
@ -77,8 +77,9 @@ cd surfsense_backend
|
|||
## TODO / out of scope (v1)
|
||||
|
||||
- Sticky-IP provider support: the fetch layer assumes a sticky exit IP per
|
||||
session (the `loid` binds to it). `anonymous_proxies` does not yet set the
|
||||
`"si"` sticky key — add it (proxy layer) before high-volume production use.
|
||||
session (the `loid` binds to it). The `dataimpulse` provider does not yet emit
|
||||
a sticky `__sid.<id>` username suffix — add it (proxy layer) before high-volume
|
||||
production use.
|
||||
- `/api/morechildren` deep-comment expansion — `more` stubs terminate the tree
|
||||
walk today.
|
||||
- Routes / `connector_service.py` / ingestion / Celery wiring.
|
||||
|
|
|
|||
|
|
@ -72,10 +72,17 @@ _MAX_ROTATIONS = 3
|
|||
_MAX_BACKOFFS = 4
|
||||
_BACKOFF_BASE_S = 5.0
|
||||
|
||||
# Reddit 429s aggressively (~60-100 req/min/IP). Pace each sticky session to
|
||||
# ~1 req/s + jitter to stay under the per-IP threshold.
|
||||
_MIN_INTERVAL_S = 1.0
|
||||
_PACE_JITTER_S = 0.5
|
||||
# Reddit 429s aggressively (~60-100 req/min/IP). Pace each sticky session so a
|
||||
# fast IP can't burst past the per-IP threshold. A live probe (12 rapid fetches
|
||||
# on one sticky IP => 0x429; scripts/_bench_reddit2.py) showed the natural
|
||||
# ~0.85s request latency already holds a session near ~1 req/s, so the old 1.0s
|
||||
# floor just ADDED ~0.4s of dead sleep to every fetch. A 0.5s floor is a no-op
|
||||
# for typical fetches yet still caps a fast IP at ~2 req/s; the 429 backoff below
|
||||
# is the real safety net if an IP's limit is tighter.
|
||||
# ponytail: 0.5s is tuned to dataimpulse residential exits. A pool with a
|
||||
# stricter per-IP cap may need it raised — watch for 429 log spam and bump it.
|
||||
_MIN_INTERVAL_S = 0.5
|
||||
_PACE_JITTER_S = 0.25
|
||||
|
||||
_HEADERS = {"Accept-Language": "en-US,en;q=0.9"}
|
||||
|
||||
|
|
|
|||
|
|
@ -56,6 +56,14 @@ _LISTING_LIMIT = 100
|
|||
_MAX_PAGES = 10
|
||||
_EMPTY_STREAK_ABORT = 2
|
||||
|
||||
# A subreddit's per-post comment fetches are independent (each is a separate
|
||||
# .json), so after paging the listing on one sticky IP we fan them across their
|
||||
# own warm sessions instead of walking them sequentially — the dominant cost of
|
||||
# a subreddit+comments scrape (~3.6x on the comment phase; scripts/_bench_reddit2).
|
||||
# Kept below the top-level fan-out width: with N concurrent subreddit targets the
|
||||
# worst case is N x this many proxy IPs, so this bounds that multiplication.
|
||||
_COMMENT_CONCURRENCY = 8
|
||||
|
||||
# Search sorts differ from listing sorts; fall back to "new" for a listing path
|
||||
# when the input carries a search-only sort.
|
||||
_LISTING_SORTS = frozenset({"hot", "new", "top", "rising", "controversial", "best"})
|
||||
|
|
@ -227,6 +235,7 @@ async def _subreddit_flow(
|
|||
if sort == "top" and input_model.time:
|
||||
params["t"] = input_model.time
|
||||
|
||||
post_ids: list[str] = []
|
||||
async for data in _paginate_listing(
|
||||
f"r/{subreddit}/{sort}",
|
||||
params,
|
||||
|
|
@ -238,14 +247,24 @@ async def _subreddit_flow(
|
|||
item = _emit(parse_post(data), include_nsfw=input_model.includeNSFW)
|
||||
if item is not None:
|
||||
yield item
|
||||
if not input_model.skipComments and isinstance(data.get("id"), str):
|
||||
async for comment in _post_flow(
|
||||
data["id"],
|
||||
# Collect ids now; fetch the comment trees in parallel below. Walking
|
||||
# them here would serialize one .json per post on this single sticky IP.
|
||||
parsed_id = data.get("id")
|
||||
if not input_model.skipComments and isinstance(parsed_id, str):
|
||||
post_ids.append(parsed_id)
|
||||
|
||||
if post_ids:
|
||||
comment_jobs = [
|
||||
_post_flow(
|
||||
pid,
|
||||
input_model=input_model,
|
||||
subreddit=subreddit,
|
||||
include_post=False,
|
||||
):
|
||||
yield comment
|
||||
)
|
||||
for pid in post_ids
|
||||
]
|
||||
async for comment in fan_out(comment_jobs, concurrency=_COMMENT_CONCURRENCY):
|
||||
yield comment
|
||||
|
||||
|
||||
async def _user_flow(
|
||||
|
|
|
|||
|
|
@ -273,7 +273,7 @@ class WebCrawlerConnector:
|
|||
endpoint on every ``get_proxy_url()`` call, so simply re-invoking the tier
|
||||
rotates the proxy. Bounded to a single extra attempt per tier — no
|
||||
unbounded fan-out on billable crawls. Single-endpoint providers
|
||||
(including the server-side-rotating ``anonymous_proxies``) skip the retry
|
||||
(including the server-side-rotating ``dataimpulse``) skip the retry
|
||||
entirely (``is_pool_backed()`` is ``False``), since retrying the same
|
||||
static endpoint would just re-hit the same dead proxy. Non-proxy errors
|
||||
(and ``NotImplementedError`` from the browser tiers) propagate unchanged
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from dataclasses import dataclass
|
|||
from typing import Any
|
||||
|
||||
from app.config import config
|
||||
from app.utils.proxy import get_active_provider
|
||||
|
||||
# Coarse region -> (locale, IANA timezone) map. Country granularity only: per the
|
||||
# 03e "Geoip accuracy" risk, wrong-but-coherent beats a default mismatch and
|
||||
|
|
@ -131,8 +132,9 @@ def location_to_locale_timezone(
|
|||
) -> tuple[str | None, str | None]:
|
||||
"""Map a free-form proxy region string -> (locale, timezone_id).
|
||||
|
||||
``location`` is the vendor-specific ``RESIDENTIAL_PROXY_LOCATION`` value
|
||||
(``03b``). Best-effort: matches an ISO-3166 alpha-2 code (e.g. ``"us"``) or a
|
||||
``location`` is the active proxy provider's exit region
|
||||
(``ProxyProvider.get_location()``). Best-effort: matches an ISO-3166 alpha-2
|
||||
code (e.g. ``"us"``) or a
|
||||
common country name (e.g. ``"Germany"``); only the leading token is honored,
|
||||
so vendor strings like ``"us:nyc"`` or ``"de region"`` still resolve. Returns
|
||||
``(None, None)`` for empty/unknown input (caller then leaves the browser
|
||||
|
|
@ -181,7 +183,7 @@ def get_stealth_config() -> StealthConfig:
|
|||
"""Build a :class:`StealthConfig` from the current process config/env."""
|
||||
return StealthConfig(
|
||||
geoip_match_enabled=config.CRAWL_GEOIP_MATCH_ENABLED,
|
||||
proxy_location=config.RESIDENTIAL_PROXY_LOCATION or "",
|
||||
proxy_location=get_active_provider().get_location(),
|
||||
block_webrtc=config.CRAWL_BLOCK_WEBRTC,
|
||||
hide_canvas=config.CRAWL_HIDE_CANVAS,
|
||||
google_search=config.CRAWL_GOOGLE_SEARCH_REFERER,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ registering it in ``registry.py``.
|
|||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
class ProxyProvider(ABC):
|
||||
|
|
@ -27,12 +28,33 @@ class ProxyProvider(ABC):
|
|||
def get_proxy_url(self) -> str | None:
|
||||
"""Return ``http://user:pass@host:port`` (no trailing slash), or ``None``.
|
||||
|
||||
This is the canonical form Scrapling/curl_cffi consume directly.
|
||||
This is the canonical form Scrapling/curl_cffi consume directly, and the
|
||||
single source every provider must supply — the ``requests`` and Playwright
|
||||
shapes below are derived from it.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_playwright_proxy(self) -> dict[str, str] | None:
|
||||
"""Return a Playwright proxy dict, or ``None`` when not configured."""
|
||||
"""Return a Playwright ``{"server","username","password"}`` dict, or ``None``.
|
||||
|
||||
Parsed from :meth:`get_proxy_url` (the canonical URL) by default, since
|
||||
every provider's credentials already live in that URL. Override only for a
|
||||
vendor whose Playwright form can't be expressed as a parse of the URL.
|
||||
"""
|
||||
proxy_url = self.get_proxy_url()
|
||||
if not proxy_url:
|
||||
return None
|
||||
parts = urlsplit(proxy_url)
|
||||
if not parts.hostname:
|
||||
return None
|
||||
server = f"{parts.scheme or 'http'}://{parts.hostname}"
|
||||
if parts.port:
|
||||
server = f"{server}:{parts.port}"
|
||||
proxy: dict[str, str] = {"server": server}
|
||||
if parts.username:
|
||||
proxy["username"] = parts.username
|
||||
if parts.password:
|
||||
proxy["password"] = parts.password
|
||||
return proxy
|
||||
|
||||
def get_requests_proxies(self) -> dict[str, str] | None:
|
||||
"""Return a ``requests``/``aiohttp`` proxies dict, or ``None``.
|
||||
|
|
@ -45,14 +67,25 @@ class ProxyProvider(ABC):
|
|||
return None
|
||||
return {"http": proxy_url, "https": proxy_url}
|
||||
|
||||
def get_location(self) -> str:
|
||||
"""Return the proxy's configured exit region (e.g. ``"us"``), or ``""``.
|
||||
|
||||
Vendor-agnostic hook the crawler's geoip-match (``03e``) uses to align the
|
||||
browser locale/timezone with the exit IP's country. Default ``""``
|
||||
(unknown) for providers that don't pin a region (e.g. BYO ``custom`` URLs,
|
||||
where the region is baked opaquely into the URL). Override in providers
|
||||
that hold the region as a discrete field.
|
||||
"""
|
||||
return ""
|
||||
|
||||
@property
|
||||
def is_pool_backed(self) -> bool:
|
||||
"""Whether this provider rotates across a *client-side* pool of endpoints.
|
||||
|
||||
``False`` for single-endpoint providers (including server-side rotating
|
||||
gateways like ``anonymous_proxies``, whose rotation happens upstream).
|
||||
The crawler performs its bounded proxy-error rotation-retry **only** when
|
||||
this is ``True`` — retrying a single static endpoint would just re-hit the
|
||||
same dead proxy.
|
||||
gateways like ``dataimpulse``, whose rotation happens upstream). The
|
||||
crawler performs its bounded proxy-error rotation-retry **only** when this
|
||||
is ``True`` — retrying a single static endpoint would just re-hit the same
|
||||
dead proxy.
|
||||
"""
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -1,65 +0,0 @@
|
|||
"""anonymous-proxies.net residential / rotating proxy provider.
|
||||
|
||||
The vendor (``rotating.dnsproxifier.com``) encodes the location and rotation
|
||||
``type`` options inside a base64-encoded JSON "password". The hostname already
|
||||
includes the port (e.g. ``rotating.dnsproxifier.com:31230``).
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
|
||||
from app.config import Config
|
||||
from app.utils.proxy.base import ProxyProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AnonymousProxiesProvider(ProxyProvider):
|
||||
"""Provider for anonymous-proxies.net credentials in ``RESIDENTIAL_PROXY_*``."""
|
||||
|
||||
name = "anonymous_proxies"
|
||||
|
||||
def _password_b64(self) -> str | None:
|
||||
"""Build the base64-encoded password dict required by the vendor.
|
||||
|
||||
Returns ``None`` when the password is not configured.
|
||||
"""
|
||||
password = Config.RESIDENTIAL_PROXY_PASSWORD
|
||||
if not password:
|
||||
return None
|
||||
|
||||
password_dict = {
|
||||
"p": password,
|
||||
"l": Config.RESIDENTIAL_PROXY_LOCATION,
|
||||
"t": Config.RESIDENTIAL_PROXY_TYPE,
|
||||
}
|
||||
return base64.b64encode(json.dumps(password_dict).encode("utf-8")).decode(
|
||||
"utf-8"
|
||||
)
|
||||
|
||||
def get_proxy_url(self) -> str | None:
|
||||
username = Config.RESIDENTIAL_PROXY_USERNAME
|
||||
hostname = Config.RESIDENTIAL_PROXY_HOSTNAME
|
||||
password_b64 = self._password_b64()
|
||||
|
||||
if not all([username, hostname, password_b64]):
|
||||
return None
|
||||
|
||||
# No trailing slash: curl_cffi (Scrapling static fetcher) expects a bare
|
||||
# ``http://user:pass@host:port`` URL.
|
||||
return f"http://{username}:{password_b64}@{hostname}"
|
||||
|
||||
def get_playwright_proxy(self) -> dict[str, str] | None:
|
||||
username = Config.RESIDENTIAL_PROXY_USERNAME
|
||||
hostname = Config.RESIDENTIAL_PROXY_HOSTNAME
|
||||
password_b64 = self._password_b64()
|
||||
|
||||
if not all([username, hostname, password_b64]):
|
||||
return None
|
||||
|
||||
return {
|
||||
"server": f"http://{hostname}",
|
||||
"username": username,
|
||||
"password": password_b64,
|
||||
}
|
||||
|
|
@ -1,16 +1,15 @@
|
|||
"""Bring-your-own (BYO) custom proxy provider.
|
||||
|
||||
Reads one or more proxy endpoints from env (``CUSTOM_PROXY_URL`` and/or the
|
||||
comma-separated ``CUSTOM_PROXY_URLS``). With a single endpoint it behaves like a
|
||||
static proxy; with a pool (>1) it rotates client-side via Scrapling's thread-safe
|
||||
Reads one or more proxy endpoints from the shared env (``PROXY_URL`` and/or the
|
||||
comma-separated ``PROXY_URLS``). With a single endpoint it behaves like a static
|
||||
proxy; with a pool (>1) it rotates client-side via Scrapling's thread-safe
|
||||
``ProxyRotator`` (cyclic), transparently to every caller of the zero-arg getters.
|
||||
|
||||
No vendor-specific auth assumptions: a user who wants a specific vendor points
|
||||
``CUSTOM_PROXY_URLS`` at that vendor's ``http://user:pass@host:port`` endpoints.
|
||||
``PROXY_URLS`` at that vendor's ``http://user:pass@host:port`` endpoints.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from scrapling.engines.toolbelt import ProxyRotator
|
||||
|
||||
|
|
@ -31,19 +30,19 @@ class CustomProxyProvider(ProxyProvider):
|
|||
self._rotator = ProxyRotator(self._urls) if len(self._urls) > 1 else None
|
||||
if not self._urls:
|
||||
logger.warning(
|
||||
"PROXY_PROVIDER='custom' selected but neither CUSTOM_PROXY_URL nor "
|
||||
"CUSTOM_PROXY_URLS is set; crawls will run without a proxy."
|
||||
"PROXY_PROVIDER='custom' selected but neither PROXY_URL nor "
|
||||
"PROXY_URLS is set; crawls will run without a proxy."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _load_urls() -> list[str]:
|
||||
"""Collect proxy URLs from env (pool first, then single), de-duplicated."""
|
||||
urls: list[str] = []
|
||||
pool = Config.CUSTOM_PROXY_URLS
|
||||
pool = Config.PROXY_URLS
|
||||
if pool:
|
||||
urls.extend(part.strip() for part in pool.split(",") if part.strip())
|
||||
|
||||
single = (Config.CUSTOM_PROXY_URL or "").strip()
|
||||
single = (Config.PROXY_URL or "").strip()
|
||||
if single and single not in urls:
|
||||
urls.append(single)
|
||||
|
||||
|
|
@ -60,23 +59,3 @@ class CustomProxyProvider(ProxyProvider):
|
|||
# Advances the cyclic index on every call (thread-safe).
|
||||
return self._rotator.get_proxy()
|
||||
return self._urls[0]
|
||||
|
||||
def get_playwright_proxy(self) -> dict[str, str] | None:
|
||||
proxy_url = self.get_proxy_url()
|
||||
if not proxy_url:
|
||||
return None
|
||||
|
||||
parts = urlsplit(proxy_url)
|
||||
if not parts.hostname:
|
||||
return None
|
||||
|
||||
server = f"{parts.scheme or 'http'}://{parts.hostname}"
|
||||
if parts.port:
|
||||
server = f"{server}:{parts.port}"
|
||||
|
||||
proxy: dict[str, str] = {"server": server}
|
||||
if parts.username:
|
||||
proxy["username"] = parts.username
|
||||
if parts.password:
|
||||
proxy["password"] = parts.password
|
||||
return proxy
|
||||
|
|
|
|||
56
surfsense_backend/app/utils/proxy/providers/dataimpulse.py
Normal file
56
surfsense_backend/app/utils/proxy/providers/dataimpulse.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
"""DataImpulse residential / rotating proxy provider.
|
||||
|
||||
Takes the shared ``PROXY_URL`` env, exactly like the BYO ``custom`` provider —
|
||||
the format is uniform, paste it straight from the vendor dashboard. What makes
|
||||
this a *named* provider rather than just ``custom`` is the vendor-specific
|
||||
knowledge it layers on top of that URL:
|
||||
|
||||
* :meth:`get_location` reads DataImpulse's ``__cr.<country>`` username suffix so
|
||||
the crawler's geoip-match can align the browser locale with the exit country
|
||||
(``custom`` can't — it treats the URL as opaque).
|
||||
|
||||
Rotation happens server-side (a fresh exit IP per request on the default pool
|
||||
port), so this is NOT :pyattr:`~ProxyProvider.is_pool_backed`.
|
||||
|
||||
Example URL::
|
||||
|
||||
http://<token>__cr.us:<password>@gw.dataimpulse.com:823
|
||||
|
||||
ponytail: sticky sessions (a stable exit IP across requests) are another
|
||||
username suffix (``__sid.<id>``) — the lever the Reddit scraper's README flags as
|
||||
a TODO for its ``loid``-per-IP flow. Not built yet: Reddit isn't wired to a
|
||||
route, so there's no caller to thread a session id through. Add a
|
||||
``get_sticky_proxy_url(session_id)`` here (rewriting the username) when it lands.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from app.config import Config
|
||||
from app.utils.proxy.base import ProxyProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# DataImpulse encodes country routing as a "__cr.<country>" username suffix; the
|
||||
# country token runs until the next "__" param (e.g. "__sid") or the end.
|
||||
_COUNTRY_MARKER = "__cr."
|
||||
|
||||
|
||||
class DataImpulseProvider(ProxyProvider):
|
||||
"""Provider for a DataImpulse proxy URL in the shared ``PROXY_URL`` env."""
|
||||
|
||||
name = "dataimpulse"
|
||||
|
||||
def get_proxy_url(self) -> str | None:
|
||||
url = (Config.PROXY_URL or "").strip()
|
||||
return url or None
|
||||
|
||||
def get_location(self) -> str:
|
||||
"""Country parsed from the ``__cr.<country>`` username suffix, or ``""``."""
|
||||
url = self.get_proxy_url()
|
||||
if not url:
|
||||
return ""
|
||||
username = urlsplit(url).username or ""
|
||||
if _COUNTRY_MARKER not in username:
|
||||
return ""
|
||||
return username.split(_COUNTRY_MARKER, 1)[1].split("__", 1)[0].lower()
|
||||
|
|
@ -9,18 +9,20 @@ import logging
|
|||
|
||||
from app.config import Config
|
||||
from app.utils.proxy.base import ProxyProvider
|
||||
from app.utils.proxy.providers.anonymous_proxies import AnonymousProxiesProvider
|
||||
from app.utils.proxy.providers.custom import CustomProxyProvider
|
||||
from app.utils.proxy.providers.dataimpulse import DataImpulseProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Registered proxy providers, keyed by their ``name``.
|
||||
_PROVIDERS: dict[str, type[ProxyProvider]] = {
|
||||
AnonymousProxiesProvider.name: AnonymousProxiesProvider,
|
||||
CustomProxyProvider.name: CustomProxyProvider,
|
||||
DataImpulseProvider.name: DataImpulseProvider,
|
||||
}
|
||||
|
||||
_DEFAULT_PROVIDER = AnonymousProxiesProvider.name
|
||||
# BYO ``custom`` is the neutral default: it needs no vendor and returns no proxy
|
||||
# until PROXY_URL(S) is set, so an unconfigured install simply runs direct.
|
||||
_DEFAULT_PROVIDER = CustomProxyProvider.name
|
||||
|
||||
_active_provider: ProxyProvider | None = None
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
import pytest
|
||||
|
||||
from app.config import config
|
||||
from app.proprietary.web_crawler import stealth
|
||||
from app.proprietary.web_crawler.stealth import (
|
||||
build_stealthy_kwargs,
|
||||
get_stealth_config,
|
||||
|
|
@ -12,6 +13,16 @@ from app.proprietary.web_crawler.stealth import (
|
|||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _set_proxy_location(monkeypatch: pytest.MonkeyPatch, location: str) -> None:
|
||||
"""Point get_stealth_config at a stub provider with the given exit region."""
|
||||
|
||||
class _StubProvider:
|
||||
def get_location(self) -> str:
|
||||
return location
|
||||
|
||||
monkeypatch.setattr(stealth, "get_active_provider", lambda: _StubProvider())
|
||||
|
||||
|
||||
class TestLocationToLocaleTimezone:
|
||||
def test_alpha2_code(self):
|
||||
assert location_to_locale_timezone("us") == ("en-US", "America/New_York")
|
||||
|
|
@ -58,7 +69,7 @@ class TestBuildStealthyKwargs:
|
|||
monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", False)
|
||||
monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", True)
|
||||
monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", False)
|
||||
monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "us")
|
||||
_set_proxy_location(monkeypatch, "us")
|
||||
|
||||
kwargs = build_stealthy_kwargs(get_stealth_config())
|
||||
|
||||
|
|
@ -77,7 +88,7 @@ class TestBuildStealthyKwargs:
|
|||
monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", True)
|
||||
monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", False)
|
||||
monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", True)
|
||||
monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "")
|
||||
_set_proxy_location(monkeypatch, "")
|
||||
|
||||
kwargs = build_stealthy_kwargs(get_stealth_config())
|
||||
|
||||
|
|
@ -92,7 +103,7 @@ class TestBuildStealthyKwargs:
|
|||
monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", False)
|
||||
monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", True)
|
||||
monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", False)
|
||||
monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "de")
|
||||
_set_proxy_location(monkeypatch, "de")
|
||||
|
||||
kwargs = build_stealthy_kwargs(get_stealth_config())
|
||||
|
||||
|
|
@ -105,7 +116,7 @@ class TestBuildStealthyKwargs:
|
|||
monkeypatch.setattr(config, "CRAWL_HIDE_CANVAS", False)
|
||||
monkeypatch.setattr(config, "CRAWL_GOOGLE_SEARCH_REFERER", True)
|
||||
monkeypatch.setattr(config, "CRAWL_DNS_OVER_HTTPS", False)
|
||||
monkeypatch.setattr(config, "RESIDENTIAL_PROXY_LOCATION", "atlantis")
|
||||
_set_proxy_location(monkeypatch, "atlantis")
|
||||
|
||||
kwargs = build_stealthy_kwargs(get_stealth_config())
|
||||
|
||||
|
|
|
|||
|
|
@ -15,12 +15,12 @@ pytestmark = pytest.mark.unit
|
|||
@pytest.fixture(autouse=True)
|
||||
def _clear_proxy_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Default both knobs to unset; individual tests set what they need."""
|
||||
monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", None)
|
||||
monkeypatch.setattr(Config, "CUSTOM_PROXY_URLS", None)
|
||||
monkeypatch.setattr(Config, "PROXY_URL", None)
|
||||
monkeypatch.setattr(Config, "PROXY_URLS", None)
|
||||
|
||||
|
||||
def test_single_url_is_static(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", "http://u:p@host:8080")
|
||||
monkeypatch.setattr(Config, "PROXY_URL", "http://u:p@host:8080")
|
||||
provider = CustomProxyProvider()
|
||||
|
||||
assert provider.is_pool_backed is False
|
||||
|
|
@ -32,7 +32,7 @@ def test_single_url_is_static(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
def test_pool_rotates_cyclically(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
Config,
|
||||
"CUSTOM_PROXY_URLS",
|
||||
"PROXY_URLS",
|
||||
"http://a:1@h1:8080, http://b:2@h2:8080 ,http://c:3@h3:8080",
|
||||
)
|
||||
provider = CustomProxyProvider()
|
||||
|
|
@ -48,9 +48,9 @@ def test_pool_rotates_cyclically(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
|
||||
|
||||
def test_single_plus_pool_dedupes(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A URL present in both CUSTOM_PROXY_URL and the pool is not duplicated."""
|
||||
monkeypatch.setattr(Config, "CUSTOM_PROXY_URLS", "http://a@h1:80,http://b@h2:80")
|
||||
monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", "http://a@h1:80")
|
||||
"""A URL present in both PROXY_URL and the pool is not duplicated."""
|
||||
monkeypatch.setattr(Config, "PROXY_URLS", "http://a@h1:80,http://b@h2:80")
|
||||
monkeypatch.setattr(Config, "PROXY_URL", "http://a@h1:80")
|
||||
provider = CustomProxyProvider()
|
||||
|
||||
assert provider.is_pool_backed is True
|
||||
|
|
@ -69,7 +69,7 @@ def test_unconfigured_returns_none() -> None:
|
|||
def test_playwright_proxy_parses_credentials(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(Config, "CUSTOM_PROXY_URL", "http://user:pass@host:8080")
|
||||
monkeypatch.setattr(Config, "PROXY_URL", "http://user:pass@host:8080")
|
||||
provider = CustomProxyProvider()
|
||||
|
||||
assert provider.get_playwright_proxy() == {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
"""Unit tests for the DataImpulseProvider.
|
||||
|
||||
Takes a single full URL (like ``custom``); the vendor-specific bit is parsing the
|
||||
``__cr.<country>`` username suffix for :meth:`get_location`. Playwright/requests
|
||||
shapes come from the shared base parse, so a couple of checks cover the wiring.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import Config
|
||||
from app.utils.proxy.providers.dataimpulse import DataImpulseProvider
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
_URL = "http://tok123__cr.us:secret@gw.dataimpulse.com:823"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(Config, "PROXY_URL", None)
|
||||
|
||||
|
||||
def test_returns_configured_url(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(Config, "PROXY_URL", _URL)
|
||||
provider = DataImpulseProvider()
|
||||
|
||||
assert provider.is_pool_backed is False
|
||||
assert provider.get_proxy_url() == _URL
|
||||
assert provider.get_requests_proxies() == {"http": _URL, "https": _URL}
|
||||
|
||||
|
||||
def test_location_parsed_from_country_suffix(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(Config, "PROXY_URL", _URL)
|
||||
assert DataImpulseProvider().get_location() == "us"
|
||||
|
||||
|
||||
def test_location_stops_at_next_param(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# A sticky/session suffix after the country must not bleed into the location.
|
||||
monkeypatch.setattr(
|
||||
Config,
|
||||
"PROXY_URL",
|
||||
"http://tok__cr.de__sid.abc123:secret@gw.dataimpulse.com:823",
|
||||
)
|
||||
assert DataImpulseProvider().get_location() == "de"
|
||||
|
||||
|
||||
def test_no_country_suffix_yields_empty_location(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
Config, "PROXY_URL", "http://tok:secret@gw.dataimpulse.com:823"
|
||||
)
|
||||
assert DataImpulseProvider().get_location() == ""
|
||||
|
||||
|
||||
def test_unconfigured_returns_none() -> None:
|
||||
provider = DataImpulseProvider()
|
||||
|
||||
assert provider.get_proxy_url() is None
|
||||
assert provider.get_requests_proxies() is None
|
||||
assert provider.get_playwright_proxy() is None
|
||||
assert provider.get_location() == ""
|
||||
|
||||
|
||||
def test_playwright_proxy_from_base_parse(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(Config, "PROXY_URL", _URL)
|
||||
|
||||
assert DataImpulseProvider().get_playwright_proxy() == {
|
||||
"server": "http://gw.dataimpulse.com:823",
|
||||
"username": "tok123__cr.us",
|
||||
"password": "secret",
|
||||
}
|
||||
|
|
@ -1,16 +1,16 @@
|
|||
"""Unit tests for proxy provider selection (Phase 3b).
|
||||
"""Unit tests for proxy provider selection.
|
||||
|
||||
``PROXY_PROVIDER`` selects the single app-wide provider; ``custom`` is now
|
||||
registered alongside ``anonymous_proxies``, and unknown values still warn and
|
||||
fall back to the default.
|
||||
``PROXY_PROVIDER`` selects the single app-wide provider; ``custom`` (the default)
|
||||
and ``dataimpulse`` are registered, and unknown values still warn and fall back
|
||||
to the default.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import Config
|
||||
from app.utils.proxy import registry
|
||||
from app.utils.proxy.providers.anonymous_proxies import AnonymousProxiesProvider
|
||||
from app.utils.proxy.providers.custom import CustomProxyProvider
|
||||
from app.utils.proxy.providers.dataimpulse import DataImpulseProvider
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
|
@ -26,11 +26,11 @@ def test_resolves_custom(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
assert isinstance(registry.get_active_provider(), CustomProxyProvider)
|
||||
|
||||
|
||||
def test_resolves_anonymous(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(Config, "PROXY_PROVIDER", "anonymous_proxies")
|
||||
assert isinstance(registry.get_active_provider(), AnonymousProxiesProvider)
|
||||
def test_resolves_dataimpulse(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(Config, "PROXY_PROVIDER", "dataimpulse")
|
||||
assert isinstance(registry.get_active_provider(), DataImpulseProvider)
|
||||
|
||||
|
||||
def test_unknown_falls_back_to_default(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(Config, "PROXY_PROVIDER", "does_not_exist")
|
||||
assert isinstance(registry.get_active_provider(), AnonymousProxiesProvider)
|
||||
assert isinstance(registry.get_active_provider(), CustomProxyProvider)
|
||||
|
|
|
|||
|
|
@ -110,17 +110,16 @@ In production, search engines may rate-limit or block your server's IP. To avoid
|
|||
|
||||
### Step 1: Build the Proxy URL
|
||||
|
||||
SurfSense uses [anonymous-proxies.net](https://anonymous-proxies.net/) style residential proxies where the password is a base64-encoded JSON object. Build the URL using your proxy credentials:
|
||||
|
||||
```bash
|
||||
# Encode the password (replace with your actual values)
|
||||
echo -n '{"p": "YOUR_PASSWORD", "l": "LOCATION", "t": PROXY_TYPE}' | base64
|
||||
```
|
||||
|
||||
The full proxy URL format is:
|
||||
Use any residential proxy that exposes a standard `http://user:pass@host:port` endpoint. For example, [DataImpulse](https://dataimpulse.com/) encodes country routing as a username suffix (`<token>__cr.<country>`):
|
||||
|
||||
```
|
||||
http://<username>:<base64_password>@<hostname>:<port>/
|
||||
http://<token>__cr.us:<password>@gw.dataimpulse.com:823
|
||||
```
|
||||
|
||||
The general proxy URL format is:
|
||||
|
||||
```
|
||||
http://<username>:<password>@<hostname>:<port>/
|
||||
```
|
||||
|
||||
### Step 2: Add to SearXNG Settings
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue