Merge remote-tracking branch 'upstream/dev' into feature/1354-embedding-base-url

# Conflicts:
#	surfsense_web/content/docs/docker-installation/docker-compose.mdx
#	surfsense_web/content/docs/manual-installation.mdx

docker/.env.example

@@ -338,16 +338,6 @@ STT_SERVICE=local/base
 # GATEWAY_DISCORD_ENABLED=FALSE
 # GATEWAY_DISCORD_REDIRECT_URI=http://localhost:3929/api/v1/gateway/discord/callback
 
-# ------------------------------------------------------------------------------
-# SearXNG (bundled web search, works out of the box with no config needed)
-# ------------------------------------------------------------------------------
-# SearXNG provides web search to all search spaces automatically.
-# To access the SearXNG UI directly in dev/deps-only compose: http://localhost:8888
-# To disable the service entirely: docker compose up --scale searxng=0
-# To point at your own SearXNG instance instead of the bundled one:
-# SEARXNG_DEFAULT_HOST=http://your-searxng:8080
-# SEARXNG_SECRET=surfsense-searxng-secret
-
 # ------------------------------------------------------------------------------
 # Daytona Sandbox (optional cloud code execution for the deep agent)
 # ------------------------------------------------------------------------------
@@ -440,6 +430,31 @@ SURFSENSE_ENABLE_DOOM_LOOP=true
 # ETL_CREDIT_BILLING_ENABLED=FALSE
 # MICROS_PER_PAGE=1000
 
+# Debit the credit wallet per *successful* web crawl. Default FALSE keeps
+# crawling effectively free for self-hosted installs. Price is config-driven:
+#   WEB_CRAWL_MICROS_PER_SUCCESS = round(USD_per_1000_crawls * 1_000)
+#   2000 == $2/1000 (default) | 1000 == $1/1000. Captcha solves bill as a
+# separate per-attempt unit (independent flag).
+# WEB_CRAWL_CREDIT_BILLING_ENABLED=FALSE
+# WEB_CRAWL_MICROS_PER_SUCCESS=2000
+# WEB_CRAWL_CAPTCHA_BILLING_ENABLED=FALSE
+# WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000
+
+# Debit the credit wallet per *item returned* by the platform-native scrapers
+# (Reddit, Google Search, Google Maps, YouTube). Default FALSE keeps scraping
+# effectively free for self-hosted installs. Each rate is micro-USD per item,
+# config-driven: <KEY> = round(USD_per_1000_items * 1_000). Defaults sit
+# at/above Apify's first-party actor rates (we charge no subscription tiers,
+# start fees, or separate proxy/compute billing). google_maps.scrape is
+# dual-metered (places + attached reviews).
+# PLATFORM_SCRAPE_BILLING_ENABLED=FALSE
+# REDDIT_SCRAPE_MICROS_PER_ITEM=3500
+# GOOGLE_SEARCH_MICROS_PER_SERP=5500
+# GOOGLE_MAPS_MICROS_PER_PLACE=3500
+# GOOGLE_MAPS_MICROS_PER_REVIEW=1500
+# YOUTUBE_MICROS_PER_VIDEO=2500
+# YOUTUBE_MICROS_PER_COMMENT=1500
+
 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default).
 # QUOTA_MAX_RESERVE_MICROS=1000000
 
@@ -472,12 +487,37 @@ NOLOGIN_MODE_ENABLED=FALSE
 # Connector indexing lock TTL in seconds (default: 28800 = 8 hours)
 # CONNECTOR_INDEXING_LOCK_TTL_SECONDS=28800
 
-# Residential proxy for web crawling
-# RESIDENTIAL_PROXY_USERNAME=
-# RESIDENTIAL_PROXY_PASSWORD=
-# RESIDENTIAL_PROXY_HOSTNAME=
-# RESIDENTIAL_PROXY_LOCATION=
-# RESIDENTIAL_PROXY_TYPE=1
+# Proxy provider selection: "custom" (default) or "dataimpulse".
+# PROXY_PROVIDER=custom
+
+# Proxy endpoint(s), shared across providers. PROXY_URL is a single full URL used
+# by every provider (for "dataimpulse", country is a "__cr.<country>" username
+# suffix the provider parses for geoip). PROXY_URLS is a comma-separated pool the
+# "custom" provider rotates client-side. Leave unset to disable proxying.
+# PROXY_URL=http://user:pass@host:port
+# PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port
+
+# Captcha solving — last-resort bypass tier via captchatools. Only fires on the
+# stealth browser tier when a sitekey is detected AND the flag is TRUE.
+# Cloudflare Turnstile is already solved free in-framework. Off by default.
+# NOTE: automated solving may violate a target site's ToS — opt-in, public
+# data only. See surfsense_backend/.env.example for the full option docs.
+# CAPTCHA_SOLVING_ENABLED=FALSE
+# CAPTCHA_SOLVER_PROVIDER=capsolver
+# CAPTCHA_SOLVER_API_KEY=
+# CAPTCHA_MAX_ATTEMPTS_PER_URL=1
+# CAPTCHA_SOLVE_TIMEOUT_S=120
+# CAPTCHA_TYPE_DEFAULT=v2
+# CAPTCHA_V3_MIN_SCORE=0.7
+# CAPTCHA_V3_ACTION=verify
+
+# Stealth hardening levers on the stealth browser tier. Defaults preserve
+# current behavior; see surfsense_backend/.env.example for per-flag docs.
+# CRAWL_GEOIP_MATCH_ENABLED=FALSE
+# CRAWL_BLOCK_WEBRTC=TRUE
+# CRAWL_HIDE_CANVAS=FALSE
+# CRAWL_GOOGLE_SEARCH_REFERER=TRUE
+# CRAWL_DNS_OVER_HTTPS=FALSE
 
 # ==============================================================================
 # DEV / DEPS-ONLY COMPOSE OVERRIDES
@@ -494,9 +534,6 @@ NOLOGIN_MODE_ENABLED=FALSE
 # -- Redis exposed port (dev/deps-only only; Redis is internal-only in prod) --
 # REDIS_PORT=6379
 
-# -- SearXNG exposed port (dev/deps-only only; internal-only in prod) --
-# SEARXNG_PORT=8888
-
 # -- WhatsApp bridge exposed port (dev/hybrid only; prod keeps it Docker-internal) --
 # WHATSAPP_BRIDGE_PORT=9929