Merge remote-tracking branch 'upstream/dev' into feature/1354-embedding-base-url

# Conflicts:
#	surfsense_web/content/docs/docker-installation/docker-compose.mdx
#	surfsense_web/content/docs/manual-installation.mdx

surfsense_backend/.env.example

@@ -55,11 +55,6 @@ WHATSAPP_WEBHOOK_VERIFY_TOKEN=
 WHATSAPP_WEBHOOK_APP_SECRET=
 WHATSAPP_BRIDGE_URL=http://whatsapp-bridge:9929
 
-# Platform Web Search (SearXNG)
-# Set this to enable built-in web search. Docker Compose sets it automatically.
-# Only uncomment if running the backend outside Docker (e.g. uvicorn on host).
-# SEARXNG_DEFAULT_HOST=http://localhost:8888
-
 # Periodic task interval
 # # Run every minute (default)
 # SCHEDULE_CHECKER_INTERVAL=1m
@@ -261,6 +256,41 @@ DEFAULT_CREDIT_MICROS_BALANCE=5000000
 ETL_CREDIT_BILLING_ENABLED=FALSE
 MICROS_PER_PAGE=1000
 
+# Debit the credit wallet per *successful* web crawl. Default FALSE keeps
+# crawling effectively free for self-hosted/OSS installs; hosted sets TRUE.
+# Price is fully config-driven (the only source of truth, no hardcoded rate):
+#   WEB_CRAWL_MICROS_PER_SUCCESS = round(USD_per_1000_crawls * 1_000)
+#   2000 == $2 / 1000 crawls (default) | 1000 == $1/1000 | 500 == $0.50/1000
+# Retune anytime with just an env change + restart (no code/migration).
+# WEB_CRAWL_CREDIT_BILLING_ENABLED=FALSE
+# WEB_CRAWL_MICROS_PER_SUCCESS=2000
+
+# Phase 3d: bill captcha solves as a SEPARATE per-attempt unit (the solver
+# charges per attempt regardless of crawl success). Independent of the crawl
+# flag above. Price config-driven (no hardcoded rate):
+#   WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE = round(USD_per_1000_solves * 1_000)
+#   3000 == $3 / 1000 solves (default) | 5000 == $5/1000
+# Set with margin over your solver vendor's per-attempt price.
+# WEB_CRAWL_CAPTCHA_BILLING_ENABLED=FALSE
+# WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000
+
+# Debit the credit wallet per *item returned* by the platform-native scrapers
+# (Reddit, Google Search, Google Maps, YouTube). Default FALSE keeps scraping
+# effectively free for self-hosted/OSS installs; hosted deployments set TRUE.
+# Each rate is micro-USD per item, fully config-driven (no hardcoded rate):
+#   <KEY> = round(USD_per_1000_items * 1_000)
+#   3500 == $3.50/1000 | 5000 == $5/1000 | 2000 == $2/1000
+# Defaults sit at/above Apify's first-party actor rates (Jul 2026); justified
+# because we charge no subscription tiers, no per-run start fees, and no
+# separate proxy/compute/storage billing. Retune anytime via env + restart.
+# PLATFORM_SCRAPE_BILLING_ENABLED=FALSE
+# REDDIT_SCRAPE_MICROS_PER_ITEM=3500
+# GOOGLE_SEARCH_MICROS_PER_SERP=5500
+# GOOGLE_MAPS_MICROS_PER_PLACE=3500
+# GOOGLE_MAPS_MICROS_PER_REVIEW=1500
+# YOUTUBE_MICROS_PER_VIDEO=2500
+# YOUTUBE_MICROS_PER_COMMENT=1500
+
 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50.
 CREDIT_LOW_BALANCE_WARNING_MICROS=500000
 
@@ -317,17 +347,57 @@ TURNSTILE_SECRET_KEY=
 
 
 # Proxy provider selection. Selects a ProxyProvider implementation registered in
-# app/utils/proxy/registry.py. Default: "anonymous_proxies". Add new vendors there.
-# PROXY_PROVIDER=anonymous_proxies
+# app/utils/proxy/registry.py. Default: "custom". Options: "custom", "dataimpulse".
+# PROXY_PROVIDER=custom
 
-# Residential Proxy Configuration (anonymous-proxies.net)
-# Used for web crawling, link previews, and YouTube transcript fetching to avoid IP bans.
-# Consumed by the "anonymous_proxies" provider. Leave commented out to disable proxying.
-# RESIDENTIAL_PROXY_USERNAME=your_proxy_username
-# RESIDENTIAL_PROXY_PASSWORD=your_proxy_password
-# RESIDENTIAL_PROXY_HOSTNAME=rotating.dnsproxifier.com:31230
-# RESIDENTIAL_PROXY_LOCATION=
-# RESIDENTIAL_PROXY_TYPE=1
+# Proxy endpoint(s), shared across providers — PROXY_PROVIDER picks the behavior,
+# not a different env name. Used for web crawling, link previews, and YouTube
+# transcript fetching to avoid IP bans. Leave unset to disable proxying.
+#
+# PROXY_URL: a single full http://user:pass@host:port endpoint (used by every
+#   provider). For "dataimpulse", country is a "__cr.<country>" username suffix the
+#   provider parses for geoip-match, e.g.:
+#   PROXY_URL=http://your_token__cr.us:your_password@gw.dataimpulse.com:823
+# PROXY_URLS: a comma-separated pool the "custom" provider rotates client-side
+#   (cyclic); server-side-rotating gateways ignore it.
+# PROXY_URL=http://user:pass@host:port
+# PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port
+
+# =====================================================================
+# Captcha solving (Phase 3d) — LAST-resort bypass tier via captchatools.
+# Only fires on the stealth browser tier when a sitekey is detected AND
+# CAPTCHA_SOLVING_ENABLED=TRUE. Cloudflare Turnstile is already solved free
+# in-framework (no config needed). Off by default => zero attempts, zero cost.
+# NOTE: automated solving may violate a target site's ToS — opt-in, public
+# data only (no logged-in bypass). captchatools is itself the vendor registry.
+# CAPTCHA_SOLVING_ENABLED=FALSE
+# solving_site: capmonster | 2captcha | anticaptcha | capsolver | captchaai
+# CAPTCHA_SOLVER_PROVIDER=capsolver
+# CAPTCHA_SOLVER_API_KEY=
+# Per-URL solve cap (bounds solver spend on a hostile page).
+# CAPTCHA_MAX_ATTEMPTS_PER_URL=1
+# CAPTCHA_SOLVE_TIMEOUT_S=120
+# Default type when detection is ambiguous: v2 | v3 | hcaptcha
+# CAPTCHA_TYPE_DEFAULT=v2
+# reCAPTCHA v3 tuning (only used for v3 challenges).
+# CAPTCHA_V3_MIN_SCORE=0.7
+# CAPTCHA_V3_ACTION=verify
+
+# =====================================================================
+# Stealth hardening (Phase 3e, Slice A) — runtime/config-level levers on the
+# stealth browser tier. Consumed by app/proprietary/web_crawler/stealth.py.
+# Defaults add no crawl-speed regression and preserve today's behavior.
+# Match browser locale/timezone to the proxy provider's exit region (no exit-IP lookup).
+# CRAWL_GEOIP_MATCH_ENABLED=FALSE
+# WebRTC respects the proxy (anti local-IP leak). Cheap + safe.
+# CRAWL_BLOCK_WEBRTC=TRUE
+# Random canvas noise (an unstable canvas hash is itself a tell) — opt-in only.
+# CRAWL_HIDE_CANVAS=FALSE
+# Send a Google referer so the first hit looks like organic arrival.
+# CRAWL_GOOGLE_SEARCH_REFERER=TRUE
+# Route DNS via Cloudflare DoH (anti DNS-leak). Adds a DNS round-trip => off by
+# default to avoid any speed regression; enable for leak-safety-first setups.
+# CRAWL_DNS_OVER_HTTPS=FALSE
 
 # File Parser Service
 ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING
@@ -476,7 +546,7 @@ LANGSMITH_PROJECT=surfsense
 # -----------------------------------------------------------------------------
 # Connector discovery TTL cache (Phase 1.4 perf optimization)
 # -----------------------------------------------------------------------------
-# Caches the per-search-space "available connectors" + "available document
+# Caches the per-workspace "available connectors" + "available document
 # types" lookups that ``create_surfsense_deep_agent`` hits on every turn.
 # ORM event listeners auto-invalidate on connector / document inserts,
 # updates and deletes — the TTL only bounds staleness for bulk-import
@@ -516,8 +586,8 @@ LANGSMITH_PROJECT=surfsense
 # Per-workspace spawn-paused kill switch — set via Redis at runtime, not
 # this env var. The env var below only disables the check itself (useful
 # for local dev without Redis). To pause a workspace in production:
-#     redis-cli SET surfsense:spawn_paused:<search_space_id> 1 EX 600
-#     redis-cli DEL surfsense:spawn_paused:<search_space_id>
+#     redis-cli SET surfsense:spawn_paused:<workspace_id> 1 EX 600
+#     redis-cli DEL surfsense:spawn_paused:<workspace_id>
 # The check is fail-open: a Redis blip never blocks ``task(...)``.
 # SURFSENSE_TASK_SPAWN_PAUSED_DISABLED=false