Merge remote-tracking branch 'upstream/dev' into feature/1354-embedding-base-url

# Conflicts:
#	surfsense_web/content/docs/docker-installation/docker-compose.mdx
#	surfsense_web/content/docs/manual-installation.mdx

docker/.env.example

@@ -338,16 +338,6 @@ STT_SERVICE=local/base
 # GATEWAY_DISCORD_ENABLED=FALSE
 # GATEWAY_DISCORD_REDIRECT_URI=http://localhost:3929/api/v1/gateway/discord/callback
 
-# ------------------------------------------------------------------------------
-# SearXNG (bundled web search, works out of the box with no config needed)
-# ------------------------------------------------------------------------------
-# SearXNG provides web search to all search spaces automatically.
-# To access the SearXNG UI directly in dev/deps-only compose: http://localhost:8888
-# To disable the service entirely: docker compose up --scale searxng=0
-# To point at your own SearXNG instance instead of the bundled one:
-# SEARXNG_DEFAULT_HOST=http://your-searxng:8080
-# SEARXNG_SECRET=surfsense-searxng-secret
-
 # ------------------------------------------------------------------------------
 # Daytona Sandbox (optional cloud code execution for the deep agent)
 # ------------------------------------------------------------------------------
@@ -440,6 +430,31 @@ SURFSENSE_ENABLE_DOOM_LOOP=true
 # ETL_CREDIT_BILLING_ENABLED=FALSE
 # MICROS_PER_PAGE=1000
 
+# Debit the credit wallet per *successful* web crawl. Default FALSE keeps
+# crawling effectively free for self-hosted installs. Price is config-driven:
+#   WEB_CRAWL_MICROS_PER_SUCCESS = round(USD_per_1000_crawls * 1_000)
+#   2000 == $2/1000 (default) | 1000 == $1/1000. Captcha solves bill as a
+# separate per-attempt unit (independent flag).
+# WEB_CRAWL_CREDIT_BILLING_ENABLED=FALSE
+# WEB_CRAWL_MICROS_PER_SUCCESS=2000
+# WEB_CRAWL_CAPTCHA_BILLING_ENABLED=FALSE
+# WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000
+
+# Debit the credit wallet per *item returned* by the platform-native scrapers
+# (Reddit, Google Search, Google Maps, YouTube). Default FALSE keeps scraping
+# effectively free for self-hosted installs. Each rate is micro-USD per item,
+# config-driven: <KEY> = round(USD_per_1000_items * 1_000). Defaults sit
+# at/above Apify's first-party actor rates (we charge no subscription tiers,
+# start fees, or separate proxy/compute billing). google_maps.scrape is
+# dual-metered (places + attached reviews).
+# PLATFORM_SCRAPE_BILLING_ENABLED=FALSE
+# REDDIT_SCRAPE_MICROS_PER_ITEM=3500
+# GOOGLE_SEARCH_MICROS_PER_SERP=5500
+# GOOGLE_MAPS_MICROS_PER_PLACE=3500
+# GOOGLE_MAPS_MICROS_PER_REVIEW=1500
+# YOUTUBE_MICROS_PER_VIDEO=2500
+# YOUTUBE_MICROS_PER_COMMENT=1500
+
 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default).
 # QUOTA_MAX_RESERVE_MICROS=1000000
 
@@ -472,12 +487,37 @@ NOLOGIN_MODE_ENABLED=FALSE
 # Connector indexing lock TTL in seconds (default: 28800 = 8 hours)
 # CONNECTOR_INDEXING_LOCK_TTL_SECONDS=28800
 
-# Residential proxy for web crawling
-# RESIDENTIAL_PROXY_USERNAME=
-# RESIDENTIAL_PROXY_PASSWORD=
-# RESIDENTIAL_PROXY_HOSTNAME=
-# RESIDENTIAL_PROXY_LOCATION=
-# RESIDENTIAL_PROXY_TYPE=1
+# Proxy provider selection: "custom" (default) or "dataimpulse".
+# PROXY_PROVIDER=custom
+
+# Proxy endpoint(s), shared across providers. PROXY_URL is a single full URL used
+# by every provider (for "dataimpulse", country is a "__cr.<country>" username
+# suffix the provider parses for geoip). PROXY_URLS is a comma-separated pool the
+# "custom" provider rotates client-side. Leave unset to disable proxying.
+# PROXY_URL=http://user:pass@host:port
+# PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port
+
+# Captcha solving — last-resort bypass tier via captchatools. Only fires on the
+# stealth browser tier when a sitekey is detected AND the flag is TRUE.
+# Cloudflare Turnstile is already solved free in-framework. Off by default.
+# NOTE: automated solving may violate a target site's ToS — opt-in, public
+# data only. See surfsense_backend/.env.example for the full option docs.
+# CAPTCHA_SOLVING_ENABLED=FALSE
+# CAPTCHA_SOLVER_PROVIDER=capsolver
+# CAPTCHA_SOLVER_API_KEY=
+# CAPTCHA_MAX_ATTEMPTS_PER_URL=1
+# CAPTCHA_SOLVE_TIMEOUT_S=120
+# CAPTCHA_TYPE_DEFAULT=v2
+# CAPTCHA_V3_MIN_SCORE=0.7
+# CAPTCHA_V3_ACTION=verify
+
+# Stealth hardening levers on the stealth browser tier. Defaults preserve
+# current behavior; see surfsense_backend/.env.example for per-flag docs.
+# CRAWL_GEOIP_MATCH_ENABLED=FALSE
+# CRAWL_BLOCK_WEBRTC=TRUE
+# CRAWL_HIDE_CANVAS=FALSE
+# CRAWL_GOOGLE_SEARCH_REFERER=TRUE
+# CRAWL_DNS_OVER_HTTPS=FALSE
 
 # ==============================================================================
 # DEV / DEPS-ONLY COMPOSE OVERRIDES
@@ -494,9 +534,6 @@ NOLOGIN_MODE_ENABLED=FALSE
 # -- Redis exposed port (dev/deps-only only; Redis is internal-only in prod) --
 # REDIS_PORT=6379
 
-# -- SearXNG exposed port (dev/deps-only only; internal-only in prod) --
-# SEARXNG_PORT=8888
-
 # -- WhatsApp bridge exposed port (dev/hybrid only; prod keeps it Docker-internal) --
 # WHATSAPP_BRIDGE_PORT=9929
 

docker/docker-compose.deps-only.yml

@@ -1,7 +1,7 @@
 # =============================================================================
 # SurfSense — Dependencies only (no backend / frontend / Celery images)
 # =============================================================================
-# Postgres, Redis, SearXNG, pgAdmin, Zero — run API + Next + Celery on the host.
+# Postgres, Redis, pgAdmin, Zero — run API + Next + Celery on the host.
 # Celery is not Dockerized here: use `uv run` from surfsense_backend/ (no extra
 # backend image build just for workers).
 #
@@ -9,7 +9,7 @@
 #   docker compose -f docker/docker-compose.deps-only.yml up -d
 #
 # Compose variable substitution uses `docker/.env` (copy from .env.example).
-# Bind mounts use ./postgresql.conf and ./searxng in this directory.
+# Bind mounts use ./postgresql.conf in this directory.
 #
 # Local Celery (from surfsense_backend/, after Redis is up):
 #   uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues=surfsense,surfsense.connectors
@@ -17,7 +17,6 @@
 #
 # Host setup:
 #   - Backend .env: DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense
-#   - Backend .env: SEARXNG_DEFAULT_HOST=http://localhost:${SEARXNG_PORT:-8888}
 #   - Backend .env: CELERY_BROKER_URL / REDIS_APP_URL → redis://localhost:6379/0
 #   - Web .env: NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:${ZERO_CACHE_PORT:-4848}
 #
@@ -80,20 +79,6 @@ services:
       timeout: 5s
       retries: 5
 
-  searxng:
-    image: searxng/searxng:2026.3.13-3c1f68c59
-    ports:
-      - "${SEARXNG_PORT:-8888}:8080"
-    volumes:
-      - ./searxng:/etc/searxng
-    environment:
-      - SEARXNG_SECRET=${SEARXNG_SECRET:-surfsense-searxng-secret}
-    healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
   # NOTE: zero-cache requires the `zero_publication` Postgres publication to
   # exist before it starts. In this deps-only stack there is no backend
   # container to run migrations, so you must run `uv run alembic upgrade head`

docker/docker-compose.dev.yml

@@ -85,20 +85,6 @@ services:
       - "${OTEL_TEMPO_PORT:-3200}:3200"
     restart: unless-stopped
 
-  searxng:
-    image: searxng/searxng:2026.3.13-3c1f68c59
-    ports:
-      - "${SEARXNG_PORT:-8888}:8080"
-    volumes:
-      - ./searxng:/etc/searxng
-    environment:
-      - SEARXNG_SECRET=${SEARXNG_SECRET:-surfsense-searxng-secret}
-    healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
   backend:
     build: *backend-build
     ports:
@@ -125,7 +111,6 @@ services:
       - LANGSMITH_TRACING=false
       - AUTH_TYPE=${AUTH_TYPE:-LOCAL}
       - NEXT_FRONTEND_URL=${NEXT_FRONTEND_URL:-http://localhost:3000}
-      - SEARXNG_DEFAULT_HOST=${SEARXNG_DEFAULT_HOST:-http://searxng:8080}
       - WHATSAPP_BRIDGE_URL=${WHATSAPP_BRIDGE_URL:-http://whatsapp-bridge:9929}
       # Daytona Sandbox – uncomment and set credentials to enable cloud code execution
       # - DAYTONA_SANDBOX_ENABLED=TRUE
@@ -138,8 +123,6 @@ services:
         condition: service_healthy
       redis:
         condition: service_healthy
-      searxng:
-        condition: service_healthy
       migrations:
         condition: service_completed_successfully
     healthcheck:
@@ -186,7 +169,6 @@ services:
       - CELERY_TASK_DEFAULT_QUEUE=surfsense
       - PYTHONPATH=/app
       - FILE_STORAGE_LOCAL_PATH=/app/.local_object_store
-      - SEARXNG_DEFAULT_HOST=${SEARXNG_DEFAULT_HOST:-http://searxng:8080}
       - SERVICE_ROLE=worker
     depends_on:
       db:

docker/docker-compose.watch-e2e.yml

@@ -0,0 +1,52 @@
+# =============================================================================
+# SurfSense — Isolated deps for the watch (Phase 6) manual E2E
+# =============================================================================
+# Fresh Postgres + Redis ONLY, with their own project name, volumes, and host
+# ports so they never collide with the day-to-day `surfsense-deps` stack.
+#
+# Backend + Celery run on the HOST (tests/e2e/run_backend.py / run_celery.py)
+# pointed at these ports via env:
+#   DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5442/surfsense
+#   CELERY_BROKER_URL / CELERY_RESULT_BACKEND / REDIS_APP_URL=redis://localhost:6389/0
+#
+# Up:    docker compose -p surfsense-watch -f docker/docker-compose.watch-e2e.yml up -d
+# Down:  docker compose -p surfsense-watch -f docker/docker-compose.watch-e2e.yml down -v
+# =============================================================================
+
+name: surfsense-watch
+
+services:
+  db:
+    image: pgvector/pgvector:pg17
+    ports:
+      - "5442:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=surfsense
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres -d surfsense"]
+      interval: 3s
+      timeout: 5s
+      retries: 20
+
+  redis:
+    image: redis:8-alpine
+    ports:
+      - "6389:6379"
+    volumes:
+      - redis_data:/data
+    command: redis-server --appendonly yes
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 5s
+      retries: 20
+
+volumes:
+  postgres_data:
+    name: surfsense-watch-postgres
+  redis_data:
+    name: surfsense-watch-redis

docker/docker-compose.yml

@@ -81,19 +81,6 @@ services:
   #     timeout: 5s
   #     retries: 3
 
-  searxng:
-    image: searxng/searxng:2026.3.13-3c1f68c59
-    volumes:
-      - ./searxng:/etc/searxng
-    environment:
-      SEARXNG_SECRET: ${SEARXNG_SECRET:-surfsense-searxng-secret}
-    restart: unless-stopped
-    healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
   # Single public entry point for the Docker stack. Comment this service out
   # only if you front SurfSense with your own reverse proxy.
   proxy:
@@ -146,7 +133,6 @@ services:
       FILE_STORAGE_LOCAL_PATH: /app/.local_object_store
       NEXT_FRONTEND_URL: ${NEXT_FRONTEND_URL:-${SURFSENSE_PUBLIC_URL:-http://localhost:${LISTEN_HTTP_PORT:-3929}}}
       BACKEND_URL: ${BACKEND_URL:-${SURFSENSE_PUBLIC_URL:-http://localhost:${LISTEN_HTTP_PORT:-3929}}}
-      SEARXNG_DEFAULT_HOST: ${SEARXNG_DEFAULT_HOST:-http://searxng:8080}
       WHATSAPP_BRIDGE_URL: ${WHATSAPP_BRIDGE_URL:-http://whatsapp-bridge:9929}
       # Daytona Sandbox – uncomment and set credentials to enable cloud code execution
       # DAYTONA_SANDBOX_ENABLED: "TRUE"
@@ -161,8 +147,6 @@ services:
         condition: service_healthy
       redis:
         condition: service_healthy
-      searxng:
-        condition: service_healthy
       migrations:
         condition: service_completed_successfully
     restart: unless-stopped
@@ -210,7 +194,6 @@ services:
       CELERY_TASK_DEFAULT_QUEUE: surfsense
       PYTHONPATH: /app
       FILE_STORAGE_LOCAL_PATH: /app/.local_object_store
-      SEARXNG_DEFAULT_HOST: ${SEARXNG_DEFAULT_HOST:-http://searxng:8080}
       SERVICE_ROLE: worker
     depends_on:
       db:

docker/scripts/install.ps1

@@ -329,7 +329,6 @@ Write-Step "Downloading SurfSense files"
 Write-Info "Installation directory: $InstallDir"
 
 New-Item -ItemType Directory -Path "$InstallDir\scripts" -Force | Out-Null
-New-Item -ItemType Directory -Path "$InstallDir\searxng" -Force | Out-Null
 New-Item -ItemType Directory -Path "$InstallDir\proxy" -Force | Out-Null
 
 $Files = @(
@@ -339,8 +338,6 @@ $Files = @(
     @{ Src = "docker/proxy/Caddyfile";                   Dest = "proxy/Caddyfile" }
     @{ Src = "docker/postgresql.conf";                   Dest = "postgresql.conf" }
     @{ Src = "docker/scripts/migrate-database.ps1";      Dest = "scripts/migrate-database.ps1" }
-    @{ Src = "docker/searxng/settings.yml";              Dest = "searxng/settings.yml" }
-    @{ Src = "docker/searxng/limiter.toml";              Dest = "searxng/limiter.toml" }
 )
 
 foreach ($f in $Files) {

docker/scripts/install.sh

@@ -332,7 +332,6 @@ SELECTED_VARIANT=$(resolve_variant)
 step "Downloading SurfSense files"
 info "Installation directory: ${INSTALL_DIR}"
 mkdir -p "${INSTALL_DIR}/scripts"
-mkdir -p "${INSTALL_DIR}/searxng"
 mkdir -p "${INSTALL_DIR}/proxy"
 
 FILES=(
@@ -342,8 +341,6 @@ FILES=(
     "docker/proxy/Caddyfile:proxy/Caddyfile"
     "docker/postgresql.conf:postgresql.conf"
     "docker/scripts/migrate-database.sh:scripts/migrate-database.sh"
-    "docker/searxng/settings.yml:searxng/settings.yml"
-    "docker/searxng/limiter.toml:searxng/limiter.toml"
 )
 
 for entry in "${FILES[@]}"; do

docker/searxng/limiter.toml

@@ -1,5 +0,0 @@
-[botdetection.ip_limit]
-link_token = false
-
-[botdetection.ip_lists]
-pass_ip = ["0.0.0.0/0"]

docker/searxng/settings.yml

@@ -1,90 +0,0 @@
-use_default_settings:
-  engines:
-    remove:
-      - ahmia
-      - torch
-      - qwant
-      - qwant news
-      - qwant images
-      - qwant videos
-      - mojeek
-      - mojeek images
-      - mojeek news
-
-server:
-  secret_key: "override-me-via-env"
-  limiter: false
-  image_proxy: false
-  method: "GET"
-  default_http_headers:
-    X-Robots-Tag: "noindex, nofollow"
-
-search:
-  formats:
-    - html
-    - json
-  default_lang: "auto"
-  autocomplete: ""
-  safe_search: 0
-  ban_time_on_fail: 5
-  max_ban_time_on_fail: 120
-  suspended_times:
-    SearxEngineAccessDenied: 3600
-    SearxEngineCaptcha: 3600
-    SearxEngineTooManyRequests: 600
-    cf_SearxEngineCaptcha: 7200
-    cf_SearxEngineAccessDenied: 3600
-    recaptcha_SearxEngineCaptcha: 7200
-
-ui:
-  static_use_hash: true
-
-outgoing:
-  request_timeout: 12.0
-  max_request_timeout: 20.0
-  pool_connections: 100
-  pool_maxsize: 20
-  enable_http2: true
-  extra_proxy_timeout: 10
-  retries: 1
-  # Uncomment and set your residential proxy URL to route search engine requests through it.
-  # Format: http://<username>:<base64_password>@<hostname>:<port>/
-  #
-  # proxies:
-  #   all://:
-  #     - http://user:pass@proxy-host:port/
-
-engines:
-  - name: google
-    disabled: false
-    weight: 1.2
-    retry_on_http_error: [429, 503]
-  - name: duckduckgo
-    disabled: false
-    weight: 1.1
-    retry_on_http_error: [429, 503]
-  - name: brave
-    disabled: false
-    weight: 1.0
-    retry_on_http_error: [429, 503]
-  - name: bing
-    disabled: false
-    weight: 0.9
-    retry_on_http_error: [429, 503]
-  - name: wikipedia
-    disabled: false
-    weight: 0.8
-  - name: stackoverflow
-    disabled: false
-    weight: 0.7
-  - name: yahoo
-    disabled: false
-    weight: 0.7
-    retry_on_http_error: [429, 503]
-  - name: wikidata
-    disabled: false
-    weight: 0.6
-  - name: currency
-    disabled: false
-  - name: ddg definitions
-    disabled: false