feat: integrate SearXNG web search service with platform capabilities

- Added SearXNG service configuration to Docker setup, including environment variables and health checks.
- Introduced new settings management for web search in the frontend, allowing users to enable/disable and configure search engines and language preferences.
- Updated backend to support web search functionality, including database schema changes and service integration.
- Implemented health check endpoint for the web search service and integrated it into the application.
- Removed legacy SearXNG API connector references in favor of the new platform service approach.

docker/.env.example

@@ -36,6 +36,7 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
 # BACKEND_PORT=8929
 # FRONTEND_PORT=3929
 # ELECTRIC_PORT=5929
+# SEARXNG_PORT=8888
 # FLOWER_PORT=5555
 
 # ==============================================================================
@@ -199,6 +200,16 @@ STT_SERVICE=local/base
 # COMPOSIO_ENABLED=TRUE
 # COMPOSIO_REDIRECT_URI=http://localhost:8000/api/v1/auth/composio/connector/callback
 
+# ------------------------------------------------------------------------------
+# SearXNG (bundled web search — works out of the box, no config needed)
+# ------------------------------------------------------------------------------
+# SearXNG provides web search to all search spaces automatically.
+# To access the SearXNG UI directly: http://localhost:8888
+# To disable the service entirely: docker compose up --scale searxng=0
+# To point at your own SearXNG instance instead of the bundled one:
+# SEARXNG_DEFAULT_HOST=http://your-searxng:8080
+# SEARXNG_SECRET=surfsense-searxng-secret
+
 # ------------------------------------------------------------------------------
 # Daytona Sandbox (optional — cloud code execution for the deep agent)
 # ------------------------------------------------------------------------------

docker/docker-compose.dev.yml

@@ -57,6 +57,23 @@ services:
       timeout: 5s
       retries: 5
 
+  searxng:
+    image: searxng/searxng:2024.12.23
+    ports:
+      - "${SEARXNG_PORT:-8888}:8080"
+    volumes:
+      - ./searxng/settings.yml:/etc/searxng/settings.yml:ro
+      - ./searxng/limiter.toml:/etc/searxng/limiter.toml:ro
+      - searxng_data:/etc/searxng
+    environment:
+      - SEARXNG_BASE_URL=http://localhost:${SEARXNG_PORT:-8888}/
+      - SEARXNG_SECRET=${SEARXNG_SECRET:-surfsense-searxng-secret}
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
   backend:
     build: ../surfsense_backend
     ports:
@@ -81,6 +98,7 @@ services:
       - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password}
       - AUTH_TYPE=${AUTH_TYPE:-LOCAL}
       - NEXT_FRONTEND_URL=${NEXT_FRONTEND_URL:-http://localhost:3000}
+      - SEARXNG_DEFAULT_HOST=${SEARXNG_DEFAULT_HOST:-http://searxng:8080}
       # Daytona Sandbox – uncomment and set credentials to enable cloud code execution
       # - DAYTONA_SANDBOX_ENABLED=TRUE
       # - DAYTONA_API_KEY=${DAYTONA_API_KEY:-}
@@ -92,6 +110,8 @@ services:
         condition: service_healthy
       redis:
         condition: service_healthy
+      searxng:
+        condition: service_healthy
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
       interval: 15s
@@ -115,6 +135,7 @@ services:
       - PYTHONPATH=/app
       - ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric}
       - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password}
+      - SEARXNG_DEFAULT_HOST=${SEARXNG_DEFAULT_HOST:-http://searxng:8080}
       - SERVICE_ROLE=worker
     depends_on:
       db:
@@ -205,3 +226,5 @@ volumes:
     name: surfsense-dev-redis
   shared_temp:
     name: surfsense-dev-shared-temp
+  searxng_data:
+    name: surfsense-dev-searxng

docker/docker-compose.yml

@@ -42,6 +42,24 @@ services:
       timeout: 5s
       retries: 5
 
+  searxng:
+    image: searxng/searxng:2024.12.23
+    volumes:
+      - ./searxng/settings.yml:/etc/searxng/settings.yml:ro
+      - ./searxng/limiter.toml:/etc/searxng/limiter.toml:ro
+      - searxng_data:/etc/searxng
+    environment:
+      SEARXNG_BASE_URL: http://localhost:${SEARXNG_PORT:-8888}/
+      SEARXNG_SECRET: ${SEARXNG_SECRET:-surfsense-searxng-secret}
+    ports:
+      - "${SEARXNG_PORT:-8888}:8080"
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
   backend:
     image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
     ports:
@@ -62,6 +80,7 @@ services:
       ELECTRIC_DB_USER: ${ELECTRIC_DB_USER:-electric}
       ELECTRIC_DB_PASSWORD: ${ELECTRIC_DB_PASSWORD:-electric_password}
       NEXT_FRONTEND_URL: ${NEXT_FRONTEND_URL:-http://localhost:${FRONTEND_PORT:-3929}}
+      SEARXNG_DEFAULT_HOST: ${SEARXNG_DEFAULT_HOST:-http://searxng:8080}
       # Daytona Sandbox – uncomment and set credentials to enable cloud code execution
       # DAYTONA_SANDBOX_ENABLED: "TRUE"
       # DAYTONA_API_KEY: ${DAYTONA_API_KEY:-}
@@ -75,6 +94,8 @@ services:
         condition: service_healthy
       redis:
         condition: service_healthy
+      searxng:
+        condition: service_healthy
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
@@ -98,6 +119,7 @@ services:
       PYTHONPATH: /app
       ELECTRIC_DB_USER: ${ELECTRIC_DB_USER:-electric}
       ELECTRIC_DB_PASSWORD: ${ELECTRIC_DB_PASSWORD:-electric_password}
+      SEARXNG_DEFAULT_HOST: ${SEARXNG_DEFAULT_HOST:-http://searxng:8080}
       SERVICE_ROLE: worker
     depends_on:
       db:
@@ -193,3 +215,5 @@ volumes:
     name: surfsense-redis
   shared_temp:
     name: surfsense-shared-temp
+  searxng_data:
+    name: surfsense-searxng

docker/searxng/limiter.toml

@@ -0,0 +1,5 @@
+[botdetection.ip_limit]
+link_token = false
+
+[botdetection.ip_lists]
+pass_ip = ["0.0.0.0/0"]

docker/searxng/settings.yml

@@ -0,0 +1,60 @@
+use_default_settings: true
+
+server:
+  secret_key: "override-me-via-env"
+  limiter: false
+  image_proxy: false
+  method: "GET"
+  default_http_headers:
+    X-Robots-Tag: "noindex, nofollow"
+
+search:
+  formats:
+    - html
+    - json
+  default_lang: "auto"
+  autocomplete: ""
+  safe_search: 0
+
+ui:
+  static_use_hash: true
+
+outgoing:
+  request_timeout: 12.0
+  pool_connections: 100
+  pool_maxsize: 20
+  enable_http2: true
+
+engines:
+  - name: google
+    disabled: false
+    weight: 1.2
+  - name: duckduckgo
+    disabled: false
+    weight: 1.1
+  - name: brave
+    disabled: false
+    weight: 1.0
+  - name: qwant
+    disabled: false
+    weight: 0.9
+  - name: mojeek
+    disabled: false
+    weight: 0.7
+  - name: wikipedia
+    disabled: false
+    weight: 0.8
+  - name: wikidata
+    disabled: false
+    weight: 0.6
+  - name: currency_convert
+    disabled: false
+  - name: duckduckgo definitions
+    disabled: false
+  - name: stackoverflow
+    disabled: false
+    weight: 0.7
+  - name: bing
+    disabled: true
+  - name: yahoo
+    disabled: true