From 41a93ca8fb83118e9c4ca306b3cf214b8637b0d2 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 8 Jun 2026 14:37:44 -0700 Subject: [PATCH] refactor(env): streamline Redis configuration and remove deprecated variables - Consolidated Redis configuration by introducing a single `REDIS_URL` variable for Celery broker, result backend, and app cache. - Removed deprecated variables related to Firecrawl and Stripe token limits from `.env.example` files. - Updated documentation to reflect changes in environment variable usage for improved clarity and maintainability. --- docker/.env.example | 5 ----- surfsense_backend/.env.example | 20 +++++++------------ surfsense_backend/app/config/__init__.py | 10 ++++++---- .../docker-installation/docker-compose.mdx | 1 - .../content/docs/manual-installation.mdx | 6 ++---- 5 files changed, 15 insertions(+), 27 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index ba81543d3..cafc74af9 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -185,7 +185,6 @@ STRIPE_PAGE_BUYING_ENABLED=FALSE # STRIPE_TOKEN_BUYING_ENABLED=FALSE # STRIPE_PREMIUM_TOKEN_PRICE_ID=price_... # STRIPE_CREDIT_MICROS_PER_UNIT=1000000 -# DEPRECATED: STRIPE_TOKENS_PER_UNIT=1000000 # ------------------------------------------------------------------------------ # TTS & STT (Text-to-Speech / Speech-to-Text) @@ -340,9 +339,6 @@ STT_SERVICE=local/base # External API Keys (optional) # ------------------------------------------------------------------------------ -# Firecrawl (web scraping) -# FIRECRAWL_API_KEY= - # Unstructured (if ETL_SERVICE=UNSTRUCTURED) # UNSTRUCTURED_API_KEY= @@ -418,7 +414,6 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # Premium turns are debited at the actual per-call provider cost reported # by LiteLLM. Only applies to models with billing_tier=premium. # PREMIUM_CREDIT_MICROS_LIMIT=5000000 -# DEPRECATED: PREMIUM_TOKEN_LIMIT=5000000 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index fafc4ef7b..cbd508bda 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -3,15 +3,15 @@ DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense # Deployment environment: dev or production SURFSENSE_ENV=dev -#Celery Config -CELERY_BROKER_URL=redis://localhost:6379/0 -CELERY_RESULT_BACKEND=redis://localhost:6379/0 +# Redis (single endpoint for Celery broker/result backend + app features) +REDIS_URL=redis://localhost:6379/0 +# Optional: override individually only to split Redis across instances. +# Each defaults to REDIS_URL when unset. +# CELERY_BROKER_URL=redis://localhost:6379/0 +# CELERY_RESULT_BACKEND=redis://localhost:6379/0 +# REDIS_APP_URL=redis://localhost:6379/0 # Optional: isolate queues when sharing Redis with other apps CELERY_TASK_DEFAULT_QUEUE=surfsense - -# Redis for app-level features (heartbeats, podcast markers) -# Defaults to CELERY_BROKER_URL when not set -REDIS_APP_URL=redis://localhost:6379/0 # Optional: TTL in seconds for connector indexing lock key # CONNECTOR_INDEXING_LOCK_TTL_SECONDS=28800 @@ -92,8 +92,6 @@ STRIPE_PAGE_BUYING_ENABLED=TRUE STRIPE_TOKEN_BUYING_ENABLED=FALSE STRIPE_PREMIUM_TOKEN_PRICE_ID=price_... STRIPE_CREDIT_MICROS_PER_UNIT=1000000 -# DEPRECATED — use STRIPE_CREDIT_MICROS_PER_UNIT (1:1 numerical mapping): -# STRIPE_TOKENS_PER_UNIT=1000000 # Periodic Stripe safety net for purchases left in PENDING (minutes old) STRIPE_RECONCILIATION_LOOKBACK_MINUTES=10 @@ -232,8 +230,6 @@ PAGES_LIMIT=500 # models bill proportionally. Applies only to models with # billing_tier=premium in global_llm_config.yaml. PREMIUM_CREDIT_MICROS_LIMIT=5000000 -# DEPRECATED — use PREMIUM_CREDIT_MICROS_LIMIT (1:1 numerical mapping): -# PREMIUM_TOKEN_LIMIT=5000000 # Safety ceiling on per-call premium reservation, in micro-USD. # stream_new_chat estimates an upper-bound cost from the model's @@ -290,8 +286,6 @@ TURNSTILE_SECRET_KEY= # RESIDENTIAL_PROXY_LOCATION= # RESIDENTIAL_PROXY_TYPE=1 -FIRECRAWL_API_KEY=fcr-01J0000000000000000000000 - # File Parser Service ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING UNSTRUCTURED_API_KEY=Tpu3P0U8iy diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index eb832c69e..e88626a40 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -546,10 +546,12 @@ class Config: DATABASE_URL = os.getenv("DATABASE_URL") # Celery / Redis - CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0") - CELERY_RESULT_BACKEND = os.getenv( - "CELERY_RESULT_BACKEND", "redis://localhost:6379/0" - ) + # Redis (single endpoint for Celery broker, result backend, and app cache). + # Legacy CELERY_BROKER_URL / CELERY_RESULT_BACKEND / REDIS_APP_URL still + # override individually when you need to split Redis across instances. + REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0") + CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", REDIS_URL) + CELERY_RESULT_BACKEND = os.getenv("CELERY_RESULT_BACKEND", REDIS_URL) CELERY_TASK_DEFAULT_QUEUE = os.getenv("CELERY_TASK_DEFAULT_QUEUE", "surfsense") REDIS_APP_URL = os.getenv("REDIS_APP_URL", CELERY_BROKER_URL) CONNECTOR_INDEXING_LOCK_TTL_SECONDS = int( diff --git a/surfsense_web/content/docs/docker-installation/docker-compose.mdx b/surfsense_web/content/docs/docker-installation/docker-compose.mdx index 8f71ec77e..60b5e67b6 100644 --- a/surfsense_web/content/docs/docker-installation/docker-compose.mdx +++ b/surfsense_web/content/docs/docker-installation/docker-compose.mdx @@ -160,7 +160,6 @@ Create credentials at the [Google Cloud Console](https://console.cloud.google.co | Variable | Description | |----------|-------------| -| `FIRECRAWL_API_KEY` | [Firecrawl](https://www.firecrawl.dev/) API key for web crawling | | `UNSTRUCTURED_API_KEY` | [Unstructured.io](https://unstructured.io/) API key (required if `ETL_SERVICE=UNSTRUCTURED`) | | `LLAMA_CLOUD_API_KEY` | [LlamaCloud](https://cloud.llamaindex.ai/) API key (required if `ETL_SERVICE=LLAMACLOUD`) | diff --git a/surfsense_web/content/docs/manual-installation.mdx b/surfsense_web/content/docs/manual-installation.mdx index 203c244c0..22a8ff5a1 100644 --- a/surfsense_web/content/docs/manual-installation.mdx +++ b/surfsense_web/content/docs/manual-installation.mdx @@ -94,12 +94,10 @@ Edit the `.env` file and set the following variables: | STT_SERVICE | Speech-to-Text API provider for Audio Files (e.g., `local/base`, `openai/whisper-1`). See [supported providers](https://docs.litellm.ai/docs/audio_transcription#supported-providers) | | STT_SERVICE_API_KEY | (Optional if local) API key for the Speech-to-Text service | | STT_SERVICE_API_BASE | (Optional) Custom API base URL for the Speech-to-Text service | -| FIRECRAWL_API_KEY | (Optional) API key for Firecrawl service for web crawling | | ETL_SERVICE | Document parsing service: `UNSTRUCTURED` (supports 34+ formats), `LLAMACLOUD` (supports 50+ formats including legacy document types), or `DOCLING` (local processing, supports PDF, Office docs, images, HTML, CSV) | | UNSTRUCTURED_API_KEY | API key for Unstructured.io service for document parsing (required if ETL_SERVICE=UNSTRUCTURED) | | LLAMA_CLOUD_API_KEY | API key for LlamaCloud service for document parsing (required if ETL_SERVICE=LLAMACLOUD) | -| CELERY_BROKER_URL | Redis connection URL for Celery broker (e.g., `redis://localhost:6379/0`) | -| CELERY_RESULT_BACKEND | Redis connection URL for Celery result backend (e.g., `redis://localhost:6379/0`) | +| REDIS_URL | Single Redis connection URL for the Celery broker, result backend, and app cache (e.g., `redis://localhost:6379/0`). Optionally override per use with `CELERY_BROKER_URL`, `CELERY_RESULT_BACKEND`, or `REDIS_APP_URL`. | | SCHEDULE_CHECKER_INTERVAL | (Optional) How often to check for scheduled connector tasks. Format: `` where unit is `m` (minutes) or `h` (hours). Examples: `1m`, `5m`, `1h`, `2h` (default: `1m`) | | REGISTRATION_ENABLED | (Optional) Enable or disable new user registration (e.g., `TRUE` or `FALSE`, default: `TRUE`) | | PAGES_LIMIT | (Optional) Maximum pages limit per user for ETL services (default: `999999999` for unlimited in OSS version) | @@ -690,7 +688,7 @@ To verify your installation: ## Troubleshooting - **Database Connection Issues**: Verify your PostgreSQL server is running and pgvector is properly installed -- **Redis Connection Issues**: Ensure Redis server is running (`redis-cli ping` should return `PONG`). Check that `CELERY_BROKER_URL` and `CELERY_RESULT_BACKEND` are correctly set in your `.env` file +- **Redis Connection Issues**: Ensure Redis server is running (`redis-cli ping` should return `PONG`). Check that `REDIS_URL` is correctly set in your `.env` file - **Celery Worker Issues**: Make sure the Celery worker is running in a separate terminal. Check worker logs for any errors - **Authentication Problems**: Check your Google OAuth configuration and ensure redirect URIs are set correctly - **LLM Errors**: Confirm your LLM API keys are valid and the selected models are accessible